1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
|
19 | |
package org.apache.giraph.bsp; |
20 | |
|
21 | |
import java.io.IOException; |
22 | |
import java.util.ArrayList; |
23 | |
import java.util.List; |
24 | |
|
25 | |
import org.apache.giraph.graph.GiraphJob; |
26 | |
import org.apache.hadoop.mapreduce.RecordReader; |
27 | |
import org.apache.hadoop.mapreduce.InputFormat; |
28 | |
import org.apache.hadoop.mapreduce.TaskAttemptContext; |
29 | |
import org.apache.hadoop.mapreduce.JobContext; |
30 | |
import org.apache.hadoop.conf.Configuration; |
31 | |
import org.apache.hadoop.io.Text; |
32 | |
import org.apache.hadoop.mapreduce.InputSplit; |
33 | |
import org.apache.log4j.Logger; |
34 | |
|
35 | |
|
36 | |
|
37 | |
|
38 | |
|
39 | |
|
40 | |
|
41 | 48 | public class BspInputFormat extends InputFormat<Text, Text> { |
42 | |
|
43 | 1 | private static final Logger LOG = Logger.getLogger(BspInputFormat.class); |
44 | |
|
45 | |
|
46 | |
|
47 | |
|
48 | |
|
49 | |
|
50 | |
|
51 | |
public static int getMaxTasks(Configuration conf) { |
52 | 48 | int maxWorkers = conf.getInt(GiraphJob.MAX_WORKERS, 0); |
53 | 48 | boolean splitMasterWorker = |
54 | |
conf.getBoolean(GiraphJob.SPLIT_MASTER_WORKER, |
55 | |
GiraphJob.SPLIT_MASTER_WORKER_DEFAULT); |
56 | 48 | int maxTasks = maxWorkers; |
57 | 48 | if (splitMasterWorker) { |
58 | 0 | int zkServers = |
59 | |
conf.getInt(GiraphJob.ZOOKEEPER_SERVER_COUNT, |
60 | |
GiraphJob.ZOOKEEPER_SERVER_COUNT_DEFAULT); |
61 | 0 | maxTasks += zkServers; |
62 | |
} |
63 | 48 | if (LOG.isDebugEnabled()) { |
64 | 0 | LOG.debug("getMaxTasks: Max workers = " + maxWorkers + |
65 | |
", split master/worker = " + splitMasterWorker + |
66 | |
", total max tasks = " + maxTasks); |
67 | |
} |
68 | 48 | return maxTasks; |
69 | |
} |
70 | |
|
71 | |
@Override |
72 | |
public List<InputSplit> getSplits(JobContext context) |
73 | |
throws IOException, InterruptedException { |
74 | 24 | Configuration conf = context.getConfiguration(); |
75 | 24 | int maxTasks = getMaxTasks(conf); |
76 | 24 | if (maxTasks <= 0) { |
77 | 0 | throw new InterruptedException( |
78 | |
"getSplits: Cannot have maxTasks <= 0 - " + maxTasks); |
79 | |
} |
80 | 24 | List<InputSplit> inputSplitList = new ArrayList<InputSplit>(); |
81 | 48 | for (int i = 0; i < maxTasks; ++i) { |
82 | 24 | inputSplitList.add(new BspInputSplit()); |
83 | |
} |
84 | 24 | return inputSplitList; |
85 | |
} |
86 | |
|
87 | |
@Override |
88 | |
public RecordReader<Text, Text> |
89 | |
createRecordReader(InputSplit split, TaskAttemptContext context) |
90 | |
throws IOException, InterruptedException { |
91 | 24 | return new BspRecordReader(); |
92 | |
} |
93 | |
} |