1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
package org.apache.giraph; |
19 | |
|
20 | |
import org.apache.commons.cli.BasicParser; |
21 | |
import org.apache.commons.cli.CommandLine; |
22 | |
import org.apache.commons.cli.CommandLineParser; |
23 | |
import org.apache.commons.cli.HelpFormatter; |
24 | |
import org.apache.commons.cli.Options; |
25 | |
import org.apache.giraph.examples.Algorithm; |
26 | |
import org.apache.giraph.graph.GiraphJob; |
27 | |
import org.apache.giraph.graph.GiraphTypeValidator; |
28 | |
import org.apache.giraph.graph.Vertex; |
29 | |
import org.apache.giraph.utils.AnnotationUtils; |
30 | |
import org.apache.hadoop.conf.Configuration; |
31 | |
import org.apache.hadoop.filecache.DistributedCache; |
32 | |
import org.apache.hadoop.fs.Path; |
33 | |
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat; |
34 | |
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; |
35 | |
import org.apache.hadoop.util.Tool; |
36 | |
import org.apache.hadoop.util.ToolRunner; |
37 | |
import org.apache.log4j.Level; |
38 | |
import org.apache.log4j.Logger; |
39 | |
import org.apache.zookeeper.ZooKeeper; |
40 | |
|
41 | |
import com.google.common.base.Splitter; |
42 | |
import com.google.common.collect.Iterables; |
43 | |
|
44 | |
import java.net.URI; |
45 | |
import java.util.List; |
46 | |
|
47 | |
|
48 | |
|
49 | |
|
50 | |
|
51 | 0 | public class GiraphRunner implements Tool { |
52 | |
static { |
53 | 0 | Configuration.addDefaultResource("giraph-site.xml"); |
54 | |
} |
55 | |
|
56 | |
|
57 | 0 | private static final Logger LOG = Logger.getLogger(GiraphRunner.class); |
58 | |
|
59 | |
private Configuration conf; |
60 | |
|
61 | |
|
62 | |
|
63 | |
|
64 | 0 | private final String [][] requiredOptions = |
65 | |
{{"w", "Need to choose the number of workers (-w)"}, |
66 | |
{"if", "Need to set inputformat (-if)"}}; |
67 | |
|
68 | |
|
69 | |
|
70 | |
|
71 | |
|
72 | |
|
73 | |
private static Options getOptions() { |
74 | 0 | Options options = new Options(); |
75 | 0 | options.addOption("h", "help", false, "Help"); |
76 | 0 | options.addOption("la", "listAlgorithms", false, "List supported " + |
77 | |
"algorithms"); |
78 | 0 | options.addOption("q", "quiet", false, "Quiet output"); |
79 | 0 | options.addOption("w", "workers", true, "Number of workers"); |
80 | 0 | options.addOption("if", "inputFormat", true, "Graph inputformat"); |
81 | 0 | options.addOption("of", "outputFormat", true, "Graph outputformat"); |
82 | 0 | options.addOption("ip", "inputPath", true, "Graph input path"); |
83 | 0 | options.addOption("op", "outputPath", true, "Graph output path"); |
84 | 0 | options.addOption("c", "combiner", true, "VertexCombiner class"); |
85 | 0 | options.addOption("wc", "workerContext", true, "WorkerContext class"); |
86 | 0 | options.addOption("aw", "aggregatorWriter", true, "AggregatorWriter class"); |
87 | 0 | options.addOption("mc", "masterCompute", true, "MasterCompute class"); |
88 | 0 | options.addOption("cf", "cacheFile", true, "Files for distributed cache"); |
89 | 0 | options.addOption("ca", "customArguments", true, "provide custom" + |
90 | |
" arguments for the job configuration in the form:" + |
91 | |
" <param1>=<value1>,<param2>=<value2> etc."); |
92 | 0 | return options; |
93 | |
} |
94 | |
|
95 | |
@Override |
96 | |
public Configuration getConf() { |
97 | 0 | return conf; |
98 | |
} |
99 | |
|
100 | |
@Override |
101 | |
public void setConf(Configuration conf) { |
102 | 0 | this.conf = conf; |
103 | 0 | } |
104 | |
|
105 | |
|
106 | |
|
107 | |
|
108 | |
private void printSupportedAlgorithms() { |
109 | 0 | Logger.getLogger(ZooKeeper.class).setLevel(Level.OFF); |
110 | |
|
111 | 0 | List<Class<?>> classes = AnnotationUtils.getAnnotatedClasses( |
112 | |
Algorithm.class, "org.apache.giraph"); |
113 | 0 | System.out.print(" Supported algorithms:\n"); |
114 | 0 | for (Class<?> clazz : classes) { |
115 | 0 | if (Vertex.class.isAssignableFrom(clazz)) { |
116 | 0 | Algorithm algorithm = clazz.getAnnotation(Algorithm.class); |
117 | 0 | StringBuilder sb = new StringBuilder(); |
118 | 0 | sb.append(algorithm.name()).append(" - ").append(clazz.getName()) |
119 | |
.append("\n"); |
120 | 0 | if (!algorithm.description().equals("")) { |
121 | 0 | sb.append(" ").append(algorithm.description()).append("\n"); |
122 | |
} |
123 | 0 | System.out.print(sb.toString()); |
124 | 0 | } |
125 | |
} |
126 | 0 | } |
127 | |
|
128 | |
@Override |
129 | |
public int run(String[] args) throws Exception { |
130 | 0 | Options options = getOptions(); |
131 | |
|
132 | 0 | CommandLineParser parser = new BasicParser(); |
133 | 0 | CommandLine cmd = parser.parse(options, args); |
134 | |
|
135 | 0 | if (args.length == 0 || cmd.hasOption("h")) { |
136 | 0 | HelpFormatter formatter = new HelpFormatter(); |
137 | 0 | formatter.printHelp(getClass().getName(), options, true); |
138 | 0 | return 0; |
139 | |
} |
140 | |
|
141 | 0 | if (cmd.hasOption("la")) { |
142 | 0 | printSupportedAlgorithms(); |
143 | 0 | return 0; |
144 | |
} |
145 | |
|
146 | 0 | String vertexClassName = args[0]; |
147 | 0 | if (LOG.isDebugEnabled()) { |
148 | 0 | LOG.debug("Attempting to run Vertex: " + vertexClassName); |
149 | |
} |
150 | |
|
151 | |
|
152 | 0 | for (String[] requiredOption : requiredOptions) { |
153 | 0 | if (!cmd.hasOption(requiredOption[0])) { |
154 | 0 | if (LOG.isInfoEnabled()) { |
155 | 0 | LOG.info(requiredOption[1]); |
156 | |
} |
157 | 0 | return -1; |
158 | |
} |
159 | |
} |
160 | |
|
161 | 0 | int workers = Integer.parseInt(cmd.getOptionValue('w')); |
162 | 0 | GiraphJob job = new GiraphJob(getConf(), "Giraph: " + vertexClassName); |
163 | 0 | job.setVertexClass(Class.forName(vertexClassName)); |
164 | 0 | job.setVertexInputFormatClass(Class.forName(cmd.getOptionValue("if"))); |
165 | 0 | job.setVertexOutputFormatClass(Class.forName(cmd.getOptionValue("of"))); |
166 | |
|
167 | 0 | if (cmd.hasOption("ip")) { |
168 | 0 | FileInputFormat.addInputPath(job.getInternalJob(), |
169 | |
new Path(cmd.getOptionValue("ip"))); |
170 | |
} else { |
171 | 0 | if (LOG.isInfoEnabled()) { |
172 | 0 | LOG.info("No input path specified. Ensure your InputFormat does" + |
173 | |
" not require one."); |
174 | |
} |
175 | |
} |
176 | |
|
177 | 0 | if (cmd.hasOption("op")) { |
178 | 0 | FileOutputFormat.setOutputPath(job.getInternalJob(), |
179 | |
new Path(cmd.getOptionValue("op"))); |
180 | |
} else { |
181 | 0 | if (LOG.isInfoEnabled()) { |
182 | 0 | LOG.info("No output path specified. Ensure your OutputFormat does" + |
183 | |
" not require one."); |
184 | |
} |
185 | |
} |
186 | |
|
187 | 0 | if (cmd.hasOption("c")) { |
188 | 0 | job.setVertexCombinerClass(Class.forName(cmd.getOptionValue("c"))); |
189 | |
} |
190 | |
|
191 | 0 | if (cmd.hasOption("wc")) { |
192 | 0 | job.setWorkerContextClass(Class.forName(cmd.getOptionValue("wc"))); |
193 | |
} |
194 | |
|
195 | 0 | if (cmd.hasOption("mc")) { |
196 | 0 | job.setMasterComputeClass(Class.forName(cmd.getOptionValue("mc"))); |
197 | |
} |
198 | |
|
199 | 0 | if (cmd.hasOption("aw")) { |
200 | 0 | job.setAggregatorWriterClass(Class.forName(cmd.getOptionValue("aw"))); |
201 | |
} |
202 | |
|
203 | 0 | if (cmd.hasOption("cf")) { |
204 | 0 | DistributedCache.addCacheFile(new URI(cmd.getOptionValue("cf")), |
205 | |
job.getConfiguration()); |
206 | |
} |
207 | |
|
208 | 0 | if (cmd.hasOption("ca")) { |
209 | 0 | Configuration jobConf = job.getConfiguration(); |
210 | |
for (String paramValue : |
211 | 0 | Splitter.on(',').split(cmd.getOptionValue("ca"))) { |
212 | 0 | String[] parts = Iterables.toArray(Splitter.on('=').split(paramValue), |
213 | |
String.class); |
214 | 0 | if (parts.length != 2) { |
215 | 0 | throw new IllegalArgumentException("Unable to parse custom " + |
216 | |
" argument: " + paramValue); |
217 | |
} |
218 | 0 | if (LOG.isInfoEnabled()) { |
219 | 0 | LOG.info("Setting custom argument [" + parts[0] + "] to [" + |
220 | |
parts[1] + "]"); |
221 | |
} |
222 | 0 | jobConf.set(parts[0], parts[1]); |
223 | 0 | } |
224 | |
} |
225 | |
|
226 | |
|
227 | |
|
228 | |
@SuppressWarnings("rawtypes") |
229 | 0 | GiraphTypeValidator<?, ?, ?, ?> validator = |
230 | |
new GiraphTypeValidator(job.getConfiguration()); |
231 | 0 | validator.validateClassTypes(); |
232 | |
|
233 | 0 | job.setWorkerConfiguration(workers, workers, 100.0f); |
234 | |
|
235 | 0 | boolean verbose = !cmd.hasOption('q'); |
236 | |
|
237 | 0 | return job.run(verbose) ? 0 : -1; |
238 | |
} |
239 | |
|
240 | |
|
241 | |
|
242 | |
|
243 | |
|
244 | |
|
245 | |
|
246 | |
public static void main(String[] args) throws Exception { |
247 | 0 | System.exit(ToolRunner.run(new GiraphRunner(), args)); |
248 | 0 | } |
249 | |
} |