1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
|
19 | |
package org.apache.giraph.graph; |
20 | |
|
21 | |
import org.apache.giraph.graph.partition.GraphPartitionerFactory; |
22 | |
import org.apache.giraph.graph.partition.HashPartitionerFactory; |
23 | |
import org.apache.giraph.graph.partition.PartitionStats; |
24 | |
import org.apache.hadoop.conf.Configuration; |
25 | |
import org.apache.hadoop.io.NullWritable; |
26 | |
import org.apache.hadoop.io.Writable; |
27 | |
import org.apache.hadoop.io.WritableComparable; |
28 | |
import org.apache.hadoop.util.ReflectionUtils; |
29 | |
|
30 | |
|
31 | |
|
32 | |
|
33 | |
|
34 | |
public class BspUtils { |
35 | |
|
36 | |
|
37 | |
|
38 | 0 | private BspUtils() { } |
39 | |
|
40 | |
|
41 | |
|
42 | |
|
43 | |
|
44 | |
|
45 | |
|
46 | |
|
47 | |
|
48 | |
|
49 | |
|
50 | |
@SuppressWarnings({ "rawtypes", "unchecked" }) |
51 | |
public static <I extends WritableComparable, V extends Writable, |
52 | |
E extends Writable, M extends Writable> |
53 | |
Class<? extends GraphPartitionerFactory<I, V, E, M>> |
54 | |
getGraphPartitionerClass(Configuration conf) { |
55 | 48 | return (Class<? extends GraphPartitionerFactory<I, V, E, M>>) |
56 | |
conf.getClass(GiraphJob.GRAPH_PARTITIONER_FACTORY_CLASS, |
57 | |
HashPartitionerFactory.class, |
58 | |
GraphPartitionerFactory.class); |
59 | |
} |
60 | |
|
61 | |
|
62 | |
|
63 | |
|
64 | |
|
65 | |
|
66 | |
|
67 | |
|
68 | |
|
69 | |
|
70 | |
|
71 | |
@SuppressWarnings("rawtypes") |
72 | |
public static <I extends WritableComparable, V extends Writable, |
73 | |
E extends Writable, M extends Writable> |
74 | |
GraphPartitionerFactory<I, V, E, M> |
75 | |
createGraphPartitioner(Configuration conf) { |
76 | |
Class<? extends GraphPartitionerFactory<I, V, E, M>> |
77 | 48 | graphPartitionerFactoryClass = getGraphPartitionerClass(conf); |
78 | 48 | return ReflectionUtils.newInstance(graphPartitionerFactoryClass, conf); |
79 | |
} |
80 | |
|
81 | |
|
82 | |
|
83 | |
|
84 | |
|
85 | |
|
86 | |
|
87 | |
|
88 | |
|
89 | |
|
90 | |
|
91 | |
@SuppressWarnings("rawtypes") |
92 | |
public static <I extends WritableComparable, V extends Writable, |
93 | |
E extends Writable, M extends Writable> |
94 | |
PartitionStats createGraphPartitionStats(Configuration conf) { |
95 | 0 | GraphPartitionerFactory<I, V, E, M> graphPartitioner = |
96 | |
createGraphPartitioner(conf); |
97 | 0 | return graphPartitioner.createMasterGraphPartitioner(). |
98 | |
createPartitionStats(); |
99 | |
} |
100 | |
|
101 | |
|
102 | |
|
103 | |
|
104 | |
|
105 | |
|
106 | |
|
107 | |
|
108 | |
|
109 | |
|
110 | |
|
111 | |
@SuppressWarnings({ "rawtypes", "unchecked" }) |
112 | |
public static <I extends WritableComparable, |
113 | |
V extends Writable, |
114 | |
E extends Writable, |
115 | |
M extends Writable> |
116 | |
Class<? extends VertexInputFormat<I, V, E, M>> |
117 | |
getVertexInputFormatClass(Configuration conf) { |
118 | 81 | return (Class<? extends VertexInputFormat<I, V, E, M>>) |
119 | |
conf.getClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS, |
120 | |
null, |
121 | |
VertexInputFormat.class); |
122 | |
} |
123 | |
|
124 | |
|
125 | |
|
126 | |
|
127 | |
|
128 | |
|
129 | |
|
130 | |
|
131 | |
|
132 | |
|
133 | |
|
134 | |
@SuppressWarnings("rawtypes") |
135 | |
public static <I extends WritableComparable, |
136 | |
V extends Writable, |
137 | |
E extends Writable, |
138 | |
M extends Writable> VertexInputFormat<I, V, E, M> |
139 | |
createVertexInputFormat(Configuration conf) { |
140 | 49 | Class<? extends VertexInputFormat<I, V, E, M>> vertexInputFormatClass = |
141 | |
getVertexInputFormatClass(conf); |
142 | 49 | VertexInputFormat<I, V, E, M> inputFormat = |
143 | |
ReflectionUtils.newInstance(vertexInputFormatClass, conf); |
144 | 49 | return inputFormat; |
145 | |
} |
146 | |
|
147 | |
|
148 | |
|
149 | |
|
150 | |
|
151 | |
|
152 | |
|
153 | |
|
154 | |
|
155 | |
|
156 | |
@SuppressWarnings({ "rawtypes", "unchecked" }) |
157 | |
public static <I extends WritableComparable, |
158 | |
V extends Writable, |
159 | |
E extends Writable> |
160 | |
Class<? extends VertexOutputFormat<I, V, E>> |
161 | |
getVertexOutputFormatClass(Configuration conf) { |
162 | 104 | return (Class<? extends VertexOutputFormat<I, V, E>>) |
163 | |
conf.getClass(GiraphJob.VERTEX_OUTPUT_FORMAT_CLASS, |
164 | |
null, |
165 | |
VertexOutputFormat.class); |
166 | |
} |
167 | |
|
168 | |
|
169 | |
|
170 | |
|
171 | |
|
172 | |
|
173 | |
|
174 | |
|
175 | |
|
176 | |
|
177 | |
@SuppressWarnings("rawtypes") |
178 | |
public static <I extends WritableComparable, V extends Writable, |
179 | |
E extends Writable> VertexOutputFormat<I, V, E> |
180 | |
createVertexOutputFormat(Configuration conf) { |
181 | 51 | Class<? extends VertexOutputFormat<I, V, E>> vertexOutputFormatClass = |
182 | |
getVertexOutputFormatClass(conf); |
183 | 51 | return ReflectionUtils.newInstance(vertexOutputFormatClass, conf); |
184 | |
} |
185 | |
|
186 | |
|
187 | |
|
188 | |
|
189 | |
|
190 | |
|
191 | |
|
192 | |
public static Class<? extends AggregatorWriter> |
193 | |
getAggregatorWriterClass(Configuration conf) { |
194 | 24 | return conf.getClass(GiraphJob.AGGREGATOR_WRITER_CLASS, |
195 | |
TextAggregatorWriter.class, |
196 | |
AggregatorWriter.class); |
197 | |
} |
198 | |
|
199 | |
|
200 | |
|
201 | |
|
202 | |
|
203 | |
|
204 | |
|
205 | |
public static AggregatorWriter createAggregatorWriter(Configuration conf) { |
206 | 24 | Class<? extends AggregatorWriter> aggregatorWriterClass = |
207 | |
getAggregatorWriterClass(conf); |
208 | 24 | return ReflectionUtils.newInstance(aggregatorWriterClass, conf); |
209 | |
} |
210 | |
|
211 | |
|
212 | |
|
213 | |
|
214 | |
|
215 | |
|
216 | |
|
217 | |
|
218 | |
|
219 | |
@SuppressWarnings({ "rawtypes", "unchecked" }) |
220 | |
public static <I extends WritableComparable, M extends Writable> |
221 | |
Class<? extends VertexCombiner<I, M>> |
222 | |
getVertexCombinerClass(Configuration conf) { |
223 | 48 | return (Class<? extends VertexCombiner<I, M>>) |
224 | |
conf.getClass(GiraphJob.VERTEX_COMBINER_CLASS, |
225 | |
null, |
226 | |
VertexCombiner.class); |
227 | |
} |
228 | |
|
229 | |
|
230 | |
|
231 | |
|
232 | |
|
233 | |
|
234 | |
|
235 | |
|
236 | |
|
237 | |
@SuppressWarnings("rawtypes") |
238 | |
public static <I extends WritableComparable, M extends Writable> |
239 | |
VertexCombiner<I, M> createVertexCombiner(Configuration conf) { |
240 | 3 | Class<? extends VertexCombiner<I, M>> vertexCombinerClass = |
241 | |
getVertexCombinerClass(conf); |
242 | 3 | return ReflectionUtils.newInstance(vertexCombinerClass, conf); |
243 | |
} |
244 | |
|
245 | |
|
246 | |
|
247 | |
|
248 | |
|
249 | |
|
250 | |
|
251 | |
|
252 | |
|
253 | |
|
254 | |
|
255 | |
|
256 | |
@SuppressWarnings({ "unchecked", "rawtypes" }) |
257 | |
public static <I extends WritableComparable, V extends Writable, |
258 | |
E extends Writable, M extends Writable> |
259 | |
Class<? extends VertexResolver<I, V, E, M>> |
260 | |
getVertexResolverClass(Configuration conf) { |
261 | 76 | return (Class<? extends VertexResolver<I, V, E, M>>) |
262 | |
conf.getClass(GiraphJob.VERTEX_RESOLVER_CLASS, |
263 | |
VertexResolver.class, |
264 | |
VertexResolver.class); |
265 | |
} |
266 | |
|
267 | |
|
268 | |
|
269 | |
|
270 | |
|
271 | |
|
272 | |
|
273 | |
|
274 | |
|
275 | |
|
276 | |
|
277 | |
|
278 | |
@SuppressWarnings("rawtypes") |
279 | |
public static <I extends WritableComparable, V extends Writable, |
280 | |
E extends Writable, M extends Writable> VertexResolver<I, V, E, M> |
281 | |
createVertexResolver(Configuration conf, |
282 | |
GraphState<I, V, E, M> graphState) { |
283 | 45 | Class<? extends VertexResolver<I, V, E, M>> vertexResolverClass = |
284 | |
getVertexResolverClass(conf); |
285 | 45 | VertexResolver<I, V, E, M> resolver = |
286 | |
ReflectionUtils.newInstance(vertexResolverClass, conf); |
287 | 45 | resolver.setGraphState(graphState); |
288 | 45 | return resolver; |
289 | |
} |
290 | |
|
291 | |
|
292 | |
|
293 | |
|
294 | |
|
295 | |
|
296 | |
|
297 | |
public static Class<? extends WorkerContext> |
298 | |
getWorkerContextClass(Configuration conf) { |
299 | 24 | return (Class<? extends WorkerContext>) |
300 | |
conf.getClass(GiraphJob.WORKER_CONTEXT_CLASS, |
301 | |
DefaultWorkerContext.class, |
302 | |
WorkerContext.class); |
303 | |
} |
304 | |
|
305 | |
|
306 | |
|
307 | |
|
308 | |
|
309 | |
|
310 | |
|
311 | |
|
312 | |
|
313 | |
|
314 | |
|
315 | |
|
316 | |
@SuppressWarnings("rawtypes") |
317 | |
public static <I extends WritableComparable, V extends Writable, |
318 | |
E extends Writable, M extends Writable> |
319 | |
WorkerContext createWorkerContext(Configuration conf, |
320 | |
GraphState<I, V, E, M> graphState) { |
321 | 24 | Class<? extends WorkerContext> workerContextClass = |
322 | |
getWorkerContextClass(conf); |
323 | 24 | WorkerContext workerContext = |
324 | |
ReflectionUtils.newInstance(workerContextClass, conf); |
325 | 24 | workerContext.setGraphState(graphState); |
326 | 24 | return workerContext; |
327 | |
} |
328 | |
|
329 | |
|
330 | |
|
331 | |
|
332 | |
|
333 | |
|
334 | |
|
335 | |
public static Class<? extends MasterCompute> |
336 | |
getMasterComputeClass(Configuration conf) { |
337 | 24 | return (Class<? extends MasterCompute>) |
338 | |
conf.getClass(GiraphJob.MASTER_COMPUTE_CLASS, |
339 | |
DefaultMasterCompute.class, |
340 | |
MasterCompute.class); |
341 | |
} |
342 | |
|
343 | |
|
344 | |
|
345 | |
|
346 | |
|
347 | |
|
348 | |
|
349 | |
public static MasterCompute |
350 | |
createMasterCompute(Configuration conf) { |
351 | 24 | Class<? extends MasterCompute> masterComputeClass = |
352 | |
getMasterComputeClass(conf); |
353 | 24 | MasterCompute masterCompute = |
354 | |
ReflectionUtils.newInstance(masterComputeClass, conf); |
355 | 24 | return masterCompute; |
356 | |
} |
357 | |
|
358 | |
|
359 | |
|
360 | |
|
361 | |
|
362 | |
|
363 | |
|
364 | |
|
365 | |
|
366 | |
|
367 | |
|
368 | |
@SuppressWarnings({ "rawtypes", "unchecked" }) |
369 | |
public static <I extends WritableComparable, V extends Writable, |
370 | |
E extends Writable, M extends Writable> |
371 | |
Class<? extends Vertex<I, V, E, M>> getVertexClass(Configuration conf) { |
372 | 579 | return (Class<? extends Vertex<I, V, E, M>>) |
373 | |
conf.getClass(GiraphJob.VERTEX_CLASS, |
374 | |
null, |
375 | |
Vertex.class); |
376 | |
} |
377 | |
|
378 | |
|
379 | |
|
380 | |
|
381 | |
|
382 | |
|
383 | |
|
384 | |
|
385 | |
|
386 | |
|
387 | |
|
388 | |
@SuppressWarnings("rawtypes") |
389 | |
public static <I extends WritableComparable, V extends Writable, |
390 | |
E extends Writable, M extends Writable> Vertex<I, V, E, M> |
391 | |
createVertex(Configuration conf) { |
392 | 523 | Class<? extends Vertex<I, V, E, M>> vertexClass = getVertexClass(conf); |
393 | 523 | Vertex<I, V, E, M> vertex = |
394 | |
ReflectionUtils.newInstance(vertexClass, conf); |
395 | 523 | return vertex; |
396 | |
} |
397 | |
|
398 | |
|
399 | |
|
400 | |
|
401 | |
|
402 | |
|
403 | |
|
404 | |
|
405 | |
@SuppressWarnings("unchecked") |
406 | |
public static <I extends Writable> Class<I> |
407 | |
getVertexIdClass(Configuration conf) { |
408 | 6306 | return (Class<I>) conf.getClass(GiraphJob.VERTEX_ID_CLASS, |
409 | |
WritableComparable.class); |
410 | |
} |
411 | |
|
412 | |
|
413 | |
|
414 | |
|
415 | |
|
416 | |
|
417 | |
|
418 | |
|
419 | |
@SuppressWarnings("rawtypes") |
420 | |
public static <I extends WritableComparable> |
421 | |
I createVertexId(Configuration conf) { |
422 | 6306 | Class<I> vertexIdClass = getVertexIdClass(conf); |
423 | |
try { |
424 | 6306 | return vertexIdClass.newInstance(); |
425 | 0 | } catch (InstantiationException e) { |
426 | 0 | throw new IllegalArgumentException( |
427 | |
"createVertexId: Failed to instantiate", e); |
428 | 0 | } catch (IllegalAccessException e) { |
429 | 0 | throw new IllegalArgumentException( |
430 | |
"createVertexId: Illegally accessed", e); |
431 | |
} |
432 | |
} |
433 | |
|
434 | |
|
435 | |
|
436 | |
|
437 | |
|
438 | |
|
439 | |
|
440 | |
|
441 | |
@SuppressWarnings("unchecked") |
442 | |
public static <V extends Writable> Class<V> |
443 | |
getVertexValueClass(Configuration conf) { |
444 | 156 | return (Class<V>) conf.getClass(GiraphJob.VERTEX_VALUE_CLASS, |
445 | |
Writable.class); |
446 | |
} |
447 | |
|
448 | |
|
449 | |
|
450 | |
|
451 | |
|
452 | |
|
453 | |
|
454 | |
|
455 | |
@SuppressWarnings("unchecked") |
456 | |
public static <V extends Writable> V |
457 | |
createVertexValue(Configuration conf) { |
458 | 156 | Class<V> vertexValueClass = getVertexValueClass(conf); |
459 | 156 | if (vertexValueClass == NullWritable.class) { |
460 | 1 | return (V) NullWritable.get(); |
461 | |
} else { |
462 | |
try { |
463 | 155 | return vertexValueClass.newInstance(); |
464 | 0 | } catch (InstantiationException e) { |
465 | 0 | throw new IllegalArgumentException( |
466 | |
"createVertexValue: Failed to instantiate", e); |
467 | 0 | } catch (IllegalAccessException e) { |
468 | 0 | throw new IllegalArgumentException( |
469 | |
"createVertexValue: Illegally accessed", e); |
470 | |
} |
471 | |
} |
472 | |
} |
473 | |
|
474 | |
|
475 | |
|
476 | |
|
477 | |
|
478 | |
|
479 | |
|
480 | |
|
481 | |
@SuppressWarnings("unchecked") |
482 | |
public static <E extends Writable> Class<E> |
483 | |
getEdgeValueClass(Configuration conf) { |
484 | 1258 | return (Class<E>) conf.getClass(GiraphJob.EDGE_VALUE_CLASS, |
485 | |
Writable.class); |
486 | |
} |
487 | |
|
488 | |
|
489 | |
|
490 | |
|
491 | |
|
492 | |
|
493 | |
|
494 | |
|
495 | |
@SuppressWarnings("unchecked") |
496 | |
public static <E extends Writable> E |
497 | |
createEdgeValue(Configuration conf) { |
498 | 1258 | Class<E> edgeValueClass = getEdgeValueClass(conf); |
499 | 1258 | if (edgeValueClass == NullWritable.class) { |
500 | 1 | return (E) NullWritable.get(); |
501 | |
} else { |
502 | |
try { |
503 | 1257 | return edgeValueClass.newInstance(); |
504 | 0 | } catch (InstantiationException e) { |
505 | 0 | throw new IllegalArgumentException( |
506 | |
"createEdgeValue: Failed to instantiate", e); |
507 | 0 | } catch (IllegalAccessException e) { |
508 | 0 | throw new IllegalArgumentException( |
509 | |
"createEdgeValue: Illegally accessed", e); |
510 | |
} |
511 | |
} |
512 | |
} |
513 | |
|
514 | |
|
515 | |
|
516 | |
|
517 | |
|
518 | |
|
519 | |
|
520 | |
|
521 | |
@SuppressWarnings("unchecked") |
522 | |
public static <M extends Writable> Class<M> |
523 | |
getMessageValueClass(Configuration conf) { |
524 | 222352 | return (Class<M>) conf.getClass(GiraphJob.MESSAGE_VALUE_CLASS, |
525 | |
Writable.class); |
526 | |
} |
527 | |
|
528 | |
|
529 | |
|
530 | |
|
531 | |
|
532 | |
|
533 | |
|
534 | |
|
535 | |
@SuppressWarnings("unchecked") |
536 | |
public static <M extends Writable> M |
537 | |
createMessageValue(Configuration conf) { |
538 | 222352 | Class<M> messageValueClass = getMessageValueClass(conf); |
539 | 222352 | if (messageValueClass == NullWritable.class) { |
540 | 1 | return (M) NullWritable.get(); |
541 | |
} else { |
542 | |
try { |
543 | 222351 | return messageValueClass.newInstance(); |
544 | 0 | } catch (InstantiationException e) { |
545 | 0 | throw new IllegalArgumentException( |
546 | |
"createMessageValue: Failed to instantiate", e); |
547 | 0 | } catch (IllegalAccessException e) { |
548 | 0 | throw new IllegalArgumentException( |
549 | |
"createMessageValue: Illegally accessed", e); |
550 | |
} |
551 | |
} |
552 | |
} |
553 | |
} |