1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
|
19 | |
package org.apache.giraph.integration; |
20 | |
|
21 | |
import java.util.ArrayList; |
22 | |
import java.util.Collection; |
23 | |
import java.util.List; |
24 | |
|
25 | |
import org.apache.giraph.graph.WorkerInfo; |
26 | |
import org.apache.giraph.graph.partition.BasicPartitionOwner; |
27 | |
import org.apache.giraph.graph.partition.HashMasterPartitioner; |
28 | |
import org.apache.giraph.graph.partition.HashPartitionerFactory; |
29 | |
import org.apache.giraph.graph.partition.MasterGraphPartitioner; |
30 | |
import org.apache.giraph.graph.partition.PartitionOwner; |
31 | |
import org.apache.giraph.graph.partition.PartitionStats; |
32 | |
import org.apache.hadoop.conf.Configuration; |
33 | |
import org.apache.hadoop.io.Writable; |
34 | |
import org.apache.hadoop.io.WritableComparable; |
35 | |
import org.apache.log4j.Logger; |
36 | |
|
37 | |
|
38 | |
|
39 | |
|
40 | |
|
41 | |
|
42 | |
|
43 | |
|
44 | |
|
45 | |
|
46 | |
|
47 | |
@SuppressWarnings("rawtypes") |
48 | 4 | public class SuperstepHashPartitionerFactory<I extends WritableComparable, |
49 | |
V extends Writable, E extends Writable, M extends Writable> |
50 | |
extends HashPartitionerFactory<I, V, E, M> { |
51 | |
|
52 | |
|
53 | |
|
54 | |
|
55 | |
|
56 | |
|
57 | |
|
58 | |
|
59 | |
|
60 | |
|
61 | 4 | private static class SuperstepMasterPartition<I extends WritableComparable, |
62 | |
V extends Writable, E extends Writable, M extends Writable> |
63 | |
extends HashMasterPartitioner<I, V, E, M> { |
64 | |
|
65 | 1 | private static Logger LOG = |
66 | |
Logger.getLogger(SuperstepMasterPartition.class); |
67 | |
|
68 | |
|
69 | |
|
70 | |
|
71 | |
|
72 | |
|
73 | |
public SuperstepMasterPartition(Configuration conf) { |
74 | 2 | super(conf); |
75 | 2 | } |
76 | |
|
77 | |
@Override |
78 | |
public Collection<PartitionOwner> generateChangedPartitionOwners( |
79 | |
Collection<PartitionStats> allPartitionStatsList, |
80 | |
Collection<WorkerInfo> availableWorkerInfos, |
81 | |
int maxWorkers, |
82 | |
long superstep) { |
83 | |
|
84 | |
|
85 | |
|
86 | |
|
87 | 16 | long workerIndex = superstep % availableWorkerInfos.size(); |
88 | 16 | int i = 0; |
89 | 16 | WorkerInfo chosenWorkerInfo = null; |
90 | 16 | for (WorkerInfo workerInfo : availableWorkerInfos) { |
91 | 16 | if (workerIndex == i) { |
92 | 16 | chosenWorkerInfo = workerInfo; |
93 | |
} |
94 | 16 | ++i; |
95 | |
} |
96 | 16 | if (LOG.isInfoEnabled()) { |
97 | 16 | LOG.info("generateChangedPartitionOwners: Chosen worker " + |
98 | |
"for superstep " + superstep + " is " + |
99 | |
chosenWorkerInfo); |
100 | |
} |
101 | |
|
102 | 16 | List<PartitionOwner> partitionOwnerList = new ArrayList<PartitionOwner>(); |
103 | |
for (PartitionOwner partitionOwner : |
104 | 16 | getCurrentPartitionOwners()) { |
105 | 16 | WorkerInfo prevWorkerinfo = |
106 | |
partitionOwner.getWorkerInfo().equals(chosenWorkerInfo) ? |
107 | |
null : partitionOwner.getWorkerInfo(); |
108 | 16 | PartitionOwner tmpPartitionOwner = |
109 | |
new BasicPartitionOwner(partitionOwner.getPartitionId(), |
110 | |
chosenWorkerInfo, |
111 | |
prevWorkerinfo, |
112 | |
null); |
113 | 16 | partitionOwnerList.add(tmpPartitionOwner); |
114 | 16 | LOG.info("partition owner was " + partitionOwner + |
115 | |
", new " + tmpPartitionOwner); |
116 | 16 | } |
117 | 16 | setPartitionOwnerList(partitionOwnerList); |
118 | 16 | return partitionOwnerList; |
119 | |
} |
120 | |
} |
121 | |
|
122 | |
@Override |
123 | |
public MasterGraphPartitioner<I, V, E, M> |
124 | |
createMasterGraphPartitioner() { |
125 | 2 | return new SuperstepMasterPartition<I, V, E, M>(getConf()); |
126 | |
} |
127 | |
} |