1 | |
|
2 | |
|
3 | |
|
4 | |
|
5 | |
|
6 | |
|
7 | |
|
8 | |
|
9 | |
|
10 | |
|
11 | |
|
12 | |
|
13 | |
|
14 | |
|
15 | |
|
16 | |
|
17 | |
|
18 | |
package org.apache.giraph.types.ops.collections; |
19 | |
|
20 | |
import it.unimi.dsi.fastutil.ints.IntIterator; |
21 | |
import it.unimi.dsi.fastutil.ints.IntOpenHashBigSet; |
22 | |
import it.unimi.dsi.fastutil.ints.IntOpenHashSet; |
23 | |
import it.unimi.dsi.fastutil.ints.IntSet; |
24 | |
import it.unimi.dsi.fastutil.longs.LongIterator; |
25 | |
import it.unimi.dsi.fastutil.longs.LongOpenHashBigSet; |
26 | |
import it.unimi.dsi.fastutil.longs.LongOpenHashSet; |
27 | |
import it.unimi.dsi.fastutil.longs.LongSet; |
28 | |
|
29 | |
import java.io.DataInput; |
30 | |
import java.io.DataOutput; |
31 | |
import java.io.IOException; |
32 | |
|
33 | |
import org.apache.giraph.types.ops.IntTypeOps; |
34 | |
import org.apache.giraph.types.ops.LongTypeOps; |
35 | |
import org.apache.giraph.types.ops.PrimitiveIdTypeOps; |
36 | |
import org.apache.hadoop.io.IntWritable; |
37 | |
import org.apache.hadoop.io.LongWritable; |
38 | |
import org.apache.hadoop.io.Writable; |
39 | |
|
40 | |
|
41 | |
|
42 | |
|
43 | |
|
44 | |
|
45 | |
|
46 | |
|
47 | |
public interface BasicSet<T> extends Writable { |
48 | |
|
49 | |
long MAX_OPEN_HASHSET_CAPACITY = 800000000; |
50 | |
|
51 | |
|
52 | |
void clear(); |
53 | |
|
54 | |
|
55 | |
|
56 | |
|
57 | |
|
58 | |
|
59 | |
long size(); |
60 | |
|
61 | |
|
62 | |
|
63 | |
|
64 | |
|
65 | |
|
66 | |
|
67 | |
void trim(long n); |
68 | |
|
69 | |
|
70 | |
|
71 | |
|
72 | |
|
73 | |
|
74 | |
|
75 | |
|
76 | |
|
77 | |
boolean add(T value); |
78 | |
|
79 | |
|
80 | |
|
81 | |
|
82 | |
|
83 | |
|
84 | |
|
85 | |
boolean contains(T value); |
86 | |
|
87 | |
|
88 | |
|
89 | |
|
90 | |
|
91 | |
|
92 | |
PrimitiveIdTypeOps<T> getElementTypeOps(); |
93 | |
|
94 | |
|
95 | 0 | public static final class BasicIntOpenHashSet |
96 | |
implements BasicSet<IntWritable> { |
97 | |
|
98 | |
private final IntSet set; |
99 | |
|
100 | |
|
101 | 0 | public BasicIntOpenHashSet() { |
102 | 0 | set = new IntOpenHashSet(); |
103 | 0 | } |
104 | |
|
105 | |
|
106 | |
|
107 | |
|
108 | |
|
109 | |
|
110 | 0 | public BasicIntOpenHashSet(long capacity) { |
111 | 0 | if (capacity <= MAX_OPEN_HASHSET_CAPACITY) { |
112 | 0 | set = new IntOpenHashSet((int) capacity); |
113 | |
} else { |
114 | 0 | set = new IntOpenHashBigSet(capacity); |
115 | |
} |
116 | 0 | } |
117 | |
|
118 | |
@Override |
119 | |
public void clear() { |
120 | 0 | set.clear(); |
121 | 0 | } |
122 | |
|
123 | |
@Override |
124 | |
public long size() { |
125 | 0 | if (set instanceof IntOpenHashBigSet) { |
126 | 0 | return ((IntOpenHashBigSet) set).size64(); |
127 | |
} |
128 | 0 | return set.size(); |
129 | |
} |
130 | |
|
131 | |
@Override |
132 | |
public void trim(long n) { |
133 | 0 | if (set instanceof IntOpenHashSet) { |
134 | 0 | ((IntOpenHashSet) set).trim((int) Math.max(set.size(), n)); |
135 | |
} else { |
136 | 0 | ((IntOpenHashBigSet) set).trim(Math.max(set.size(), n)); |
137 | |
} |
138 | 0 | } |
139 | |
|
140 | |
@Override |
141 | |
public boolean add(IntWritable value) { |
142 | 0 | return set.add(value.get()); |
143 | |
} |
144 | |
|
145 | |
@Override |
146 | |
public boolean contains(IntWritable value) { |
147 | 0 | return set.contains(value.get()); |
148 | |
} |
149 | |
|
150 | |
@Override |
151 | |
public PrimitiveIdTypeOps<IntWritable> getElementTypeOps() { |
152 | 0 | return IntTypeOps.INSTANCE; |
153 | |
} |
154 | |
|
155 | |
@Override |
156 | |
public void write(DataOutput out) throws IOException { |
157 | 0 | out.writeInt(set.size()); |
158 | 0 | IntIterator iter = set.iterator(); |
159 | 0 | while (iter.hasNext()) { |
160 | 0 | out.writeInt(iter.nextInt()); |
161 | |
} |
162 | 0 | } |
163 | |
|
164 | |
@Override |
165 | |
public void readFields(DataInput in) throws IOException { |
166 | 0 | long size = in.readLong(); |
167 | 0 | set.clear(); |
168 | 0 | trim(size); |
169 | 0 | for (long i = 0; i < size; ++i) { |
170 | 0 | set.add(in.readInt()); |
171 | |
} |
172 | 0 | } |
173 | |
} |
174 | |
|
175 | |
|
176 | 0 | public static final class BasicLongOpenHashSet |
177 | |
implements BasicSet<LongWritable> { |
178 | |
|
179 | |
private final LongSet set; |
180 | |
|
181 | |
|
182 | 0 | public BasicLongOpenHashSet() { |
183 | 0 | set = new LongOpenHashSet(); |
184 | 0 | } |
185 | |
|
186 | |
|
187 | |
|
188 | |
|
189 | |
|
190 | |
|
191 | 0 | public BasicLongOpenHashSet(long capacity) { |
192 | 0 | if (capacity <= MAX_OPEN_HASHSET_CAPACITY) { |
193 | 0 | set = new LongOpenHashSet((int) capacity); |
194 | |
} else { |
195 | 0 | set = new LongOpenHashBigSet(capacity); |
196 | |
} |
197 | 0 | } |
198 | |
|
199 | |
@Override |
200 | |
public void clear() { |
201 | 0 | set.clear(); |
202 | 0 | } |
203 | |
|
204 | |
@Override |
205 | |
public long size() { |
206 | 0 | if (set instanceof LongOpenHashBigSet) { |
207 | 0 | return ((LongOpenHashBigSet) set).size64(); |
208 | |
} |
209 | 0 | return set.size(); |
210 | |
} |
211 | |
|
212 | |
@Override |
213 | |
public void trim(long n) { |
214 | 0 | if (set instanceof LongOpenHashSet) { |
215 | 0 | ((LongOpenHashSet) set).trim((int) Math.max(set.size(), n)); |
216 | |
} else { |
217 | 0 | ((LongOpenHashBigSet) set).trim(Math.max(set.size(), n)); |
218 | |
} |
219 | 0 | } |
220 | |
|
221 | |
@Override |
222 | |
public boolean add(LongWritable value) { |
223 | 0 | return set.add(value.get()); |
224 | |
} |
225 | |
|
226 | |
@Override |
227 | |
public boolean contains(LongWritable value) { |
228 | 0 | return set.contains(value.get()); |
229 | |
} |
230 | |
|
231 | |
@Override |
232 | |
public PrimitiveIdTypeOps<LongWritable> getElementTypeOps() { |
233 | 0 | return LongTypeOps.INSTANCE; |
234 | |
} |
235 | |
|
236 | |
@Override |
237 | |
public void write(DataOutput out) throws IOException { |
238 | 0 | out.writeLong(set.size()); |
239 | 0 | LongIterator iter = set.iterator(); |
240 | 0 | while (iter.hasNext()) { |
241 | 0 | out.writeLong(iter.nextLong()); |
242 | |
} |
243 | 0 | } |
244 | |
|
245 | |
@Override |
246 | |
public void readFields(DataInput in) throws IOException { |
247 | 0 | long size = in.readLong(); |
248 | 0 | set.clear(); |
249 | 0 | trim(size); |
250 | 0 | for (long i = 0; i < size; ++i) { |
251 | 0 | set.add(in.readLong()); |
252 | |
} |
253 | 0 | } |
254 | |
} |
255 | |
} |