1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.accumulo.core.util;
18
19 import java.util.ArrayList;
20 import java.util.HashSet;
21 import java.util.Iterator;
22 import java.util.List;
23 import java.util.Map;
24 import java.util.Map.Entry;
25 import java.util.SortedMap;
26 import java.util.SortedSet;
27 import java.util.TreeMap;
28
29 import org.apache.accumulo.core.Constants;
30 import org.apache.accumulo.core.client.AccumuloException;
31 import org.apache.accumulo.core.client.AccumuloSecurityException;
32 import org.apache.accumulo.core.client.Instance;
33 import org.apache.accumulo.core.client.Scanner;
34 import org.apache.accumulo.core.client.TableNotFoundException;
35 import org.apache.accumulo.core.client.impl.ScannerImpl;
36 import org.apache.accumulo.core.client.impl.Tables;
37 import org.apache.accumulo.core.data.Key;
38 import org.apache.accumulo.core.data.KeyExtent;
39 import org.apache.accumulo.core.data.PartialKey;
40 import org.apache.accumulo.core.data.Range;
41 import org.apache.accumulo.core.data.Value;
42 import org.apache.accumulo.core.security.CredentialHelper;
43 import org.apache.accumulo.core.security.thrift.Credential;
44 import org.apache.hadoop.io.Text;
45
46 public class MetadataTable {
47 public static class DataFileValue {
48 private long size;
49 private long numEntries;
50 private long time = -1;
51
52 public DataFileValue(long size, long numEntries, long time) {
53 this.size = size;
54 this.numEntries = numEntries;
55 this.time = time;
56 }
57
58 public DataFileValue(long size, long numEntries) {
59 this.size = size;
60 this.numEntries = numEntries;
61 this.time = -1;
62 }
63
64 public DataFileValue(byte[] encodedDFV) {
65 String[] ba = new String(encodedDFV).split(",");
66
67 size = Long.parseLong(ba[0]);
68 numEntries = Long.parseLong(ba[1]);
69
70 if (ba.length == 3)
71 time = Long.parseLong(ba[2]);
72 else
73 time = -1;
74 }
75
76 public long getSize() {
77 return size;
78 }
79
80 public long getNumEntries() {
81 return numEntries;
82 }
83
84 public boolean isTimeSet() {
85 return time >= 0;
86 }
87
88 public long getTime() {
89 return time;
90 }
91
92 public byte[] encode() {
93 if (time >= 0)
94 return ("" + size + "," + numEntries + "," + time).getBytes();
95 return ("" + size + "," + numEntries).getBytes();
96 }
97
98 public boolean equals(Object o) {
99 if (o instanceof DataFileValue) {
100 DataFileValue odfv = (DataFileValue) o;
101
102 return size == odfv.size && numEntries == odfv.numEntries;
103 }
104
105 return false;
106 }
107
108 public int hashCode() {
109 return Long.valueOf(size + numEntries).hashCode();
110 }
111
112 public String toString() {
113 return size + " " + numEntries;
114 }
115
116 public void setTime(long time) {
117 if (time < 0)
118 throw new IllegalArgumentException();
119 this.time = time;
120 }
121 }
122
123 public static Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> getMetadataLocationEntries(SortedMap<Key,Value> entries) {
124 Key key;
125 Value val;
126 Text location = null;
127 Value prevRow = null;
128 KeyExtent ke;
129
130 SortedMap<KeyExtent,Text> results = new TreeMap<KeyExtent,Text>();
131 ArrayList<KeyExtent> locationless = new ArrayList<KeyExtent>();
132
133 Text lastRowFromKey = new Text();
134
135
136 Text colf = new Text();
137 Text colq = new Text();
138
139 for (Entry<Key,Value> entry : entries.entrySet()) {
140 key = entry.getKey();
141 val = entry.getValue();
142
143 if (key.compareRow(lastRowFromKey) != 0) {
144 prevRow = null;
145 location = null;
146 key.getRow(lastRowFromKey);
147 }
148
149 colf = key.getColumnFamily(colf);
150 colq = key.getColumnQualifier(colq);
151
152
153 if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) || colf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY))
154 location = new Text(val.toString());
155 else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
156 prevRow = new Value(val);
157
158 if (prevRow != null) {
159 ke = new KeyExtent(key.getRow(), prevRow);
160 if (location != null)
161 results.put(ke, location);
162 else
163 locationless.add(ke);
164
165 location = null;
166 prevRow = null;
167 }
168 }
169
170 return new Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>>(results, locationless);
171 }
172
173 public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(Instance instance, KeyExtent ke, List<ColumnFQ> columns, Credential credentials) {
174 TreeMap<Key,Value> tkv = new TreeMap<Key,Value>();
175 getTabletAndPrevTabletKeyValues(instance, tkv, ke, columns, credentials);
176 return getTabletEntries(tkv, columns);
177 }
178
179 public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
180 TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
181
182 HashSet<ColumnFQ> colSet = null;
183 if (columns != null) {
184 colSet = new HashSet<ColumnFQ>(columns);
185 }
186
187 for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
188
189 if (columns != null && !colSet.contains(new ColumnFQ(entry.getKey()))) {
190 continue;
191 }
192
193 Text row = entry.getKey().getRow();
194
195 SortedMap<ColumnFQ,Value> colVals = tabletEntries.get(row);
196 if (colVals == null) {
197 colVals = new TreeMap<ColumnFQ,Value>();
198 tabletEntries.put(row, colVals);
199 }
200
201 colVals.put(new ColumnFQ(entry.getKey()), entry.getValue());
202 }
203
204 return tabletEntries;
205 }
206
207 public static void getTabletAndPrevTabletKeyValues(Instance instance, SortedMap<Key,Value> tkv, KeyExtent ke, List<ColumnFQ> columns, Credential credentials) {
208 Text startRow;
209 Text endRow = ke.getMetadataEntry();
210
211 if (ke.getPrevEndRow() == null) {
212 startRow = new Text(KeyExtent.getMetadataEntry(ke.getTableId(), new Text()));
213 } else {
214 startRow = new Text(KeyExtent.getMetadataEntry(ke.getTableId(), ke.getPrevEndRow()));
215 }
216
217 Scanner scanner = new ScannerImpl(instance, credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
218
219 if (columns != null) {
220 for (ColumnFQ column : columns)
221 column.fetch(scanner);
222 }
223
224 scanner.setRange(new Range(new Key(startRow), true, new Key(endRow).followingKey(PartialKey.ROW), false));
225
226 tkv.clear();
227 boolean successful = false;
228 try {
229 for (Entry<Key,Value> entry : scanner) {
230 tkv.put(entry.getKey(), entry.getValue());
231 }
232 successful = true;
233 } finally {
234 if (!successful) {
235 tkv.clear();
236 }
237 }
238 }
239
240 public static void getEntries(Instance instance, Credential credentials, String table, boolean isTid, Map<KeyExtent,String> locations,
241 SortedSet<KeyExtent> tablets) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
242 String tableId = isTid ? table : Tables.getNameToIdMap(instance).get(table);
243
244 Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
245
246 Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
247 scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
248
249
250 KeyExtent ke = new KeyExtent(new Text(tableId), new Text(), null);
251 Key startKey = new Key(ke.getMetadataEntry());
252 ke = new KeyExtent(new Text(tableId), null, null);
253 Key endKey = new Key(ke.getMetadataEntry()).followingKey(PartialKey.ROW);
254 scanner.setRange(new Range(startKey, endKey));
255
256 Text colf = new Text();
257 Text colq = new Text();
258
259 KeyExtent currentKeyExtent = null;
260 String location = null;
261 Text row = null;
262
263 boolean haveExtent = false;
264 boolean haveLocation = false;
265 for (Entry<Key,Value> entry : scanner) {
266 if (row != null) {
267 if (!row.equals(entry.getKey().getRow())) {
268 currentKeyExtent = null;
269 haveExtent = false;
270 haveLocation = false;
271 row = entry.getKey().getRow();
272 }
273 } else
274 row = entry.getKey().getRow();
275
276 colf = entry.getKey().getColumnFamily(colf);
277 colq = entry.getKey().getColumnQualifier(colq);
278
279
280 if (!(new KeyExtent(entry.getKey().getRow(), (Text) null)).getTableId().toString().equals(tableId))
281 break;
282
283 if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq)) {
284 currentKeyExtent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
285 tablets.add(currentKeyExtent);
286 haveExtent = true;
287 } else if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
288 location = entry.getValue().toString();
289 haveLocation = true;
290 }
291
292 if (haveExtent && haveLocation) {
293 locations.put(currentKeyExtent, location);
294 haveExtent = false;
295 haveLocation = false;
296 currentKeyExtent = null;
297 }
298 }
299
300 validateEntries(tableId, tablets);
301 }
302
303 public static void validateEntries(String tableId, SortedSet<KeyExtent> tablets) throws AccumuloException {
304
305
306 if (tablets.size() == 0)
307 throw new AccumuloException("No entries found in metadata table for table " + tableId);
308
309 if (tablets.first().getPrevEndRow() != null)
310 throw new AccumuloException("Problem with metadata table, first entry for table " + tableId + "- " + tablets.first() + " - has non null prev end row");
311
312 if (tablets.last().getEndRow() != null)
313 throw new AccumuloException("Problem with metadata table, last entry for table " + tableId + "- " + tablets.first() + " - has non null end row");
314
315 Iterator<KeyExtent> tabIter = tablets.iterator();
316 Text lastEndRow = tabIter.next().getEndRow();
317 while (tabIter.hasNext()) {
318 KeyExtent tabke = tabIter.next();
319
320 if (tabke.getPrevEndRow() == null)
321 throw new AccumuloException("Problem with metadata table, it has null prev end row in middle of table " + tabke);
322
323 if (!tabke.getPrevEndRow().equals(lastEndRow))
324 throw new AccumuloException("Problem with metadata table, it has a hole " + tabke.getPrevEndRow() + " != " + lastEndRow);
325
326 lastEndRow = tabke.getEndRow();
327 }
328
329
330 }
331
332 public static boolean isContiguousRange(KeyExtent ke, SortedSet<KeyExtent> children) {
333 if (children.size() == 0)
334 return false;
335
336 if (children.size() == 1)
337 return children.first().equals(ke);
338
339 Text per = children.first().getPrevEndRow();
340 Text er = children.last().getEndRow();
341
342 boolean perEqual = (per == ke.getPrevEndRow() || per != null && ke.getPrevEndRow() != null && ke.getPrevEndRow().compareTo(per) == 0);
343
344 boolean erEqual = (er == ke.getEndRow() || er != null && ke.getEndRow() != null && ke.getEndRow().compareTo(er) == 0);
345
346 if (!perEqual || !erEqual)
347 return false;
348
349 Iterator<KeyExtent> iter = children.iterator();
350
351 Text lastEndRow = iter.next().getEndRow();
352
353 while (iter.hasNext()) {
354 KeyExtent cke = iter.next();
355
356 per = cke.getPrevEndRow();
357
358
359
360 if (per == null || lastEndRow == null)
361 return false;
362
363 if (per.compareTo(lastEndRow) != 0)
364 return false;
365
366 lastEndRow = cke.getEndRow();
367 }
368
369 return true;
370 }
371
372 }