View Javadoc

1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements.  See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache License, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License.  You may obtain a copy of the License at
8    *
9    *     http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the License for the specific language governing permissions and
15   * limitations under the License.
16   */
17  package org.apache.accumulo.core.util;
18  
19  import java.util.ArrayList;
20  import java.util.HashSet;
21  import java.util.Iterator;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Map.Entry;
25  import java.util.SortedMap;
26  import java.util.SortedSet;
27  import java.util.TreeMap;
28  
29  import org.apache.accumulo.core.Constants;
30  import org.apache.accumulo.core.client.AccumuloException;
31  import org.apache.accumulo.core.client.AccumuloSecurityException;
32  import org.apache.accumulo.core.client.Instance;
33  import org.apache.accumulo.core.client.Scanner;
34  import org.apache.accumulo.core.client.TableNotFoundException;
35  import org.apache.accumulo.core.client.impl.ScannerImpl;
36  import org.apache.accumulo.core.client.impl.Tables;
37  import org.apache.accumulo.core.data.Key;
38  import org.apache.accumulo.core.data.KeyExtent;
39  import org.apache.accumulo.core.data.PartialKey;
40  import org.apache.accumulo.core.data.Range;
41  import org.apache.accumulo.core.data.Value;
42  import org.apache.accumulo.core.security.CredentialHelper;
43  import org.apache.accumulo.core.security.thrift.Credential;
44  import org.apache.hadoop.io.Text;
45  
46  public class MetadataTable {
47    public static class DataFileValue {
48      private long size;
49      private long numEntries;
50      private long time = -1;
51      
52      public DataFileValue(long size, long numEntries, long time) {
53        this.size = size;
54        this.numEntries = numEntries;
55        this.time = time;
56      }
57      
58      public DataFileValue(long size, long numEntries) {
59        this.size = size;
60        this.numEntries = numEntries;
61        this.time = -1;
62      }
63      
64      public DataFileValue(byte[] encodedDFV) {
65        String[] ba = new String(encodedDFV).split(",");
66        
67        size = Long.parseLong(ba[0]);
68        numEntries = Long.parseLong(ba[1]);
69        
70        if (ba.length == 3)
71          time = Long.parseLong(ba[2]);
72        else
73          time = -1;
74      }
75      
76      public long getSize() {
77        return size;
78      }
79      
80      public long getNumEntries() {
81        return numEntries;
82      }
83      
84      public boolean isTimeSet() {
85        return time >= 0;
86      }
87      
88      public long getTime() {
89        return time;
90      }
91      
92      public byte[] encode() {
93        if (time >= 0)
94          return ("" + size + "," + numEntries + "," + time).getBytes();
95        return ("" + size + "," + numEntries).getBytes();
96      }
97      
98      public boolean equals(Object o) {
99        if (o instanceof DataFileValue) {
100         DataFileValue odfv = (DataFileValue) o;
101         
102         return size == odfv.size && numEntries == odfv.numEntries;
103       }
104       
105       return false;
106     }
107     
108     public int hashCode() {
109       return Long.valueOf(size + numEntries).hashCode();
110     }
111     
112     public String toString() {
113       return size + " " + numEntries;
114     }
115     
116     public void setTime(long time) {
117       if (time < 0)
118         throw new IllegalArgumentException();
119       this.time = time;
120     }
121   }
122   
123   public static Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>> getMetadataLocationEntries(SortedMap<Key,Value> entries) {
124     Key key;
125     Value val;
126     Text location = null;
127     Value prevRow = null;
128     KeyExtent ke;
129     
130     SortedMap<KeyExtent,Text> results = new TreeMap<KeyExtent,Text>();
131     ArrayList<KeyExtent> locationless = new ArrayList<KeyExtent>();
132     
133     Text lastRowFromKey = new Text();
134     
135     // text obj below is meant to be reused in loop for efficiency
136     Text colf = new Text();
137     Text colq = new Text();
138     
139     for (Entry<Key,Value> entry : entries.entrySet()) {
140       key = entry.getKey();
141       val = entry.getValue();
142       
143       if (key.compareRow(lastRowFromKey) != 0) {
144         prevRow = null;
145         location = null;
146         key.getRow(lastRowFromKey);
147       }
148       
149       colf = key.getColumnFamily(colf);
150       colq = key.getColumnQualifier(colq);
151       
152       // interpret the row id as a key extent
153       if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY) || colf.equals(Constants.METADATA_FUTURE_LOCATION_COLUMN_FAMILY))
154         location = new Text(val.toString());
155       else if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq))
156         prevRow = new Value(val);
157       
158       if (prevRow != null) {
159         ke = new KeyExtent(key.getRow(), prevRow);
160         if (location != null)
161           results.put(ke, location);
162         else
163           locationless.add(ke);
164 
165         location = null;
166         prevRow = null;
167       }
168     }
169     
170     return new Pair<SortedMap<KeyExtent,Text>,List<KeyExtent>>(results, locationless);
171   }
172   
173   public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(Instance instance, KeyExtent ke, List<ColumnFQ> columns, Credential credentials) {
174     TreeMap<Key,Value> tkv = new TreeMap<Key,Value>();
175     getTabletAndPrevTabletKeyValues(instance, tkv, ke, columns, credentials);
176     return getTabletEntries(tkv, columns);
177   }
178   
179   public static SortedMap<Text,SortedMap<ColumnFQ,Value>> getTabletEntries(SortedMap<Key,Value> tabletKeyValues, List<ColumnFQ> columns) {
180     TreeMap<Text,SortedMap<ColumnFQ,Value>> tabletEntries = new TreeMap<Text,SortedMap<ColumnFQ,Value>>();
181     
182     HashSet<ColumnFQ> colSet = null;
183     if (columns != null) {
184       colSet = new HashSet<ColumnFQ>(columns);
185     }
186     
187     for (Entry<Key,Value> entry : tabletKeyValues.entrySet()) {
188       
189       if (columns != null && !colSet.contains(new ColumnFQ(entry.getKey()))) {
190         continue;
191       }
192       
193       Text row = entry.getKey().getRow();
194       
195       SortedMap<ColumnFQ,Value> colVals = tabletEntries.get(row);
196       if (colVals == null) {
197         colVals = new TreeMap<ColumnFQ,Value>();
198         tabletEntries.put(row, colVals);
199       }
200       
201       colVals.put(new ColumnFQ(entry.getKey()), entry.getValue());
202     }
203     
204     return tabletEntries;
205   }
206   
207   public static void getTabletAndPrevTabletKeyValues(Instance instance, SortedMap<Key,Value> tkv, KeyExtent ke, List<ColumnFQ> columns, Credential credentials) {
208     Text startRow;
209     Text endRow = ke.getMetadataEntry();
210     
211     if (ke.getPrevEndRow() == null) {
212       startRow = new Text(KeyExtent.getMetadataEntry(ke.getTableId(), new Text()));
213     } else {
214       startRow = new Text(KeyExtent.getMetadataEntry(ke.getTableId(), ke.getPrevEndRow()));
215     }
216     
217     Scanner scanner = new ScannerImpl(instance, credentials, Constants.METADATA_TABLE_ID, Constants.NO_AUTHS);
218     
219     if (columns != null) {
220       for (ColumnFQ column : columns)
221         column.fetch(scanner);
222     }
223     
224     scanner.setRange(new Range(new Key(startRow), true, new Key(endRow).followingKey(PartialKey.ROW), false));
225     
226     tkv.clear();
227     boolean successful = false;
228     try {
229       for (Entry<Key,Value> entry : scanner) {
230         tkv.put(entry.getKey(), entry.getValue());
231       }
232       successful = true;
233     } finally {
234       if (!successful) {
235         tkv.clear();
236       }
237     }
238   }
239   
240   public static void getEntries(Instance instance, Credential credentials, String table, boolean isTid, Map<KeyExtent,String> locations,
241       SortedSet<KeyExtent> tablets) throws AccumuloException, AccumuloSecurityException, TableNotFoundException {
242     String tableId = isTid ? table : Tables.getNameToIdMap(instance).get(table);
243     
244     Scanner scanner = instance.getConnector(credentials.getPrincipal(), CredentialHelper.extractToken(credentials)).createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS);
245     
246     Constants.METADATA_PREV_ROW_COLUMN.fetch(scanner);
247     scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);
248     
249     // position at first entry in metadata table for given table
250     KeyExtent ke = new KeyExtent(new Text(tableId), new Text(), null);
251     Key startKey = new Key(ke.getMetadataEntry());
252     ke = new KeyExtent(new Text(tableId), null, null);
253     Key endKey = new Key(ke.getMetadataEntry()).followingKey(PartialKey.ROW);
254     scanner.setRange(new Range(startKey, endKey));
255     
256     Text colf = new Text();
257     Text colq = new Text();
258     
259     KeyExtent currentKeyExtent = null;
260     String location = null;
261     Text row = null;
262     // acquire this tables METADATA table entries
263     boolean haveExtent = false;
264     boolean haveLocation = false;
265     for (Entry<Key,Value> entry : scanner) {
266       if (row != null) {
267         if (!row.equals(entry.getKey().getRow())) {
268           currentKeyExtent = null;
269           haveExtent = false;
270           haveLocation = false;
271           row = entry.getKey().getRow();
272         }
273       } else
274         row = entry.getKey().getRow();
275       
276       colf = entry.getKey().getColumnFamily(colf);
277       colq = entry.getKey().getColumnQualifier(colq);
278       
279       // stop scanning metadata table when another table is reached
280       if (!(new KeyExtent(entry.getKey().getRow(), (Text) null)).getTableId().toString().equals(tableId))
281         break;
282       
283       if (Constants.METADATA_PREV_ROW_COLUMN.equals(colf, colq)) {
284         currentKeyExtent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
285         tablets.add(currentKeyExtent);
286         haveExtent = true;
287       } else if (colf.equals(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY)) {
288         location = entry.getValue().toString();
289         haveLocation = true;
290       }
291       
292       if (haveExtent && haveLocation) {
293         locations.put(currentKeyExtent, location);
294         haveExtent = false;
295         haveLocation = false;
296         currentKeyExtent = null;
297       }
298     }
299     
300     validateEntries(tableId, tablets);
301   }
302   
303   public static void validateEntries(String tableId, SortedSet<KeyExtent> tablets) throws AccumuloException {
304     // sanity check of metadata table entries
305     // make sure tablets has no holes, and that it starts and ends w/ null
306     if (tablets.size() == 0)
307       throw new AccumuloException("No entries found in metadata table for table " + tableId);
308     
309     if (tablets.first().getPrevEndRow() != null)
310       throw new AccumuloException("Problem with metadata table, first entry for table " + tableId + "- " + tablets.first() + " - has non null prev end row");
311     
312     if (tablets.last().getEndRow() != null)
313       throw new AccumuloException("Problem with metadata table, last entry for table " + tableId + "- " + tablets.first() + " - has non null end row");
314     
315     Iterator<KeyExtent> tabIter = tablets.iterator();
316     Text lastEndRow = tabIter.next().getEndRow();
317     while (tabIter.hasNext()) {
318       KeyExtent tabke = tabIter.next();
319       
320       if (tabke.getPrevEndRow() == null)
321         throw new AccumuloException("Problem with metadata table, it has null prev end row in middle of table " + tabke);
322       
323       if (!tabke.getPrevEndRow().equals(lastEndRow))
324         throw new AccumuloException("Problem with metadata table, it has a hole " + tabke.getPrevEndRow() + " != " + lastEndRow);
325       
326       lastEndRow = tabke.getEndRow();
327     }
328     
329     // end METADATA table sanity check
330   }
331   
332   public static boolean isContiguousRange(KeyExtent ke, SortedSet<KeyExtent> children) {
333     if (children.size() == 0)
334       return false;
335     
336     if (children.size() == 1)
337       return children.first().equals(ke);
338     
339     Text per = children.first().getPrevEndRow();
340     Text er = children.last().getEndRow();
341     
342     boolean perEqual = (per == ke.getPrevEndRow() || per != null && ke.getPrevEndRow() != null && ke.getPrevEndRow().compareTo(per) == 0);
343     
344     boolean erEqual = (er == ke.getEndRow() || er != null && ke.getEndRow() != null && ke.getEndRow().compareTo(er) == 0);
345     
346     if (!perEqual || !erEqual)
347       return false;
348     
349     Iterator<KeyExtent> iter = children.iterator();
350     
351     Text lastEndRow = iter.next().getEndRow();
352     
353     while (iter.hasNext()) {
354       KeyExtent cke = iter.next();
355       
356       per = cke.getPrevEndRow();
357       
358       // something in the middle should not be null
359       
360       if (per == null || lastEndRow == null)
361         return false;
362       
363       if (per.compareTo(lastEndRow) != 0)
364         return false;
365       
366       lastEndRow = cke.getEndRow();
367     }
368     
369     return true;
370   }
371   
372 }