1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase.regionserver; 19 20 import java.io.IOException; 21 import java.util.Collection; 22 import java.util.List; 23 import java.util.NavigableSet; 24 25 import org.apache.hadoop.hbase.classification.InterfaceAudience; 26 import org.apache.hadoop.hbase.classification.InterfaceStability; 27 import org.apache.hadoop.fs.FileSystem; 28 import org.apache.hadoop.fs.Path; 29 import org.apache.hadoop.hbase.Cell; 30 import org.apache.hadoop.hbase.CellComparator; 31 import org.apache.hadoop.hbase.HBaseInterfaceAudience; 32 import org.apache.hadoop.hbase.HColumnDescriptor; 33 import org.apache.hadoop.hbase.HRegionInfo; 34 import org.apache.hadoop.hbase.TableName; 35 import org.apache.hadoop.hbase.client.Scan; 36 import org.apache.hadoop.hbase.conf.PropagatingConfigurationObserver; 37 import org.apache.hadoop.hbase.io.HeapSize; 38 import org.apache.hadoop.hbase.io.compress.Compression; 39 import org.apache.hadoop.hbase.io.hfile.CacheConfig; 40 import org.apache.hadoop.hbase.io.hfile.HFileDataBlockEncoder; 41 import org.apache.hadoop.hbase.protobuf.generated.WALProtos.CompactionDescriptor; 42 import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext; 43 import org.apache.hadoop.hbase.regionserver.compactions.CompactionProgress; 44 import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; 45 import org.apache.hadoop.hbase.regionserver.compactions.CompactionThroughputController; 46 import org.apache.hadoop.hbase.security.User; 47 48 /** 49 * Interface for objects that hold a column family in a Region. Its a memstore and a set of zero or 50 * more StoreFiles, which stretch backwards over time. 51 */ 52 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) 53 @InterfaceStability.Evolving 54 public interface Store extends HeapSize, StoreConfigInformation, PropagatingConfigurationObserver { 55 56 /* The default priority for user-specified compaction requests. 57 * The user gets top priority unless we have blocking compactions. (Pri <= 0) 58 */ int PRIORITY_USER = 1; 59 int NO_PRIORITY = Integer.MIN_VALUE; 60 61 // General Accessors 62 CellComparator getComparator(); 63 64 Collection<StoreFile> getStorefiles(); 65 66 /** 67 * Close all the readers We don't need to worry about subsequent requests because the Region 68 * holds a write lock that will prevent any more reads or writes. 69 * @return the {@link StoreFile StoreFiles} that were previously being used. 70 * @throws IOException on failure 71 */ 72 Collection<StoreFile> close() throws IOException; 73 74 /** 75 * Return a scanner for both the memstore and the HStore files. Assumes we are not in a 76 * compaction. 77 * @param scan Scan to apply when scanning the stores 78 * @param targetCols columns to scan 79 * @return a scanner over the current key values 80 * @throws IOException on failure 81 */ 82 KeyValueScanner getScanner(Scan scan, final NavigableSet<byte[]> targetCols, long readPt) 83 throws IOException; 84 85 /** 86 * Get all scanners with no filtering based on TTL (that happens further down 87 * the line). 88 * @param cacheBlocks 89 * @param isGet 90 * @param usePread 91 * @param isCompaction 92 * @param matcher 93 * @param startRow 94 * @param stopRow 95 * @param readPt 96 * @return all scanners for this store 97 */ 98 List<KeyValueScanner> getScanners( 99 boolean cacheBlocks, 100 boolean isGet, 101 boolean usePread, 102 boolean isCompaction, 103 ScanQueryMatcher matcher, 104 byte[] startRow, 105 byte[] stopRow, 106 long readPt 107 ) throws IOException; 108 109 ScanInfo getScanInfo(); 110 111 /** 112 * Adds or replaces the specified KeyValues. 113 * <p> 114 * For each KeyValue specified, if a cell with the same row, family, and qualifier exists in 115 * MemStore, it will be replaced. Otherwise, it will just be inserted to MemStore. 116 * <p> 117 * This operation is atomic on each KeyValue (row/family/qualifier) but not necessarily atomic 118 * across all of them. 119 * @param cells 120 * @param readpoint readpoint below which we can safely remove duplicate KVs 121 * @return memstore size delta 122 * @throws IOException 123 */ 124 long upsert(Iterable<Cell> cells, long readpoint) throws IOException; 125 126 /** 127 * Adds a value to the memstore 128 * @param cell 129 * @return memstore size delta 130 */ 131 long add(Cell cell); 132 133 /** 134 * When was the last edit done in the memstore 135 */ 136 long timeOfOldestEdit(); 137 138 /** 139 * Removes a Cell from the memstore. The Cell is removed only if its key 140 * & memstoreTS match the key & memstoreTS value of the cell 141 * parameter. 142 * @param cell 143 */ 144 void rollback(final Cell cell); 145 146 FileSystem getFileSystem(); 147 148 149 /** 150 * @param maxKeyCount 151 * @param compression Compression algorithm to use 152 * @param isCompaction whether we are creating a new file in a compaction 153 * @param includeMVCCReadpoint whether we should out the MVCC readpoint 154 * @return Writer for a new StoreFile in the tmp dir. 155 */ 156 StoreFile.Writer createWriterInTmp( 157 long maxKeyCount, 158 Compression.Algorithm compression, 159 boolean isCompaction, 160 boolean includeMVCCReadpoint, 161 boolean includesTags 162 ) throws IOException; 163 164 /** 165 * @param maxKeyCount 166 * @param compression Compression algorithm to use 167 * @param isCompaction whether we are creating a new file in a compaction 168 * @param includeMVCCReadpoint whether we should out the MVCC readpoint 169 * @param shouldDropBehind should the writer drop caches behind writes 170 * @return Writer for a new StoreFile in the tmp dir. 171 */ 172 StoreFile.Writer createWriterInTmp( 173 long maxKeyCount, 174 Compression.Algorithm compression, 175 boolean isCompaction, 176 boolean includeMVCCReadpoint, 177 boolean includesTags, 178 boolean shouldDropBehind 179 ) throws IOException; 180 181 182 183 184 // Compaction oriented methods 185 186 boolean throttleCompaction(long compactionSize); 187 188 /** 189 * getter for CompactionProgress object 190 * @return CompactionProgress object; can be null 191 */ 192 CompactionProgress getCompactionProgress(); 193 194 CompactionContext requestCompaction() throws IOException; 195 196 /** 197 * @deprecated see requestCompaction(int, CompactionRequest, User) 198 */ 199 @Deprecated 200 CompactionContext requestCompaction(int priority, CompactionRequest baseRequest) 201 throws IOException; 202 203 CompactionContext requestCompaction(int priority, CompactionRequest baseRequest, User user) 204 throws IOException; 205 206 void cancelRequestedCompaction(CompactionContext compaction); 207 208 /** 209 * @deprecated see compact(CompactionContext, CompactionThroughputController, User) 210 */ 211 @Deprecated 212 List<StoreFile> compact(CompactionContext compaction, 213 CompactionThroughputController throughputController) throws IOException; 214 215 List<StoreFile> compact(CompactionContext compaction, 216 CompactionThroughputController throughputController, User user) throws IOException; 217 218 /** 219 * @return true if we should run a major compaction. 220 */ 221 boolean isMajorCompaction() throws IOException; 222 223 void triggerMajorCompaction(); 224 225 /** 226 * See if there's too much store files in this store 227 * @return true if number of store files is greater than the number defined in minFilesToCompact 228 */ 229 boolean needsCompaction(); 230 231 int getCompactPriority(); 232 233 StoreFlushContext createFlushContext(long cacheFlushId); 234 235 /** 236 * Call to complete a compaction. Its for the case where we find in the WAL a compaction 237 * that was not finished. We could find one recovering a WAL after a regionserver crash. 238 * See HBASE-2331. 239 * @param compaction the descriptor for compaction 240 * @param pickCompactionFiles whether or not pick up the new compaction output files and 241 * add it to the store 242 * @param removeFiles whether to remove/archive files from filesystem 243 */ 244 void replayCompactionMarker(CompactionDescriptor compaction, boolean pickCompactionFiles, 245 boolean removeFiles) 246 throws IOException; 247 248 // Split oriented methods 249 250 boolean canSplit(); 251 252 /** 253 * Determines if Store should be split 254 * @return byte[] if store should be split, null otherwise. 255 */ 256 byte[] getSplitPoint(); 257 258 // Bulk Load methods 259 260 /** 261 * This throws a WrongRegionException if the HFile does not fit in this region, or an 262 * InvalidHFileException if the HFile is not valid. 263 */ 264 void assertBulkLoadHFileOk(Path srcPath) throws IOException; 265 266 /** 267 * This method should only be called from Region. It is assumed that the ranges of values in the 268 * HFile fit within the stores assigned region. (assertBulkLoadHFileOk checks this) 269 * 270 * @param srcPathStr 271 * @param sequenceId sequence Id associated with the HFile 272 */ 273 Path bulkLoadHFile(String srcPathStr, long sequenceId) throws IOException; 274 275 // General accessors into the state of the store 276 // TODO abstract some of this out into a metrics class 277 278 /** 279 * @return <tt>true</tt> if the store has any underlying reference files to older HFiles 280 */ 281 boolean hasReferences(); 282 283 /** 284 * @return The size of this store's memstore, in bytes 285 */ 286 long getMemStoreSize(); 287 288 /** 289 * @return The amount of memory we could flush from this memstore; usually this is equal to 290 * {@link #getMemStoreSize()} unless we are carrying snapshots and then it will be the size of 291 * outstanding snapshots. 292 */ 293 long getFlushableSize(); 294 295 /** 296 * Returns the memstore snapshot size 297 * @return size of the memstore snapshot 298 */ 299 long getSnapshotSize(); 300 301 HColumnDescriptor getFamily(); 302 303 /** 304 * @return The maximum sequence id in all store files. 305 */ 306 long getMaxSequenceId(); 307 308 /** 309 * @return The maximum memstoreTS in all store files. 310 */ 311 long getMaxMemstoreTS(); 312 313 /** 314 * @return the data block encoder 315 */ 316 HFileDataBlockEncoder getDataBlockEncoder(); 317 318 /** @return aggregate size of all HStores used in the last compaction */ 319 long getLastCompactSize(); 320 321 /** @return aggregate size of HStore */ 322 long getSize(); 323 324 /** 325 * @return Count of store files 326 */ 327 int getStorefilesCount(); 328 329 /** 330 * @return The size of the store files, in bytes, uncompressed. 331 */ 332 long getStoreSizeUncompressed(); 333 334 /** 335 * @return The size of the store files, in bytes. 336 */ 337 long getStorefilesSize(); 338 339 /** 340 * @return The size of the store file indexes, in bytes. 341 */ 342 long getStorefilesIndexSize(); 343 344 /** 345 * Returns the total size of all index blocks in the data block indexes, including the root level, 346 * intermediate levels, and the leaf level for multi-level indexes, or just the root level for 347 * single-level indexes. 348 * @return the total size of block indexes in the store 349 */ 350 long getTotalStaticIndexSize(); 351 352 /** 353 * Returns the total byte size of all Bloom filter bit arrays. For compound Bloom filters even the 354 * Bloom blocks currently not loaded into the block cache are counted. 355 * @return the total size of all Bloom filters in the store 356 */ 357 long getTotalStaticBloomSize(); 358 359 // Test-helper methods 360 361 /** 362 * Used for tests. 363 * @return cache configuration for this Store. 364 */ 365 CacheConfig getCacheConfig(); 366 367 /** 368 * @return the parent region info hosting this store 369 */ 370 HRegionInfo getRegionInfo(); 371 372 RegionCoprocessorHost getCoprocessorHost(); 373 374 boolean areWritesEnabled(); 375 376 /** 377 * @return The smallest mvcc readPoint across all the scanners in this 378 * region. Writes older than this readPoint, are included in every 379 * read operation. 380 */ 381 long getSmallestReadPoint(); 382 383 String getColumnFamilyName(); 384 385 TableName getTableName(); 386 387 /** 388 * @return The number of cells flushed to disk 389 */ 390 long getFlushedCellsCount(); 391 392 /** 393 * @return The total size of data flushed to disk, in bytes 394 */ 395 long getFlushedCellsSize(); 396 397 /** 398 * @return The number of cells processed during minor compactions 399 */ 400 long getCompactedCellsCount(); 401 402 /** 403 * @return The total amount of data processed during minor compactions, in bytes 404 */ 405 long getCompactedCellsSize(); 406 407 /** 408 * @return The number of cells processed during major compactions 409 */ 410 long getMajorCompactedCellsCount(); 411 412 /** 413 * @return The total amount of data processed during major compactions, in bytes 414 */ 415 long getMajorCompactedCellsSize(); 416 417 /* 418 * @param o Observer who wants to know about changes in set of Readers 419 */ 420 void addChangedReaderObserver(ChangedReadersObserver o); 421 422 /* 423 * @param o Observer no longer interested in changes in set of Readers. 424 */ 425 void deleteChangedReaderObserver(ChangedReadersObserver o); 426 427 /** 428 * @return Whether this store has too many store files. 429 */ 430 boolean hasTooManyStoreFiles(); 431 432 /** 433 * Checks the underlying store files, and opens the files that have not 434 * been opened, and removes the store file readers for store files no longer 435 * available. Mainly used by secondary region replicas to keep up to date with 436 * the primary region files. 437 * @throws IOException 438 */ 439 void refreshStoreFiles() throws IOException; 440 441 /** 442 * This value can represent the degree of emergency of compaction for this store. It should be 443 * greater than or equal to 0.0, any value greater than 1.0 means we have too many store files. 444 * <ul> 445 * <li>if getStorefilesCount <= getMinFilesToCompact, return 0.0</li> 446 * <li>return (getStorefilesCount - getMinFilesToCompact) / (blockingFileCount - 447 * getMinFilesToCompact)</li> 448 * </ul> 449 * <p> 450 * And for striped stores, we should calculate this value by the files in each stripe separately 451 * and return the maximum value. 452 * <p> 453 * It is similar to {@link #getCompactPriority()} except that it is more suitable to use in a 454 * linear formula. 455 */ 456 double getCompactionPressure(); 457 458 /** 459 * Replaces the store files that the store has with the given files. Mainly used by 460 * secondary region replicas to keep up to date with 461 * the primary region files. 462 * @throws IOException 463 */ 464 void refreshStoreFiles(Collection<String> newFiles) throws IOException; 465 466 void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException; 467 468 boolean isPrimaryReplicaStore(); 469 }