public class AcidUtils extends Object
Modifier and Type | Class and Description |
---|---|
static class |
AcidUtils.AcidBaseFileInfo
A simple wrapper class that stores the information about a base file and its type.
|
static class |
AcidUtils.AcidBaseFileType |
static class |
AcidUtils.AcidOperationalProperties |
static interface |
AcidUtils.Directory |
static class |
AcidUtils.Operation |
static class |
AcidUtils.ParsedDelta |
Modifier and Type | Field and Description |
---|---|
static String |
BASE_PREFIX |
static org.apache.hadoop.fs.PathFilter |
baseFileFilter |
static Pattern |
BUCKET_DIGIT_PATTERN |
static String |
BUCKET_DIGITS |
static String |
BUCKET_PREFIX |
static org.apache.hadoop.fs.PathFilter |
bucketFileFilter |
static String |
CONF_ACID_KEY |
static String |
DELETE_DELTA_PREFIX |
static org.apache.hadoop.fs.PathFilter |
deleteEventDeltaDirFilter |
static String |
DELTA_DIGITS |
static String |
DELTA_PREFIX |
static String |
DELTA_SIDE_FILE_SUFFIX |
static org.apache.hadoop.fs.PathFilter |
deltaFileFilter |
static org.apache.hadoop.fs.PathFilter |
hiddenFileFilter |
static Pattern |
LEGACY_BUCKET_DIGIT_PATTERN |
static String |
LEGACY_FILE_BUCKET_DIGITS |
static int |
MAX_STATEMENTS_PER_TXN
This must be in sync with
STATEMENT_DIGITS |
static org.apache.hadoop.fs.PathFilter |
originalBucketFilter |
static String |
STATEMENT_DIGITS
10K statements per tx.
|
Modifier and Type | Method and Description |
---|---|
static String |
baseDir(long txnId) |
static org.apache.hadoop.fs.Path |
createBucketFile(org.apache.hadoop.fs.Path subdir,
int bucket)
Create the bucket filename.
|
static org.apache.hadoop.fs.Path |
createFilename(org.apache.hadoop.fs.Path directory,
AcidOutputFormat.Options options)
Create a filename for a bucket file.
|
static HadoopShims.HdfsFileStatusWithId |
createOriginalObj(HadoopShims.HdfsFileStatusWithId childWithId,
org.apache.hadoop.fs.FileStatus child) |
static String |
deltaSubdir(long min,
long max)
This is format of delta dir name prior to Hive 1.3.x
|
static String |
deltaSubdir(long min,
long max,
int statementId)
Each write statement in a transaction creates its own delta dir.
|
static org.apache.hadoop.fs.Path[] |
deserializeDeleteDeltas(org.apache.hadoop.fs.Path root,
List<AcidInputFormat.DeltaMetaData> deleteDeltas)
Convert the list of begin/end transaction id pairs to a list of delete delta
directories.
|
static org.apache.hadoop.fs.Path[] |
deserializeDeltas(org.apache.hadoop.fs.Path root,
List<AcidInputFormat.DeltaMetaData> deltas)
Convert the list of begin/end transaction id pairs to a list of delta
directories.
|
static AcidUtils.AcidOperationalProperties |
getAcidOperationalProperties(org.apache.hadoop.conf.Configuration conf)
Returns the acidOperationalProperties for a given configuration.
|
static AcidUtils.AcidOperationalProperties |
getAcidOperationalProperties(Map<String,String> parameters)
Returns the acidOperationalProperties for a given map.
|
static AcidUtils.AcidOperationalProperties |
getAcidOperationalProperties(Properties props)
Returns the acidOperationalProperties for a given set of properties.
|
static AcidUtils.AcidOperationalProperties |
getAcidOperationalProperties(Table table)
Returns the acidOperationalProperties for a given table.
|
static AcidUtils.Directory |
getAcidState(org.apache.hadoop.fs.Path directory,
org.apache.hadoop.conf.Configuration conf,
ValidTxnList txnList) |
static AcidUtils.Directory |
getAcidState(org.apache.hadoop.fs.Path directory,
org.apache.hadoop.conf.Configuration conf,
ValidTxnList txnList,
boolean useFileIds,
boolean ignoreEmptyFiles)
Get the ACID state of the given directory.
|
static AcidUtils.Directory |
getAcidState(org.apache.hadoop.fs.Path directory,
org.apache.hadoop.conf.Configuration conf,
ValidTxnList txnList,
Ref<Boolean> useFileIds,
boolean ignoreEmptyFiles) |
static org.apache.hadoop.fs.Path[] |
getPaths(List<AcidUtils.ParsedDelta> deltas)
Convert a list of deltas to a list of delta directories.
|
static boolean |
isAcid(org.apache.hadoop.fs.Path directory,
org.apache.hadoop.conf.Configuration conf)
Is the given directory in ACID format?
|
static boolean |
isAcidTable(Table table)
Checks if a table is a valid ACID table.
|
static boolean |
isTablePropertyTransactional(org.apache.hadoop.conf.Configuration conf) |
static boolean |
isTablePropertyTransactional(Map<String,String> parameters) |
static boolean |
isTablePropertyTransactional(Properties props) |
static AcidOutputFormat.Options |
parseBaseOrDeltaBucketFilename(org.apache.hadoop.fs.Path bucketFile,
org.apache.hadoop.conf.Configuration conf)
Parse a bucket filename back into the options that would have created
the file.
|
static AcidUtils.ParsedDelta |
parsedDelta(org.apache.hadoop.fs.Path deltaDir) |
static AcidUtils.ParsedDelta |
parsedDelta(org.apache.hadoop.fs.Path deltaDir,
String deltaPrefix) |
static List<AcidInputFormat.DeltaMetaData> |
serializeDeltas(List<AcidUtils.ParsedDelta> deltas)
Convert the list of deltas into an equivalent list of begin/end
transaction id pairs.
|
static void |
setAcidOperationalProperties(org.apache.hadoop.conf.Configuration conf,
AcidUtils.AcidOperationalProperties properties)
Sets the acidOperationalProperties in the configuration object argument.
|
static void |
setAcidOperationalProperties(Map<String,String> parameters,
AcidUtils.AcidOperationalProperties properties)
Sets the acidOperationalProperties in the map object argument.
|
static void |
setTransactionalTableScan(org.apache.hadoop.conf.Configuration conf,
boolean isAcidTable) |
static void |
setTransactionalTableScan(Map<String,String> parameters,
boolean isAcidTable) |
static DataOperationType |
toDataOperationType(AcidUtils.Operation op)
Logically this should have been defined in Operation but that causes a dependency
on metastore package from exec jar (from the cluster) which is not allowed.
|
public static final String CONF_ACID_KEY
public static final String BASE_PREFIX
public static final org.apache.hadoop.fs.PathFilter baseFileFilter
public static final String DELTA_PREFIX
public static final String DELETE_DELTA_PREFIX
public static final String DELTA_SIDE_FILE_SUFFIX
public static final org.apache.hadoop.fs.PathFilter deltaFileFilter
public static final org.apache.hadoop.fs.PathFilter deleteEventDeltaDirFilter
public static final String BUCKET_PREFIX
public static final org.apache.hadoop.fs.PathFilter bucketFileFilter
public static final String BUCKET_DIGITS
public static final String LEGACY_FILE_BUCKET_DIGITS
public static final String DELTA_DIGITS
public static final String STATEMENT_DIGITS
public static final int MAX_STATEMENTS_PER_TXN
STATEMENT_DIGITS
public static final Pattern BUCKET_DIGIT_PATTERN
public static final Pattern LEGACY_BUCKET_DIGIT_PATTERN
public static final org.apache.hadoop.fs.PathFilter originalBucketFilter
public static final org.apache.hadoop.fs.PathFilter hiddenFileFilter
public static org.apache.hadoop.fs.Path createBucketFile(org.apache.hadoop.fs.Path subdir, int bucket)
subdir
- the subdirectory for the bucket.bucket
- the bucket numberpublic static String deltaSubdir(long min, long max)
public static String deltaSubdir(long min, long max, int statementId)
public static String baseDir(long txnId)
public static org.apache.hadoop.fs.Path createFilename(org.apache.hadoop.fs.Path directory, AcidOutputFormat.Options options)
directory
- the partition directoryoptions
- the options for writing the bucketpublic static AcidOutputFormat.Options parseBaseOrDeltaBucketFilename(org.apache.hadoop.fs.Path bucketFile, org.apache.hadoop.conf.Configuration conf)
bucketFile
- the path to a bucket fileconf
- the configurationpublic static DataOperationType toDataOperationType(AcidUtils.Operation op)
public static org.apache.hadoop.fs.Path[] getPaths(List<AcidUtils.ParsedDelta> deltas)
deltas
- the list of deltas out of a Directory object.public static List<AcidInputFormat.DeltaMetaData> serializeDeltas(List<AcidUtils.ParsedDelta> deltas)
deltas
is sorted.deltas
- public static org.apache.hadoop.fs.Path[] deserializeDeltas(org.apache.hadoop.fs.Path root, List<AcidInputFormat.DeltaMetaData> deltas) throws IOException
deltaSubdir(long, long, int)
root
- the root directorydeltas
- list of begin/end transaction id pairsIOException
public static org.apache.hadoop.fs.Path[] deserializeDeleteDeltas(org.apache.hadoop.fs.Path root, List<AcidInputFormat.DeltaMetaData> deleteDeltas) throws IOException
deltaSubdir(long, long, int)
root
- the root directorydeleteDeltas
- list of begin/end transaction id pairsIOException
public static AcidUtils.ParsedDelta parsedDelta(org.apache.hadoop.fs.Path deltaDir)
public static AcidUtils.ParsedDelta parsedDelta(org.apache.hadoop.fs.Path deltaDir, String deltaPrefix)
public static boolean isAcid(org.apache.hadoop.fs.Path directory, org.apache.hadoop.conf.Configuration conf) throws IOException
directory
- the partition directory to checkconf
- the query configurationIOException
public static AcidUtils.Directory getAcidState(org.apache.hadoop.fs.Path directory, org.apache.hadoop.conf.Configuration conf, ValidTxnList txnList) throws IOException
IOException
public static AcidUtils.Directory getAcidState(org.apache.hadoop.fs.Path directory, org.apache.hadoop.conf.Configuration conf, ValidTxnList txnList, boolean useFileIds, boolean ignoreEmptyFiles) throws IOException
directory
- the partition directory to analyzeconf
- the configurationtxnList
- the list of transactions that we are readingIOException
public static AcidUtils.Directory getAcidState(org.apache.hadoop.fs.Path directory, org.apache.hadoop.conf.Configuration conf, ValidTxnList txnList, Ref<Boolean> useFileIds, boolean ignoreEmptyFiles) throws IOException
IOException
public static HadoopShims.HdfsFileStatusWithId createOriginalObj(HadoopShims.HdfsFileStatusWithId childWithId, org.apache.hadoop.fs.FileStatus child)
public static boolean isTablePropertyTransactional(Properties props)
public static boolean isTablePropertyTransactional(Map<String,String> parameters)
public static boolean isTablePropertyTransactional(org.apache.hadoop.conf.Configuration conf)
public static void setTransactionalTableScan(Map<String,String> parameters, boolean isAcidTable)
public static void setTransactionalTableScan(org.apache.hadoop.conf.Configuration conf, boolean isAcidTable)
public static boolean isAcidTable(Table table)
table
- tablepublic static void setAcidOperationalProperties(org.apache.hadoop.conf.Configuration conf, AcidUtils.AcidOperationalProperties properties)
conf
- Mutable configuration objectproperties
- An acidOperationalProperties object to initialize from.public static void setAcidOperationalProperties(Map<String,String> parameters, AcidUtils.AcidOperationalProperties properties)
parameters
- Mutable map objectproperties
- An acidOperationalProperties object to initialize from.public static AcidUtils.AcidOperationalProperties getAcidOperationalProperties(Table table)
table
- A table objectpublic static AcidUtils.AcidOperationalProperties getAcidOperationalProperties(org.apache.hadoop.conf.Configuration conf)
conf
- A configuration objectpublic static AcidUtils.AcidOperationalProperties getAcidOperationalProperties(Properties props)
props
- A properties objectpublic static AcidUtils.AcidOperationalProperties getAcidOperationalProperties(Map<String,String> parameters)
parameters
- A parameters objectCopyright © 2021 The Apache Software Foundation. All rights reserved.