public class MetaStoreUtils extends Object
Modifier and Type | Class and Description |
---|---|
static class |
MetaStoreUtils.ColStatsObjWithSourceInfo |
static class |
MetaStoreUtils.FullTableName |
Modifier and Type | Field and Description |
---|---|
static int |
CAT_NAME
Position in the array returned by
parseDbName(java.lang.String, org.apache.hadoop.conf.Configuration) that has the catalog name. |
static String |
CATALOG_DB_SEPARATOR
This String is used to seaprate the catalog name from the database name.
|
static char |
CATALOG_DB_THRIFT_NAME_MARKER
This character is used to mark a database name as having a catalog name prepended.
|
static String |
DB_EMPTY_MARKER
Mark a database as being empty (as distinct from null).
|
static int |
DB_NAME
Position in the array returned by
parseDbName(java.lang.String, org.apache.hadoop.conf.Configuration) that has the database name. |
static ThreadLocal<DateFormat> |
PARTITION_DATE_FORMAT
A fixed date format to be used for hive partition column values.
|
static String |
TYPE_FROM_DESERIALIZER |
Constructor and Description |
---|
MetaStoreUtils() |
Modifier and Type | Method and Description |
---|---|
static ClassLoader |
addToClassPath(ClassLoader cloader,
String[] newPaths)
Add new elements to the classpath.
|
static List<ColumnStatisticsObj> |
aggrPartitionStats(List<ColumnStatistics> partStats,
String catName,
String dbName,
String tableName,
List<String> partNames,
List<String> colNames,
boolean areAllPartsFound,
boolean useDensityFunctionForNDVEstimation,
double ndvTuner) |
static List<ColumnStatisticsObj> |
aggrPartitionStats(Map<ColumnStatsAggregator,List<MetaStoreUtils.ColStatsObjWithSourceInfo>> colStatsMap,
List<String> partNames,
boolean areAllPartsFound,
boolean useDensityFunctionForNDVEstimation,
double ndvTuner) |
static boolean |
areSameColumns(List<FieldSchema> oldCols,
List<FieldSchema> newCols) |
static boolean |
checkUserHasHostProxyPrivileges(String user,
org.apache.hadoop.conf.Configuration conf,
String ipAddress)
Verify if the user is allowed to make DB notification related calls.
|
static void |
clearQuickStats(Map<String,String> params) |
static boolean |
columnsIncludedByNameType(List<FieldSchema> oldCols,
List<FieldSchema> newCols) |
static boolean |
compareFieldColumns(List<FieldSchema> schema1,
List<FieldSchema> schema2) |
static double |
decimalToDouble(Decimal decimal) |
static String |
encodeTableName(String name) |
static int |
findFreePort()
Finds a free port on the machine.
|
static int |
findFreePortExcepting(int portToExclude)
Finds a free port on the machine, but allow the
ability to specify a port number to not use, no matter what.
|
static int |
getArchivingLevel(Partition part) |
static String |
getColumnCommentsFromFieldSchema(List<FieldSchema> fieldSchemas) |
static String |
getColumnNameDelimiter(List<FieldSchema> fieldSchemas) |
static List<String> |
getColumnNames(List<FieldSchema> schema) |
static List<String> |
getColumnNamesForPartition(Partition partition) |
static List<String> |
getColumnNamesForTable(Table table) |
static String |
getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas)
Convert FieldSchemas to columnNames.
|
static String |
getColumnTypesFromFieldSchema(List<FieldSchema> fieldSchemas)
Convert FieldSchemas to columnTypes.
|
static String |
getDDLFromFieldSchema(String structName,
List<FieldSchema> fieldSchemas)
Convert FieldSchemas to Thrift DDL.
|
static String |
getDefaultCatalog(org.apache.hadoop.conf.Configuration conf) |
static String |
getIndexTableName(String dbName,
String baseTblName,
String indexName) |
static void |
getMergableCols(ColumnStatistics csNew,
Map<String,String> parameters) |
static <T> List<T> |
getMetaStoreListeners(Class<T> clazz,
org.apache.hadoop.conf.Configuration conf,
String listenerImplList)
create listener instances as per the configuration.
|
static Map<String,String> |
getMetaStoreSaslProperties(org.apache.hadoop.conf.Configuration conf,
boolean useSSL)
Read and return the meta store Sasl configuration.
|
static org.apache.hadoop.fs.Path |
getOriginalLocation(Partition part) |
static Properties |
getPartitionMetadata(Partition partition,
Table table) |
static Properties |
getPartSchemaFromTableSchema(StorageDescriptor sd,
Map<String,String> parameters,
Properties tblSchema)
Get partition level schema from table level schema.
|
static List<String> |
getPvals(List<FieldSchema> partCols,
Map<String,String> partSpec)
Given a list of partition columns and a partial mapping from
some partition columns to values the function returns the values
for the column.
|
static Properties |
getSchema(Partition part,
Table table) |
static Properties |
getSchema(StorageDescriptor sd,
StorageDescriptor tblsd,
Map<String,String> parameters,
String databaseName,
String tableName,
List<FieldSchema> partitionKeys) |
static Properties |
getSchemaWithoutCols(StorageDescriptor sd,
Map<String,String> parameters,
String databaseName,
String tableName,
List<FieldSchema> partitionKeys) |
static Properties |
getTableMetadata(Table table) |
static byte[] |
hashStorageDescriptor(StorageDescriptor sd,
MessageDigest md)
Produce a hash for the storage descriptor
|
static boolean |
isArchived(Partition part) |
static boolean |
isExternal(Map<String,String> tableParams) |
static boolean |
isExternalTable(Table table)
Determines whether a table is an external table.
|
static boolean |
isFastStatsSame(Partition oldPart,
Partition newPart)
Determines whether the "fast stats" for the passed partitions are the same.
|
static boolean |
isInsertOnlyTableParam(Map<String,String> params)
Duplicates AcidUtils; used in a couple places in metastore.
|
static boolean |
isMaterializedViewTable(Table table) |
static boolean |
isNonNativeTable(Table table) |
static boolean |
isTransactionalTable(Map<String,String> params)
Duplicates AcidUtils; used in a couple places in metastore.
|
static boolean |
isValidSchedulingPolicy(String str) |
static boolean |
isView(Table table) |
static void |
logAndThrowMetaException(Exception e)
Catches exceptions that can't be handled and bundles them to MetaException
|
static String |
makePartNameMatcher(Table table,
List<String> partVals) |
static void |
mergeColStats(ColumnStatistics csNew,
ColumnStatistics csOld) |
static MetaException |
newMetaException(Exception e)
convert Exception to MetaException, which sets the cause to such exception
|
static MetaException |
newMetaException(String errorMessage,
Exception e)
convert Exception to MetaException, which sets the cause to such exception
|
static String[] |
parseDbName(String dbName,
org.apache.hadoop.conf.Configuration conf)
Parse the catalog name out of the database name.
|
static WMPoolSchedulingPolicy |
parseSchedulingPolicy(String schedulingPolicy) |
static boolean |
partitionNameHasValidCharacters(List<String> partVals,
Pattern partitionValidationPattern) |
static void |
populateQuickStats(List<org.apache.hadoop.fs.FileStatus> fileStatus,
Map<String,String> params)
This method is invalid for MM and ACID tables unless fileStatus comes from AcidUtils.
|
static String |
prependCatalogToDbName(String dbName,
org.apache.hadoop.conf.Configuration conf)
Prepend the default 'hive' catalog onto the database name.
|
static String |
prependCatalogToDbName(String catalogName,
String dbName,
org.apache.hadoop.conf.Configuration conf)
Given a catalog name and database name cram them together into one string.
|
static String |
prependNotNullCatToDbName(String catalogName,
String dbName)
Given a catalog name and database name, cram them together into one string.
|
static boolean |
requireCalStats(Partition oldPart,
Partition newPart,
Table tbl,
EnvironmentContext environmentContext) |
static int |
startMetaStore() |
static int |
startMetaStore(org.apache.hadoop.conf.Configuration conf) |
static int |
startMetaStore(HadoopThriftAuthBridge bridge,
org.apache.hadoop.conf.Configuration conf) |
static void |
startMetaStore(int port,
HadoopThriftAuthBridge bridge) |
static void |
startMetaStore(int port,
HadoopThriftAuthBridge bridge,
org.apache.hadoop.conf.Configuration hiveConf) |
static Map<String,String> |
trimMapNulls(Map<String,String> dnMap,
boolean retrieveMapNullsAsEmptyStrings)
We have a need to sanity-check the map before conversion from persisted objects to
metadata thrift objects because null values in maps will cause a NPE if we send
across thrift.
|
static void |
updateBasicState(EnvironmentContext environmentContext,
Map<String,String> params) |
static boolean |
updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part,
Table table,
Warehouse wh,
boolean madeDir,
boolean forceRecompute,
EnvironmentContext environmentContext,
boolean isCreate)
Updates the numFiles and totalSize parameters for the passed Partition by querying
the warehouse if the passed Partition does not already have values for these parameters.
|
static boolean |
updatePartitionStatsFast(Partition part,
Table tbl,
Warehouse wh,
boolean madeDir,
boolean forceRecompute,
EnvironmentContext environmentContext,
boolean isCreate)
Updates the numFiles and totalSize parameters for the passed Partition by querying
the warehouse if the passed Partition does not already have values for these parameters.
|
static void |
updateTableStatsSlow(Database db,
Table tbl,
Warehouse wh,
boolean newDir,
boolean forceRecompute,
EnvironmentContext environmentContext)
Updates the numFiles and totalSize parameters for the passed Table by querying
the warehouse if the passed Table does not already have values for these parameters.
|
static boolean |
validateColumnName(String name) |
static boolean |
validateName(String name,
org.apache.hadoop.conf.Configuration conf)
validateName
Checks the name conforms to our standars which are: "[a-zA-z_0-9]+".
|
static void |
validatePartitionNameCharacters(List<String> partVals,
Pattern partitionValidationPattern) |
static String |
validateSkewedColNames(List<String> cols) |
static String |
validateSkewedColNamesSubsetCol(List<String> skewedColNames,
List<FieldSchema> cols) |
static String |
validateTblColumns(List<FieldSchema> cols) |
public static final ThreadLocal<DateFormat> PARTITION_DATE_FORMAT
public static final String TYPE_FROM_DESERIALIZER
public static final char CATALOG_DB_THRIFT_NAME_MARKER
public static final String CATALOG_DB_SEPARATOR
CATALOG_DB_THRIFT_NAME_MARKER
. # is
chosen because it is not used in regular expressions. this is only intended for use when
making old Thrift calls that do not support catalog names.public static final String DB_EMPTY_MARKER
public static final int CAT_NAME
parseDbName(java.lang.String, org.apache.hadoop.conf.Configuration)
that has the catalog name.public static final int DB_NAME
parseDbName(java.lang.String, org.apache.hadoop.conf.Configuration)
that has the database name.public static void logAndThrowMetaException(Exception e) throws MetaException
e
- exception to wrap.MetaException
- wrapper for the exceptionpublic static MetaException newMetaException(Exception e)
e
- cause of the exceptionpublic static MetaException newMetaException(String errorMessage, Exception e)
errorMessage
- the error message for this MetaExceptione
- cause of the exceptionpublic static Map<String,String> trimMapNulls(Map<String,String> dnMap, boolean retrieveMapNullsAsEmptyStrings)
public static List<ColumnStatisticsObj> aggrPartitionStats(List<ColumnStatistics> partStats, String catName, String dbName, String tableName, List<String> partNames, List<String> colNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException
MetaException
public static List<ColumnStatisticsObj> aggrPartitionStats(Map<ColumnStatsAggregator,List<MetaStoreUtils.ColStatsObjWithSourceInfo>> colStatsMap, List<String> partNames, boolean areAllPartsFound, boolean useDensityFunctionForNDVEstimation, double ndvTuner) throws MetaException
MetaException
public static double decimalToDouble(Decimal decimal)
public static void validatePartitionNameCharacters(List<String> partVals, Pattern partitionValidationPattern) throws MetaException
MetaException
public static byte[] hashStorageDescriptor(StorageDescriptor sd, MessageDigest md)
sd
- storage descriptor to hashmd
- message descriptor to use to generate the hashpublic static List<String> getColumnNamesForPartition(Partition partition)
public static boolean validateName(String name, org.apache.hadoop.conf.Configuration conf)
name
- the name to validateconf
- hive configurationpublic static boolean validateColumnName(String name)
public static String validateTblColumns(List<FieldSchema> cols)
public static boolean isExternalTable(Table table)
table
- table of interestpublic static boolean requireCalStats(Partition oldPart, Partition newPart, Table tbl, EnvironmentContext environmentContext)
public static boolean isView(Table table)
public static boolean isFastStatsSame(Partition oldPart, Partition newPart)
oldPart
- Old partition to compare.newPart
- New partition to compare.public static void updateTableStatsSlow(Database db, Table tbl, Warehouse wh, boolean newDir, boolean forceRecompute, EnvironmentContext environmentContext) throws MetaException
newDir
- if true, the directory was just created and can be assumed to be emptyforceRecompute
- Recompute stats even if the passed Table already has
these parameters setMetaException
public static void populateQuickStats(List<org.apache.hadoop.fs.FileStatus> fileStatus, Map<String,String> params)
public static boolean areSameColumns(List<FieldSchema> oldCols, List<FieldSchema> newCols)
public static void updateBasicState(EnvironmentContext environmentContext, Map<String,String> params)
public static boolean updatePartitionStatsFast(Partition part, Table tbl, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext, boolean isCreate) throws MetaException
part
- wh
- madeDir
- if true, the directory was just created and can be assumed to be emptyforceRecompute
- Recompute stats even if the passed Partition already has
these parameters setMetaException
public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part, Table table, Warehouse wh, boolean madeDir, boolean forceRecompute, EnvironmentContext environmentContext, boolean isCreate) throws MetaException
part
- wh
- madeDir
- if true, the directory was just created and can be assumed to be emptyforceRecompute
- Recompute stats even if the passed Partition already has
these parameters setMetaException
public static boolean columnsIncludedByNameType(List<FieldSchema> oldCols, List<FieldSchema> newCols)
public static boolean isTransactionalTable(Map<String,String> params)
public static boolean isInsertOnlyTableParam(Map<String,String> params)
public static <T> List<T> getMetaStoreListeners(Class<T> clazz, org.apache.hadoop.conf.Configuration conf, String listenerImplList) throws MetaException
clazz
- Class of the listenerconf
- configuration objectlistenerImplList
- Implementation class nameMetaException
- if there is any failure instantiating the classpublic static String validateSkewedColNamesSubsetCol(List<String> skewedColNames, List<FieldSchema> cols)
public static boolean isNonNativeTable(Table table)
public static List<String> getPvals(List<FieldSchema> partCols, Map<String,String> partSpec)
partCols
- the list of table partition columnspartSpec
- the partial mapping from partition column to valuespublic static String makePartNameMatcher(Table table, List<String> partVals) throws MetaException
MetaException
public static boolean compareFieldColumns(List<FieldSchema> schema1, List<FieldSchema> schema2)
schema1:
- The first schema to be comparedschema2:
- The second schema to be comparedpublic static boolean isArchived(Partition part)
public static org.apache.hadoop.fs.Path getOriginalLocation(Partition part)
public static int getArchivingLevel(Partition part) throws MetaException
MetaException
public static boolean partitionNameHasValidCharacters(List<String> partVals, Pattern partitionValidationPattern)
public static void getMergableCols(ColumnStatistics csNew, Map<String,String> parameters)
public static void mergeColStats(ColumnStatistics csNew, ColumnStatistics csOld) throws InvalidObjectException
InvalidObjectException
public static Map<String,String> getMetaStoreSaslProperties(org.apache.hadoop.conf.Configuration conf, boolean useSSL)
conf
- public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception
newPaths
- Array of classpath elementsException
public static boolean checkUserHasHostProxyPrivileges(String user, org.apache.hadoop.conf.Configuration conf, String ipAddress)
user
- the short user nameconf
- that contains the proxy user settingspublic static String getDDLFromFieldSchema(String structName, List<FieldSchema> fieldSchemas)
public static Properties getTableMetadata(Table table)
public static Properties getPartitionMetadata(Partition partition, Table table)
public static Properties getSchema(Partition part, Table table)
public static Properties getPartSchemaFromTableSchema(StorageDescriptor sd, Map<String,String> parameters, Properties tblSchema)
sd
- The Partition level Storage Descriptor.parameters
- partition level parameterstblSchema
- The table level schema from which this partition should be copied.public static Properties getSchemaWithoutCols(StorageDescriptor sd, Map<String,String> parameters, String databaseName, String tableName, List<FieldSchema> partitionKeys)
public static Properties getSchema(StorageDescriptor sd, StorageDescriptor tblsd, Map<String,String> parameters, String databaseName, String tableName, List<FieldSchema> partitionKeys)
public static String getColumnNameDelimiter(List<FieldSchema> fieldSchemas)
public static String getColumnNamesFromFieldSchema(List<FieldSchema> fieldSchemas)
public static String getColumnTypesFromFieldSchema(List<FieldSchema> fieldSchemas)
public static String getColumnCommentsFromFieldSchema(List<FieldSchema> fieldSchemas)
public static int startMetaStore(HadoopThriftAuthBridge bridge, org.apache.hadoop.conf.Configuration conf) throws Exception
Exception
public static int startMetaStore(org.apache.hadoop.conf.Configuration conf) throws Exception
Exception
public static void startMetaStore(int port, HadoopThriftAuthBridge bridge) throws Exception
Exception
public static void startMetaStore(int port, HadoopThriftAuthBridge bridge, org.apache.hadoop.conf.Configuration hiveConf) throws Exception
Exception
public static int findFreePort() throws IOException
IOException
public static int findFreePortExcepting(int portToExclude) throws IOException
IOException
public static String getIndexTableName(String dbName, String baseTblName, String indexName)
public static boolean isMaterializedViewTable(Table table)
public static List<String> getColumnNames(List<FieldSchema> schema)
public static boolean isValidSchedulingPolicy(String str)
public static WMPoolSchedulingPolicy parseSchedulingPolicy(String schedulingPolicy)
public static String prependCatalogToDbName(@Nullable String catalogName, @Nullable String dbName, org.apache.hadoop.conf.Configuration conf)
parseDbName(String, Configuration)
.catalogName
- catalog name, can be null if no known.dbName
- database name, can be null or empty.conf
- configuration object, used to determine default catalog if catalogName is nullpublic static String prependNotNullCatToDbName(String catalogName, String dbName)
parseDbName(String, Configuration)
.catalogName
- catalog name. This cannot be null. If this might be null use
prependCatalogToDbName(String, String, Configuration)
instead.dbName
- database name.public static String prependCatalogToDbName(String dbName, org.apache.hadoop.conf.Configuration conf)
dbName
- database nameconf
- configuration object, used to determine default catalogpublic static String[] parseDbName(String dbName, org.apache.hadoop.conf.Configuration conf) throws MetaException
dbName
- name of the database. This may or may not contain the catalog name.conf
- configuration object, used to determine the default catalog if it is not present
in the database name.MetaException
- if the name is not either just a database name or a catalog plus
database name with the proper delimiters.public static String getDefaultCatalog(org.apache.hadoop.conf.Configuration conf)
Copyright © 2022 The Apache Software Foundation. All rights reserved.