org.apache.hadoop.hive.ql.parse

Class BaseSemanticAnalyzer

java.lang.Object

org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer

Direct Known Subclasses:

DDLSemanticAnalyzer, ExplainSemanticAnalyzer, ExplainSQRewriteSemanticAnalyzer, ExportSemanticAnalyzer, FunctionSemanticAnalyzer, ImportSemanticAnalyzer, LoadSemanticAnalyzer, MacroSemanticAnalyzer, SemanticAnalyzer

public abstract class BaseSemanticAnalyzer
extends Object

BaseSemanticAnalyzer.

Nested Class Summary

Nested Classes

Modifier and Type	Class and Description
class	BaseSemanticAnalyzer.AnalyzeRewriteContext
static class	BaseSemanticAnalyzer.TableSpec TableSpec.

Field Summary

Fields

Modifier and Type	Field and Description
protected Set<FileSinkDesc>	acidFileSinks A set of FileSinkOperators being written to in an ACID compliant way.
protected ColumnAccessInfo	columnAccessInfo
protected HiveConf	conf
protected SessionState.LogHelper	console
protected Context	ctx
protected Hive	db
protected FetchTask	fetchTask
static int	HIVE_COLUMN_ORDER_ASC
static int	HIVE_COLUMN_ORDER_DESC
protected HashMap<String,String>	idToTableNameMap
protected HashSet<ReadEntity>	inputs ReadEntitites that are passed to the hooks.
protected LineageInfo	linfo Lineage information for the query.
protected org.apache.commons.logging.Log	LOG
protected HashSet<WriteEntity>	outputs List of WriteEntities that are passed to the hooks.
protected QueryProperties	queryProperties
protected List<Task<? extends Serializable>>	rootTasks
protected static org.apache.commons.logging.Log	STATIC_LOG
protected TableAccessInfo	tableAccessInfo
protected ColumnAccessInfo	updateColumnAccessInfo Columns accessed by updates

Constructor Summary

Constructors

Constructor and Description
BaseSemanticAnalyzer(HiveConf conf)
BaseSemanticAnalyzer(HiveConf conf, Hive db)

Method Summary

Methods

Modifier and Type	Method and Description
void	analyze(ASTNode ast, Context ctx)
protected void	analyzeDDLSkewedValues(List<List<String>> skewedValues, ASTNode child) Handle skewed values in DDL.
abstract void	analyzeInternal(ASTNode ast)
protected List<String>	analyzeSkewedTablDDLColNames(List<String> skewedColNames, ASTNode child) Analyze list bucket column names
protected boolean	analyzeStoredAdDirs(ASTNode child) process stored as directories
static String	charSetString(String charSetName, String charSetString)
protected ListBucketingCtx	constructListBucketingCtx(List<String> skewedColNames, List<List<String>> skewedValues, Map<List<String>,String> skewedColValueLocationMaps, boolean isStoredAsSubDirectories, HiveConf conf) Construct list bucketing context.
protected static Hive	createHiveDB(HiveConf conf)
static String	escapeSQLString(String b) Escapes the string for AST; doesn't enclose it in quotes, however.
Set<FileSinkDesc>	getAcidFileSinks()
ColumnAccessInfo	getColumnAccessInfo() Gets the column access information.
static List<String>	getColumnNames(ASTNode ast)
protected List<Order>	getColumnNamesOrder(ASTNode ast)
protected List<FieldSchema>	getColumns(ASTNode ast)
static List<FieldSchema>	getColumns(ASTNode ast, boolean lowerCase) Get the list of FieldSchema out of the ASTNode.
protected Database	getDatabase(String dbName)
protected Database	getDatabase(String dbName, boolean throwException)
Hive	getDb()
static Map.Entry<String,String>	getDbTableNamePair(ASTNode tableNameNode)
static String	getDotName(String[] qname)
FetchTask	getFetchTask()
HashMap<String,String>	getIdToTableNameMap()
HashSet<ReadEntity>	getInputs()
LineageInfo	getLineageInfo() Gets the lineage information.
HashSet<WriteEntity>	getOutputs()
protected Partition	getPartition(Table table, Map<String,String> partSpec, boolean throwException)
protected List<Partition>	getPartitions(Table table, Map<String,String> partSpec, boolean throwException)
static String[]	getQualifiedTableName(ASTNode tabNameNode)
QueryProperties	getQueryProperties()
List<FieldSchema>	getResultSchema()
List<Task<? extends Serializable>>	getRootTasks()
protected List<String>	getSkewedValueFromASTNode(ASTNode ast) Given a ASTNode, return list of values.
protected List<String>	getSkewedValuesFromASTNode(Node node) Retrieve skewed values from ASTNode.
protected Table	getTable(String tblName)
protected Table	getTable(String[] qualified)
protected Table	getTable(String[] qualified, boolean throwException)
protected Table	getTable(String tblName, boolean throwException)
protected Table	getTable(String database, String tblName, boolean throwException)
TableAccessInfo	getTableAccessInfo() Gets the table access information.
protected static String	getTypeStringFromAST(ASTNode typeNode)
static String	getUnescapedName(ASTNode tableOrColumnNode) Get dequoted name from a table/column node.
static String	getUnescapedName(ASTNode tableOrColumnNode, String currentDatabase)
static String	getUnescapedUnqualifiedTableName(ASTNode node) Get the unqualified name from a table node.
ColumnAccessInfo	getUpdateColumnAccessInfo()
void	init(boolean clearPartsCache)
void	initCtx(Context ctx)
boolean	isValidPrefixSpec(Table tTable, Map<String,String> spec) Checks if given specification is proper specification for prefix of partition cols, for table partitioned by ds, hr, min valid ones are (ds='2008-04-08'), (ds='2008-04-08', hr='12'), (ds='2008-04-08', hr='12', min='30') invalid one is for example (ds='2008-04-08', min='30')
static void	readProps(ASTNode prop, Map<String,String> mapProp) Converts parsed key/value properties pairs into a map.
protected void	reset(boolean clearPartsCache)
void	setColumnAccessInfo(ColumnAccessInfo columnAccessInfo) Sets the column access information.
void	setFetchTask(FetchTask fetchTask)
void	setLineageInfo(LineageInfo linfo) Sets the lineage information.
void	setTableAccessInfo(TableAccessInfo tableAccessInfo) Sets the table access information.
void	setUpdateColumnAccessInfo(ColumnAccessInfo updateColumnAccessInfo)
boolean	skipAuthorization()
static String	stripIdentifierQuotes(String val)
static String	stripQuotes(String val)
protected String	toMessage(ErrorMsg message, Object detail)
protected ReadEntity	toReadEntity(org.apache.hadoop.fs.Path location)
protected ReadEntity	toReadEntity(String location)
protected WriteEntity	toWriteEntity(org.apache.hadoop.fs.Path location)
protected WriteEntity	toWriteEntity(String location)
static String	unescapeIdentifier(String val) Remove the encapsulating "`" pair from the identifier.
static String	unescapeSQLString(String b)
void	validate()
static void	validatePartColumnType(Table tbl, Map<String,String> partSpec, ASTNode astNode, HiveConf conf)
static void	validatePartSpec(Table tbl, Map<String,String> partSpec, ASTNode astNode, HiveConf conf, boolean shouldBeFull)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

STATIC_LOG

protected static final org.apache.commons.logging.Log STATIC_LOG

db

protected final Hive db

conf

protected final HiveConf conf

rootTasks

protected List<Task<? extends Serializable>> rootTasks

fetchTask

protected FetchTask fetchTask

LOG

protected final org.apache.commons.logging.Log LOG

console

protected final SessionState.LogHelper console

ctx

protected Context ctx

idToTableNameMap

protected HashMap<String,String> idToTableNameMap

queryProperties

protected QueryProperties queryProperties

acidFileSinks

protected Set<FileSinkDesc> acidFileSinks

A set of FileSinkOperators being written to in an ACID compliant way. We need to remember them here because when we build them we don't yet know the transaction id. We need to go back and set it once we actually start running the query.

HIVE_COLUMN_ORDER_ASC

public static int HIVE_COLUMN_ORDER_ASC

HIVE_COLUMN_ORDER_DESC

public static int HIVE_COLUMN_ORDER_DESC

inputs

protected HashSet<ReadEntity> inputs

ReadEntitites that are passed to the hooks.

outputs

protected HashSet<WriteEntity> outputs

List of WriteEntities that are passed to the hooks.

linfo

protected LineageInfo linfo

Lineage information for the query.

tableAccessInfo

protected TableAccessInfo tableAccessInfo

columnAccessInfo

protected ColumnAccessInfo columnAccessInfo

updateColumnAccessInfo

protected ColumnAccessInfo updateColumnAccessInfo

Columns accessed by updates

Constructor Detail

BaseSemanticAnalyzer

public BaseSemanticAnalyzer(HiveConf conf)
                     throws SemanticException

Throws:

SemanticException

BaseSemanticAnalyzer

public BaseSemanticAnalyzer(HiveConf conf,
                    Hive db)
                     throws SemanticException

Throws:

SemanticException

Method Detail

skipAuthorization

public boolean skipAuthorization()

createHiveDB

protected static Hive createHiveDB(HiveConf conf)
                            throws SemanticException

Throws:

SemanticException

getIdToTableNameMap

public HashMap<String,String> getIdToTableNameMap()

analyzeInternal

public abstract void analyzeInternal(ASTNode ast)
                              throws SemanticException

Throws:

SemanticException

init

public void init(boolean clearPartsCache)

initCtx

public void initCtx(Context ctx)

analyze

public void analyze(ASTNode ast,
           Context ctx)
             throws SemanticException

Throws:

SemanticException

validate

public void validate()
              throws SemanticException

Throws:

SemanticException

getRootTasks

public List<Task<? extends Serializable>> getRootTasks()

getFetchTask

public FetchTask getFetchTask()

Returns:

the fetchTask

setFetchTask

public void setFetchTask(FetchTask fetchTask)

Parameters:

fetchTask - the fetchTask to set

reset

protected void reset(boolean clearPartsCache)

stripIdentifierQuotes

public static String stripIdentifierQuotes(String val)

stripQuotes

public static String stripQuotes(String val)

charSetString

public static String charSetString(String charSetName,
                   String charSetString)
                            throws SemanticException

Throws:

SemanticException

getUnescapedName

public static String getUnescapedName(ASTNode tableOrColumnNode)

Get dequoted name from a table/column node.

Parameters:

tableOrColumnNode - the table or column node

Returns:

for table node, db.tab or tab. for column node column.

getDbTableNamePair

public static Map.Entry<String,String> getDbTableNamePair(ASTNode tableNameNode)

getUnescapedName

public static String getUnescapedName(ASTNode tableOrColumnNode,
                      String currentDatabase)

getQualifiedTableName

public static String[] getQualifiedTableName(ASTNode tabNameNode)
                                      throws SemanticException

Throws:

SemanticException

getDotName

public static String getDotName(String[] qname)
                         throws SemanticException

Throws:

SemanticException

getUnescapedUnqualifiedTableName

public static String getUnescapedUnqualifiedTableName(ASTNode node)

Get the unqualified name from a table node. This method works for table names qualified with their schema (e.g., "db.table") and table names without schema qualification. In both cases, it returns the table name without the schema.

Parameters:

node - the table node

Returns:

the table name without schema qualification (i.e., if name is "db.table" or "table", returns "table")

unescapeIdentifier

public static String unescapeIdentifier(String val)

Remove the encapsulating "`" pair from the identifier. We allow users to use "`" to escape identifier for table names, column names and aliases, in case that coincide with Hive language keywords.

readProps

public static void readProps(ASTNode prop,
             Map<String,String> mapProp)

Converts parsed key/value properties pairs into a map.

Parameters:

prop - ASTNode parent of the key/value pairs

mapProp - property map which receives the mappings

unescapeSQLString

public static String unescapeSQLString(String b)

escapeSQLString

public static String escapeSQLString(String b)

Escapes the string for AST; doesn't enclose it in quotes, however.

getInputs

public HashSet<ReadEntity> getInputs()

getOutputs

public HashSet<WriteEntity> getOutputs()

getResultSchema

public List<FieldSchema> getResultSchema()

Returns:

the schema for the fields which will be produced when the statement is executed, or null if not known

getColumns

protected List<FieldSchema> getColumns(ASTNode ast)
                                throws SemanticException

Throws:

SemanticException

getColumns

public static List<FieldSchema> getColumns(ASTNode ast,
                           boolean lowerCase)
                                    throws SemanticException

Get the list of FieldSchema out of the ASTNode.

Throws:

SemanticException

getColumnNames

public static List<String> getColumnNames(ASTNode ast)

getColumnNamesOrder

protected List<Order> getColumnNamesOrder(ASTNode ast)

getTypeStringFromAST

protected static String getTypeStringFromAST(ASTNode typeNode)
                                      throws SemanticException

Throws:

SemanticException

getLineageInfo

public LineageInfo getLineageInfo()

Gets the lineage information.

Returns:

LineageInfo associated with the query.

setLineageInfo

public void setLineageInfo(LineageInfo linfo)

Sets the lineage information.

Parameters:

linfo - The LineageInfo structure that is set in the optimization phase.

getTableAccessInfo

public TableAccessInfo getTableAccessInfo()

Gets the table access information.

Returns:

TableAccessInfo associated with the query.

setTableAccessInfo

public void setTableAccessInfo(TableAccessInfo tableAccessInfo)

Sets the table access information.

Parameters:

tableAccessInfo - The TableAccessInfo structure that is set in the optimization phase.

getColumnAccessInfo

public ColumnAccessInfo getColumnAccessInfo()

Gets the column access information.

Returns:

ColumnAccessInfo associated with the query.

setColumnAccessInfo

public void setColumnAccessInfo(ColumnAccessInfo columnAccessInfo)

Sets the column access information.

Parameters:

columnAccessInfo - The ColumnAccessInfo structure that is set immediately after the optimization phase.

getUpdateColumnAccessInfo

public ColumnAccessInfo getUpdateColumnAccessInfo()

setUpdateColumnAccessInfo

public void setUpdateColumnAccessInfo(ColumnAccessInfo updateColumnAccessInfo)

isValidPrefixSpec

public final boolean isValidPrefixSpec(Table tTable,
                        Map<String,String> spec)
                                throws HiveException

Checks if given specification is proper specification for prefix of partition cols, for table partitioned by ds, hr, min valid ones are (ds='2008-04-08'), (ds='2008-04-08', hr='12'), (ds='2008-04-08', hr='12', min='30') invalid one is for example (ds='2008-04-08', min='30')

Parameters:

spec - specification key-value map

Returns:

true if the specification is prefix; never returns false, but throws

Throws:

HiveException

getDb

public Hive getDb()

getQueryProperties

public QueryProperties getQueryProperties()

getAcidFileSinks

public Set<FileSinkDesc> getAcidFileSinks()

constructListBucketingCtx

protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames,
                                         List<List<String>> skewedValues,
                                         Map<List<String>,String> skewedColValueLocationMaps,
                                         boolean isStoredAsSubDirectories,
                                         HiveConf conf)

Construct list bucketing context.

Parameters:

skewedColNames -

skewedValues -

skewedColValueLocationMaps -

isStoredAsSubDirectories -

Returns:

getSkewedValueFromASTNode

protected List<String> getSkewedValueFromASTNode(ASTNode ast)

Given a ASTNode, return list of values. use case: create table xyz list bucketed (col1) with skew (1,2,5) AST Node is for (1,2,5)

Parameters:

ast -

Returns:

getSkewedValuesFromASTNode

protected List<String> getSkewedValuesFromASTNode(Node node)
                                           throws SemanticException

Retrieve skewed values from ASTNode.

Parameters:

node -

Returns:

Throws:

SemanticException

analyzeSkewedTablDDLColNames

protected List<String> analyzeSkewedTablDDLColNames(List<String> skewedColNames,
                                        ASTNode child)
                                             throws SemanticException

Analyze list bucket column names

Parameters:

skewedColNames -

child -

Returns:

Throws:

SemanticException

analyzeDDLSkewedValues

protected void analyzeDDLSkewedValues(List<List<String>> skewedValues,
                          ASTNode child)
                               throws SemanticException

Handle skewed values in DDL. It can be used by both skewed by ... on () and set skewed location ().

Parameters:

skewedValues -

child -

Throws:

SemanticException

analyzeStoredAdDirs

protected boolean analyzeStoredAdDirs(ASTNode child)

process stored as directories

Parameters:

child -

Returns:

validatePartSpec

public static void validatePartSpec(Table tbl,
                    Map<String,String> partSpec,
                    ASTNode astNode,
                    HiveConf conf,
                    boolean shouldBeFull)
                             throws SemanticException

Throws:

SemanticException

validatePartColumnType

public static void validatePartColumnType(Table tbl,
                          Map<String,String> partSpec,
                          ASTNode astNode,
                          HiveConf conf)
                                   throws SemanticException

Throws:

SemanticException

toWriteEntity

protected WriteEntity toWriteEntity(String location)
                             throws SemanticException

Throws:

SemanticException

toWriteEntity

protected WriteEntity toWriteEntity(org.apache.hadoop.fs.Path location)
                             throws SemanticException

Throws:

SemanticException

toReadEntity

protected ReadEntity toReadEntity(String location)
                           throws SemanticException

Throws:

SemanticException

toReadEntity

protected ReadEntity toReadEntity(org.apache.hadoop.fs.Path location)
                           throws SemanticException

Throws:

SemanticException

getDatabase

protected Database getDatabase(String dbName)
                        throws SemanticException

Throws:

SemanticException

getDatabase

protected Database getDatabase(String dbName,
                   boolean throwException)
                        throws SemanticException

Throws:

SemanticException

getTable

protected Table getTable(String[] qualified)
                  throws SemanticException

Throws:

SemanticException

getTable

protected Table getTable(String[] qualified,
             boolean throwException)
                  throws SemanticException

Throws:

SemanticException

getTable

protected Table getTable(String tblName)
                  throws SemanticException

Throws:

SemanticException

getTable

protected Table getTable(String tblName,
             boolean throwException)
                  throws SemanticException

Throws:

SemanticException

getTable

protected Table getTable(String database,
             String tblName,
             boolean throwException)
                  throws SemanticException

Throws:

SemanticException

getPartition

protected Partition getPartition(Table table,
                     Map<String,String> partSpec,
                     boolean throwException)
                          throws SemanticException

Throws:

SemanticException

getPartitions

protected List<Partition> getPartitions(Table table,
                            Map<String,String> partSpec,
                            boolean throwException)
                                 throws SemanticException

Throws:

SemanticException

toMessage

protected String toMessage(ErrorMsg message,
               Object detail)