public class SemanticAnalyzer extends BaseSemanticAnalyzer
Modifier and Type | Class and Description |
---|---|
static class |
SemanticAnalyzer.GenericUDAFInfo
Class to store GenericUDAF related information.
|
protected static class |
SemanticAnalyzer.MaterializationRebuildMode |
protected static interface |
SemanticAnalyzer.PlannerContextFactory |
BaseSemanticAnalyzer.AnalyzeRewriteContext, BaseSemanticAnalyzer.TableSpec
Modifier and Type | Field and Description |
---|---|
protected BaseSemanticAnalyzer.AnalyzeRewriteContext |
analyzeRewrite |
protected CreateViewDesc |
createVwDesc |
protected boolean |
defaultJoinMerge |
protected boolean |
disableJoinMerge |
static String |
DUMMY_DATABASE |
static String |
DUMMY_TABLE |
protected MaterializedViewDesc |
materializedViewUpdateDesc |
protected String |
mvRebuildDbName |
protected SemanticAnalyzer.MaterializationRebuildMode |
mvRebuildMode |
protected String |
mvRebuildName |
protected boolean |
noscan |
protected LinkedHashMap<Operator<? extends OperatorDesc>,OpParseContext> |
opParseCtx |
protected List<FieldSchema> |
resultSchema |
static String |
SUBQUERY_TAG_1 |
static String |
SUBQUERY_TAG_2 |
protected TableMask |
tableMask |
protected HashMap<String,TableScanOperator> |
topOps |
protected UnparseTranslator |
unparseTranslator |
static String |
VALUES_TMP_TABLE_NAME_PREFIX |
protected ASTNode |
viewSelect |
protected ArrayList<String> |
viewsExpanded |
acidFileSinks, cacheUsage, cContext, columnAccessInfo, conf, console, ctx, db, fetchTask, HIVE_COLUMN_NULLS_FIRST, HIVE_COLUMN_NULLS_LAST, HIVE_COLUMN_ORDER_ASC, HIVE_COLUMN_ORDER_DESC, idToTableNameMap, inputs, linfo, LOG, outputs, queryProperties, queryState, rootTasks, STATIC_LOG, tableAccessInfo, transactionalInQuery, txnManager, updateColumnAccessInfo
Constructor and Description |
---|
SemanticAnalyzer(QueryState queryState) |
Modifier and Type | Method and Description |
---|---|
protected void |
addPartitionColsToInsert(List<FieldSchema> partCols,
Map<String,String> partSpec,
StringBuilder rewrittenQueryStr)
Append list of partition columns to Insert statement.
|
protected void |
addPartitionColsToInsert(List<FieldSchema> partCols,
StringBuilder rewrittenQueryStr)
Append list of partition columns to Insert statement, i.e.
|
protected ASTNode |
analyzeCreateView(ASTNode ast,
QB qb,
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContext plannerCtx) |
void |
analyzeInternal(ASTNode ast) |
protected void |
checkAcidTxnManager(Table table) |
protected boolean |
deleting(String destination) |
protected List<ExprNodeDesc> |
determineSprayKeys(QBParseInfo qbp,
String dest,
RowResolver inputRR) |
boolean |
doPhase1(ASTNode ast,
QB qb,
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx ctx_1,
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContext plannerCtx)
Phase 1: (including, but not limited to):
1.
|
void |
doPhase1QBExpr(ASTNode ast,
QBExpr qbexpr,
String id,
String alias) |
void |
doPhase1QBExpr(ASTNode ast,
QBExpr qbexpr,
String id,
String alias,
boolean insideView) |
Map<ASTNode,ExprNodeDesc> |
genAllExprNodeDesc(ASTNode expr,
RowResolver input)
Generates an expression node descriptors for the expression and children of it
with default TypeCheckCtx.
|
Map<ASTNode,ExprNodeDesc> |
genAllExprNodeDesc(ASTNode expr,
RowResolver input,
TypeCheckCtx tcCtx)
Generates all of the expression node descriptors for the expression and children of it
passed in the arguments.
|
static String |
generateErrorMessage(ASTNode ast,
String message) |
ExprNodeDesc |
genExprNodeDesc(ASTNode expr,
RowResolver input)
Generates an expression node descriptor for the expression with TypeCheckCtx.
|
ExprNodeDesc |
genExprNodeDesc(ASTNode expr,
RowResolver input,
boolean useCaching) |
ExprNodeDesc |
genExprNodeDesc(ASTNode expr,
RowResolver input,
boolean useCaching,
boolean foldExpr) |
ExprNodeDesc |
genExprNodeDesc(ASTNode expr,
RowResolver input,
RowResolver outerRR,
Map<ASTNode,org.apache.calcite.rel.RelNode> subqueryToRelNode,
boolean useCaching) |
ExprNodeDesc |
genExprNodeDesc(ASTNode expr,
RowResolver input,
TypeCheckCtx tcCtx)
Returns expression node descriptor for the expression.
|
protected Operator |
genFileSinkPlan(String dest,
QB qb,
Operator input) |
String |
genPartValueString(String partColType,
String partVal) |
Operator |
genPlan(QB qb) |
Operator |
genPlan(QB qb,
boolean skipAmbiguityCheck) |
static ASTNode |
genSelectDIAST(RowResolver rr) |
protected String |
getAliasId(String alias,
QB qb) |
HashSet<ReadEntity> |
getAllInputs() |
HashSet<WriteEntity> |
getAllOutputs() |
List<Task<? extends Serializable>> |
getAllRootTasks() |
static String |
getColumnInternalName(int pos) |
protected Table |
getDummyTable() |
protected String |
getFullTableNameForSQL(ASTNode n) |
static GenericUDAFEvaluator |
getGenericUDAFEvaluator(String aggName,
ArrayList<ExprNodeDesc> aggParameters,
ASTNode aggTree,
boolean isDistinct,
boolean isAllColumns)
Returns the GenericUDAFEvaluator for the aggregation.
|
static SemanticAnalyzer.GenericUDAFInfo |
getGenericUDAFInfo(GenericUDAFEvaluator evaluator,
GenericUDAFEvaluator.Mode emode,
ArrayList<ExprNodeDesc> aggParameters)
Returns the GenericUDAFInfo struct for the aggregation.
|
protected List<Long> |
getGroupingSets(List<ASTNode> groupByExpr,
QBParseInfo parseInfo,
String dest) |
protected List<Long> |
getGroupingSetsForCube(int size) |
protected List<Long> |
getGroupingSetsForRollup(int size) |
void |
getHintsFromQB(QBExpr qbExpr,
List<ASTNode> hints) |
void |
getHintsFromQB(QB qb,
List<ASTNode> hints) |
String |
getInvalidQueryMaterializationReason() |
List<LoadFileDesc> |
getLoadFileWork() |
void |
getMaterializationMetadata(QB qb) |
void |
getMetaData(QB qb) |
void |
getMetaData(QB qb,
boolean enableMaterialization) |
CompilationOpContext |
getOpContext() |
ParseContext |
getParseContext() |
QB |
getQB() |
List<FieldSchema> |
getResultSchema() |
protected Table |
getTableObjectByName(String tableName,
boolean throwException) |
static GenericUDAFEvaluator.Mode |
groupByDescModeToUDAFMode(GroupByDesc.Mode mode,
boolean isDistinct) |
RowResolver |
handleInsertStatementSpec(List<ExprNodeDesc> col_list,
String dest,
RowResolver outputRR,
RowResolver inputRR,
QB qb,
ASTNode selExprList)
This modifies the Select projections when the Select is part of an insert statement and
the insert statement specifies a column list for the target table, e.g.
|
void |
init(boolean clearPartsCache) |
void |
initParseCtx(ParseContext pctx) |
org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx |
initPhase1Ctx() |
static ExprNodeDesc |
isConstantParameterInAggregationParameters(String internalName,
List<ExprNodeDesc> reduceValues)
Check if the given internalName represents a constant parameter in aggregation parameters
of an aggregation tree.
|
boolean |
isValidQueryMaterialization() |
static ArrayList<WindowingSpec.WindowExpressionSpec> |
parseSelect(String selectExprStr) |
protected void |
processNoScanCommand(ASTNode tree)
process analyze ...
|
void |
processPositionAlias(ASTNode ast) |
<T extends OperatorDesc> |
putOpInsertMap(Operator<T> op,
RowResolver rr) |
protected static IllegalArgumentException |
raiseWrongType(String expectedTokName,
ASTNode n) |
static String |
replaceDefaultKeywordForMerge(String valueClause,
Table targetTable) |
protected void |
reset(boolean clearCache) |
protected static ASTNode |
rewriteASTWithMaskAndFilter(TableMask tableMask,
ASTNode ast,
org.antlr.runtime.TokenRewriteStream tokenRewriteStream,
Context ctx,
Hive db,
Map<String,Table> tabNameToTabObject,
Set<Integer> ignoredTokens) |
protected static ASTNode |
rewriteGroupingFunctionAST(List<ASTNode> grpByAstExprs,
ASTNode targetNode,
boolean noneSet) |
protected String |
rewriteQueryWithQualifiedNames(ASTNode ast,
org.antlr.runtime.TokenRewriteStream tokenRewriteStream) |
protected void |
saveViewDefinition() |
protected void |
setAST(ASTNode newAST) |
static long |
setBit(long bitmap,
int bitIdx) |
void |
setInvalidQueryMaterializationReason(String invalidQueryMaterializationReason) |
void |
setLoadFileWork(List<LoadFileDesc> loadFileWork) |
void |
setQB(QB qb) |
static long |
unsetBit(long bitmap,
int bitIdx) |
protected boolean |
updating(String destination) |
void |
validate() |
analyze, analyzeDDLSkewedValues, analyzeSkewedTablDDLColNames, analyzeStoredAdDirs, charSetString, constructListBucketingCtx, createFetchTask, createHiveDB, escapeSQLString, getAcidDdlDesc, getAcidFileSinks, getAutoCommitValue, getCacheUsage, getColumnAccessInfo, getColumnNames, getColumnNamesOrder, getColumns, getColumns, getColumns, getDatabase, getDatabase, getDb, getDbTableNamePair, getDotName, getFetchTask, getIdToTableNameMap, getInputs, getLineageInfo, getOutputs, getPartition, getPartitions, getQualifiedTableName, getQueryProperties, getQueryState, getRootTasks, getSkewedValueFromASTNode, getSkewedValuesFromASTNode, getTable, getTable, getTable, getTable, getTable, getTableAccessInfo, getTxnMgr, getTypeStringFromAST, getUnescapedName, getUnescapedName, getUnescapedUnqualifiedTableName, getUpdateColumnAccessInfo, hasEnabledOrValidatedConstraints, hasTransactionalInQuery, initCtx, isValidPrefixSpec, processCheckConstraints, processDefaultConstraints, processForeignKeys, processNotNullConstraints, processPrimaryKeys, processPrimaryKeys, processUniqueConstraints, processUniqueConstraints, readProps, setCacheUsage, setColumnAccessInfo, setFetchTask, setLineageInfo, setTableAccessInfo, setUpdateColumnAccessInfo, skipAuthorization, stripIdentifierQuotes, stripQuotes, toMessage, toReadEntity, toReadEntity, toReadEntity, toWriteEntity, toWriteEntity, toWriteEntity, tryQualifyPath, unescapeIdentifier, unescapeSQLString, validateCheckConstraint, validatePartColumnType, validatePartSpec
public static final String DUMMY_DATABASE
public static final String DUMMY_TABLE
public static final String SUBQUERY_TAG_1
public static final String SUBQUERY_TAG_2
public static final String VALUES_TMP_TABLE_NAME_PREFIX
protected HashMap<String,TableScanOperator> topOps
protected LinkedHashMap<Operator<? extends OperatorDesc>,OpParseContext> opParseCtx
protected List<FieldSchema> resultSchema
protected CreateViewDesc createVwDesc
protected MaterializedViewDesc materializedViewUpdateDesc
protected ASTNode viewSelect
protected final UnparseTranslator unparseTranslator
protected boolean noscan
protected SemanticAnalyzer.MaterializationRebuildMode mvRebuildMode
protected String mvRebuildDbName
protected String mvRebuildName
protected volatile boolean disableJoinMerge
protected final boolean defaultJoinMerge
protected TableMask tableMask
protected BaseSemanticAnalyzer.AnalyzeRewriteContext analyzeRewrite
public SemanticAnalyzer(QueryState queryState) throws SemanticException
SemanticException
protected void reset(boolean clearCache)
reset
in class BaseSemanticAnalyzer
public void initParseCtx(ParseContext pctx)
public ParseContext getParseContext()
public CompilationOpContext getOpContext()
public String genPartValueString(String partColType, String partVal) throws SemanticException
SemanticException
public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias) throws SemanticException
SemanticException
public void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias, boolean insideView) throws SemanticException
SemanticException
public static String replaceDefaultKeywordForMerge(String valueClause, Table targetTable) throws SemanticException
SemanticException
protected void setAST(ASTNode newAST)
public List<Task<? extends Serializable>> getAllRootTasks()
getAllRootTasks
in class BaseSemanticAnalyzer
public HashSet<ReadEntity> getAllInputs()
getAllInputs
in class BaseSemanticAnalyzer
public HashSet<WriteEntity> getAllOutputs()
getAllOutputs
in class BaseSemanticAnalyzer
public boolean doPhase1(ASTNode ast, QB qb, org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx ctx_1, org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContext plannerCtx) throws SemanticException
ast
- qb
- ctx_1
- SemanticException
public void getMaterializationMetadata(QB qb) throws SemanticException
SemanticException
public void getMetaData(QB qb) throws SemanticException
SemanticException
public void getMetaData(QB qb, boolean enableMaterialization) throws SemanticException
SemanticException
public <T extends OperatorDesc> Operator<T> putOpInsertMap(Operator<T> op, RowResolver rr)
protected static ASTNode rewriteGroupingFunctionAST(List<ASTNode> grpByAstExprs, ASTNode targetNode, boolean noneSet) throws SemanticException
SemanticException
public static String getColumnInternalName(int pos)
protected List<Long> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo, String dest) throws SemanticException
SemanticException
public static long setBit(long bitmap, int bitIdx)
public static long unsetBit(long bitmap, int bitIdx)
public RowResolver handleInsertStatementSpec(List<ExprNodeDesc> col_list, String dest, RowResolver outputRR, RowResolver inputRR, QB qb, ASTNode selExprList) throws SemanticException
public static GenericUDAFEvaluator getGenericUDAFEvaluator(String aggName, ArrayList<ExprNodeDesc> aggParameters, ASTNode aggTree, boolean isDistinct, boolean isAllColumns) throws SemanticException
SemanticException
public static SemanticAnalyzer.GenericUDAFInfo getGenericUDAFInfo(GenericUDAFEvaluator evaluator, GenericUDAFEvaluator.Mode emode, ArrayList<ExprNodeDesc> aggParameters) throws SemanticException
evaluator
- emode
- aggParameters
- The exprNodeDesc of the original parametersSemanticException
- when the UDAF is not found or has problems.public static GenericUDAFEvaluator.Mode groupByDescModeToUDAFMode(GroupByDesc.Mode mode, boolean isDistinct)
public static ExprNodeDesc isConstantParameterInAggregationParameters(String internalName, List<ExprNodeDesc> reduceValues)
reduceValues
- value columns of the corresponding ReduceSinkOperatorinternalName
- the internal name of this parameterprotected Operator genFileSinkPlan(String dest, QB qb, Operator input) throws SemanticException
SemanticException
protected List<ExprNodeDesc> determineSprayKeys(QBParseInfo qbp, String dest, RowResolver inputRR) throws SemanticException
SemanticException
public Operator genPlan(QB qb) throws SemanticException
SemanticException
public Operator genPlan(QB qb, boolean skipAmbiguityCheck) throws SemanticException
SemanticException
protected Table getDummyTable() throws SemanticException
SemanticException
public org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx initPhase1Ctx()
public void init(boolean clearPartsCache)
init
in class BaseSemanticAnalyzer
public void analyzeInternal(ASTNode ast) throws SemanticException
analyzeInternal
in class BaseSemanticAnalyzer
SemanticException
protected Table getTableObjectByName(String tableName, boolean throwException) throws HiveException
HiveException
protected String rewriteQueryWithQualifiedNames(ASTNode ast, org.antlr.runtime.TokenRewriteStream tokenRewriteStream) throws SemanticException
SemanticException
protected static ASTNode rewriteASTWithMaskAndFilter(TableMask tableMask, ASTNode ast, org.antlr.runtime.TokenRewriteStream tokenRewriteStream, Context ctx, Hive db, Map<String,Table> tabNameToTabObject, Set<Integer> ignoredTokens) throws SemanticException
SemanticException
public List<FieldSchema> getResultSchema()
getResultSchema
in class BaseSemanticAnalyzer
protected void saveViewDefinition() throws SemanticException
SemanticException
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input) throws SemanticException
SemanticException
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, RowResolver outerRR, Map<ASTNode,org.apache.calcite.rel.RelNode> subqueryToRelNode, boolean useCaching) throws SemanticException
SemanticException
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching) throws SemanticException
SemanticException
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, boolean useCaching, boolean foldExpr) throws SemanticException
SemanticException
public Map<ASTNode,ExprNodeDesc> genAllExprNodeDesc(ASTNode expr, RowResolver input) throws SemanticException
SemanticException
public ExprNodeDesc genExprNodeDesc(ASTNode expr, RowResolver input, TypeCheckCtx tcCtx) throws SemanticException
SemanticException
public Map<ASTNode,ExprNodeDesc> genAllExprNodeDesc(ASTNode expr, RowResolver input, TypeCheckCtx tcCtx) throws SemanticException
expr
- The expressioninput
- The row resolvertcCtx
- Customized type-checking contextSemanticException
- Failed to evaluate expressionpublic void validate() throws SemanticException
validate
in class BaseSemanticAnalyzer
SemanticException
protected ASTNode analyzeCreateView(ASTNode ast, QB qb, org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.PlannerContext plannerCtx) throws SemanticException
SemanticException
public void processPositionAlias(ASTNode ast) throws SemanticException
SemanticException
protected void processNoScanCommand(ASTNode tree) throws SemanticException
tree
- SemanticException
public QB getQB()
public void setQB(QB qb)
public static ArrayList<WindowingSpec.WindowExpressionSpec> parseSelect(String selectExprStr) throws SemanticException
SemanticException
protected boolean updating(String destination)
protected boolean deleting(String destination)
protected void checkAcidTxnManager(Table table) throws SemanticException
SemanticException
public static ASTNode genSelectDIAST(RowResolver rr)
public List<LoadFileDesc> getLoadFileWork()
public void setLoadFileWork(List<LoadFileDesc> loadFileWork)
public String getInvalidQueryMaterializationReason()
public void setInvalidQueryMaterializationReason(String invalidQueryMaterializationReason)
public boolean isValidQueryMaterialization()
protected String getFullTableNameForSQL(ASTNode n) throws SemanticException
SemanticException
protected static IllegalArgumentException raiseWrongType(String expectedTokName, ASTNode n)
protected void addPartitionColsToInsert(List<FieldSchema> partCols, StringBuilder rewrittenQueryStr)
protected void addPartitionColsToInsert(List<FieldSchema> partCols, Map<String,String> partSpec, StringBuilder rewrittenQueryStr)
Copyright © 2022 The Apache Software Foundation. All rights reserved.