public abstract class AbstractFileMergeOperator<T extends FileMergeDesc> extends Operator<T> implements Serializable
OrcFileMergeOperator
or RCFileMergeOperator
for more details.Operator.Counter, Operator.OperatorFunc, Operator.State
Modifier and Type | Field and Description |
---|---|
static String |
BACKUP_PREFIX |
protected org.apache.hadoop.fs.FileSystem |
fs |
protected org.apache.hadoop.mapred.JobConf |
jc |
static org.slf4j.Logger |
LOG |
static String |
UNION_SUDBIR_PREFIX |
abortOp, alias, asyncInitOperations, bucketingVersion, cContext, childOperators, childOperatorsArray, childOperatorsTag, conf, CONTEXT_NAME_KEY, done, groupKeyObject, HIVE_COUNTER_CREATED_DYNAMIC_PARTITIONS, HIVE_COUNTER_CREATED_FILES, HIVE_COUNTER_FATAL, id, indexForTezUnion, inputObjInspectors, numRows, operatorId, out, outputObjInspector, parentOperators, reporter, runTimeNumRows, state, statsMap
Modifier | Constructor and Description |
---|---|
protected |
AbstractFileMergeOperator()
Kryo ctor.
|
|
AbstractFileMergeOperator(CompilationOpContext ctx) |
Modifier and Type | Method and Description |
---|---|
protected void |
addIncompatibleFile(org.apache.hadoop.fs.Path path) |
protected void |
checkPartitionsMatch(org.apache.hadoop.fs.Path inputPath)
Validates that each input path belongs to the same partition since each
mapper merges the input to a single output directory
|
void |
closeOp(boolean abort)
Operator specific close routine.
|
protected void |
fixTmpPath(org.apache.hadoop.fs.Path path) |
protected void |
fixTmpPath(org.apache.hadoop.fs.Path inputPath,
int depthDiff)
Fixes tmpPath to point to the correct partition.
|
String |
getName()
Gets the name of the node.
|
static String |
getOperatorName() |
protected org.apache.hadoop.fs.Path |
getOutPath() |
void |
initializeOp(org.apache.hadoop.conf.Configuration hconf)
Operator specific initialization.
|
void |
jobCloseOp(org.apache.hadoop.conf.Configuration hconf,
boolean success) |
abort, acceptLimitPushdown, allInitializedParentsAreClosed, areAllParentsInitialized, augmentPlan, cleanUpInputFileChanged, cleanUpInputFileChangedOp, clone, cloneOp, cloneRecursiveChildren, close, columnNamesRowResolvedCanBeObtained, completeInitializationOp, createDummy, defaultEndGroup, defaultStartGroup, dump, dump, endGroup, flush, flushRecursive, forward, forward, forward, getAdditionalCounters, getBucketingVersion, getChildOperators, getChildren, getColumnExprMap, getCompilationOpContext, getConf, getConfiguration, getCounterName, getDone, getExecContext, getGroupKeyObject, getIdentifier, getIndexForTezUnion, getInputObjInspectors, getIsReduceSink, getMarker, getNextCntr, getNumChild, getNumParent, getOperatorId, getOpTraits, getOutputObjInspector, getParentOperators, getReduceOutputName, getSchema, getStatistics, getStats, getType, initEvaluators, initEvaluators, initEvaluatorsAndReturnStruct, initialize, initialize, initializeChildren, initializeLocalWork, initOperatorId, isUseBucketizedHiveInputFormat, jobClose, logicalEquals, logicalEqualsTree, logStats, opAllowedAfterMapJoin, opAllowedBeforeMapJoin, opAllowedBeforeSortMergeJoin, opAllowedConvertMapJoin, passExecContext, preorderMap, process, processGroup, removeChild, removeChildAndAdoptItsChildren, removeParent, removeParents, replaceChild, replaceParent, reset, setAlias, setBucketingVersion, setChildOperators, setColumnExprMap, setCompilationOpContext, setConf, setDone, setExecContext, setGroupKeyObject, setIndexForTezUnion, setInputContext, setInputObjInspectors, setMarker, setNextVectorBatchGroupStatus, setOpTraits, setOutputCollector, setParentOperators, setReporter, setSchema, setStatistics, setUseBucketizedHiveInputFormat, startGroup, supportAutomaticSortMergeJoin, supportSkewJoinOptimization, supportUnionRemoveOptimization, toString, toString
public static final String BACKUP_PREFIX
public static final String UNION_SUDBIR_PREFIX
public static final org.slf4j.Logger LOG
protected org.apache.hadoop.mapred.JobConf jc
protected org.apache.hadoop.fs.FileSystem fs
protected AbstractFileMergeOperator()
public AbstractFileMergeOperator(CompilationOpContext ctx)
public void initializeOp(org.apache.hadoop.conf.Configuration hconf) throws HiveException
Operator
initializeOp
in class Operator<T extends FileMergeDesc>
HiveException
protected void fixTmpPath(org.apache.hadoop.fs.Path inputPath, int depthDiff) throws IOException
inputPath
- - input pathIOException
protected void checkPartitionsMatch(org.apache.hadoop.fs.Path inputPath) throws IOException
inputPath
- - input pathIOException
protected void fixTmpPath(org.apache.hadoop.fs.Path path) throws IOException
IOException
public void closeOp(boolean abort) throws HiveException
Operator
closeOp
in class Operator<T extends FileMergeDesc>
HiveException
public void jobCloseOp(org.apache.hadoop.conf.Configuration hconf, boolean success) throws HiveException
jobCloseOp
in class Operator<T extends FileMergeDesc>
HiveException
public String getName()
Node
public static String getOperatorName()
protected final org.apache.hadoop.fs.Path getOutPath()
protected final void addIncompatibleFile(org.apache.hadoop.fs.Path path)
Copyright © 2022 The Apache Software Foundation. All rights reserved.