public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implements Serializable
Operator.OperatorFunc, Operator.State
Modifier and Type | Field and Description |
---|---|
protected HybridHashTableContainer |
firstSmallTable |
protected MapJoinTableContainer.ReusableGetAdaptor[] |
hashMapRowGetters |
protected HashTableLoader |
loader |
protected MapJoinTableContainer[] |
mapJoinTables |
protected MapJoinBytesTableContainer[] |
spilledMapJoinTables |
emptyList, joinKeys, joinKeysObjectInspectors, posBigTable
alias, aliasFilterTags, condn, countAfterReport, dummyObj, dummyObjVectors, filterMaps, filterTags, forwardCache, heartbeatInterval, intermediate, joinFilterObjectInspectors, joinFilters, joinValues, joinValuesObjectInspectors, joinValuesStandardObjectInspectors, needsPostEvaluation, noOuterJoin, NOTSKIPBIGTABLE, nullsafes, numAliases, offsets, order, residualJoinFilters, residualJoinFiltersOIs, rowContainerPostFilteredOuterJoin, rowContainerStandardObjectInspectors, skipVectors, spillTableDesc, totalSz
abortOp, asyncInitOperations, cContext, childOperators, childOperatorsArray, childOperatorsTag, colExprMap, conf, CONTEXT_NAME_KEY, done, groupKeyObject, HIVECOUNTERCREATEDFILES, HIVECOUNTERFATAL, id, indexForTezUnion, inputObjInspectors, isLogDebugEnabled, isLogInfoEnabled, isLogTraceEnabled, operatorId, out, outputObjInspector, parentOperators, PLOG, reporter, runTimeNumRows, state, statsMap
Modifier | Constructor and Description |
---|---|
protected |
MapJoinOperator()
Kryo ctor.
|
|
MapJoinOperator(AbstractMapJoinOperator<? extends MapJoinDesc> mjop) |
|
MapJoinOperator(CompilationOpContext ctx) |
Modifier and Type | Method and Description |
---|---|
protected boolean |
canSkipJoinProcessing(ExecMapperContext mapContext) |
protected boolean |
canSkipReload(ExecMapperContext mapContext) |
void |
cleanUpInputFileChangedOp() |
void |
closeOp(boolean abort)
All done.
|
protected void |
completeInitializationOp(Object[] os)
This method can be used to retrieve the results from async operations
started at init time - before the operator pipeline is started.
|
void |
endGroup()
Forward a record of join results.
|
void |
generateMapMetaData() |
protected HashTableLoader |
getHashTableLoader(org.apache.hadoop.conf.Configuration hconf) |
String |
getName()
Implements the getName function for the Node Interface.
|
static String |
getOperatorName() |
protected MapJoinKey |
getRefKey(byte alias) |
OperatorType |
getType()
Return the type of the specific operator among the
types in OperatorType.
|
protected List<ObjectInspector> |
getValueObjectInspectors(byte alias,
List<ObjectInspector>[] aliasToObjectInspectors) |
protected void |
initializeOp(org.apache.hadoop.conf.Configuration hconf)
Operator specific initialization.
|
protected boolean |
isInputFileChangeSensitive(ExecMapperContext mapContext) |
protected org.apache.commons.lang3.tuple.Pair<MapJoinTableContainer[],MapJoinTableContainerSerDe[]> |
loadHashTable(ExecMapperContext mapContext,
MapredContext mrContext) |
void |
process(Object row,
int tag)
Process the row.
|
protected void |
reloadHashTable(byte pos,
int partitionId)
Reload hashtable from the hash partition.
|
protected void |
reProcessBigTable(int partitionId)
Iterate over the big table row container and feed process() with leftover rows
|
protected JoinUtil.JoinResult |
setMapJoinKey(MapJoinTableContainer.ReusableGetAdaptor dest,
Object row,
byte alias) |
protected void |
spillBigTableRow(MapJoinTableContainer hybridHtContainer,
Object row)
Postpone processing the big table row temporarily by spilling it to a row container
|
void |
startGroup() |
checkAndGenObject, getFilteredValue, getFilterTag, getNextSize, getPosToAliasMap, hasFilter, internalForward, opAllowedAfterMapJoin, opAllowedBeforeMapJoin, reportProgress, setPosToAliasMap
abort, acceptLimitPushdown, allInitializedParentsAreClosed, areAllParentsInitialized, augmentPlan, cleanUpInputFileChanged, clone, cloneOp, cloneRecursiveChildren, close, columnNamesRowResolvedCanBeObtained, createDummy, defaultEndGroup, defaultStartGroup, dump, dump, flush, forward, getAdditionalCounters, getChildOperators, getChildren, getColumnExprMap, getCompilationOpContext, getConf, getConfiguration, getDone, getExecContext, getGroupKeyObject, getIdentifier, getIndexForTezUnion, getInputObjInspectors, getIsReduceSink, getNextCntr, getNumChild, getNumParent, getOperatorId, getOpTraits, getOutputObjInspector, getParentOperators, getReduceOutputName, getSchema, getStatistics, getStats, initEvaluators, initEvaluators, initEvaluatorsAndReturnStruct, initialize, initialize, initializeChildren, initializeLocalWork, initOperatorId, isUseBucketizedHiveInputFormat, jobClose, jobCloseOp, logStats, opAllowedBeforeSortMergeJoin, opAllowedConvertMapJoin, passExecContext, preorderMap, processGroup, removeChild, removeChildAndAdoptItsChildren, removeParent, removeParents, replaceChild, replaceParent, reset, resetStats, setAlias, setChildOperators, setColumnExprMap, setCompilationOpContext, setConf, setDone, setExecContext, setGroupKeyObject, setId, setIndexForTezUnion, setInputContext, setInputObjInspectors, setOperatorId, setOpTraits, setOutputCollector, setParentOperators, setReporter, setSchema, setStatistics, setUseBucketizedHiveInputFormat, supportAutomaticSortMergeJoin, supportSkewJoinOptimization, supportUnionRemoveOptimization, toString, toString
protected HashTableLoader loader
protected transient MapJoinTableContainer[] mapJoinTables
protected transient MapJoinTableContainer.ReusableGetAdaptor[] hashMapRowGetters
protected transient MapJoinBytesTableContainer[] spilledMapJoinTables
protected HybridHashTableContainer firstSmallTable
protected MapJoinOperator()
public MapJoinOperator(CompilationOpContext ctx)
public MapJoinOperator(AbstractMapJoinOperator<? extends MapJoinDesc> mjop)
public void endGroup() throws HiveException
CommonJoinOperator
endGroup
in class CommonJoinOperator<MapJoinDesc>
HiveException
public void startGroup() throws HiveException
startGroup
in class CommonJoinOperator<MapJoinDesc>
HiveException
protected HashTableLoader getHashTableLoader(org.apache.hadoop.conf.Configuration hconf)
protected void initializeOp(org.apache.hadoop.conf.Configuration hconf) throws HiveException
Operator
initializeOp
in class AbstractMapJoinOperator<MapJoinDesc>
HiveException
protected void completeInitializationOp(Object[] os) throws HiveException
Operator
completeInitializationOp
in class Operator<MapJoinDesc>
HiveException
protected List<ObjectInspector> getValueObjectInspectors(byte alias, List<ObjectInspector>[] aliasToObjectInspectors)
getValueObjectInspectors
in class AbstractMapJoinOperator<MapJoinDesc>
public void generateMapMetaData() throws HiveException
HiveException
protected org.apache.commons.lang3.tuple.Pair<MapJoinTableContainer[],MapJoinTableContainerSerDe[]> loadHashTable(ExecMapperContext mapContext, MapredContext mrContext) throws HiveException
HiveException
public void cleanUpInputFileChangedOp() throws HiveException
cleanUpInputFileChangedOp
in class Operator<MapJoinDesc>
HiveException
protected JoinUtil.JoinResult setMapJoinKey(MapJoinTableContainer.ReusableGetAdaptor dest, Object row, byte alias) throws HiveException
HiveException
protected MapJoinKey getRefKey(byte alias)
public void process(Object row, int tag) throws HiveException
Operator
process
in class Operator<MapJoinDesc>
row
- The object representing the row.tag
- The tag of the row usually means which parent this row comes from.
Rows with the same tag should have exactly the same rowInspector
all the time.HiveException
protected void spillBigTableRow(MapJoinTableContainer hybridHtContainer, Object row) throws HiveException
hybridHtContainer
- Hybrid hashtable containerrow
- big table rowHiveException
public void closeOp(boolean abort) throws HiveException
CommonJoinOperator
closeOp
in class AbstractMapJoinOperator<MapJoinDesc>
HiveException
protected void reloadHashTable(byte pos, int partitionId) throws IOException, HiveException, SerDeException, ClassNotFoundException
pos
- position of small tablepartitionId
- the partition of the small table to be reloaded fromIOException
HiveException
SerDeException
ClassNotFoundException
protected void reProcessBigTable(int partitionId) throws HiveException
partitionId
- the partition from which to take out spilled big table rowsHiveException
public String getName()
getName
in interface Node
getName
in class CommonJoinOperator<MapJoinDesc>
public static String getOperatorName()
public OperatorType getType()
Operator
getType
in class AbstractMapJoinOperator<MapJoinDesc>
protected boolean isInputFileChangeSensitive(ExecMapperContext mapContext)
protected boolean canSkipReload(ExecMapperContext mapContext)
protected boolean canSkipJoinProcessing(ExecMapperContext mapContext)
Copyright © 2017 The Apache Software Foundation. All rights reserved.