public class GenSparkUtils extends Object
Modifier | Constructor and Description |
---|---|
protected |
GenSparkUtils() |
Modifier and Type | Method and Description |
---|---|
void |
annotateMapWork(GenSparkProcContext context)
Fill MapWork with 'local' work and bucket information for SMB Join.
|
MapWork |
createMapWork(GenSparkProcContext context,
Operator<?> root,
SparkWork sparkWork,
PrunedPartitionList partitions) |
MapWork |
createMapWork(GenSparkProcContext context,
Operator<?> root,
SparkWork sparkWork,
PrunedPartitionList partitions,
boolean deferSetup) |
static org.apache.hadoop.fs.Path |
createMoveTask(Task<? extends Serializable> currTask,
boolean chDir,
FileSinkOperator fsOp,
ParseContext parseCtx,
List<Task<MoveWork>> mvTasks,
HiveConf hconf,
DependencyCollectionTask dependencyTask)
Create and add any dependent move tasks.
|
ReduceWork |
createReduceWork(GenSparkProcContext context,
Operator<?> root,
SparkWork sparkWork) |
static <T> T |
getChildOperator(Operator<?> root,
Class<T> klazz) |
static SparkEdgeProperty |
getEdgeProperty(HiveConf conf,
ReduceSinkOperator reduceSink,
ReduceWork reduceWork) |
int |
getNextSeqNumber() |
static GenSparkUtils |
getUtils() |
void |
processFileSink(GenSparkProcContext context,
FileSinkOperator fileSink) |
void |
processPartitionPruningSink(GenSparkProcContext context,
SparkPartitionPruningSinkOperator pruningSink)
Populate partition pruning information from the pruning sink operator to the
target MapWork (the MapWork for the big table side).
|
void |
removeUnionOperators(GenSparkProcContext context,
BaseWork work) |
void |
resetSequenceNumber() |
protected void |
setupMapWork(MapWork mapWork,
GenSparkProcContext context,
PrunedPartitionList partitions,
TableScanOperator root,
String alias_id) |
protected void |
setupReduceSink(GenSparkProcContext context,
ReduceWork reduceWork,
ReduceSinkOperator reduceSink) |
public static GenSparkUtils getUtils()
public void resetSequenceNumber()
public ReduceWork createReduceWork(GenSparkProcContext context, Operator<?> root, SparkWork sparkWork) throws SemanticException
SemanticException
protected void setupReduceSink(GenSparkProcContext context, ReduceWork reduceWork, ReduceSinkOperator reduceSink)
public MapWork createMapWork(GenSparkProcContext context, Operator<?> root, SparkWork sparkWork, PrunedPartitionList partitions) throws SemanticException
SemanticException
public MapWork createMapWork(GenSparkProcContext context, Operator<?> root, SparkWork sparkWork, PrunedPartitionList partitions, boolean deferSetup) throws SemanticException
SemanticException
protected void setupMapWork(MapWork mapWork, GenSparkProcContext context, PrunedPartitionList partitions, TableScanOperator root, String alias_id) throws SemanticException
SemanticException
public void removeUnionOperators(GenSparkProcContext context, BaseWork work) throws SemanticException
SemanticException
public void processFileSink(GenSparkProcContext context, FileSinkOperator fileSink) throws SemanticException
SemanticException
public static org.apache.hadoop.fs.Path createMoveTask(Task<? extends Serializable> currTask, boolean chDir, FileSinkOperator fsOp, ParseContext parseCtx, List<Task<MoveWork>> mvTasks, HiveConf hconf, DependencyCollectionTask dependencyTask)
GenMapRedUtils
. The difference is that it doesn't check
'isLinkedFileSink' and does not set parent dir for the linked file sinks.public void processPartitionPruningSink(GenSparkProcContext context, SparkPartitionPruningSinkOperator pruningSink)
context
- the spark context.pruningSink
- the pruner sink operator being processed.public static SparkEdgeProperty getEdgeProperty(HiveConf conf, ReduceSinkOperator reduceSink, ReduceWork reduceWork) throws SemanticException
SemanticException
public static <T> T getChildOperator(Operator<?> root, Class<T> klazz) throws SemanticException
SemanticException
public void annotateMapWork(GenSparkProcContext context) throws SemanticException
context
- context, containing references to MapWorks and their SMB information.SemanticException
public int getNextSeqNumber()
Copyright © 2022 The Apache Software Foundation. All rights reserved.