org.apache.hadoop.hive.ql.io.merge

Class MergeFileWork

java.lang.Object

org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

org.apache.hadoop.hive.ql.plan.BaseWork

org.apache.hadoop.hive.ql.plan.MapWork

org.apache.hadoop.hive.ql.io.merge.MergeFileWork

All Implemented Interfaces:

Serializable, Cloneable, OperatorDesc

public class MergeFileWork
extends MapWork

See Also:

Serialized Form

Field Summary

Fields inherited from class org.apache.hadoop.hive.ql.plan.MapWork

SAMPLING_ON_PREV_MR, SAMPLING_ON_START

Fields inherited from class org.apache.hadoop.hive.ql.plan.BaseWork

vectorColumnNameMap, vectorColumnTypeMap, vectorScratchColumnTypeMap

Fields inherited from class org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

opProps, opTraits, statistics, vectorMode

Constructor Summary

Constructors

Constructor and Description
MergeFileWork(List<org.apache.hadoop.fs.Path> inputPaths, org.apache.hadoop.fs.Path outputDir, boolean hasDynamicPartitions, String srcTblInputFormat)
MergeFileWork(List<org.apache.hadoop.fs.Path> inputPaths, org.apache.hadoop.fs.Path outputDir, String srcTblInputFormat)

Method Summary

Methods

Modifier and Type	Method and Description
String	getInputformat()
Class<? extends org.apache.hadoop.mapred.InputFormat>	getInputformatClass()
List<org.apache.hadoop.fs.Path>	getInputPaths()
ListBucketingCtx	getListBucketingCtx()
String	getMergeLevel()
Long	getMinSplitSize()
org.apache.hadoop.fs.Path	getOutputDir()
String	getSourceTableInputFormat()
boolean	hasDynamicPartitions()
boolean	isGatheringStats()
boolean	isListBucketingAlterTableConcatenate()
void	resolveConcatenateMerge(HiveConf conf) alter table ...
void	resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf, org.apache.hadoop.fs.Path path, TableDesc tblDesc, ArrayList<String> aliases, PartitionDesc partDesc)
void	setHasDynamicPartitions(boolean hasDynamicPartitions)
void	setInputPaths(List<org.apache.hadoop.fs.Path> inputPaths)
void	setListBucketingCtx(ListBucketingCtx listBucketingCtx)
void	setOutputDir(org.apache.hadoop.fs.Path outputDir)
void	setSourceTableInputFormat(String srcTblInputFormat)

Methods inherited from class org.apache.hadoop.hive.ql.plan.MapWork

addIndexIntermediateFile, addMapWork, configureJobConf, deriveExplainAttributes, getAliases, getAliasToPartnInfo, getAliasToWork, getAllRootOperators, getBaseSrc, getBucketedColsByDirectory, getDoSplitsGrouping, getDummyTableScan, getEventSourceColumnNameMap, getEventSourcePartKeyExprMap, getEventSourceTableDescMap, getHadoopSupportsSplittable, getIndexIntermediateFile, getMapAliases, getMaxSplitSize, getMinSplitSizePerNode, getMinSplitSizePerRack, getNameToSplitSample, getNumMapTasks, getPartitionDescs, getPaths, getPathToAliases, getPathToPartitionInfo, getSamplingType, getSamplingTypeString, getSortedColsByDirectory, getTmpHDFSPath, getTruncatedPathToAliases, getVectorModeOn, getWorks, initialize, internTable, isInputFormatSorted, isLeftInputJoin, isMapperCannotSpanPartns, isUseBucketizedHiveInputFormat, isUseOneNullRowInputFormat, logPathToAliases, mergeAliasedInput, mergingInto, replaceRoots, setAliasToPartnInfo, setAliasToWork, setBaseSrc, setDoSplitsGrouping, setDummyTableScan, setEventSourceColumnNameMap, setEventSourcePartKeyExprMap, setEventSourceTableDescMap, setHadoopSupportsSplittable, setInputformat, setInputFormatSorted, setLeftInputJoin, setMapAliases, setMapperCannotSpanPartns, setMaxSplitSize, setMinSplitSize, setMinSplitSizePerNode, setMinSplitSizePerRack, setNameToSplitSample, setNumMapTasks, setPathToAliases, setPathToPartitionInfo, setSamplingType, setTmpHDFSPath, setUseBucketizedHiveInputFormat, setUseOneNullRowInputFormat

Methods inherited from class org.apache.hadoop.hive.ql.plan.BaseWork

addDummyOp, addSortCols, getAllLeafOperators, getAllOperators, getDummyOps, getMapRedLocalWork, getName, getSortCols, getTag, getVectorColumnNameMap, getVectorColumnTypeMap, getVectorScratchColumnTypeMap, setDummyOps, setGatheringStats, setMapRedLocalWork, setName, setTag, setVectorColumnNameMap, setVectorColumnTypeMap, setVectorScratchColumnTypeMap

Methods inherited from class org.apache.hadoop.hive.ql.plan.AbstractOperatorDesc

clone, getOpProps, getStatistics, getTraits, getVectorMode, setOpProps, setStatistics, setTraits, setVectorMode

Methods inherited from class java.lang.Object

equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

MergeFileWork

public MergeFileWork(List<org.apache.hadoop.fs.Path> inputPaths,
             org.apache.hadoop.fs.Path outputDir,
             String srcTblInputFormat)

MergeFileWork

public MergeFileWork(List<org.apache.hadoop.fs.Path> inputPaths,
             org.apache.hadoop.fs.Path outputDir,
             boolean hasDynamicPartitions,
             String srcTblInputFormat)

Method Detail

getInputPaths

public List<org.apache.hadoop.fs.Path> getInputPaths()

setInputPaths

public void setInputPaths(List<org.apache.hadoop.fs.Path> inputPaths)

getOutputDir

public org.apache.hadoop.fs.Path getOutputDir()

setOutputDir

public void setOutputDir(org.apache.hadoop.fs.Path outputDir)

getMinSplitSize

public Long getMinSplitSize()

Overrides:

getMinSplitSize in class MapWork

getInputformat

public String getInputformat()

Overrides:

getInputformat in class MapWork

getInputformatClass

public Class<? extends org.apache.hadoop.mapred.InputFormat> getInputformatClass()

isGatheringStats

public boolean isGatheringStats()

Overrides:

isGatheringStats in class BaseWork

hasDynamicPartitions

public boolean hasDynamicPartitions()

setHasDynamicPartitions

public void setHasDynamicPartitions(boolean hasDynamicPartitions)

resolveDynamicPartitionStoredAsSubDirsMerge

public void resolveDynamicPartitionStoredAsSubDirsMerge(HiveConf conf,
                                               org.apache.hadoop.fs.Path path,
                                               TableDesc tblDesc,
                                               ArrayList<String> aliases,
                                               PartitionDesc partDesc)

Overrides:

resolveDynamicPartitionStoredAsSubDirsMerge in class MapWork

resolveConcatenateMerge

public void resolveConcatenateMerge(HiveConf conf)

alter table ... concatenate

If it is skewed table, use subdirectories in inputpaths.

getListBucketingCtx

public ListBucketingCtx getListBucketingCtx()

Returns:

the listBucketingCtx

setListBucketingCtx

public void setListBucketingCtx(ListBucketingCtx listBucketingCtx)

Parameters:

listBucketingCtx - the listBucketingCtx to set

isListBucketingAlterTableConcatenate

public boolean isListBucketingAlterTableConcatenate()

Returns:

the isListBucketingAlterTableConcatenate

getSourceTableInputFormat

public String getSourceTableInputFormat()

setSourceTableInputFormat

public void setSourceTableInputFormat(String srcTblInputFormat)

getMergeLevel

public String getMergeLevel()