public class HiveInputFormat<K extends org.apache.hadoop.io.WritableComparable,V extends org.apache.hadoop.io.Writable> extends Object implements org.apache.hadoop.mapred.InputFormat<K,V>, org.apache.hadoop.mapred.JobConfigurable
Modifier and Type | Class and Description |
---|---|
static class |
HiveInputFormat.HiveInputSplit
HiveInputSplit encapsulates an InputSplit with its corresponding
inputFormatClass.
|
static class |
HiveInputFormat.HiveInputSplitComparator |
Modifier and Type | Field and Description |
---|---|
protected MapWork |
mrwork |
protected Map<org.apache.hadoop.fs.Path,PartitionDesc> |
pathToPartitionInfo |
Constructor and Description |
---|
HiveInputFormat() |
Modifier and Type | Method and Description |
---|---|
static boolean |
canInjectCaches(Class<? extends org.apache.hadoop.mapred.InputFormat> clazz) |
static boolean |
canWrapForLlap(Class<? extends org.apache.hadoop.mapred.InputFormat> clazz,
boolean checkVector) |
void |
configure(org.apache.hadoop.mapred.JobConf job) |
static org.apache.hadoop.mapred.JobConf |
createConfForMmOriginalsSplit(org.apache.hadoop.mapred.JobConf conf,
List<org.apache.hadoop.fs.Path> dirsWithFileOriginals) |
static org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> |
getInputFormatFromCache(Class inputFormatClass,
org.apache.hadoop.mapred.JobConf job)
NOTE: this no longer wraps the IF for LLAP.
|
protected org.apache.hadoop.hive.common.ValidWriteIdList |
getMmValidWriteIds(org.apache.hadoop.mapred.JobConf conf,
TableDesc table,
org.apache.hadoop.hive.common.ValidWriteIdList validWriteIdList) |
protected static PartitionDesc |
getPartitionDescFromPath(Map<org.apache.hadoop.fs.Path,PartitionDesc> pathToPartitionInfo,
org.apache.hadoop.fs.Path dir) |
org.apache.hadoop.mapred.RecordReader |
getRecordReader(org.apache.hadoop.mapred.InputSplit split,
org.apache.hadoop.mapred.JobConf job,
org.apache.hadoop.mapred.Reporter reporter) |
org.apache.hadoop.mapred.InputSplit[] |
getSplits(org.apache.hadoop.mapred.JobConf job,
int numSplits) |
protected void |
init(org.apache.hadoop.mapred.JobConf job) |
static void |
injectLlapCaches(org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> inputFormat,
LlapIo<org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch> llapIo) |
static void |
processPathsForMmRead(List<org.apache.hadoop.fs.Path> dirs,
org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.hive.common.ValidWriteIdList validWriteIdList,
List<org.apache.hadoop.fs.Path> finalPaths,
List<org.apache.hadoop.fs.Path> pathsWithFileOriginals) |
static void |
pushFilters(org.apache.hadoop.mapred.JobConf jobConf,
TableScanOperator tableScan,
MapWork mrwork) |
protected void |
pushProjectionsAndFilters(org.apache.hadoop.mapred.JobConf jobConf,
Class inputFormatClass,
org.apache.hadoop.fs.Path splitPath) |
protected void |
pushProjectionsAndFilters(org.apache.hadoop.mapred.JobConf jobConf,
Class inputFormatClass,
org.apache.hadoop.fs.Path splitPath,
boolean nonNative) |
static org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> |
wrapForLlap(org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> inputFormat,
org.apache.hadoop.conf.Configuration conf,
PartitionDesc part) |
protected Map<org.apache.hadoop.fs.Path,PartitionDesc> pathToPartitionInfo
protected MapWork mrwork
public void configure(org.apache.hadoop.mapred.JobConf job)
configure
in interface org.apache.hadoop.mapred.JobConfigurable
public static org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> wrapForLlap(org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> inputFormat, org.apache.hadoop.conf.Configuration conf, PartitionDesc part) throws HiveException
HiveException
public static void injectLlapCaches(org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> inputFormat, LlapIo<org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch> llapIo)
public static boolean canWrapForLlap(Class<? extends org.apache.hadoop.mapred.InputFormat> clazz, boolean checkVector)
public static boolean canInjectCaches(Class<? extends org.apache.hadoop.mapred.InputFormat> clazz)
public static org.apache.hadoop.mapred.InputFormat<org.apache.hadoop.io.WritableComparable,org.apache.hadoop.io.Writable> getInputFormatFromCache(Class inputFormatClass, org.apache.hadoop.mapred.JobConf job) throws IOException
IOException
public org.apache.hadoop.mapred.RecordReader getRecordReader(org.apache.hadoop.mapred.InputSplit split, org.apache.hadoop.mapred.JobConf job, org.apache.hadoop.mapred.Reporter reporter) throws IOException
getRecordReader
in interface org.apache.hadoop.mapred.InputFormat<K extends org.apache.hadoop.io.WritableComparable,V extends org.apache.hadoop.io.Writable>
IOException
protected void init(org.apache.hadoop.mapred.JobConf job)
public static org.apache.hadoop.mapred.JobConf createConfForMmOriginalsSplit(org.apache.hadoop.mapred.JobConf conf, List<org.apache.hadoop.fs.Path> dirsWithFileOriginals)
protected org.apache.hadoop.hive.common.ValidWriteIdList getMmValidWriteIds(org.apache.hadoop.mapred.JobConf conf, TableDesc table, org.apache.hadoop.hive.common.ValidWriteIdList validWriteIdList) throws IOException
IOException
public static void processPathsForMmRead(List<org.apache.hadoop.fs.Path> dirs, org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.hive.common.ValidWriteIdList validWriteIdList, List<org.apache.hadoop.fs.Path> finalPaths, List<org.apache.hadoop.fs.Path> pathsWithFileOriginals) throws IOException
IOException
public org.apache.hadoop.mapred.InputSplit[] getSplits(org.apache.hadoop.mapred.JobConf job, int numSplits) throws IOException
getSplits
in interface org.apache.hadoop.mapred.InputFormat<K extends org.apache.hadoop.io.WritableComparable,V extends org.apache.hadoop.io.Writable>
IOException
protected static PartitionDesc getPartitionDescFromPath(Map<org.apache.hadoop.fs.Path,PartitionDesc> pathToPartitionInfo, org.apache.hadoop.fs.Path dir) throws IOException
IOException
public static void pushFilters(org.apache.hadoop.mapred.JobConf jobConf, TableScanOperator tableScan, MapWork mrwork)
protected void pushProjectionsAndFilters(org.apache.hadoop.mapred.JobConf jobConf, Class inputFormatClass, org.apache.hadoop.fs.Path splitPath)
protected void pushProjectionsAndFilters(org.apache.hadoop.mapred.JobConf jobConf, Class inputFormatClass, org.apache.hadoop.fs.Path splitPath, boolean nonNative)
Copyright © 2022 The Apache Software Foundation. All rights reserved.