@InterfaceAudience.Private @InterfaceStability.Unstable public class MergeManager extends Object
Constructor and Description |
---|
MergeManager(org.apache.hadoop.conf.Configuration conf,
org.apache.hadoop.fs.FileSystem localFS,
org.apache.hadoop.fs.LocalDirAllocator localDirAllocator,
org.apache.tez.runtime.api.InputContext inputContext,
Combiner combiner,
org.apache.tez.common.counters.TezCounter spilledRecordsCounter,
org.apache.tez.common.counters.TezCounter reduceCombineInputCounter,
org.apache.tez.common.counters.TezCounter mergedMapOutputsCounter,
org.apache.tez.runtime.library.common.shuffle.orderedgrouped.ExceptionReporter exceptionReporter,
long initialMemoryAvailable,
org.apache.hadoop.io.compress.CompressionCodec codec,
boolean ifileReadAheadEnabled,
int ifileReadAheadLength)
Construct the MergeManager.
|
Modifier and Type | Method and Description |
---|---|
TezRawKeyValueIterator |
close() |
void |
closeInMemoryFile(org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput mapOutput) |
void |
closeInMemoryMergedFile(org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput mapOutput) |
void |
closeOnDiskFile(FileChunk file) |
boolean |
isMergeComplete()
Should only be used after the Shuffle phaze is complete, otherwise can
return an invalid state since a merge may not be in progress dur to
inadequate inputs
|
org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput |
reserve(InputAttemptIdentifier srcAttemptIdentifier,
long requestedSize,
long compressedLength,
int fetcher) |
void |
waitForInMemoryMerge() |
void |
waitForShuffleToMergeMemory() |
public MergeManager(org.apache.hadoop.conf.Configuration conf, org.apache.hadoop.fs.FileSystem localFS, org.apache.hadoop.fs.LocalDirAllocator localDirAllocator, org.apache.tez.runtime.api.InputContext inputContext, Combiner combiner, org.apache.tez.common.counters.TezCounter spilledRecordsCounter, org.apache.tez.common.counters.TezCounter reduceCombineInputCounter, org.apache.tez.common.counters.TezCounter mergedMapOutputsCounter, org.apache.tez.runtime.library.common.shuffle.orderedgrouped.ExceptionReporter exceptionReporter, long initialMemoryAvailable, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAheadEnabled, int ifileReadAheadLength)
public void waitForInMemoryMerge() throws InterruptedException
InterruptedException
public void waitForShuffleToMergeMemory() throws InterruptedException
InterruptedException
public org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput reserve(InputAttemptIdentifier srcAttemptIdentifier, long requestedSize, long compressedLength, int fetcher) throws IOException
IOException
public void closeInMemoryFile(org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput mapOutput)
public void closeInMemoryMergedFile(org.apache.tez.runtime.library.common.shuffle.orderedgrouped.MapOutput mapOutput)
public void closeOnDiskFile(FileChunk file)
@InterfaceAudience.Private public boolean isMergeComplete()
public TezRawKeyValueIterator close() throws Throwable
Throwable
Copyright © 2015 Apache Software Foundation. All rights reserved.