org.apache.tez.runtime.library.common.shuffle.impl

Class ShuffleManager

java.lang.Object

org.apache.tez.runtime.library.common.shuffle.impl.ShuffleManager

All Implemented Interfaces:

FetcherCallback

public class ShuffleManager
extends Object
implements FetcherCallback

Constructor Summary

Constructors

Constructor and Description
ShuffleManager(org.apache.tez.runtime.api.InputContext inputContext, org.apache.hadoop.conf.Configuration conf, int numInputs, int bufferSize, boolean ifileReadAheadEnabled, int ifileReadAheadLength, org.apache.hadoop.io.compress.CompressionCodec codec, FetchedInputAllocator inputAllocator)

Method Summary

Methods

Modifier and Type	Method and Description
void	addCompletedInputWithData(InputAttemptIdentifier srcAttemptIdentifier, FetchedInput fetchedInput)
void	addCompletedInputWithNoData(InputAttemptIdentifier srcAttemptIdentifier)
void	addKnownInput(String hostName, int port, InputAttemptIdentifier srcAttemptIdentifier, int srcPhysicalIndex)
boolean	allInputsFetched()
void	fetchFailed(String host, InputAttemptIdentifier srcAttemptIdentifier, boolean connectFailed)
void	fetchSucceeded(String host, InputAttemptIdentifier srcAttemptIdentifier, FetchedInput fetchedInput, long fetchedBytes, long decompressedLength, long copyDuration)
FetchedInput	getNextInput()
boolean	newInputAvailable()
void	obsoleteKnownInput(InputAttemptIdentifier srcAttemptIdentifier)
void	run()
void	shutdown()
protected void	updateEventReceivedTime()

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Constructor Detail

ShuffleManager

public ShuffleManager(org.apache.tez.runtime.api.InputContext inputContext,
              org.apache.hadoop.conf.Configuration conf,
              int numInputs,
              int bufferSize,
              boolean ifileReadAheadEnabled,
              int ifileReadAheadLength,
              org.apache.hadoop.io.compress.CompressionCodec codec,
              FetchedInputAllocator inputAllocator)
               throws IOException

Throws:

IOException

Method Detail

run

public void run()
         throws IOException

Throws:

IOException

addKnownInput

public void addKnownInput(String hostName,
                 int port,
                 InputAttemptIdentifier srcAttemptIdentifier,
                 int srcPhysicalIndex)

addCompletedInputWithNoData

public void addCompletedInputWithNoData(InputAttemptIdentifier srcAttemptIdentifier)

updateEventReceivedTime

protected void updateEventReceivedTime()

addCompletedInputWithData

public void addCompletedInputWithData(InputAttemptIdentifier srcAttemptIdentifier,
                             FetchedInput fetchedInput)
                               throws IOException

Throws:

IOException

obsoleteKnownInput

public void obsoleteKnownInput(InputAttemptIdentifier srcAttemptIdentifier)

fetchSucceeded

public void fetchSucceeded(String host,
                  InputAttemptIdentifier srcAttemptIdentifier,
                  FetchedInput fetchedInput,
                  long fetchedBytes,
                  long decompressedLength,
                  long copyDuration)
                    throws IOException

Specified by:

fetchSucceeded in interface FetcherCallback

Throws:

IOException

fetchFailed

public void fetchFailed(String host,
               InputAttemptIdentifier srcAttemptIdentifier,
               boolean connectFailed)

Specified by:

fetchFailed in interface FetcherCallback

shutdown

public void shutdown()
              throws InterruptedException

Throws:

InterruptedException

newInputAvailable

public boolean newInputAvailable()

Returns:

true if there is another input ready for consumption.

allInputsFetched

public boolean allInputsFetched()

Returns:

true if all of the required inputs have been fetched.

getNextInput

public FetchedInput getNextInput()
                          throws InterruptedException

Returns:

the next available input, or null if there are no available inputs. This method will block if there are currently no available inputs, but more may become available.

Throws:

InterruptedException