org.apache.tez.runtime.library.common.shuffle

Class ShuffleUtils

java.lang.Object

org.apache.tez.runtime.library.common.shuffle.ShuffleUtils

public class ShuffleUtils
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
static String	SHUFFLE_HANDLER_SERVICE_ID

Constructor Summary

Constructors

Constructor and Description
ShuffleUtils()

Method Summary

Methods

Modifier and Type	Method and Description
static StringBuilder	constructBaseURIForShuffleHandler(String host, int port, int partition, String appId, boolean sslShuffle)
static StringBuilder	constructBaseURIForShuffleHandler(String hostIdentifier, int partition, String appId, boolean sslShuffle)
static HttpConnection.HttpConnectionParams	constructHttpShuffleConnectionParams(org.apache.hadoop.conf.Configuration conf)
static URL	constructInputURL(String baseURI, List<InputAttemptIdentifier> inputs, boolean keepAlive)
static ByteBuffer	convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken)
static int	deserializeShuffleProviderMetaData(ByteBuffer meta)
static void	generateEventsForNonStartedOutput(List<org.apache.tez.runtime.api.Event> eventList, int numPhysicalOutputs, org.apache.tez.runtime.api.OutputContext context, boolean generateVmEvent, boolean isCompositeEvent) Generate events for outputs which have not been started.
static SecretKey	getJobTokenSecretFromTokenBytes(ByteBuffer meta)
static void	shuffleToDisk(OutputStream output, String hostIdentifier, InputStream input, long compressedLength, org.apache.commons.logging.Log LOG, String identifier)
static void	shuffleToMemory(byte[] shuffleData, InputStream input, int decompressedLength, int compressedLength, org.apache.hadoop.io.compress.CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, org.apache.commons.logging.Log LOG, String identifier)
static String	stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

SHUFFLE_HANDLER_SERVICE_ID

public static String SHUFFLE_HANDLER_SERVICE_ID

Constructor Detail

ShuffleUtils

public ShuffleUtils()

Method Detail

getJobTokenSecretFromTokenBytes

public static SecretKey getJobTokenSecretFromTokenBytes(ByteBuffer meta)
                                                 throws IOException

Throws:

IOException

convertJobTokenToBytes

public static ByteBuffer convertJobTokenToBytes(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken)
                                         throws IOException

Throws:

IOException

deserializeShuffleProviderMetaData

public static int deserializeShuffleProviderMetaData(ByteBuffer meta)
                                              throws IOException

Throws:

IOException

shuffleToMemory

public static void shuffleToMemory(byte[] shuffleData,
                   InputStream input,
                   int decompressedLength,
                   int compressedLength,
                   org.apache.hadoop.io.compress.CompressionCodec codec,
                   boolean ifileReadAhead,
                   int ifileReadAheadLength,
                   org.apache.commons.logging.Log LOG,
                   String identifier)
                            throws IOException

Throws:

IOException

shuffleToDisk

public static void shuffleToDisk(OutputStream output,
                 String hostIdentifier,
                 InputStream input,
                 long compressedLength,
                 org.apache.commons.logging.Log LOG,
                 String identifier)
                          throws IOException

Throws:

IOException

constructBaseURIForShuffleHandler

public static StringBuilder constructBaseURIForShuffleHandler(String host,
                                              int port,
                                              int partition,
                                              String appId,
                                              boolean sslShuffle)

constructBaseURIForShuffleHandler

public static StringBuilder constructBaseURIForShuffleHandler(String hostIdentifier,
                                              int partition,
                                              String appId,
                                              boolean sslShuffle)

constructInputURL

public static URL constructInputURL(String baseURI,
                    List<InputAttemptIdentifier> inputs,
                    boolean keepAlive)
                             throws MalformedURLException

Throws:

MalformedURLException

constructHttpShuffleConnectionParams

public static HttpConnection.HttpConnectionParams constructHttpShuffleConnectionParams(org.apache.hadoop.conf.Configuration conf)

generateEventsForNonStartedOutput

public static void generateEventsForNonStartedOutput(List<org.apache.tez.runtime.api.Event> eventList,
                                     int numPhysicalOutputs,
                                     org.apache.tez.runtime.api.OutputContext context,
                                     boolean generateVmEvent,
                                     boolean isCompositeEvent)
                                              throws IOException

Generate events for outputs which have not been started.

Parameters:

eventList -

numPhysicalOutputs -

context -

generateVmEvent - whether to generate a vm event or not

isCompositeEvent - whether to generate a CompositeDataMovementEvent or a DataMovementEvent

Throws:

IOException

stringify

public static String stringify(ShuffleUserPayloads.DataMovementEventPayloadProto dmProto)