public class ShuffleHandler extends Object implements AttemptRegistrationListener
Modifier and Type | Method and Description |
---|---|
static int |
deserializeMetaData(ByteBuffer meta)
A helper function to deserialize the metadata returned by ShuffleHandler.
|
static ShuffleHandler |
get() |
int |
getPort() |
protected org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.Shuffle |
getShuffle(org.apache.hadoop.conf.Configuration conf) |
static void |
initializeAndStart(org.apache.hadoop.conf.Configuration conf) |
void |
registerAttemptDirs(org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.AttemptPathIdentifier identifier,
org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.AttemptPathInfo pathInfo) |
void |
registerDag(String applicationIdString,
int dagIdentifier,
org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> appToken,
String user,
String[] appDirs)
Register an application and it's associated credentials and user information.
|
static ByteBuffer |
serializeMetaData(int port)
Serialize the shuffle port into a ByteBuffer for use later on.
|
static ByteBuffer |
serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken)
A helper function to serialize the JobTokenIdentifier to be sent to the
ShuffleHandler as ServiceData.
|
static void |
shutdown() |
void |
start() |
protected void |
stop() |
void |
unregisterDag(String dir,
String applicationIdString,
int dagIdentifier)
Unregister a specific dag
This method and registerDag must be synchronized externally to prevent races in shuffle token registration/unregistration
|
public static final String SHUFFLE_HANDLER_LOCAL_DIRS
public static final String SHUFFLE_MANAGE_OS_CACHE
public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE
public static final String SHUFFLE_READAHEAD_BYTES
public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES
public static final String SHUFFLE_DIR_WATCHER_ENABLED
public static final boolean SHUFFLE_DIR_WATCHER_ENABLED_DEFAULT
protected org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.HttpPipelineFactory pipelineFact
public static final String SHUFFLE_PORT_CONFIG_KEY
public static final int DEFAULT_SHUFFLE_PORT
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT
public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE
public static final String CONNECTION_CLOSE
public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY
public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE
public static final String MAX_SHUFFLE_CONNECTIONS
public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS
public static final String MAX_SHUFFLE_THREADS
public static final int DEFAULT_MAX_SHUFFLE_THREADS
public static final String SHUFFLE_BUFFER_SIZE
public static final int DEFAULT_SHUFFLE_BUFFER_SIZE
public static final String SHUFFLE_TRANSFERTO_ALLOWED
public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED
public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED
public void registerAttemptDirs(org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.AttemptPathIdentifier identifier, org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.AttemptPathInfo pathInfo)
registerAttemptDirs
in interface AttemptRegistrationListener
public static void initializeAndStart(org.apache.hadoop.conf.Configuration conf) throws Exception
Exception
public static ShuffleHandler get()
public static ByteBuffer serializeMetaData(int port) throws IOException
port
- the port to be sent to the ApplciationMasterIOException
public static int deserializeMetaData(ByteBuffer meta) throws IOException
meta
- the metadata returned by the ShuffleHandlerIOException
public static ByteBuffer serializeServiceData(org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> jobToken) throws IOException
jobToken
- the job token to be used for authentication of
shuffle data requests.IOException
public int getPort()
public void registerDag(String applicationIdString, int dagIdentifier, org.apache.hadoop.security.token.Token<org.apache.tez.common.security.JobTokenIdentifier> appToken, String user, String[] appDirs)
applicationIdString
- dagIdentifier
- appToken
- user
- public void unregisterDag(String dir, String applicationIdString, int dagIdentifier)
dir
- applicationIdString
- dagIdentifier
- protected org.apache.hadoop.hive.llap.shufflehandler.ShuffleHandler.Shuffle getShuffle(org.apache.hadoop.conf.Configuration conf)
Copyright © 2016 The Apache Software Foundation. All rights reserved.