public class SparkUtilities extends Object
Constructor and Description |
---|
SparkUtilities() |
Modifier and Type | Method and Description |
---|---|
static org.apache.hadoop.io.BytesWritable |
copyBytesWritable(org.apache.hadoop.io.BytesWritable bw) |
static HiveKey |
copyHiveKey(HiveKey key) |
static SparkSession |
getSparkSession(HiveConf conf,
SparkSessionManager sparkSessionManager) |
static URI |
getURI(String path) |
static boolean |
needUploadToHDFS(URI source,
org.apache.spark.SparkConf sparkConf) |
static URI |
uploadToHDFS(URI source,
HiveConf conf)
Uploads a local file to HDFS
|
public static org.apache.hadoop.io.BytesWritable copyBytesWritable(org.apache.hadoop.io.BytesWritable bw)
public static URI getURI(String path) throws URISyntaxException
URISyntaxException
public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException
source
- conf
- IOException
public static boolean needUploadToHDFS(URI source, org.apache.spark.SparkConf sparkConf)
public static SparkSession getSparkSession(HiveConf conf, SparkSessionManager sparkSessionManager) throws HiveException
HiveException
Copyright © 2017 The Apache Software Foundation. All rights reserved.