public class HiveStreamingConnection extends Object implements StreamingConnection
// create delimited record writer whose schema exactly matches table schema
StrictDelimitedInputWriter writer = StrictDelimitedInputWriter.newBuilder()
.withFieldDelimiter(',')
.build();
// create and open streaming connection (default.src table has to exist already)
StreamingConnection connection = HiveStreamingConnection.newBuilder()
.withDatabase("default")
.withTable("src")
.withAgentInfo("nifi-agent")
.withRecordWriter(writer)
.withHiveConf(hiveConf)
.connect();
// begin a transaction, write records and commit 1st transaction
connection.beginTransaction();
connection.write("key1,val1".getBytes());
connection.write("key2,val2".getBytes());
connection.commitTransaction();
// begin another transaction, write more records and commit 2nd transaction
connection.beginTransaction();
connection.write("key3,val3".getBytes());
connection.write("key4,val4".getBytes());
connection.commitTransaction();
// close the streaming connection
connection.close();
Modifier and Type | Class and Description |
---|---|
static class |
HiveStreamingConnection.Builder |
static class |
HiveStreamingConnection.TxnState |
Modifier and Type | Method and Description |
---|---|
void |
abortTransaction()
Manually abort the opened transaction.
|
void |
beginTransaction()
Begin a transaction for writing.
|
void |
close()
Close connection
|
void |
commitTransaction()
Commit a transaction to make the writes visible for readers.
|
PartitionInfo |
createPartitionIfNotExists(List<String> partitionValues)
Creates a partition if it does not exist.
|
String |
getAgentInfo()
Get agent info that is set during streaming connection.
|
ConnectionStats |
getConnectionStats()
Gets stats about the streaming connection.
|
HiveConf |
getHiveConf()
Returns hive configuration object used during connection creation.
|
String |
getMetastoreUri()
Get metastore URI that metastore client uses.
|
List<String> |
getStaticPartitionValues()
Get any static partitions specified during streaming connection creation.
|
Table |
getTable()
Get the table used by streaming connection.
|
boolean |
isDynamicPartitioning()
Get if dynamic partitioning is used.
|
boolean |
isPartitionedTable()
Get if the specified table is partitioned table or not.
|
static HiveStreamingConnection.Builder |
newBuilder() |
String |
toString() |
void |
write(byte[] record)
Write record using RecordWriter.
|
void |
write(InputStream inputStream)
Write record using RecordWriter.
|
public static HiveStreamingConnection.Builder newBuilder()
public PartitionInfo createPartitionIfNotExists(List<String> partitionValues) throws StreamingException
PartitionHandler
createPartitionIfNotExists
in interface PartitionHandler
partitionValues
- - partition valuesStreamingException
- - any metastore related exceptionspublic void beginTransaction() throws StreamingException
StreamingConnection
beginTransaction
in interface StreamingConnection
StreamingException
- - if there are errors when beginning transactionpublic void commitTransaction() throws StreamingException
StreamingConnection
commitTransaction
in interface StreamingConnection
StreamingException
- - if there are errors when committing the open transactionpublic void abortTransaction() throws StreamingException
StreamingConnection
abortTransaction
in interface StreamingConnection
StreamingException
- - if there are errors when aborting the transactionpublic void write(byte[] record) throws StreamingException
StreamingConnection
write
in interface StreamingConnection
record
- - the data to be writtenStreamingException
- - if there are errors when writingpublic void write(InputStream inputStream) throws StreamingException
StreamingConnection
write
in interface StreamingConnection
inputStream
- - input stream of recordsStreamingException
- - if there are errors when writingpublic void close()
close
in interface StreamingConnection
public ConnectionStats getConnectionStats()
StreamingConnection
getConnectionStats
in interface StreamingConnection
public HiveConf getHiveConf()
StreamingConnection
getHiveConf
in interface StreamingConnection
public String getMetastoreUri()
ConnectionInfo
getMetastoreUri
in interface ConnectionInfo
public Table getTable()
ConnectionInfo
getTable
in interface ConnectionInfo
public List<String> getStaticPartitionValues()
ConnectionInfo
getStaticPartitionValues
in interface ConnectionInfo
public String getAgentInfo()
ConnectionInfo
getAgentInfo
in interface ConnectionInfo
public boolean isPartitionedTable()
ConnectionInfo
isPartitionedTable
in interface ConnectionInfo
public boolean isDynamicPartitioning()
ConnectionInfo
isDynamicPartitioning
in interface ConnectionInfo
Copyright © 2022 The Apache Software Foundation. All rights reserved.