public class WriterImpl extends Object implements Writer, MemoryManager.Callback
Modifier and Type | Field and Description |
---|---|
static String |
BASE_TIMESTAMP_STRING |
static long |
MILLIS_PER_DAY |
static int |
MILLIS_PER_SECOND |
static long |
NANOS_PER_MILLI |
protected PhysicalWriter |
physWriter |
Constructor and Description |
---|
WriterImpl(org.apache.hadoop.fs.FileSystem fs,
org.apache.hadoop.fs.Path path,
OrcFile.WriterOptions opts) |
WriterImpl(PhysicalWriter writer,
org.apache.hadoop.fs.Path pathForMem,
OrcFile.WriterOptions opts) |
Modifier and Type | Method and Description |
---|---|
void |
addRowBatch(org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch batch)
Add a row batch to the ORC file.
|
void |
addUserMetadata(String name,
ByteBuffer value)
Add arbitrary meta-data to the ORC file.
|
void |
appendStripe(byte[] stripe,
int offset,
int length,
StripeInformation stripeInfo,
OrcProto.StripeStatistics stripeStatistics)
Fast stripe append to ORC file.
|
void |
appendUserMetadata(List<OrcProto.UserMetadataItem> userMetadata)
When fast stripe append is used for merging ORC stripes, after appending
the last stripe from a file, this interface must be used to merge any
user metadata.
|
boolean |
checkMemory(double newScale)
The writer needs to check its memory usage
|
void |
close()
Flush all of the buffers and close the file.
|
void |
ensureStream() |
long |
getNumberOfRows()
Row count gets updated when flushing the stripes.
|
long |
getRawDataSize()
Raw data size will be compute when writing the file footer.
|
TypeDescription |
getSchema()
Get the schema for this writer
|
long |
writeIntermediateFooter()
Write an intermediate footer on the file such that if the file is
truncated to the returned offset, it would be a valid ORC file.
|
protected final PhysicalWriter physWriter
public static long MILLIS_PER_DAY
public static long NANOS_PER_MILLI
public static final int MILLIS_PER_SECOND
public static final String BASE_TIMESTAMP_STRING
public WriterImpl(org.apache.hadoop.fs.FileSystem fs, org.apache.hadoop.fs.Path path, OrcFile.WriterOptions opts) throws IOException
IOException
public WriterImpl(PhysicalWriter writer, org.apache.hadoop.fs.Path pathForMem, OrcFile.WriterOptions opts) throws IOException
IOException
public boolean checkMemory(double newScale) throws IOException
MemoryManager.Callback
checkMemory
in interface MemoryManager.Callback
newScale
- the current scale factor for memory allocationsIOException
public void ensureStream() throws IOException
IOException
public TypeDescription getSchema()
Writer
public void addUserMetadata(String name, ByteBuffer value)
Writer
addUserMetadata
in interface Writer
name
- a key to label the data with.value
- the contents of the metadata.public void addRowBatch(org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch batch) throws IOException
Writer
addRowBatch
in interface Writer
batch
- the rows to addIOException
public void close() throws IOException
Writer
close
in interface Writer
IOException
public long getRawDataSize()
getRawDataSize
in interface Writer
public long getNumberOfRows()
getNumberOfRows
in interface Writer
public long writeIntermediateFooter() throws IOException
Writer
writeIntermediateFooter
in interface Writer
IOException
public void appendStripe(byte[] stripe, int offset, int length, StripeInformation stripeInfo, OrcProto.StripeStatistics stripeStatistics) throws IOException
Writer
appendStripe
in interface Writer
stripe
- - stripe as byte arrayoffset
- - offset within byte arraylength
- - length of stripe within byte arraystripeInfo
- - stripe informationstripeStatistics
- - stripe statistics (Protobuf objects can be
merged directly)IOException
public void appendUserMetadata(List<OrcProto.UserMetadataItem> userMetadata)
Writer
appendUserMetadata
in interface Writer
userMetadata
- - user metadataCopyright © 2017 The Apache Software Foundation. All rights reserved.