public class SolrOutputFormat<K,V>
extends org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<K,V>
Modifier and Type | Field and Description |
---|---|
static String |
OUTPUT_ZIP_FILE
The key used to pass the boolean configuration parameter that instructs for
regular or zip file output
|
static String |
SETUP_OK
The parameter used to pass the solr config zip file information.
|
static String |
SOLR_RECORD_WRITER_BATCH_SIZE |
static String |
SOLR_RECORD_WRITER_MAX_SEGMENTS |
static String |
SOLR_WRITER_QUEUE_SIZE |
static String |
SOLR_WRITER_THREAD_COUNT |
static String |
ZIP_FILE_BASE_NAME
The base name of the zip file containing the configuration information.
|
static String |
ZIP_NAME
The key used to pass the zip file name through the configuration.
|
Constructor and Description |
---|
SolrOutputFormat() |
Modifier and Type | Method and Description |
---|---|
static void |
addSolrConfToDistributedCache(org.apache.hadoop.mapreduce.Job job,
File solrHomeZip) |
void |
checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext job) |
static File |
createSolrHomeZip(File solrHomeDir) |
static int |
getBatchSize(org.apache.hadoop.conf.Configuration jobConf) |
static String |
getOutputName(org.apache.hadoop.mapreduce.JobContext job) |
org.apache.hadoop.mapreduce.RecordWriter<K,V> |
getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext context) |
static String |
getSetupOk() |
static int |
getSolrWriterQueueSize(org.apache.hadoop.conf.Configuration conf)
Return the maximum size for the number of documents pending index writing.
|
static int |
getSolrWriterThreadCount(org.apache.hadoop.conf.Configuration conf)
Set the number of threads used for index writing
|
static String |
getZipName(org.apache.hadoop.conf.Configuration conf)
Return the file name portion of the configuration zip file, from the
configuration.
|
static boolean |
isOutputZipFormat(org.apache.hadoop.conf.Configuration conf)
return true if the output should be a zip file of the index, rather than
the raw index
|
static void |
setBatchSize(int count,
org.apache.hadoop.conf.Configuration jobConf) |
static void |
setOutputZipFormat(boolean output,
org.apache.hadoop.conf.Configuration conf)
configure the job to output zip files of the output index, or full
directory trees.
|
static void |
setSolrWriterQueueSize(int count,
org.apache.hadoop.conf.Configuration conf)
Set the maximum size of the the queue for documents to be written to the
index.
|
static void |
setSolrWriterThreadCount(int count,
org.apache.hadoop.conf.Configuration conf)
Get the number of threads used for index writing
|
static void |
setupSolrHomeCache(File solrHomeDir,
org.apache.hadoop.mapreduce.Job job) |
getCompressOutput, getDefaultWorkFile, getOutputCommitter, getOutputCompressorClass, getOutputPath, getPathForWorkFile, getUniqueFile, getWorkOutputPath, setCompressOutput, setOutputCompressorClass, setOutputName, setOutputPath
public static final String SETUP_OK
public static final String ZIP_NAME
public static final String ZIP_FILE_BASE_NAME
getZipName(Configuration jobConf)
.public static final String OUTPUT_ZIP_FILE
public static final String SOLR_WRITER_THREAD_COUNT
public static final String SOLR_WRITER_QUEUE_SIZE
public static final String SOLR_RECORD_WRITER_BATCH_SIZE
public static final String SOLR_RECORD_WRITER_MAX_SEGMENTS
public static String getSetupOk()
public static void setSolrWriterThreadCount(int count, org.apache.hadoop.conf.Configuration conf)
public static int getSolrWriterThreadCount(org.apache.hadoop.conf.Configuration conf)
public static void setSolrWriterQueueSize(int count, org.apache.hadoop.conf.Configuration conf)
public static int getSolrWriterQueueSize(org.apache.hadoop.conf.Configuration conf)
public static String getZipName(org.apache.hadoop.conf.Configuration conf)
public static void setOutputZipFormat(boolean output, org.apache.hadoop.conf.Configuration conf)
output
- true if should output zip filesconf
- to usepublic static boolean isOutputZipFormat(org.apache.hadoop.conf.Configuration conf)
conf
- to usepublic static String getOutputName(org.apache.hadoop.mapreduce.JobContext job)
public void checkOutputSpecs(org.apache.hadoop.mapreduce.JobContext job) throws IOException
checkOutputSpecs
in class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<K,V>
IOException
public org.apache.hadoop.mapreduce.RecordWriter<K,V> getRecordWriter(org.apache.hadoop.mapreduce.TaskAttemptContext context) throws IOException, InterruptedException
getRecordWriter
in class org.apache.hadoop.mapreduce.lib.output.FileOutputFormat<K,V>
IOException
InterruptedException
public static void setupSolrHomeCache(File solrHomeDir, org.apache.hadoop.mapreduce.Job job) throws IOException
IOException
public static File createSolrHomeZip(File solrHomeDir) throws IOException
IOException
public static void addSolrConfToDistributedCache(org.apache.hadoop.mapreduce.Job job, File solrHomeZip) throws IOException
IOException
public static int getBatchSize(org.apache.hadoop.conf.Configuration jobConf)
public static void setBatchSize(int count, org.apache.hadoop.conf.Configuration jobConf)
Copyright © 2000-2017 Apache Software Foundation. All Rights Reserved.