public class OldFetcher extends org.apache.hadoop.conf.Configured implements org.apache.hadoop.util.Tool, org.apache.hadoop.mapred.MapRunnable<org.apache.hadoop.io.WritableComparable<?>,org.apache.hadoop.io.Writable,org.apache.hadoop.io.Text,NutchWritable>
Modifier and Type | Class and Description |
---|---|
static class |
OldFetcher.InputFormat |
Modifier and Type | Field and Description |
---|---|
static String |
CONTENT_REDIR |
static org.slf4j.Logger |
LOG |
static int |
PERM_REFRESH_TIME |
static String |
PROTOCOL_REDIR |
Constructor and Description |
---|
OldFetcher() |
OldFetcher(org.apache.hadoop.conf.Configuration conf) |
Modifier and Type | Method and Description |
---|---|
void |
close() |
void |
configure(org.apache.hadoop.mapred.JobConf job) |
void |
fetch(org.apache.hadoop.fs.Path segment,
int threads) |
static boolean |
isParsing(org.apache.hadoop.conf.Configuration conf) |
static boolean |
isStoringContent(org.apache.hadoop.conf.Configuration conf) |
static void |
main(String[] args)
Run the fetcher.
|
void |
run(org.apache.hadoop.mapred.RecordReader<org.apache.hadoop.io.WritableComparable<?>,org.apache.hadoop.io.Writable> input,
org.apache.hadoop.mapred.OutputCollector<org.apache.hadoop.io.Text,NutchWritable> output,
org.apache.hadoop.mapred.Reporter reporter) |
int |
run(String[] args) |
public static final org.slf4j.Logger LOG
public static final int PERM_REFRESH_TIME
public static final String CONTENT_REDIR
public static final String PROTOCOL_REDIR
public OldFetcher()
public OldFetcher(org.apache.hadoop.conf.Configuration conf)
public void configure(org.apache.hadoop.mapred.JobConf job)
configure
in interface org.apache.hadoop.mapred.JobConfigurable
public void close()
public static boolean isParsing(org.apache.hadoop.conf.Configuration conf)
public static boolean isStoringContent(org.apache.hadoop.conf.Configuration conf)
public void run(org.apache.hadoop.mapred.RecordReader<org.apache.hadoop.io.WritableComparable<?>,org.apache.hadoop.io.Writable> input, org.apache.hadoop.mapred.OutputCollector<org.apache.hadoop.io.Text,NutchWritable> output, org.apache.hadoop.mapred.Reporter reporter) throws IOException
run
in interface org.apache.hadoop.mapred.MapRunnable<org.apache.hadoop.io.WritableComparable<?>,org.apache.hadoop.io.Writable,org.apache.hadoop.io.Text,NutchWritable>
IOException
public void fetch(org.apache.hadoop.fs.Path segment, int threads) throws IOException
IOException
Copyright © 2014 The Apache Software Foundation