public static class Generator.Selector extends Object implements Mapper<Text,CrawlDatum,FloatWritable,Generator.SelectorEntry>, Partitioner<FloatWritable,Writable>, Reducer<FloatWritable,Generator.SelectorEntry,FloatWritable,Generator.SelectorEntry>
Constructor and Description |
---|
Generator.Selector() |
Modifier and Type | Method and Description |
---|---|
void |
close() |
void |
configure(JobConf job) |
int |
getPartition(FloatWritable key,
Writable value,
int numReduceTasks)
Partition by host / domain or IP.
|
void |
map(Text key,
CrawlDatum value,
OutputCollector<FloatWritable,Generator.SelectorEntry> output,
Reporter reporter)
Select & invert subset due for fetch.
|
void |
reduce(FloatWritable key,
Iterator<Generator.SelectorEntry> values,
OutputCollector<FloatWritable,Generator.SelectorEntry> output,
Reporter reporter)
Collect until limit is reached.
|
public void configure(JobConf job)
configure
in interface JobConfigurable
public void close()
close
in interface Closeable
close
in interface AutoCloseable
public void map(Text key, CrawlDatum value, OutputCollector<FloatWritable,Generator.SelectorEntry> output, Reporter reporter) throws IOException
map
in interface Mapper<Text,CrawlDatum,FloatWritable,Generator.SelectorEntry>
IOException
public int getPartition(FloatWritable key, Writable value, int numReduceTasks)
getPartition
in interface Partitioner<FloatWritable,Writable>
public void reduce(FloatWritable key, Iterator<Generator.SelectorEntry> values, OutputCollector<FloatWritable,Generator.SelectorEntry> output, Reporter reporter) throws IOException
reduce
in interface Reducer<FloatWritable,Generator.SelectorEntry,FloatWritable,Generator.SelectorEntry>
IOException
Copyright © 2015 The Apache Software Foundation