public class IndexerMapReduce extends Configured implements Mapper<Text,Writable,Text,NutchWritable>, Reducer<Text,NutchWritable,Text,NutchIndexAction>
Modifier and Type | Field and Description |
---|---|
static String |
INDEXER_DELETE |
static String |
INDEXER_DELETE_ROBOTS_NOINDEX |
static String |
INDEXER_PARAMS |
static String |
INDEXER_SKIP_NOTMODIFIED |
static org.slf4j.Logger |
LOG |
static String |
URL_FILTERING |
static String |
URL_NORMALIZING |
Constructor and Description |
---|
IndexerMapReduce() |
Modifier and Type | Method and Description |
---|---|
void |
close() |
void |
configure(JobConf job) |
static void |
initMRJob(Path crawlDb,
Path linkDb,
Collection<Path> segments,
JobConf job) |
void |
map(Text key,
Writable value,
OutputCollector<Text,NutchWritable> output,
Reporter reporter) |
void |
reduce(Text key,
Iterator<NutchWritable> values,
OutputCollector<Text,NutchIndexAction> output,
Reporter reporter) |
getConf, setConf
public static final org.slf4j.Logger LOG
public static final String INDEXER_PARAMS
public static final String INDEXER_DELETE
public static final String INDEXER_DELETE_ROBOTS_NOINDEX
public static final String INDEXER_SKIP_NOTMODIFIED
public static final String URL_FILTERING
public static final String URL_NORMALIZING
public void configure(JobConf job)
configure
in interface JobConfigurable
public void map(Text key, Writable value, OutputCollector<Text,NutchWritable> output, Reporter reporter) throws IOException
map
in interface Mapper<Text,Writable,Text,NutchWritable>
IOException
public void reduce(Text key, Iterator<NutchWritable> values, OutputCollector<Text,NutchIndexAction> output, Reporter reporter) throws IOException
reduce
in interface Reducer<Text,NutchWritable,Text,NutchIndexAction>
IOException
public void close() throws IOException
close
in interface Closeable
close
in interface AutoCloseable
IOException
public static void initMRJob(Path crawlDb, Path linkDb, Collection<Path> segments, JobConf job)
Copyright © 2015 The Apache Software Foundation