public class CrawlDbReader extends Object implements Closeable
Modifier and Type | Class and Description |
---|---|
static class |
CrawlDbReader.CrawlDatumCsvOutputFormat |
static class |
CrawlDbReader.CrawlDbDumpMapper |
static class |
CrawlDbReader.CrawlDbStatCombiner |
static class |
CrawlDbReader.CrawlDbStatMapper |
static class |
CrawlDbReader.CrawlDbStatReducer |
static class |
CrawlDbReader.CrawlDbTopNMapper |
static class |
CrawlDbReader.CrawlDbTopNReducer |
Modifier and Type | Field and Description |
---|---|
static org.slf4j.Logger |
LOG |
Constructor and Description |
---|
CrawlDbReader() |
Modifier and Type | Method and Description |
---|---|
void |
close() |
CrawlDatum |
get(String crawlDb,
String url,
org.apache.hadoop.conf.Configuration config) |
static void |
main(String[] args) |
void |
processDumpJob(String crawlDb,
String output,
org.apache.hadoop.conf.Configuration config,
String format,
String regex,
String status,
Integer retry) |
void |
processStatJob(String crawlDb,
org.apache.hadoop.conf.Configuration config,
boolean sort) |
void |
processTopNJob(String crawlDb,
long topN,
float min,
String output,
org.apache.hadoop.conf.Configuration config) |
void |
readUrl(String crawlDb,
String url,
org.apache.hadoop.conf.Configuration config) |
public void close()
close
in interface Closeable
close
in interface AutoCloseable
public void processStatJob(String crawlDb, org.apache.hadoop.conf.Configuration config, boolean sort) throws IOException
IOException
public CrawlDatum get(String crawlDb, String url, org.apache.hadoop.conf.Configuration config) throws IOException
IOException
public void readUrl(String crawlDb, String url, org.apache.hadoop.conf.Configuration config) throws IOException
IOException
public void processDumpJob(String crawlDb, String output, org.apache.hadoop.conf.Configuration config, String format, String regex, String status, Integer retry) throws IOException
IOException
public void processTopNJob(String crawlDb, long topN, float min, String output, org.apache.hadoop.conf.Configuration config) throws IOException
IOException
public static void main(String[] args) throws IOException
IOException
Copyright © 2014 The Apache Software Foundation