org.apache.nutch.tools

Class AbstractCommonCrawlFormat

java.lang.Object

org.apache.nutch.tools.AbstractCommonCrawlFormat

All Implemented Interfaces:

CommonCrawlFormat

Direct Known Subclasses:

CommonCrawlFormatJackson, CommonCrawlFormatJettinson, CommonCrawlFormatSimple

public abstract class AbstractCommonCrawlFormat
extends Object
implements CommonCrawlFormat

Abstract class that implements CommonCrawlFormat interface.

Field Summary

Fields

Modifier and Type	Field and Description
protected Configuration	conf
protected byte[]	content
protected boolean	jsonArray
protected String	keyPrefix
protected Metadata	metadata
protected boolean	reverseKey
protected String	reverseKeyValue
protected boolean	simpleDateFormat
protected String	url

Constructor Summary

Constructors

Constructor and Description
AbstractCommonCrawlFormat(String url, byte[] content, Metadata metadata, Configuration nutchConf, CommonCrawlConfig config)

Method Summary

Methods

Modifier and Type	Method and Description
protected abstract void	closeArray(String key, boolean nested, boolean newline)
protected abstract void	closeObject(String key)
protected abstract String	generateJson()
protected String	getImported()
String	getJsonData()
protected String	getKey()
protected String	getMethod()
protected String	getRequestAccept()
protected String	getRequestAcceptEncoding()
protected String	getRequestAcceptLanguage()
protected String	getRequestContactEmail()
protected String	getRequestContactName()
protected String	getRequestHostAddress()
protected String	getRequestHostName()
protected String	getRequestRobots()
protected String	getRequestSoftware()
protected String	getRequestUserAgent()
protected String	getResponseAddress()
protected String	getResponseContent()
protected String	getResponseContentEncoding()
protected String	getResponseContentType()
protected String	getResponseDate()
protected String	getResponseHostName()
protected String	getResponseServer()
protected String	getResponseStatus()
protected String	getTimestamp()
protected String	getUrl()
protected abstract void	startArray(String key, boolean nested, boolean newline)
protected abstract void	startObject(String key)
protected abstract void	writeArrayValue(String value)
protected abstract void	writeKeyNull(String key)
protected abstract void	writeKeyValue(String key, String value)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

url

protected String url

content

protected byte[] content

metadata

protected Metadata metadata

conf

protected Configuration conf

keyPrefix

protected String keyPrefix

simpleDateFormat

protected boolean simpleDateFormat

jsonArray

protected boolean jsonArray

reverseKey

protected boolean reverseKey

reverseKeyValue

protected String reverseKeyValue

Constructor Detail

AbstractCommonCrawlFormat

public AbstractCommonCrawlFormat(String url,
                         byte[] content,
                         Metadata metadata,
                         Configuration nutchConf,
                         CommonCrawlConfig config)
                          throws IOException

Throws:

IOException

Method Detail

getJsonData

public String getJsonData()
                   throws IOException

Specified by:

getJsonData in interface CommonCrawlFormat

Returns:

the JSON data

Throws:

IOException

writeKeyValue

protected abstract void writeKeyValue(String key,
                 String value)
                               throws IOException

Throws:

IOException

writeKeyNull

protected abstract void writeKeyNull(String key)
                              throws IOException

Throws:

IOException

startArray

protected abstract void startArray(String key,
              boolean nested,
              boolean newline)
                            throws IOException

Throws:

IOException

closeArray

protected abstract void closeArray(String key,
              boolean nested,
              boolean newline)
                            throws IOException

Throws:

IOException

writeArrayValue

protected abstract void writeArrayValue(String value)
                                 throws IOException

Throws:

IOException

startObject

protected abstract void startObject(String key)
                             throws IOException

Throws:

IOException

closeObject

protected abstract void closeObject(String key)
                             throws IOException

Throws:

IOException

generateJson

protected abstract String generateJson()
                                throws IOException

Throws:

IOException

getUrl

protected String getUrl()

getTimestamp

protected String getTimestamp()

getMethod

protected String getMethod()

getRequestHostName

protected String getRequestHostName()

getRequestHostAddress

protected String getRequestHostAddress()

getRequestSoftware

protected String getRequestSoftware()

getRequestRobots

protected String getRequestRobots()

getRequestContactName

protected String getRequestContactName()

getRequestContactEmail

protected String getRequestContactEmail()

getRequestAccept

protected String getRequestAccept()

getRequestAcceptEncoding

protected String getRequestAcceptEncoding()

getRequestAcceptLanguage

protected String getRequestAcceptLanguage()

getRequestUserAgent

protected String getRequestUserAgent()

getResponseStatus

protected String getResponseStatus()

getResponseHostName

protected String getResponseHostName()

getResponseAddress

protected String getResponseAddress()

getResponseContentEncoding

protected String getResponseContentEncoding()

getResponseContentType

protected String getResponseContentType()

getResponseDate

protected String getResponseDate()

getResponseServer

protected String getResponseServer()

getResponseContent

protected String getResponseContent()

getKey

protected String getKey()

getImported

protected String getImported()