org.apache.hadoop.mrunit

Class TestDriver<K,V,T extends TestDriver<K,V,T>>

java.lang.Object

org.apache.hadoop.mrunit.TestDriver<K,V,T>

Direct Known Subclasses:

MapDriverBase, MapReduceDriverBase, org.apache.hadoop.mrunit.internal.driver.MultipleInputsMapReduceDriverBase, PipelineMapReduceDriver, ReduceDriverBase

public abstract class TestDriver<K,V,T extends TestDriver<K,V,T>>
extends Object

Field Summary

Fields

Modifier and Type	Field and Description
protected org.apache.hadoop.mrunit.internal.counters.CounterWrapper	counterWrapper
protected List<Pair<Enum<?>,Long>>	expectedEnumCounters
protected Map<String,List<Pair<?,?>>>	expectedMultipleOutputs
protected List<Pair<K,V>>	expectedOutputs
protected Map<String,List<Pair<?,?>>>	expectedPathOutputs
protected List<Pair<Pair<String,String>,Long>>	expectedStringCounters
static org.apache.commons.logging.Log	LOG
protected org.apache.hadoop.mrunit.internal.output.MockMultipleOutputs	mos

Constructor Summary

Constructors

Constructor and Description
TestDriver()

Method Summary

Methods

Modifier and Type	Method and Description
void	addAllOutput(List<Pair<K,V>> outputRecords) Adds output (k, v)* pairs we expect
void	addCacheArchive(String path) Adds an archive to be put on the distributed cache.
void	addCacheArchive(URI uri) Adds an archive to be put on the distributed cache.
void	addCacheFile(String path) Adds a file to be put on the distributed cache.
void	addCacheFile(URI uri) Adds a file to be put on the distributed cache.
<K,V> void	addMultiOutput(String namedOutput, K key, V val) add a (k, v) pair we expect as Multiple output
<K,V> void	addMultiOutput(String namedOutput, Pair<K,V> outputRecord) Adds an output (k, v) pair we expect as Multiple output
void	addOutput(K key, V val) Adds a (k, v) pair we expect as output
void	addOutput(Pair<K,V> outputRecord) Adds an output (k, v) pair we expect
void	addOutputFromString(String output) Deprecated. No replacement due to lack of type safety and incompatibility with non Text Writables
protected void	cleanupDistributedCache() Cleans up the distributed cache test by deleting the temporary directory and any extracted cache archives contained within
protected <E> E	copy(E object)
protected <S,E> Pair<S,E>	copyPair(S first, E second)
protected boolean	driverReused() Check to see if this driver is being reused
protected static <KEYIN,VALUEIN> void	formatPairList(List<Pair<KEYIN,VALUEIN>> pairs, StringBuilder sb)
static void	formatValueList(List<?> values, StringBuilder sb)
org.apache.hadoop.conf.Configuration	getConfiguration()
List<Pair<Enum<?>,Long>>	getExpectedEnumCounters()
List<Pair<K,V>>	getExpectedOutputs()
List<Pair<Pair<String,String>,Long>>	getExpectedStringCounters()
Comparator<K>	getKeyComparator()
org.apache.hadoop.conf.Configuration	getOutputSerializationConfiguration() Get the Configuration to use when copying output for use with run* methods or for the InputFormat when reading output back in when setting a real OutputFormat.
Comparator<V>	getValueComparator()
protected void	initDistributedCache() Initialises the test distributed cache if required.
protected static List<org.apache.hadoop.io.Text>	parseCommaDelimitedList(String commaDelimList) Split "val,val,val,val..." into a List of Text(val) objects.
static Pair<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text>	parseTabbedPair(String tabSeparatedPair) Split "key \t val" into Pair(Text(key), Text(val))
protected void	printPreTestDebugLog() Overridable hook for printing pre-test debug information
void	resetExpectedCounters() Clears the list of expected counters from this driver
void	resetOutput() Clears the list of outputs expected from this driver
abstract List<Pair<K,V>>	run() Runs the test but returns the result set instead of validating it (ignores any addOutput(), etc calls made before this)
List<Pair<K,V>>	run(boolean validateCounters) Runs the test but returns the result set instead of validating it (ignores any addOutput(), etc calls made before this).
void	runTest() Runs the test and validates the results
void	runTest(boolean orderMatters) Runs the test and validates the results
void	setCacheArchives(URI[] archives) Set the list of archives to put on the distributed cache
void	setCacheFiles(URI[] files) Set the list of files to put on the distributed cache
void	setConfiguration(org.apache.hadoop.conf.Configuration configuration) Deprecated. Use getConfiguration() to set configuration items as opposed to overriding the entire configuration object as it's used internally.
void	setKeyComparator(Comparator<K> keyComparator) Set the output key comparator
void	setOutputSerializationConfiguration(org.apache.hadoop.conf.Configuration configuration) Set the Configuration to use when copying output for use with run* methods or for the InputFormat when reading output back in when setting a real OutputFormat.
protected void	setUsedOnceStatus() Set to true when run() is called to prevent driver reuse
void	setValueComparator(Comparator<V> valueComparator) Set the output value comparator
protected T	thisAsTestDriver()
protected void	validate(org.apache.hadoop.mrunit.internal.counters.CounterWrapper counterWrapper) Check counters.
protected void	validate(List<Pair<K,V>> outputs, boolean orderMatters) check the outputs against the expected inputs in record
protected void	validate(org.apache.hadoop.mrunit.internal.output.MockMultipleOutputs mos)
protected void	validateOutputList(String name, org.apache.hadoop.mrunit.internal.util.Errors errors, Map<String,List<Pair<?,?>>> actuals, Map<String,List<Pair<?,?>>> expects) Check Multiple Outputs.
T	withAllOutput(List<Pair<K,V>> outputRecords) Functions like addAllOutput() but returns self for fluent programming style
T	withCacheArchive(String archive) Adds an archive to be put on the distributed cache.
T	withCacheArchive(URI archive) Adds an archive to be put on the distributed cache.
T	withCacheFile(String file) Adds a file to be put on the distributed cache.
T	withCacheFile(URI file) Adds a file to be put on the distributed cache.
T	withConfiguration(org.apache.hadoop.conf.Configuration configuration) Deprecated. Use getConfiguration() to set configuration items as opposed to overriding the entire configuration object as it's used internally.
T	withCounter(Enum<?> e, long expectedValue) Register expected enumeration based counter value
T	withCounter(String group, String name, long expectedValue) Register expected name based counter value
<K extends Comparable,V extends Comparable> T	withMultiOutput(String namedOutput, K key, V value) works like addMultiOutput() but returns self for fluent programming style
<K,V> T	withMultiOutput(String namedOutput, Pair<K,V> outputRecord) Works like addMultiOutput(), but returns self for fluent programming style
T	withOutput(K key, V val) Works like addOutput() but returns self for fluent programming style
T	withOutput(Pair<K,V> outputRecord) Works like addOutput(), but returns self for fluent style
T	withOutputFromString(String output) Deprecated. No replacement due to lack of type safety and incompatibility with non Text Writables
T	withOutputSerializationConfiguration(org.apache.hadoop.conf.Configuration configuration) Set the Configuration to use when copying output for use with run* methods or for the InputFormat when reading output back in when setting a real OutputFormat.
<K,V> T	withPathOutput(K key, V value, String path)
<K,V> T	withPathOutput(Pair<K,V> outputRecord, String path)
T	withStrictCounterChecking() Change counter checking.

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

LOG

public static final org.apache.commons.logging.Log LOG

expectedOutputs

protected List<Pair<K,V>> expectedOutputs

expectedEnumCounters

protected List<Pair<Enum<?>,Long>> expectedEnumCounters

expectedStringCounters

protected List<Pair<Pair<String,String>,Long>> expectedStringCounters

counterWrapper

protected org.apache.hadoop.mrunit.internal.counters.CounterWrapper counterWrapper

mos

protected org.apache.hadoop.mrunit.internal.output.MockMultipleOutputs mos

expectedMultipleOutputs

protected Map<String,List<Pair<?,?>>> expectedMultipleOutputs

expectedPathOutputs

protected Map<String,List<Pair<?,?>>> expectedPathOutputs

Constructor Detail

TestDriver

public TestDriver()

Method Detail

driverReused

protected boolean driverReused()

Check to see if this driver is being reused

Returns:

boolean - true if run() has been called more than once

setUsedOnceStatus

protected void setUsedOnceStatus()

Set to true when run() is called to prevent driver reuse

addAllOutput

public void addAllOutput(List<Pair<K,V>> outputRecords)

Adds output (k, v)* pairs we expect

Parameters:

outputRecords - The (k, v)* pairs to add

withAllOutput

public T withAllOutput(List<Pair<K,V>> outputRecords)

Functions like addAllOutput() but returns self for fluent programming style

Parameters:

outputRecords -

Returns:

this

addOutput

public void addOutput(Pair<K,V> outputRecord)

Adds an output (k, v) pair we expect

Parameters:

outputRecord - The (k, v) pair to add

addOutput

public void addOutput(K key,
             V val)

Adds a (k, v) pair we expect as output

Parameters:

key - the key

val - the value

withOutput

public T withOutput(Pair<K,V> outputRecord)

Works like addOutput(), but returns self for fluent style

Parameters:

outputRecord -

Returns:

this

withOutput

public T withOutput(K key,
           V val)

Works like addOutput() but returns self for fluent programming style

Returns:

this

addOutputFromString

@Deprecated
public void addOutputFromString(String output)

Deprecated. No replacement due to lack of type safety and incompatibility with non Text Writables

Expects an input of the form "key \t val" Forces the output types to Text.

Parameters:

output - A string of the form "key \t val". Trims any whitespace.

withOutputFromString

@Deprecated
public T withOutputFromString(String output)

Deprecated. No replacement due to lack of type safety and incompatibility with non Text Writables

Identical to addOutputFromString, but with a fluent programming style

Parameters:

output - A string of the form "key \t val". Trims any whitespace.

Returns:

this

getExpectedOutputs

public List<Pair<K,V>> getExpectedOutputs()

Returns:

the list of (k, v) pairs expected as output from this driver

resetOutput

public void resetOutput()

Clears the list of outputs expected from this driver

getExpectedEnumCounters

public List<Pair<Enum<?>,Long>> getExpectedEnumCounters()

Returns:

expected counters from this driver

getExpectedStringCounters

public List<Pair<Pair<String,String>,Long>> getExpectedStringCounters()

Returns:

expected counters from this driver

resetExpectedCounters

public void resetExpectedCounters()

Clears the list of expected counters from this driver

thisAsTestDriver

protected T thisAsTestDriver()

withCounter

public T withCounter(Enum<?> e,
            long expectedValue)

Register expected enumeration based counter value

Parameters:

e - Enumeration based counter

expectedValue - Expected value

Returns:

this

withCounter

public T withCounter(String group,
            String name,
            long expectedValue)

Register expected name based counter value

Parameters:

group - Counter group

name - Counter name

expectedValue - Expected value

Returns:

this

withStrictCounterChecking

public T withStrictCounterChecking()

Change counter checking. After this method is called, the test will fail if an actual counter is not matched by an expected counter. By default, the test only check that every expected counter is there. This mode allows you to ensure that no unexpected counters has been declared.

getConfiguration

public org.apache.hadoop.conf.Configuration getConfiguration()

Returns:

The configuration object that will given to the mapper and/or reducer associated with the driver

getKeyComparator

public Comparator<K> getKeyComparator()

Returns:

The comparator for output keys or null of none has been set

getValueComparator

public Comparator<V> getValueComparator()

Returns:

The comparator for output values or null of none has been set

setConfiguration

@Deprecated
public void setConfiguration(org.apache.hadoop.conf.Configuration configuration)

Deprecated. Use getConfiguration() to set configuration items as opposed to overriding the entire configuration object as it's used internally.

Parameters:

configuration - The configuration object that will given to the mapper and/or reducer associated with the driver. This method should only be called directly after the constructor as the internal state of the driver depends on the configuration object

withConfiguration

@Deprecated
public T withConfiguration(org.apache.hadoop.conf.Configuration configuration)

Deprecated. Use getConfiguration() to set configuration items as opposed to overriding the entire configuration object as it's used internally.

Parameters:

configuration - The configuration object that will given to the mapper associated with the driver. This method should only be called directly after the constructor as the internal state of the driver depends on the configuration object

Returns:

this object for fluent coding

getOutputSerializationConfiguration

public org.apache.hadoop.conf.Configuration getOutputSerializationConfiguration()

Get the Configuration to use when copying output for use with run* methods or for the InputFormat when reading output back in when setting a real OutputFormat.

Returns:

outputSerializationConfiguration, null when no outputSerializationConfiguration is set

setOutputSerializationConfiguration

public void setOutputSerializationConfiguration(org.apache.hadoop.conf.Configuration configuration)

Set the Configuration to use when copying output for use with run* methods or for the InputFormat when reading output back in when setting a real OutputFormat. When this configuration is not set, MRUnit will use the configuration set with withConfiguration(Configuration) or setConfiguration(Configuration)

Parameters:

configuration -

withOutputSerializationConfiguration

public T withOutputSerializationConfiguration(org.apache.hadoop.conf.Configuration configuration)

Set the Configuration to use when copying output for use with run* methods or for the InputFormat when reading output back in when setting a real OutputFormat. When this configuration is not set, MRUnit will use the configuration set with withConfiguration(Configuration) or setConfiguration(Configuration)

Parameters:

configuration -

Returns:

this for fluent style

addCacheFile

public void addCacheFile(String path)

Adds a file to be put on the distributed cache. The path may be relative and will try to be resolved from the classpath of the test.

Parameters:

path - path to the file

addCacheFile

public void addCacheFile(URI uri)

Adds a file to be put on the distributed cache.

Parameters:

uri - uri of the file

setCacheFiles

public void setCacheFiles(URI[] files)

Set the list of files to put on the distributed cache

Parameters:

files - list of URIs

setKeyComparator

public void setKeyComparator(Comparator<K> keyComparator)

Set the output key comparator

Parameters:

keyComparator - the key comparator

setValueComparator

public void setValueComparator(Comparator<V> valueComparator)

Set the output value comparator

Parameters:

valueComparator - the value comparator

addCacheArchive

public void addCacheArchive(String path)

Adds an archive to be put on the distributed cache. The path may be relative and will try to be resolved from the classpath of the test.

Parameters:

path - path to the archive

addCacheArchive

public void addCacheArchive(URI uri)

Adds an archive to be put on the distributed cache.

Parameters:

uri - uri of the archive

setCacheArchives

public void setCacheArchives(URI[] archives)

Set the list of archives to put on the distributed cache

Parameters:

archives - list of URIs

withCacheFile

public T withCacheFile(String file)

Adds a file to be put on the distributed cache. The path may be relative and will try to be resolved from the classpath of the test.

Parameters:

file - path to the file

Returns:

the driver

withCacheFile

public T withCacheFile(URI file)

Adds a file to be put on the distributed cache.

Parameters:

file - uri of the file

Returns:

the driver

withCacheArchive

public T withCacheArchive(String archive)

Adds an archive to be put on the distributed cache. The path may be relative and will try to be resolved from the classpath of the test.

Parameters:

archive - path to the archive

Returns:

the driver

withCacheArchive

public T withCacheArchive(URI archive)

Adds an archive to be put on the distributed cache.

Parameters:

archive - uri of the archive

Returns:

the driver

run

public List<Pair<K,V>> run(boolean validateCounters)
                    throws IOException

Runs the test but returns the result set instead of validating it (ignores any addOutput(), etc calls made before this). Also optionally performs counter validation.

Parameters:

validateCounters - whether to run automatic counter validation

Returns:

the list of (k, v) pairs returned as output from the test

Throws:

IOException

initDistributedCache

protected void initDistributedCache()
                             throws IOException

Initialises the test distributed cache if required. This process is referred to as "localizing" by Hadoop, but since this is a unit test all files/archives are already local. Cached files are not moved but cached archives are extracted into a temporary directory.

Throws:

IOException

cleanupDistributedCache

protected void cleanupDistributedCache()
                                throws IOException

Cleans up the distributed cache test by deleting the temporary directory and any extracted cache archives contained within

Throws:

IOException - if the local fs handle cannot be retrieved

run

public abstract List<Pair<K,V>> run()
                             throws IOException

Runs the test but returns the result set instead of validating it (ignores any addOutput(), etc calls made before this)

Returns:

the list of (k, v) pairs returned as output from the test

Throws:

IOException

runTest

public void runTest()
             throws IOException

Runs the test and validates the results

Throws:

IOException

runTest

public void runTest(boolean orderMatters)
             throws IOException

Runs the test and validates the results

Parameters:

orderMatters - Whether or not output ordering is important

Throws:

IOException

printPreTestDebugLog

protected void printPreTestDebugLog()

Overridable hook for printing pre-test debug information

parseTabbedPair

public static Pair<org.apache.hadoop.io.Text,org.apache.hadoop.io.Text> parseTabbedPair(String tabSeparatedPair)

Split "key \t val" into Pair(Text(key), Text(val))

Parameters:

tabSeparatedPair -

Returns:

(k,v)

parseCommaDelimitedList

protected static List<org.apache.hadoop.io.Text> parseCommaDelimitedList(String commaDelimList)

Split "val,val,val,val..." into a List of Text(val) objects.

Parameters:

commaDelimList - A list of values separated by commas

copy

protected <E> E copy(E object)

copyPair

protected <S,E> Pair<S,E> copyPair(S first,
                       E second)

validate

protected void validate(List<Pair<K,V>> outputs,
            boolean orderMatters)

check the outputs against the expected inputs in record

Parameters:

outputs - The actual output (k, v) pairs

orderMatters - Whether or not output ordering is important when validating test result

validate

protected void validate(org.apache.hadoop.mrunit.internal.counters.CounterWrapper counterWrapper)

Check counters.

validateOutputList

protected void validateOutputList(String name,
                      org.apache.hadoop.mrunit.internal.util.Errors errors,
                      Map<String,List<Pair<?,?>>> actuals,
                      Map<String,List<Pair<?,?>>> expects)

Check Multiple Outputs.

validate

protected void validate(org.apache.hadoop.mrunit.internal.output.MockMultipleOutputs mos)

formatValueList

public static void formatValueList(List<?> values,
                   StringBuilder sb)

formatPairList

protected static <KEYIN,VALUEIN> void formatPairList(List<Pair<KEYIN,VALUEIN>> pairs,
                                  StringBuilder sb)

addMultiOutput

public <K,V> void addMultiOutput(String namedOutput,
                        Pair<K,V> outputRecord)

Adds an output (k, v) pair we expect as Multiple output

Parameters:

namedOutput -

outputRecord -

addMultiOutput

public <K,V> void addMultiOutput(String namedOutput,
                        K key,
                        V val)

add a (k, v) pair we expect as Multiple output

Parameters:

namedOutput -

key -

val -

withMultiOutput

public <K extends Comparable,V extends Comparable> T withMultiOutput(String namedOutput,
                                                            K key,
                                                            V value)

works like addMultiOutput() but returns self for fluent programming style

Parameters:

namedOutput -

key -

value -

Returns:

this

withMultiOutput

public <K,V> T withMultiOutput(String namedOutput,
                      Pair<K,V> outputRecord)

Works like addMultiOutput(), but returns self for fluent programming style

Parameters:

namedOutput -

outputRecord -

Returns:

this

withPathOutput

public <K,V> T withPathOutput(K key,
                     V value,
                     String path)

withPathOutput

public <K,V> T withPathOutput(Pair<K,V> outputRecord,
                     String path)