mrunit测试

Map/Reduce 单元测试

About MRUnit

world count 测试

package com.irwin.hadoop;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
import org.apache.hadoop.mrunit.mapreduce.MapDriver;
import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver;
import org.apache.hadoop.mrunit.mapreduce.ReduceDriver;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;

import com.irwin.hadoop.WordCount.TokenizerMapper;
import com.irwin.hadoop.WordCount.IntSumReducer;


public class WordCountTest extends Assert{

	private MapDriver<Object, Text, Text, IntWritable> mapDriver;
	private ReduceDriver<Text, IntWritable, Text, IntWritable> reduceDriver;
	private static final IntWritable one = new IntWritable(1);
	MapReduceDriver<Object, Text, Text, IntWritable, Text, IntWritable> mapReduceDriver;
	
	@Before
	public void setUp(){
		Mapper<Object, Text, Text, IntWritable> mapper = new TokenizerMapper();
		mapDriver = MapDriver.newMapDriver(mapper);
		Reducer<Text, IntWritable, Text, IntWritable> reducer = new IntSumReducer();
		reduceDriver = ReduceDriver.newReduceDriver(reducer);
		mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
	}
	
	@Test
	public void testReducer() throws Exception{
		List<IntWritable> values = new ArrayList<IntWritable>();
	    values.add(new IntWritable(1));
	    values.add(new IntWritable(1));
	    reduceDriver.withInput(new Text("hello"), values);
	    reduceDriver.withOutput(new Text("hello"), new IntWritable(2));
	    reduceDriver.runTest();
	}
	
	  @Test
	  public void testMapReduce() throws IOException {
	    mapReduceDriver.withInput(new LongWritable(1), new Text("cat cat dog"));
	    mapReduceDriver.addOutput(new Text("cat"), new IntWritable(2));
	    mapReduceDriver.addOutput(new Text("dog"), new IntWritable(1));
	    mapReduceDriver.runTest();
	  }
	
	@Test
	public void testMapper() throws Exception {
		LineRecordReader reader = getBytesRecordReader();
		int counter = 0;
		while (reader.nextKeyValue()) {
			mapDriver.withInput(reader.getCurrentKey(),reader.getCurrentValue());
			StringTokenizer itr = new StringTokenizer(reader.getCurrentValue().toString());
			while (itr.hasMoreElements()) {
				mapDriver.withOutput(new Text(itr.nextToken()), one);
				
			}
			counter++;
		}
		assertEquals(3, counter);
		// auto assert output 
		mapDriver.runTest();
	}
	
	private static LineRecordReader getBytesRecordReader()throws IOException, InterruptedException {
		Configuration conf = new Configuration(false);
		conf.set("fs.default.name", "file:///");

		File testFile = new File("src\\test\\resources\\test_data\\text\\hello.text");
		Path path = new Path(testFile.getAbsoluteFile().toURI());
		FileSplit split = new FileSplit(path, 0, testFile.length(), null);

		TextInputFormat inputFormat = ReflectionUtils.newInstance(TextInputFormat.class, conf);
		TaskAttemptContext context = new TaskAttemptContextImpl(conf,new TaskAttemptID());
		LineRecordReader reader = (LineRecordReader) inputFormat.createRecordReader(split, context);

		reader.initialize(split, context);
		return reader;
	}

}