简单实现CombineFileInputFormat

import java.io.DataOutput;

import java.io.IOException;



import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.FileSystem;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.io.Writable;

import org.apache.hadoop.mapreduce.InputSplit;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.RecordReader;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.Reducer.Context;

import org.apache.hadoop.mapreduce.TaskAttemptContext;

import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;

import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.input.FileSplit;

import org.apache.hadoop.mapreduce.lib.input.LineRecordReader;

import org.apache.hadoop.mapreduce.lib.input.SequenceFileRecordReader;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.util.ReflectionUtils;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;



public class TestCombine extends Configured implements Tool {

	private static class ProvinceMapper extends

			Mapper<Object, Text, Text, Text> {

		@Override

		protected void map(Object key, Text value, Context context)

				throws IOException, InterruptedException {

			System.out.println("value : " + value + " Context " + context);

			context.write(value, value);

		}

	}



	private static class ProvinceReducer extends

			Reducer<Text, Text, Text, Text> {

		@Override

		protected void reduce(Text key, Iterable<Text> values, Context context)

				throws IOException, InterruptedException {

			for (Text va : values) {

			    System.out.println("reduce " + key);

				context.write(key, key);

			}

		}

	}

	

	public static class CombineSequenceFileInputFormat<K, V> extends CombineFileInputFormat<K, V> {  

	    @SuppressWarnings({ "unchecked", "rawtypes" })  

	    @Override  

	    public RecordReader<K, V> createRecordReader(InputSplit split, TaskAttemptContext context) throws IOException {  

	        return new CombineFileRecordReader((CombineFileSplit)split, context, CombineLineRecordReader.class);  

	    }  

	}  

	

	public static class CombineLineRecordReader<K, V> extends RecordReader<K, V> {  

	    private CombineFileSplit split;  

	    private TaskAttemptContext context;  

	    private int index;  

	    private RecordReader<K, V> rr;  

	  

	    @SuppressWarnings("unchecked")  

	    public CombineLineRecordReader(CombineFileSplit split, TaskAttemptContext context, Integer index) throws IOException, InterruptedException {  

	        this.index = index;

	        this.split = (CombineFileSplit) split;  

	        this.context = context;  

	  

	        this.rr = (RecordReader<K, V>) ReflectionUtils.newInstance(LineRecordReader.class, context.getConfiguration());  

	    }  

	  

	    @SuppressWarnings("unchecked")  

	    @Override  

	    public void initialize(InputSplit curSplit, TaskAttemptContext curContext) throws IOException, InterruptedException {  

	        this.split = (CombineFileSplit) curSplit;  

	        this.context = curContext;  

	  

	        if (null == rr) {  

	            rr = ReflectionUtils.newInstance(SequenceFileRecordReader.class, context.getConfiguration());  

	        }  

	  

	        FileSplit fileSplit = new FileSplit(this.split.getPath(index),  

	                this.split.getOffset(index), this.split.getLength(index),  

	                this.split.getLocations());  

	          

	        this.rr.initialize(fileSplit, this.context);  

	    }  

	  

	    @Override  

	    public float getProgress() throws IOException, InterruptedException {  

	        return rr.getProgress();  

	    }  

	  

	    @Override  

	    public void close() throws IOException {  

	        if (null != rr) {  

	            rr.close();  

	            rr = null;  

	        }  

	    }  

	  

	    @Override  

	    public K getCurrentKey()  

	    throws IOException, InterruptedException {  

	        return rr.getCurrentKey();  

	    }  

	  

	    @Override  

	    public V getCurrentValue()  

	    throws IOException, InterruptedException {  

	        return rr.getCurrentValue();  

	    }  

	  

	    @Override  

	    public boolean nextKeyValue() throws IOException, InterruptedException {  

	        return rr.nextKeyValue();  

	    }  

	}  



	

	public int run(String[] args) throws Exception {

		Configuration conf = new Configuration();

		

		Job job = new Job(conf);

		job.setJobName("TestCombine");

		job.setJarByClass(TestCombine.class);



		job.setMapperClass(ProvinceMapper.class);

		job.setReducerClass(ProvinceReducer.class);

		

		job.setInputFormatClass(CombineSequenceFileInputFormat.class);

		

		job.setOutputKeyClass(Text.class);

		job.setOutputValueClass(Text.class);

		

		String inpath = "/home/hadoop/tmp/combine";

		String outpath = "/home/hadoop/tmp/combineout";

		Path p = new Path(outpath);

		

		FileSystem fs = FileSystem.get(conf);

		if (fs.exists(p)){

			fs.delete(p);

		}

		FileInputFormat.addInputPaths(job, inpath);

		FileOutputFormat.setOutputPath(job, p);



		return job.waitForCompletion(true) ? 0 : 1;

	} 



	public static void main(String[] args) throws Exception {

		int ret = ToolRunner.run(new TestCombine(), args);

		System.exit(ret);

	} 

}