[置顶] 在Hadoop的streaming中使用自定义的inputformat和outputformat
在Hadoop的streaming中有一个选项是指定输入输出格式化的
-inputformat TextInputFormat(default)|SequenceFileAsTextInputFormat|JavaClassName Optional. -outputformat TextOutputFormat(default)|JavaClassName Optional.
但是在0.14版本之后，hadoop不再支持带多个jar包文件，所以，如果要使用自己定义的Inputformat或者Outputformat，就得将对应的class文件加入到hadoop-streaming-1.0.1.jar中去，比如：
jar uf ../../contrib/streaming/hadoop-streaming-1.0.1.jar org/apache/hadoop/streaming/*.class
然后在-inputformat后面就可以直接带类名了。
下面通过一个例子来说明下，实现Map的输入<key,value>,key为文件名,value为文档的整篇内容：
1.定义自己的InputFormat:
ContentRecordReder.java
package org.apache.hadoop.streaming;
import java.io.IOException;
//import org.apache.commons.logging.Log;
//import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.RecordReader;
import com.sun.org.apache.commons.logging.Log;
import com.sun.org.apache.commons.logging.LogFactory;
public class ContentRecordReder implements RecordReader<Text,Text> {
private static final Log LOG = LogFactory.getLog(ContentRecordReder.class.getName());
private CompressionCodecFactory compressionCodecs = null;
private long start;
private long pos;
private long end;
private byte[] buffer;
private String keyName;
private FSDataInputStream fileIn;
public ContentRecordReder(Configuration job,FileSplit split) throws IOException{
start = split.getStart(); //从中可以看出每个文件是作为一个split的
end = split.getLength() + start;
final Path path = split.getPath();
keyName = path.toString();
LOG.info("filename in hdfs is : " + keyName);
System.out.println("filename in hdfs is : " + keyName);
final FileSystem fs = path.getFileSystem(job);
fileIn = fs.open(path);
fileIn.seek(start);
buffer = new byte[(int)(end - start)];
this.pos = start;
}
public Text createKey() {
return new Text();
}
public Text createValue() {
return new Text();
}
public long getPos() throws IOException{
return pos;
}
public float getProgress() {
if (start == end) {
return 0.0f;
} else {
return Math.min(1.0f, (pos - start) / (float)(end - start));
}
}
public boolean next(Text key, Text value) throws IOException{
while(pos < end) {
key.set(keyName);
value.clear();
fileIn.readFully(pos,buffer);
value.set(buffer);
LOG.info("---内容: " + value.toString());
System.out.println("---内容: " + value.toString());
pos += buffer.length;
LOG.info("end is : " + end + " pos is : " + pos);
return true;
}
return false;
}
public void close() throws IOException{
if(fileIn != null) {
fileIn.close();
}
}
}
ContentInputFormat.java
package org.apache.hadoop.streaming;
import java.io.IOException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobConfigurable;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.FileInputFormat;
public class ContentInputFormat extends FileInputFormat<Text,Text>{
private long mySplitSize = 1024*1024;
private CompressionCodecFactory compressionCodecs = null;
public void configure(JobConf conf) {
compressionCodecs = new CompressionCodecFactory(conf);
}
/**
* @brief isSplitable 不对文件进行切分，必须对文件整体进行处理
*
* @param fs
* @param file
*
* @return false
*/
protected boolean isSplitable(FileSystem fs, Path file) {
return false;
}
public RecordReader<Text,Text> getRecordReader(InputSplit genericSplit,
JobConf job, Reporter reporter) throws IOException{
reporter.setStatus(genericSplit.toString());
ContentRecordReder contentRecordReder = new ContentRecordReder(job,(FileSplit)genericSplit);
return (RecordReader<Text, Text>) contentRecordReder;
}
}
2.编译
javac -classpath ~/hadoop-1.0.1/hadoop-core-1.0.1.jar:~/hadoop-1.0.1/lib/*:./content-record-reader.jar ./*.java -Xlint:deprecation
3.并加入stream的jar包
jar uf ../../contrib/streaming/hadoop-streaming-1.0.1.jar org/apache/hadoop/streaming/*.class
4.Mapper.cpp
#include <iostream>
#include <string>
using namespace std;
int main()
{
string key,value;
char ch;
cin>>key;
value = "";
while(cin>>ch&&!cin.eof()){
value.append(1,ch);
}
cout<<key<<"\t"<<value<<endl;
return 0;
}
5.Reducer.cpp
#include <iostream>
#include <map>
using namespace std;
int main() {
map<string,string> wordMap;
map<string,string>::iterator it;
string key;
string value;
while(cin>>key>>value) {
//可以在这里对value即文档做处理...
wordMap[key] +=value;
}
for(it=wordMap.begin();it != wordMap.end();it++) {//输出
cout<<it->first<<"\t"<<it->second<<endl;
}
return 0;
}
6.执行
bin/hadoop jar contrib/streaming/hadoop-streaming-1.0.1.jar \ -mapper /home/guoguo/hadoop-1.0.1/cTest/C++/Mapper \ -file /home/guoguo/hadoop-1.0.1/cTest/C++/Mapper \ -inputformat ContentInputFormat \ -reducer /home/guoguo/hadoop-1.0.1/cTest/C++/Reducer \ -file /home/guoguo/hadoop-1.0.1/cTest/C++/Reducer \ -input input \ -output stream-output
7.ok~
8.补充：如果文档是XML格式，可以用StreamXmlRecordReader，具体做法就是用到hadoop-streaming.jar的inputreader选项，如：
-inputreader "StreamXmlRecordReader,begin=<Store>,end=</Store>"
其中“<Store>”，“</Store>”为XML文件的开始标签和结束标签
9.参考资料
http://hadoop.apache.org/common/docs/r1.0.1/streaming.html
http://dongxicheng.org/mapreduce/hadoop-streaming-advanced-programming/
http://blog.csdn.net/j3smile/article/details/7371209
http://blog.csdn.net/anbo724/article/details/6955175