跨语言序列化-protobuf/thrift/avro性能测试

1. 编写Schema

1.1 student.proto

 package protobuf;
 option java_package = "com.topsec.trd";
 option java_outer_classname = "StudentProto";
 message Student  {
   required string name = 1;
   required int32 age = 2;
   required int32 sex = 3;
   optional string alias = 4;
   repeated string interest=5;

 }

1.2 studnet.avsc

 namespace java com.topsec.trd
 struct StudentThrift {
   1: string name,
   2: i32 age,
   3: i32 sex,
   4: optional string aliass,
   5: list interest
 }

1.3 studnet.avsc

 {"namespace": "com.topsec.trd",
  "type": "record",
  "name": "StudentAvro",
  "fields": [
      {"name": "name", "type": "string"},
      {"name": "alias",  "type": ["string", "null"]},
      {"name": "age",  "type": ["int", "null"]},
      {"name": "sex", "type": ["string", "null"]},
      {"name": "interet", "type": {"type": "array", "items": "string"}}
  ]
 }

2. 生成bean

生成protobuf bean

protoc --java_out=. student.proto

生成thrift bean

thrift-0.10.0.exe -gen java student.thrift

生成avro bean

java -jar lib/avro-tools-1.7.7.jar compile schema src/main/resource/student.avsc src/main/java

3. 编写测试代码

 public class ProtoBuf {
     private final static int TIMES = 10000000;
     public static void main(String[] args) throws IOException {
          long start = System.currentTimeMillis();
          for(int i = 0; i < TIMES; i++) {
              deserialize();
          }
          long end = System.currentTimeMillis();
          System.out.println("ProtoBuf total time \t" + (end -start));
     }

     public static byte [] serializeAsBytes() {
         return makeStudent().build().toByteArray();
     }

     public static InputStream serializeAsStream() throws IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         makeStudent().build().writeDelimitedTo(baos);
         return new ByteArrayInputStream(baos.toByteArray());
     }

     private static Builder makeStudent() {
         StudentProto.Student.Builder student = StudentProto.Student.newBuilder();
         student.setName("小明");
         student.setSex(0);
         student.setAge(18);
         List interests = new ArrayList();
         interests.add("吃饭");
         interests.add("睡觉");
         interests.add("打豆豆");
         student.addAllInterest(interests);
         return student;
     }

     public static void deserialize() throws IOException {
         byte [] bytes = serializeAsBytes();
         StudentProto.Student student = StudentProto.Student.parseFrom(bytes);
         System.out.println(student.getName());
     }
 }

 public class Thrift {
     private final static int TIMES = 10000000;
     private final static TSerializer SERIALIZER = new TSerializer(new TBinaryProtocol.Factory());
     private final static TDeserializer DESERIALIZER = new TDeserializer(new TBinaryProtocol.Factory());

     public static void main(String[] args) throws TException {
          long start = System.currentTimeMillis();
          for(int i = 0; i < TIMES; i++) {
              deserialize();
          }
          long end = System.currentTimeMillis();
          System.out.println("Thrift total time \t" + (end -start));
     }

     public static byte [] serialize() throws TException {
         StudentThrift stu = new StudentThrift();
         stu.setName("小明");
         stu.setSex(0);
         stu.setAge(18);
         List interests = new ArrayList();
         interests.add("吃饭");
         interests.add("睡觉");
         interests.add("打豆豆");
         stu.setInterest(interests);

         return SERIALIZER.serialize(stu);
     }

     public static void deserialize() throws TException {
         byte [] bytes = serialize();
         StudentThrift stu = new StudentThrift();
         DESERIALIZER.deserialize(stu, bytes);
 //      System.out.println(stu);
     }

 }

 public class AVRO {
     private final static int TIMES = 10000000;
      public static void main(String[] args) throws IOException {
          long start = System.currentTimeMillis();
          deserializeAsBytes();
          long end = System.currentTimeMillis();
          System.out.println("Avro total time \t" + (end -start));
      }

      public static byte [] serializeAsBytes() throws IOException {
          StudentAvro student = new StudentAvro();
          student.setName("小明");
          student.setSex("女");
          student.setAge(18);
          List interests = new ArrayList();
          interests.add("吃饭");
          interests.add("睡觉");
          interests.add("打豆豆");
          student.setInteret(interests);
          ByteArrayOutputStream baos = new ByteArrayOutputStream();
          DatumWriter  userDatumWriter = new  SpecificDatumWriter(StudentAvro.class);
          DataFileWriter  dataFileWriter = new  DataFileWriter(userDatumWriter);
          dataFileWriter.create(student.getSchema(), baos);
          dataFileWriter.append(student);
          dataFileWriter.close();
          return baos.toByteArray();
      }

      public static byte [] serializeAsBytes(int times) throws IOException {
          ByteArrayOutputStream baos = new ByteArrayOutputStream();
          DatumWriter  userDatumWriter = new  SpecificDatumWriter(StudentAvro.class);
          DataFileWriter  dataFileWriter = new  DataFileWriter(userDatumWriter);
          dataFileWriter.create(new StudentAvro().getSchema(), baos);
          for(int i = 0; i < times; i++) {
              StudentAvro student = new StudentAvro();
              student.setName("小明");
              student.setSex("女");
              student.setAge(18);
              List interests = new ArrayList();
              interests.add("吃饭");
              interests.add("睡觉");
              interests.add("打豆豆");
              student.setInteret(interests);

              dataFileWriter.append(student);
          }
          dataFileWriter.close();
          return baos.toByteArray();
      }

      public static void deserializeAsBytes() throws IOException {
          SeekableByteArrayInput sbai = new SeekableByteArrayInput(serializeAsBytes(TIMES));
          DatumReader datumReader = new SpecificDatumReader(StudentAvro.class);
          DataFileReader dataFileReader = new DataFileReader(sbai, datumReader);
          StudentAvro user = null;
          while (dataFileReader.hasNext()) {
              user = dataFileReader.next(user);
              System.out.println(user.getName());
          }
          dataFileReader.close();
      }

 }

4. 测试结果

private final static int TIMES = 100000;
ProtoBuf total time  282

Thrift total time  229
Avro total time  694

private final static int TIMES = 1000000;
ProtoBuf total time  988

Thrift total time  1248
Avro total time 2079

private final static int TIMES = 10000000;
ProtoBuf total time  7368

Thrift total time  10675
Avro total time  15025

4.1 小结

项/技术	avro	thrift	protobuf
速度	慢	中等	快
序列化到1个stream	是	否	否

5.工程下载地址

(包含测试代码及三种schema和生成bean的工具等)

源代码下载地址-点击下载

6.protobuf分析

6.1 protobuf特点

(a)占用空间小

一条消息数据，用protobuf序列化后的大小是json的10分之一，xml格式的20分之一，是二进制序列化的10分之一（极端情况下，会大于等于直接序列化），总体看来ProtoBuf的优势还是很明显的。

(b)解析速度快

解析速度快，主要归功于protobuf对message 没有动态解析，没有了动态解析的处理序列化速度自然快了。就比如xml ，获取文件之后，还需要解析标签、节点、字段，每一个都需要遍历，而protobuf不需要，直接将field装入流。

(c)兼容性好

fieldNumber 为每个field定义一个编号，其一保证不重复，其二保证其在流中的位置。如若当前数据流中有某个字段，而解析方没有相关的解析代码，解析放会直接skip 吊这个field，而且读数据的position也会后移，保证后续读取不出问题。

参考文章：

http://www.jianshu.com/p/ec39f79c0412

https://www.ibm.com/developerworks/cn/linux/l-cn-gpb

6.2  字节流分析

对象序列化

 private StudentProto.Student request(int age) {
         StudentProto.Student.Builder builder = StudentProto.Student.newBuilder();
         builder.setName("小明");
         builder.setSex(0);
         builder.setAge(age);
         return builder.build();
     }

protobuf计算tag源码

 static final int TAG_TYPE_BITS = 3;
 /** Makes a tag value given a field number and wire type. */
  static int makeTag(final int fieldNumber, final int wireType) {
    return (fieldNumber << TAG_TYPE_BITS) | wireType;
  }

数字类型存储：tag+value,

字符串存储    ：leg+value,leg是字符串的长度

分析字节流：