【Avro一】Avro入门

编程技术  /  houtizong 发布于 3年前   62

本文的目的主要是总结下基于Avro Schema代码生成，然后进行序列化和反序列化开发的基本流程。需要指出的是，Avro并不要求一定得根据Schema文件生成代码，这对于动态类型语言很有用。

 

1. 添加Maven依赖

 

<?xml version="1.0" encoding="UTF-8"?><project xmlns="http://maven.apache.org/POM/4.0.0"         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">    <modelVersion>4.0.0</modelVersion>    <groupId>learn</groupId>    <artifactId>learn.avro</artifactId>    <version>1.0-SNAPSHOT</version>    <dependencies>        <!--avro core-->        <dependency>            <groupId>org.apache.avro</groupId>            <artifactId>avro</artifactId>            <version>1.7.7</version>        </dependency>    </dependencies>    <build>        <plugins>            <plugin>                <groupId>org.apache.avro</groupId>                <artifactId>avro-maven-plugin</artifactId>                <version>1.7.7</version>                <executions>                    <execution>                        <phase>generate-sources</phase>                        <goals>                            <!--Maven goal that helps for code generation-->                            <goal>schema</goal>                            <goal>protocol</goal>                            <goal>idl-protocol</goal>                        </goals>                        <configuration>                            <!--sourceDirectory: where to find the schema file, schema: .avsc; protocol: .avpr-->                            <sourceDirectory>${project.basedir}/src/main/avro/</sourceDirectory>                            <outputDirectory>${project.basedir}/src/main/java/</outputDirectory>                        </configuration>                    </execution>                </executions>            </plugin>            <plugin>                <groupId>org.apache.maven.plugins</groupId>                <artifactId>maven-compiler-plugin</artifactId>                <configuration>                    <source>1.7</source>                    <target>1.7</target>                </configuration>            </plugin>        </plugins>    </build></project>

 

2.定义schema文件(users.avsc)

放置到src/main/avro目录下

{"namespace": "examples.avro.simple", "type": "record", "name": "User", "fields": [     {"name": "name", "type": "string"},     {"name": "favorite_number",  "type": ["int", "null"]},     {"name": "favorite_color", "type": ["string", "null"]} ]}

namespace:包名

type: record表示记录，可以理解为几个field构成的一个record，类似于Java的JavaBean

name:类名

fiels:成员变量，favorite_number最后会解释成getFavoriteNumber和setFavoriteNumber

 

3. 生成User类

在Intellij Idea的Maven视图中,learn avro->Plugins->avro->avro:schema,右击avro:schema，执行Run Maven Build，生成schema对应的Java实体类

 

/** * Autogenerated by Avro *  * DO NOT EDIT DIRECTLY */package examples.avro.simple;@SuppressWarnings("all")@org.apache.avro.specific.AvroGeneratedpublic class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {  public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"example.avro\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"favorite_number\",\"type\":[\"int\",\"null\"]},{\"name\":\"favorite_color\",\"type\":[\"string\",\"null\"]}]}");  public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }  @Deprecated public java.lang.CharSequence name;  @Deprecated public java.lang.Integer favorite_number;  @Deprecated public java.lang.CharSequence favorite_color;  /**   * Default constructor.  Note that this does not initialize fields   * to their default values from the schema.  If that is desired then   * one should use <code>newBuilder()</code>.    */  public User() {}  /**   * All-args constructor.   */  public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) {    this.name = name;    this.favorite_number = favorite_number;    this.favorite_color = favorite_color;  }  public org.apache.avro.Schema getSchema() { return SCHEMA$; }  // Used by DatumWriter.  Applications should not call.   public java.lang.Object get(int field$) {    switch (field$) {    case 0: return name;    case 1: return favorite_number;    case 2: return favorite_color;    default: throw new org.apache.avro.AvroRuntimeException("Bad index");    }  }  // Used by DatumReader.  Applications should not call.   @SuppressWarnings(value="unchecked")  public void put(int field$, java.lang.Object value$) {    switch (field$) {    case 0: name = (java.lang.CharSequence)value$; break;    case 1: favorite_number = (java.lang.Integer)value$; break;    case 2: favorite_color = (java.lang.CharSequence)value$; break;    default: throw new org.apache.avro.AvroRuntimeException("Bad index");    }  }  /**   * Gets the value of the 'name' field.   */  public java.lang.CharSequence getName() {    return name;  }  /**   * Sets the value of the 'name' field.   * @param value the value to set.   */  public void setName(java.lang.CharSequence value) {    this.name = value;  }  /**   * Gets the value of the 'favorite_number' field.   */  public java.lang.Integer getFavoriteNumber() {    return favorite_number;  }  /**   * Sets the value of the 'favorite_number' field.   * @param value the value to set.   */  public void setFavoriteNumber(java.lang.Integer value) {    this.favorite_number = value;  }  /**   * Gets the value of the 'favorite_color' field.   */  public java.lang.CharSequence getFavoriteColor() {    return favorite_color;  }  /**   * Sets the value of the 'favorite_color' field.   * @param value the value to set.   */  public void setFavoriteColor(java.lang.CharSequence value) {    this.favorite_color = value;  }  /** Creates a new User RecordBuilder */  public static User.Builder newBuilder() {    return new User.Builder();  }    /** Creates a new User RecordBuilder by copying an existing Builder */  public static User.Builder newBuilder(User.Builder other) {    return new User.Builder(other);  }    /** Creates a new User RecordBuilder by copying an existing User instance */  public static User.Builder newBuilder(User other) {    return new User.Builder(other);  }    /**   * RecordBuilder for User instances.   */  public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User>    implements org.apache.avro.data.RecordBuilder<User> {    private java.lang.CharSequence name;    private java.lang.Integer favorite_number;    private java.lang.CharSequence favorite_color;    /** Creates a new Builder */    private Builder() {      super(User.SCHEMA$);    }        /** Creates a Builder by copying an existing Builder */    private Builder(User.Builder other) {      super(other);      if (isValidValue(fields()[0], other.name)) {        this.name = data().deepCopy(fields()[0].schema(), other.name);        fieldSetFlags()[0] = true;      }      if (isValidValue(fields()[1], other.favorite_number)) {        this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);        fieldSetFlags()[1] = true;      }      if (isValidValue(fields()[2], other.favorite_color)) {        this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);        fieldSetFlags()[2] = true;      }    }        /** Creates a Builder by copying an existing User instance */    private Builder(User other) {            super(User.SCHEMA$);      if (isValidValue(fields()[0], other.name)) {        this.name = data().deepCopy(fields()[0].schema(), other.name);        fieldSetFlags()[0] = true;      }      if (isValidValue(fields()[1], other.favorite_number)) {        this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number);        fieldSetFlags()[1] = true;      }      if (isValidValue(fields()[2], other.favorite_color)) {        this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color);        fieldSetFlags()[2] = true;      }    }    /** Gets the value of the 'name' field */    public java.lang.CharSequence getName() {      return name;    }        /** Sets the value of the 'name' field */    public User.Builder setName(java.lang.CharSequence value) {      validate(fields()[0], value);      this.name = value;      fieldSetFlags()[0] = true;      return this;     }        /** Checks whether the 'name' field has been set */    public boolean hasName() {      return fieldSetFlags()[0];    }        /** Clears the value of the 'name' field */    public User.Builder clearName() {      name = null;      fieldSetFlags()[0] = false;      return this;    }    /** Gets the value of the 'favorite_number' field */    public java.lang.Integer getFavoriteNumber() {      return favorite_number;    }        /** Sets the value of the 'favorite_number' field */    public User.Builder setFavoriteNumber(java.lang.Integer value) {      validate(fields()[1], value);      this.favorite_number = value;      fieldSetFlags()[1] = true;      return this;     }        /** Checks whether the 'favorite_number' field has been set */    public boolean hasFavoriteNumber() {      return fieldSetFlags()[1];    }        /** Clears the value of the 'favorite_number' field */    public User.Builder clearFavoriteNumber() {      favorite_number = null;      fieldSetFlags()[1] = false;      return this;    }    /** Gets the value of the 'favorite_color' field */    public java.lang.CharSequence getFavoriteColor() {      return favorite_color;    }        /** Sets the value of the 'favorite_color' field */    public User.Builder setFavoriteColor(java.lang.CharSequence value) {      validate(fields()[2], value);      this.favorite_color = value;      fieldSetFlags()[2] = true;      return this;     }        /** Checks whether the 'favorite_color' field has been set */    public boolean hasFavoriteColor() {      return fieldSetFlags()[2];    }        /** Clears the value of the 'favorite_color' field */    public User.Builder clearFavoriteColor() {      favorite_color = null;      fieldSetFlags()[2] = false;      return this;    }    @Override    public User build() {      try {        User record = new User();        record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]);        record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]);        record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]);        return record;      } catch (Exception e) {        throw new org.apache.avro.AvroRuntimeException(e);      }    }  }}

 

4. User类序列化和反序列化测试

package examples.avro.simple;import java.io.File;import java.io.Serializable;import org.apache.avro.file.DataFileReader;import org.apache.avro.file.DataFileWriter;import org.apache.avro.io.DatumReader;import org.apache.avro.io.DatumWriter;import org.apache.avro.specific.SpecificDatumReader;import org.apache.avro.specific.SpecificDatumWriter;public class UserTest {    public static void main(String[] args) throws Exception {                ///无参构造函数，依赖设值注入        User user1 = new User();        user1.setName("Alyssa");        user1.setFavoriteNumber(256);        //user1没有对favorite color赋值         ///注意：User并没有实现Java的Serializable接口        System.out.println(user1 instanceof Serializable);        ///重载的构造方法        User user2 = new User("Ben", 7, "red");        //使用builder构建对象        User user3 = User.newBuilder()                .setName("Charlie")                .setFavoriteColor("blue")                .setFavoriteNumber(null)                .build();        System.out.println(user3.getName() + "," + user3.getFavoriteColor());        ///1. 序列化包含Schema和3个User对象到c:/users.avro文件        DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);        DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);           dataFileWriter.create(user1.getSchema(), new File("c:/users.avro"));        dataFileWriter.append(user1);        dataFileWriter.append(user2);        dataFileWriter.append(user3);        dataFileWriter.close();        ///2. 从c:/users.avro文件反序列化对象        DatumReader<User> userDatumReader = new SpecificDatumReader<User>(User.class);        DataFileReader<User> dataFileReader = new DataFileReader<User>(new File("c:/users.avro"), userDatumReader);        User user = null;        while (dataFileReader.hasNext()) {            // Reuse user object by passing it to next(). This saves us from            // allocating and garbage collecting many objects for files with            // many items.            user = dataFileReader.next(user);            System.out.println(user);  ///输出JSON格式的数据，Avro对User类的toString进行了改写        }    }}

 

 

 

 

 

 

 

 

 

上一篇：【Avro三】Hadoop MapReduce读写Avro文件
下一篇：【Avro二】Avro RPC框架

请勿发布不友善或者负能量的内容。与人为善，比聪明更重要！

留言需要登陆哦

技术博客集 - 网站简介：
前后端技术：
后端基于Hyperf2.1框架开发,前端使用Bootstrap可视化布局系统生成
网站主要作用：
1.编程技术分享及讨论交流，内置聊天系统;
2.测试交流框架问题，比如：Hyperf、Laravel、TP、beego;
3.本站数据是基于大数据采集等爬虫技术为基础助力分享知识，如有侵权请发邮件到站长邮箱，站长会尽快处理;
4.站长邮箱：[email protected];

      订阅博客周刊 去订阅

文章归档

文章标签

友情链接

首页
关于我们
Auther ·HouTiZong
侯体宗的博客
© 2020 zongscan.com
版权所有ICP证 : 粤ICP备20027696号
PHP交流群 也可以扫右边的二维码
侯体宗的博客