Java读写hdfs上的avro文件

1、通过Java往hdfs写avro文件

 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 
 import org.apache.avro.Schema;
 import org.apache.avro.file.CodecFactory;
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumWriter;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
 
 public class HdfsAvroTest {
 
     public static final String SCHEMA_JSON = "{\"type\": \"record\",\"name\": \"SmallFilesTest\", "
             + "\"fields\": ["
             + "{\"name\":\""
             + "username"
             + "\",\"type\":\"string\"},"
             + "{\"name\":\""
             + "password"
             + "\", \"type\":\"string\"}]}";
     public static final Schema SCHEMA = new Schema.Parser().parse(SCHEMA_JSON);
 
     public static void writeToAvro(File srcPath, OutputStream outputStream)
             throws IOException {
         DataFileWriter<Object> writer = new DataFileWriter<Object>(
                 new GenericDatumWriter<Object>()).setSyncInterval(100);
         writer.setCodec(CodecFactory.snappyCodec());
         writer.create(SCHEMA, outputStream);
         for (Object obj : FileUtils.listFiles(srcPath, null, false)) {
             File file = (File) obj;
             String filename = file.getAbsolutePath();
             byte content[] = FileUtils.readFileToByteArray(file);
             GenericRecord record = new GenericData.Record(SCHEMA);
             record.put("username", filename);
             record.put("password", ByteBuffer.wrap(content));
             writer.append(record);
         }
         IOUtils.cleanup(null, writer);
         IOUtils.cleanup(null, outputStream);
     }
 
     public static void main(String[] args) throws Exception {
         Configuration config = new Configuration();
         FileSystem hdfs = FileSystem.get(config);
         File sourceDir = new File(args[0]);
         Path destFile = new Path(args[1]);
         OutputStream os = hdfs.create(destFile);
         writeToAvro(sourceDir, os);
     }
 }

2、Java读hdfs上的avro文件

 import java.io.IOException;
 import java.io.InputStream;
 
 import org.apache.avro.file.DataFileStream;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IOUtils;
 
 public class HdfsReadAvro {
 
     public static void readFromAvro(InputStream is) throws IOException {
         DataFileStream<Object> reader = new DataFileStream<Object>(is,
                 new GenericDatumReader<Object>());
         for (Object o : reader) {
             GenericRecord r = (GenericRecord) o;
             System.out.println(r.get("username")+ ":"+r.get("password"));
         }
         IOUtils.cleanup(null, is);
         IOUtils.cleanup(null, reader);
     }
 
     public static void main(String[] args) throws Exception {
         Configuration config = new Configuration();
         FileSystem hdfs = FileSystem.get(config);
         Path destFile = new Path(args[0]);
         InputStream is = hdfs.open(destFile);
         readFromAvro(is);
     }
 }