hadoop 使用map将SequenFile里的小文件解压出来

上例中将HDFS里小文件通过mapper压缩到一个文件中，本例将这些小文件解压出来。

mapreduce可以按SequenceFile的key进行分片。

1、mapper

public class MultiOutputMapper extends Mapper<Text,BytesWritable,NullWritable,Text> {
    private MultipleOutputs<NullWritable,Text> multipleOutputs;
    private long splitLength;

    /**
     * Called once at the beginning of the task.
     *
     * @param context
     */
    @Override
    protected void setup(Context context) throws IOException, InterruptedException {
        multipleOutputs = new MultipleOutputs<NullWritable, Text>(context);
        InputSplit split = context.getInputSplit();
        splitLength = split.getLength();
    }

    /**
     * Called once for each key/value pair in the input split. Most applications
     * should override this, but the default is the identity function.
     *
     * @param key
     * @param value
     * @param context
     */
    @Override
    protected void map(Text key, BytesWritable value, Context context) throws IOException, InterruptedException {
        System.out.printf("split length:%s,value length:%s,bytes length:%s",splitLength,value.getLength(),value.getBytes().length);
        int length = value.getLength();
        byte[] bytes = new byte[length];
        System.arraycopy(value.getBytes(),0,bytes,0,length);

        Text contents = new Text(bytes);
        //根据SequenceFile里key的原路径路径生成文件
//        multipleOutputs.write(NullWritable.get(),new Text(bytes),key.toString());

        //在output里输出文件
        Path path = new Path(key.toString());
        String outputFileName = String.format("%s/%s","2019",path.getName());
        multipleOutputs.write(NullWritable.get(),new Text(bytes),outputFileName);
//
//        multipleOutputs.write(NullWritable.get(),new Text(value.getBytes()),key.toString());//这句是错的。
//        通过测试，对于SequenceFile，是按key进入分片，value的length是实际长度,value.getbytes的长度是value的buff长度，两个不一定相等
//        split length:88505,value length:4364,bytes length:6546
    }

    /**
     * Called once at the end of the task.
     *
     * @param context
     */
    @Override
    protected void cleanup(Context context) throws IOException, InterruptedException {
        multipleOutputs.close();
        super.cleanup(context);
    }
}

2、job

public class SequenceFileToSmallFileConverter {

    public static void main(String[] args) throws Exception{

        long startTime = System.currentTimeMillis();

        Configuration conf = new Configuration();

        Path outPath = new Path(args[1]);
        FileSystem fileSystem = outPath.getFileSystem(conf);
        //删除输出路径
        if(fileSystem.exists(outPath))
        {
            fileSystem.delete(outPath,true);
        }

        Job job = Job.getInstance(conf,"SequenceFileToSmallFileConverter");
        job.setJarByClass(SequenceFileToSmallFileConverter.class);

        job.setMapperClass(MultiOutputMapper.class);
        job.setNumReduceTasks(0);

        job.setInputFormatClass(SequenceFileInputFormat.class);
        //TextOutputFormat会在每行文本后面加入换行符号，如果是这个文本作为一个整体来处理，最后就会比预期多一个换行符号
//        job.setOutputFormatClass(TextOutputFormat.class);

        //WholeTextOutputFormat与TextOutputFormat的区别就是没有在每行写入换行符
        job.setOutputFormatClass(WholeTextOutputFormat.class);

        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(BytesWritable.class);

        FileInputFormat.addInputPath(job,new Path(args[0]));
        FileOutputFormat.setOutputPath(job,new Path(args[1]));

        int exitCode = job.waitForCompletion(true) ? 0:1;

        long endTime = System.currentTimeMillis();
        long timeSpan = endTime - startTime;
        System.out.println("运行耗时："+timeSpan+"毫秒。");

        System.exit(exitCode);
    }
}

public class WholeTextOutputFormat<K, V> extends FileOutputFormat<K, V> {
    public static String SEPARATOR = "mapreduce.output.textoutputformat.separator";
    /**
     * @deprecated Use {@link #SEPARATOR}
     */
    @Deprecated
    public static String SEPERATOR = SEPARATOR;
    protected static class LineRecordWriter<K, V>
            extends RecordWriter<K, V> {
        private static final byte[] NEWLINE =
                "\n".getBytes(StandardCharsets.UTF_8);

        protected DataOutputStream out;
        private final byte[] keyValueSeparator;

        public LineRecordWriter(DataOutputStream out, String keyValueSeparator) {
            this.out = out;
            this.keyValueSeparator =
                    keyValueSeparator.getBytes(StandardCharsets.UTF_8);
        }

        public LineRecordWriter(DataOutputStream out) {
            this(out, "\t");
        }

        /**
         * Write the object to the byte stream, handling Text as a special
         * case.
         * @param o the object to print
         * @throws IOException if the write throws, we pass it on
         */
        private void writeObject(Object o) throws IOException {
            if (o instanceof Text) {
                Text to = (Text) o;
                out.write(to.getBytes(), 0, to.getLength());
            } else {
                out.write(o.toString().getBytes(StandardCharsets.UTF_8));
            }
        }

        public synchronized void write(K key, V value)
                throws IOException {

            boolean nullKey = key == null || key instanceof NullWritable;
            boolean nullValue = value == null || value instanceof NullWritable;
            if (nullKey && nullValue) {
                return;
            }
            if (!nullKey) {
                writeObject(key);
            }
            if (!(nullKey || nullValue)) {
                out.write(keyValueSeparator);
            }
            if (!nullValue) {
                writeObject(value);
            }
//            out.write(NEWLINE);将文本当做整体，各key之间不主动加入换行符号
        }

        public synchronized
        void close(TaskAttemptContext context) throws IOException {
            out.close();
        }
    }

    public RecordWriter<K, V>
    getRecordWriter(TaskAttemptContext job
    ) throws IOException, InterruptedException {
        Configuration conf = job.getConfiguration();
        boolean isCompressed = getCompressOutput(job);
        String keyValueSeparator= conf.get(SEPARATOR, "\t");
        CompressionCodec codec = null;
        String extension = "";
        if (isCompressed) {
            Class<? extends CompressionCodec> codecClass =
                    getOutputCompressorClass(job, GzipCodec.class);
            codec = ReflectionUtils.newInstance(codecClass, conf);
            extension = codec.getDefaultExtension();
        }
        Path file = getDefaultWorkFile(job, extension);
        FileSystem fs = file.getFileSystem(conf);
        FSDataOutputStream fileOut = fs.create(file, false);
        if (isCompressed) {
            return new WholeTextOutputFormat.LineRecordWriter<>(
                    new DataOutputStream(codec.createOutputStream(fileOut)),
                    keyValueSeparator);
        } else {
            return new WholeTextOutputFormat.LineRecordWriter<>(fileOut, keyValueSeparator);
        }
    }
}

3、验证，压入SequenceFile和解压后的文件完全相同。

[hadoop@bigdata-senior01 ~]$ hadoop fs -checksum /demo/1.txt-m-00000 /demo3/1.txt
/demo/1.txt-m-00000    MD5-of-0MD5-of-512CRC32C    0000020000000000000000007b6bd9c9f517a6ea12ede79fd43700ca
/demo3/1.txt    MD5-of-0MD5-of-512CRC32C    0000020000000000000000007b6bd9c9f517a6ea12ede79fd43700ca