MapReduce小文件处理之CombineFileInputFormat实现
在MapReduce使用过程中。一般会遇到输入文件特别小(几百KB、几十MB)。而Hadoop默认会为每一个文件向yarn申请一个container启动map,container的启动关闭是很耗时的。
Hadoop提供了CombineFileInputFormat。一个抽象类。作用是将多个小文件合并到一个map中,我们仅仅需实现三个类:
CompressedCombineFileInputFormat
CompressedCombineFileRecordReader
CompressedCombineFileWritable
maven
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.5.0-cdh5.2.1</version>
</dependency>
CompressedCombineFileInputFormat.java
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.CombineFileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.CombineFileRecordReader;
import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit; import java.io.IOException; public class CompressedCombineFileInputFormat
extends CombineFileInputFormat<CompressedCombineFileWritable, Text> { public CompressedCombineFileInputFormat() {
super(); } public RecordReader<CompressedCombineFileWritable, Text>
createRecordReader(InputSplit split,
TaskAttemptContext context) throws IOException {
return new
CombineFileRecordReader<CompressedCombineFileWritable,
Text>((CombineFileSplit) split, context,
CompressedCombineFileRecordReader.class);
} @Override
protected boolean isSplitable(JobContext context, Path file) {
return false;
} }
CompressedCombineFileRecordReader.java
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;
import org.apache.hadoop.util.LineReader; import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream; public class CompressedCombineFileRecordReader
extends RecordReader<CompressedCombineFileWritable, Text> { private long startOffset;
private long end;
private long pos;
private FileSystem fs;
private Path path;
private Path dPath;
private CompressedCombineFileWritable key = new CompressedCombineFileWritable();
private Text value;
private long rlength;
private FSDataInputStream fileIn;
private LineReader reader; public CompressedCombineFileRecordReader(CombineFileSplit split,
TaskAttemptContext context, Integer index) throws IOException { Configuration currentConf = context.getConfiguration();
this.path = split.getPath(index);
boolean isCompressed = findCodec(currentConf, path);
if (isCompressed)
codecWiseDecompress(context.getConfiguration()); fs = this.path.getFileSystem(currentConf); this.startOffset = split.getOffset(index); if (isCompressed) {
this.end = startOffset + rlength;
} else {
this.end = startOffset + split.getLength(index);
dPath = path;
} boolean skipFirstLine = false; fileIn = fs.open(dPath); if (isCompressed) fs.deleteOnExit(dPath); if (startOffset != 0) {
skipFirstLine = true;
--startOffset;
fileIn.seek(startOffset);
}
reader = new LineReader(fileIn);
if (skipFirstLine) {
startOffset += reader.readLine(new Text(), 0,
(int) Math.min((long) Integer.MAX_VALUE, end - startOffset));
}
this.pos = startOffset;
} public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
} public void close() throws IOException {
} public float getProgress() throws IOException {
if (startOffset == end) {
return 0.0f;
} else {
return Math.min(1.0f, (pos - startOffset) / (float)
(end - startOffset));
}
} public boolean nextKeyValue() throws IOException {
if (key.fileName == null) {
key = new CompressedCombineFileWritable();
key.fileName = dPath.getName();
}
key.offset = pos;
if (value == null) {
value = new Text();
}
int newSize = 0;
if (pos < end) {
newSize = reader.readLine(value);
pos += newSize;
}
if (newSize == 0) {
key = null;
value = null;
return false;
} else {
return true;
}
} public CompressedCombineFileWritable getCurrentKey()
throws IOException, InterruptedException {
return key;
} public Text getCurrentValue() throws IOException, InterruptedException {
return value;
} private void codecWiseDecompress(Configuration conf) throws IOException { CompressionCodecFactory factory = new CompressionCodecFactory(conf);
CompressionCodec codec = factory.getCodec(path); if (codec == null) {
System.err.println("No Codec Found For " + path);
System.exit(1);
} String outputUri =
CompressionCodecFactory.removeSuffix(path.toString(),
codec.getDefaultExtension());
dPath = new Path(outputUri); InputStream in = null;
OutputStream out = null;
fs = this.path.getFileSystem(conf); try {
in = codec.createInputStream(fs.open(path));
out = fs.create(dPath);
IOUtils.copyBytes(in, out, conf);
} finally {
IOUtils.closeStream(in);
IOUtils.closeStream(out);
rlength = fs.getFileStatus(dPath).getLen();
}
} private boolean findCodec(Configuration conf, Path p) { CompressionCodecFactory factory = new CompressionCodecFactory(conf);
CompressionCodec codec = factory.getCodec(path); if (codec == null)
return false;
else
return true; } }
CompressedCombineFileWritable.java
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable; import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException; public class CompressedCombineFileWritable implements WritableComparable { public long offset;
public String fileName; public CompressedCombineFileWritable() {
super();
} public CompressedCombineFileWritable(long offset, String fileName) {
super();
this.offset = offset;
this.fileName = fileName;
} public void readFields(DataInput in) throws IOException {
this.offset = in.readLong();
this.fileName = Text.readString(in);
} public void write(DataOutput out) throws IOException {
out.writeLong(offset);
Text.writeString(out, fileName);
} public int compareTo(Object o) {
CompressedCombineFileWritable that = (CompressedCombineFileWritable) o; int f = this.fileName.compareTo(that.fileName);
if (f == 0) {
return (int) Math.signum((double) (this.offset - that.offset));
}
return f;
} @Override
public boolean equals(Object obj) {
if (obj instanceof CompressedCombineFileWritable)
return this.compareTo(obj) == 0;
return false;
} @Override
public int hashCode() { final int hashPrime = 47;
int hash = 13;
hash = hashPrime * hash + (this.fileName != null ? this.fileName.hashCode() : 0);
hash = hashPrime * hash + (int) (this.offset ^ (this.offset >>> 16)); return hash;
} @Override
public String toString() {
return this.fileName + "-" + this.offset;
} }
MR測试类
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapred.lib.CombineFileInputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.reduce.IntSumReducer;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner; import java.io.IOException;
import java.util.StringTokenizer; public class CFWordCount extends Configured implements Tool { /**
* @param args
* @throws Exception
*/
public static void main(String[] args) throws Exception {
System.exit(ToolRunner.run(new Configuration(), new CFWordCount(), args));
} public int run(String[] args) throws Exception {
Configuration conf = getConf();
conf.setLong(CombineFileInputFormat.SPLIT_MAXSIZE, 128 * 1024 * 1024);
conf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, true);
conf.setClass(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, GzipCodec.class, CompressionCodec.class);
Job job = new Job(conf);
job.setJobName("CombineFile Demo");
job.setJarByClass(CFWordCount.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
job.setInputFormatClass(CompressedCombineFileInputFormat.class);
job.setMapperClass(TestMapper.class);
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(IntWritable.class);
job.setReducerClass(IntSumReducer.class);
job.setNumReduceTasks(1);
FileOutputFormat.setOutputPath(job, new Path(args[1]));
job.submit();
job.waitForCompletion(true); return 0;
} public static class TestMapper extends Mapper<CompressedCombineFileWritable, Text, Text, IntWritable> {
private Text txt = new Text();
private IntWritable count = new IntWritable(1); public void map(CompressedCombineFileWritable key, Text val, Context context) throws IOException, InterruptedException {
StringTokenizer st = new StringTokenizer(val.toString());
while (st.hasMoreTokens()) {
txt.set(st.nextToken());
context.write(txt, count);
}
}
}
}
注意:使用CombineFileInputFormat过程中发现不管小文件积累到多大,甚至超过HDFS BlockSize后。仍然仅仅有一个map split,查看 hadoop 的源代码发现,使用CombineFileInputFormat时。假设没有显示指定CombineFileInputFormat.SPLIT_MAXSIZE,默认不会切分map split,解决方法例如以下:
conf.setLong(CombineFileInputFormat.SPLIT_MAXSIZE, 128 * 1024 * 1024);
MapReduce小文件处理之CombineFileInputFormat实现的更多相关文章
- MapReduce小文件优化与分区
一.小文件优化 1.Mapper类 package com.css.combine; import java.io.IOException; import org.apache.hadoop.io.I ...
- Hadoop MapReduce编程 API入门系列之小文件合并(二十九)
不多说,直接上代码. Hadoop 自身提供了几种机制来解决相关的问题,包括HAR,SequeueFile和CombineFileInputFormat. Hadoop 自身提供的几种小文件合并机制 ...
- [大牛翻译系列]Hadoop(17)MapReduce 文件处理:小文件
5.1 小文件 大数据这个概念似乎意味着处理GB级乃至更大的文件.实际上大数据可以是大量的小文件.比如说,日志文件通常增长到MB级时就会存档.这一节中将介绍在HDFS中有效地处理小文件的技术. 技术2 ...
- mapreduce 关于小文件导致任务缓慢的问题
小文件导致任务执行缓慢的原因: 1.很容易想到的是map task 任务启动太多,而每个文件的实际输入量很小,所以导致了任务缓慢 这个可以通过 CombineTextInputFormat,解决,主要 ...
- [转载]mapreduce合并小文件成sequencefile
mapreduce合并小文件成sequencefile http://blog.csdn.net/xiao_jun_0820/article/details/42747537
- 第3节 mapreduce高级:5、6、通过inputformat实现小文件合并成为sequenceFile格式
1.1 需求 无论hdfs还是mapreduce,对于小文件都有损效率,实践中,又难免面临处理大量小文件的场景,此时,就需要有相应解决方案 1.2 分析 小文件的优化无非以下几种方式: 1. 在数据 ...
- Hadoop对小文件的解决方式
小文件指的是那些size比HDFS的block size(默认64M)小的多的文件.不论什么一个文件,文件夹和block,在HDFS中都会被表示为一个object存储在namenode的内存中, 每一 ...
- 基于Hadoop Sequencefile的小文件解决方案
一.概述 小文件是指文件size小于HDFS上block大小的文件.这样的文件会给hadoop的扩展性和性能带来严重问题.首先,在HDFS中,任何block,文件或者目录在内存中均以对象的形式存储,每 ...
- Hadoop小文件存储方案
原文地址:https://www.cnblogs.com/ballwql/p/8944025.html HDFS总体架构 在介绍文件存储方案之前,我觉得有必要先介绍下关于HDFS存储架构方面的一些知识 ...
随机推荐
- 利用MAP动态创建C++类对象
MFC的CRuntimeClass利用链表实现了C++类的动态创建.但是如果项目中对动态创建的要求比较低,我们完全可以利用map实现简单的动态创建. http://blog.csdn.net/yeji ...
- nginx的location匹配
http://www.cnblogs.com/lidabo/p/4169396.html 这个博主写的非常好 location: 先匹配普通location,再匹配正则表达式 1.而且选择了最大前缀匹 ...
- Vue之$set使用
背景 后端参与前端开发的小白,在开发过程中遇到了如下情况:当vue的data里边声明或者已经赋值过的对象或者数组(数组里边的值是对象)时,向对象中添加新的属性,如果更新此属性的值,是不会更新视图的. ...
- 【NOIP模拟赛】【乱搞AC】【奇技淫巧】【乘法原理】回文串计数
回文串计数 (calc.pas/calc.c/calc.cpp) [题目描述] 虽然是一名理科生,Mcx常常声称自己是一名真正的文科生.不知为何,他对于背诵总有一种莫名的热爱,这也促使他走向了以记忆量 ...
- java.util.List.subList ,开区间和闭区间
比如集合中的内容为1,2,3,4,5list.sublist(2,4)就返回一个子集合:它的内容包括从下标为2到下标为4,而且这是左闭右开的就是说是从大于等于2到小于4那子集内容就是3,4(集合的下标 ...
- MessageFormat.format 字符串的模板替换
项目目前在消息的模版,模版中需要替换很多参数,比方说“用户名”,“日期”等等.不过目前没有想到更好的替换参数的方法,所以目前只能使用一个比较简单的方式来实现.这个方式太死板,参数对应必须要在代码中写死 ...
- Kubernentes中的身份验证
Kubernentes中的身份验证 kubernetes 系统的各组件需要使用 TLS 证书对通信进行加密,本文档使用 CloudFlare 的 PKI 工具集 cfssl 来生成 Certifica ...
- 3、列表 list
列表 >>> list=['aaa','bbb','ccc'] >>> print list ['aaa', 'bbb', 'ccc'] >>> ...
- 针对访问uri 限制ip
在虚拟主机配置文件中加入如下字段: <filesmatch "(.*)admin(.*)"> Order deny,allow ...
- IO流知识点
如何判断是输入还是输出?答:以程序为中心.如何判断是解码还是编码?答:以程序为中心.程序只懂二进制,所以,以二进制转换成字符是解码,字符转换成二进制是编码.1. 首先,File 它是给程序跟文件或文件 ...