Hbase和mapreduce结合

为什么需要用mapreduce去访问hbase的数据?

——加快分析速度和扩展分析能力

Mapreduce访问hbase数据作分析一定是在离线分析的场景下应用

案例1、HBase表数据的转移

在Hadoop阶段,我们编写的MR任务分别进程了Mapper和Reducer两个类,而在HBase中我们需要继承的是TableMapper和TableReducer两个类。

目标:将fruit表中的一部分数据,通过MR迁入到fruit_mr表中

Step1、构建ReadFruitMapper类,用于读取fruit表中的数据


import java.io.IOException;

import org.apache.hadoop.hbase.Cell;

import org.apache.hadoop.hbase.CellUtil;

import org.apache.hadoop.hbase.client.Put;

import org.apache.hadoop.hbase.client.Result;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.hbase.mapreduce.TableMapper;

import org.apache.hadoop.hbase.util.Bytes;

public class ReadFruitMapper extends TableMapper<ImmutableBytesWritable, Put> {

@Override

protected void map(ImmutableBytesWritable key, Result value, Context context)

throws IOException, InterruptedException {

//将fruit的name和color提取出来,相当于将每一行数据读取出来放入到Put对象中。

Put put = new Put(key.get());

//遍历添加column行

for(Cell cell: value.rawCells()){

//添加/克隆列族:info

if("info".equals(Bytes.toString(CellUtil.cloneFamily(cell)))){

//添加/克隆列:name

if("name".equals(Bytes.toString(CellUtil.cloneQualifier(cell)))){

//将该列cell加入到put对象中

put.add(cell);

//添加/克隆列:color

}else if("color".equals(Bytes.toString(CellUtil.cloneQualifier(cell)))){

//向该列cell加入到put对象中

put.add(cell);

}

}

}

//将从fruit读取到的每行数据写入到context中作为map的输出

context.write(key, put);

}

}


Step2、构建WriteFruitMRReducer类,用于将读取到的fruit表中的数据写入到fruit_mr表中


import java.io.IOException;

import org.apache.hadoop.hbase.client.Put;

import org.apache.hadoop.hbase.io.ImmutableBytesWritable;

import org.apache.hadoop.hbase.mapreduce.TableReducer;

import org.apache.hadoop.io.NullWritable;

public class WriteFruitMRReducer extends TableReducer<ImmutableBytesWritable, Put, NullWritable> {

@Override

protected void reduce(ImmutableBytesWritable key, Iterable<Put> values, Context context)

throws IOException, InterruptedException {

//读出来的每一行数据写入到fruit_mr表中

for(Put put: values){

context.write(NullWritable.get(), put);

}

}

}


Step3、构建Fruit2FruitMRJob extends Configured implements Tool,用于组装运行Job任务


//组装Job

public int run(String[] args) throws Exception {

//得到Configuration

Configuration conf = this.getConf();

//创建Job任务

Job job = Job.getInstance(conf, this.getClass().getSimpleName());

job.setJarByClass(Fruit2FruitMRJob.class);

//配置Job

Scan scan = new Scan();

scan.setCacheBlocks(false);

scan.setCaching(500);

//设置Mapper,注意导入的是mapreduce包下的,不是mapred包下的,后者是老版本

TableMapReduceUtil.initTableMapperJob(

"fruit", //数据源的表名

scan, //scan扫描控制器

ReadFruitMapper.class,//设置Mapper类

ImmutableBytesWritable.class,//设置Mapper输出key类型

Put.class,//设置Mapper输出value值类型

job//设置给哪个JOB

);

//设置Reducer

TableMapReduceUtil.initTableReducerJob("fruit_mr", WriteFruitMRReducer.class, job);

//设置Reduce数量,最少1个

job.setNumReduceTasks(1);

boolean isSuccess = job.waitForCompletion(true);

if(!isSuccess){

throw new IOException("Job running with error");

}

return isSuccess ? 0 : 1;

}


Step4、主函数中调用运行该Job任务


public static void main( String[] args ) throws Exception{

Configuration conf = HBaseConfiguration.create();

int status = ToolRunner.run(conf, new Fruit2FruitMRJob(), args);

System.exit(status);

}


案例2:从Hbase中读取数据、分析,写入hdfs

/**

public abstract class TableMapper<KEYOUT, VALUEOUT>

extends Mapper<ImmutableBytesWritable, Result, KEYOUT, VALUEOUT> {

}

* @author duanhaitao@gec.cn

*

*/

public class HbaseReader {

public static String flow_fields_import = "flow_fields_import";

static class HdfsSinkMapper extends TableMapper<Text, NullWritable>{

@Override

protected void map(ImmutableBytesWritable key, Result value, Context context) throws IOException, InterruptedException {

byte[] bytes = key.copyBytes();

String phone = new String(bytes);

byte[] urlbytes = value.getValue("f1".getBytes(), "url".getBytes());

String url = new String(urlbytes);

context.write(new Text(phone + "\t" + url), NullWritable.get());

}

}

static class HdfsSinkReducer extends Reducer<Text, NullWritable, Text, NullWritable>{

@Override

protected void reduce(Text key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

context.write(key, NullWritable.get());

}

}

public static void main(String[] args) throws Exception {

Configuration conf = HBaseConfiguration.create();

conf.set("hbase.zookeeper.quorum", "spark01");

Job job = Job.getInstance(conf);

job.setJarByClass(HbaseReader.class);

//            job.setMapperClass(HdfsSinkMapper.class);

Scan scan = new Scan();

TableMapReduceUtil.initTableMapperJob(flow_fields_import, scan, HdfsSinkMapper.class, Text.class, NullWritable.class, job);

job.setReducerClass(HdfsSinkReducer.class);

FileOutputFormat.setOutputPath(job, new Path("c:/hbasetest/output"));

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(NullWritable.class);

job.waitForCompletion(true);

}

}

2.3.2 从hdfs中读取数据写入Hbase

q

/**

public abstract class TableReducer<KEYIN, VALUEIN, KEYOUT>

extends Reducer<KEYIN, VALUEIN, KEYOUT, Writable> {

}

* @author duanhaitao@gec.cn

*

*/

public class HbaseSinker {

public static String flow_fields_import = "flow_fields_import";

static class HbaseSinkMrMapper extends Mapper<LongWritable, Text, FlowBean, NullWritable>{

@Override

protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

String line = value.toString();

String[] fields = line.split("\t");

String phone = fields[0];

String url = fields[1];

FlowBean bean = new FlowBean(phone,url);

context.write(bean, NullWritable.get());

}

}

static class HbaseSinkMrReducer extends TableReducer<FlowBean, NullWritable, ImmutableBytesWritable>{

@Override

protected void reduce(FlowBean key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {

Put put = new Put(key.getPhone().getBytes());

put.add("f1".getBytes(), "url".getBytes(), key.getUrl().getBytes());

context.write(new ImmutableBytesWritable(key.getPhone().getBytes()), put);

}

}

public static void main(String[] args) throws Exception {

Configuration conf = HBaseConfiguration.create();

conf.set("hbase.zookeeper.quorum", "spark01");

HBaseAdmin hBaseAdmin = new HBaseAdmin(conf);

boolean tableExists = hBaseAdmin.tableExists(flow_fields_import);

if(tableExists){

hBaseAdmin.disableTable(flow_fields_import);

hBaseAdmin.deleteTable(flow_fields_import);

}

HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(flow_fields_import));

HColumnDescriptor hColumnDescriptor = new HColumnDescriptor ("f1".getBytes());

desc.addFamily(hColumnDescriptor);

hBaseAdmin.createTable(desc);

Job job = Job.getInstance(conf);

job.setJarByClass(HbaseSinker.class);

job.setMapperClass(HbaseSinkMrMapper.class);

TableMapReduceUtil.initTableReducerJob(flow_fields_import, HbaseSinkMrReducer.class, job);

FileInputFormat.setInputPaths(job, new Path("c:/hbasetest/data"));

job.setMapOutputKeyClass(FlowBean.class);

job.setMapOutputValueClass(NullWritable.class);

job.setOutputKeyClass(ImmutableBytesWritable.class);

job.setOutputValueClass(Mutation.class);

job.waitForCompletion(true);

}

}

Hbase 与mapreduce结合的更多相关文章

  1. 《OD大数据实战》HBase整合MapReduce和Hive

    一.HBase整合MapReduce环境搭建 1. 搭建步骤1)在etc/hadoop目录中创建hbase-site.xml的软连接.在真正的集群环境中的时候,hadoop运行mapreduce会通过 ...

  2. HBase概念学习(七)HBase与Mapreduce集成

    这篇文章是看了HBase权威指南之后,依据上面的解说搬下来的样例,可是略微有些不一样. HBase与mapreduce的集成无非就是mapreduce作业以HBase表作为输入,或者作为输出,也或者作 ...

  3. hbase运行mapreduce设置及基本数据加载方法

    hbase与mapreduce集成后,运行mapreduce程序,同时需要mapreduce jar和hbase jar文件的支持,这时我们需要通过特殊设置使任务可以同时读取到hadoop jar和h ...

  4. Hbase框架原理及相关的知识点理解、Hbase访问MapReduce、Hbase访问Java API、Hbase shell及Hbase性能优化总结

    转自:http://blog.csdn.net/zhongwen7710/article/details/39577431 本blog的内容包含: 第一部分:Hbase框架原理理解 第二部分:Hbas ...

  5. HBase结合MapReduce批量导入(HDFS中的数据导入到HBase)

    HBase结合MapReduce批量导入 package hbase; import java.text.SimpleDateFormat; import java.util.Date; import ...

  6. 《HBase in Action》 第三章节的学习总结 ---- 如何编写和运行基于HBase的MapReduce程序

    HBase之所以与Hadoop是最好的伙伴,我理解就因为两点:1.HADOOP的HDFS,为HBase提供了分布式的存储方式:2.HADOOP的MR为HBase提供的分布式的计算方法.u 其中第一点, ...

  7. 2.8-2.10 HBase集成MapReduce

    一.HBase集成MapReduce 1.查看HBase集成MapReduce需要的jar包 [root@hadoop-senior hbase-0.98.6-hadoop2]# bin/hbase ...

  8. Hadoop之——HBASE结合MapReduce批量导入数据

    转载请注明出处:http://blog.csdn.net/l1028386804/article/details/46463889 废话不多说.直接上代码,你懂得 package hbase; imp ...

  9. HBase 与 MapReduce 集成

    6. HBase 与 MapReduce 集成 6.1 官方 HBase 与 MapReduce 集成 查看 HBase 的 MapReduce 任务的执行:bin/hbase mapredcp; 环 ...

随机推荐

  1. Resnet-34框架

    import torch import torch.nn as nn import torch.nn.functional as F class ResidualBlock(nn.Module): ' ...

  2. WingIDE 常用快捷键

    Ctrl+N        新建文件 Ctrl+O   打开文件夹 Ctrl+W            关闭当前文件 Ctrl+S              保存文件 Ctrl+shif+S      ...

  3. 构建工具build tools

    构建工具是从源代码自动创建可执行应用程序的程序(例如.apk for android app).构建包括将代码编译,链接和打包成可用或可执行的形式. 基本上,构建自动化是脚本或自动化软件开发人员在日常 ...

  4. inline元素、block元素、inline-block元素

    inline 内联元素:是不可以控制宽和高.margin等:并且在同一行显示,不换行,直到该行排满. block 块级元素:是可以控制宽和高.margin等,并且会换行.块级对象元素会单独占一行显示, ...

  5. MapServer Tutorial——MapServer7.2.1教程学习——第一节用例实践:Example1.2 Static Map with Two Layers

    MapServer Tutorial——MapServer7.2.1教程学习——第一节用例实践:Example1.2 Static Map with Two Layers 一.前言 上一篇博客< ...

  6. Mac上svn报错解决方案

    具体的报错信息为:xcrun: error: invalid active developer path (/Library/Developer/CommandLineTools), missing ...

  7. nopi 简洁笔记

    导出excel /// <summary> /// 增加二维码 /// </summary> /// <param name="dt">< ...

  8. 真机*Appium

    一.真机连接电脑123 1.一般android手机的的USB调试模式在设置里的开发者选项中,找到开发者选项,打开USB调试 2.cmd命令→[adb devices]→回车,得到真机设备 可能存在问题 ...

  9. 网络编程 多线程/socketserver模块/ threading.local

    线程:进程中负责程序执行的执行单元. 多线程:在1个进程中存在多个线程. 进程只是用来把资源集中在一起,而线程才是cpu上的执行单位. 每个进程都会默认有一个控制线程也叫作主线程. 进程之间是竞争关系 ...

  10. servlet编程操作

    所谓servlet指:服务器处理来自Web浏览器或其他客户端的HTTP请求的服务器程序.客户端向服务器发送Http请求,经Tomcat封装处理转给Servlet容器,Servlet容器在把请求或回应交 ...