HBase常用操作-HBaseUtil
package com.zhen.hbase; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger; public class HBaseUtil { //@Value("${hbase.zookeeper.quorum}")
private String addr="HDP1,HDP2,HDP3"; //@Value("${hbase.zookeeper.property.clientPort}")
private String port="2181"; Logger logger = Logger.getLogger(getClass()); private static Connection connection; @SuppressWarnings("static-access")
public void getConnection(){
Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum",addr);
conf.set("hbase.zookeeper.property.clientPort", port);
try {
this.connection = ConnectionFactory.createConnection(conf);
} catch (IOException e) {
e.printStackTrace();
}
} public HBaseUtil() {
getConnection();
} /*
* 判断表是否存在
*
*
* @tableName 表名
*/ public boolean isExist(String tableName) throws IOException {
TableName table_name = TableName.valueOf(tableName);
Admin admin = connection.getAdmin();
boolean exit = admin.tableExists(table_name);
admin.close();
return exit;
} /**
* 单行添加
* @param tableName
* @param rowKey
* @param family
* @param keyValue
* @throws IOException
*/
@SuppressWarnings("unused")
private static void addRow(String tableName, String rowKey, String family, Map<String, String> keyValue) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
for (Entry<String, String> entry : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), Bytes.toBytes(entry.getValue()));
}
table.put(put);
table.close();
keyValue.clear();
} public static void addRows(String tableName, String rowFamilySeparator, Map<String, Map<String, String>> keyValues) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
List<Put> puts = new ArrayList<Put>();
for (Entry<String, Map<String, String>> entry : keyValues.entrySet()) {
String key = entry.getKey();
if (null == rowFamilySeparator || rowFamilySeparator.isEmpty()) {
rowFamilySeparator = "_";
}
String rowKey = key.split(rowFamilySeparator)[0];
String family = key.split(rowFamilySeparator)[1];
Map<String, String> keyValue = entry.getValue();
Put put = new Put(Bytes.toBytes(rowKey), System.currentTimeMillis());
for (Entry<String, String> entry2 : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry2.getKey()), Bytes.toBytes(entry2.getValue()));
}
puts.add(put);
}
table.put(puts);
table.close();
keyValues.clear();
} /**
* 单行删除
* @param tableName
* @param rowKey
* @param family
* @throws IOException
*/
public static void deleteByRowKey(String tableName, String rowKey) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
table.close();
} /**
* 查询所有
* @param tableName
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName, String family) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 查询所有rowKey
* @param tableName
* @return
* @throws IOException
*/
public static List<String> queryAllRowKeyForScan(String tableName) throws IOException {
List<String> result = new ArrayList<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
if (null == cells || cells.length <= 0) {
continue;
}
Cell cell = cells[0];
String rowKey = new String(CellUtil.cloneRow(cell));
result.add(rowKey);
}
} finally {
rs.close();
}
return result;
} /**
* 查询所有字段
* @param tableName
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据时间范围
* @param tableName
* @param family
* @param minStamp
* @param maxStamp
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForTimeRange(String tableName, String family, long minStamp, long maxStamp) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setTimeRange(minStamp, maxStamp);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据RowKey查询
* @param tableName
* @param rowKey
* @param family
* @return
* @throws IOException
*/
public static Map<String, String> queryForRowKey(String tableName, String rowKey, String family) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
Scan scan = new Scan(get);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
}
}
} finally {
rs.close();
}
return row;
} /**
* 根据多个RowKey查询
* @param tableName
* @param rowKeys
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeys(String tableName, List<String> rowKeys, String family) throws IOException {
List<Map<String, String>> resultList = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
List<Get> getList = new ArrayList();
for (String rowKey : rowKeys) {
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
getList.add(get);
}
Result[] results = table.get(getList);
for (Result result : results){//对返回的结果集进行操作
Map<String, String> row = new HashMap<String, String>();
for (Cell kv : result.rawCells()) {
row.put(new String(CellUtil.cloneQualifier(kv)), new String(CellUtil.cloneValue(kv),"UTF-8"));
}
resultList.add(row);
}
return resultList;
} /**
* 根据RowKey范围查询
* @param tableName
* @param family
* @param startRow
* @param stopRow
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeyRange(String tableName, String family, String startRow, String stopRow) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setStartRow(Bytes.toBytes(startRow));
scan.setStopRow(Bytes.toBytes(stopRow));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据指定列名匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfier(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
PageFilter p = new PageFilter(5);
scan.setFilter(filter);
scan.setFilter(p);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
} /**
* 根据指定列名完全匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfierExactly(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL,value.getBytes());
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
} /**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter filter = new QualifierFilter(CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes(qualifier)));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList filterList = new FilterList();
filterList.addFilter(filter);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} /**
* 获取多个列值匹配的rowkey
* @param tableName
* @param family
* @param qualifierMap 列名:列值
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForMultiQuilfierName(String tableName, String family, Map<String, String> qualifierMap, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
for (Entry<String, String> entry : qualifierMap.entrySet()) {
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), entry.getKey().getBytes(), CompareOp.EQUAL,entry.getValue().getBytes());
filter.setFilterIfMissing(true);
filterList.addFilter(filter);
}
PageFilter pageFilter = new PageFilter(pageSize);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} public static void qualifierFilter() throws IOException {
Table mTable = connection.getTable(TableName.valueOf("portrait"));
QualifierFilter columnsNameFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator("basictag-14".getBytes()));
Scan scan = new Scan();
scan.setFilter(columnsNameFilter);
ResultScanner rs = mTable.getScanner(scan);
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
}
}
} /**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String family, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(qualifier.getBytes()));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList list = new FilterList();
list.addFilter(pageFilter);
list.addFilter(qualifierFilter);
scan.addFamily(Bytes.toBytes(family));
scan.setFilter(list);
System.out.println(scan.toJSON());
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} /**
* 根据指定列名匹配列值条数
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static long queryForQuilfierCount(String tableName, String family, String column, String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
long count = 0;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
count++;
}
} finally {
rs.close();
}
return count;
} /*
* 关闭连接
*
*/
public void close() {
/**
* close connection
**/
if (connection != null) {
try {
connection.close();
} catch (IOException e) {
e.printStackTrace();
}
}
} }
HBase常用操作-HBaseUtil的更多相关文章
- Hbase常用操作记录
Hbase常用操作记录 Hbase 创建表 查看表结构 修改表结构 删除表 创建表 语法:create <table>, {NAME => <family>, VERSI ...
- Hbase常用操作(增删改查)
Hbase常用操作(增删改查) [日期:2014-01-03] 来源:Linux社区 作者:net19880504 [字体:大 中 小] 运行Eclipse,创建一个新的Java工程“HBa ...
- HBase常用操作之namespace
1.介绍 在HBase中,namespace命名空间指对一组表的逻辑分组,类似RDBMS中的database,方便对表在业务上划分.Apache HBase从0.98.0, 0.95.2两个版本开始支 ...
- Hbase常用操作
下面我们看看HBase Shell的一些基本操作命令,我列出了几个常用的HBase Shell命令,如 名称 命令表达式 创建表 create '表名称', '列名称1','列名称2','列名称N' ...
- HBASE常用操作增删改查
http://javacrazyer.iteye.com/blog/1186881 http://www.cnblogs.com/invban/p/5667701.html
- Hbase到Solr同步常用操作
Hbase到Solr同步常用操作 1. 整体流程 2. 常用操作 Hbase常用操作 Solr常用操作 hbase-index常用操作 3. 其他资料 Lily HBase Indexer使用整理 h ...
- HBase Shell操作
Hbase 是一个分布式的.面向列的开源数据库,其实现是建立在google 的bigTable 理论之上,并基于hadoop HDFS文件系统. Hbase不同于一般的关系型数据库(RDBMS ...
- HBase常用操作命令
HBase常用操作命令 1.进入HBase脚本客户端 #hbase shell #进入HBase脚本客户端 > whoami #查看当前登录用户 > status ...
- HBase伪分布式安装(HDFS)+ZooKeeper安装+HBase数据操作+HBase架构体系
HBase1.2.2伪分布式安装(HDFS)+ZooKeeper-3.4.8安装配置+HBase表和数据操作+HBase的架构体系+单例安装,记录了在Ubuntu下对HBase1.2.2的实践操作,H ...
随机推荐
- Struts2 入门篇
Struts2简介 Struts2 是一个开源.免费的Web框架 官网:http://struts.apache.org/ Struts2.5.2新版本搭建环境更新 导入jar包,必备jar包如下:尤 ...
- android开发游记：meterial design 5.0 开源控件整套合集 及使用demo
android 的5.0公布不光google官方给出了一些新控件,同一时候还给出了一套符合material design风格的设计标准,这套标准将未来将覆盖google全部产品包括pc端,站点,移动端 ...
- vivado2013.4和modelsim联合仿真
vivado2013.4和modelsim联合仿真 Hello,Panda 最近在做Zynq的项目,曾经尝试使用ISE+PlanAhe ...
- linux sublime python
(三)配置python3编译环境 1.点击上部菜单栏Tools->Build System ->new Build System 2.点击之后,会出现一个空的配置文件,此时,往这个空配置文 ...
- URL重写：Rewirte模块原理详解
Apache+PHP+MySQL Rewirte主要的功能就是实现URL的跳转和隐藏真实地址,基于Perl语言的正则表达式规范.平时帮助我们实现拟静态,拟目录,域名跳转,防止盗链等.本文将针对mod_ ...
- java比较字符串长度
1.一种:str1.length()>=str.length():这种只比较字符串中的计算字符的数量,不管是中文还是英文 2:另一种:str1.getBytes().length>=str ...
- oracle分区表和分区索引概述
㈠ 分区表技术概述 ⑴ Range 分区 ① 例子 create table t (...列定义...) ...
- vmware workstation导出ovf
ovf tool路径 /Applications/VMware Fusion.app/Contents/Library/VMware OVF Tool 上面红色的部分,需要右键点击应用程序中的VMwa ...
- git学习之安装（二）
安装Git 最早Git是在Linux上开发的,很长一段时间内,Git也只能在Linux和Unix系统上跑.不过,慢慢地有人把它移植到了Windows上.现在,Git可以在Linux.Unix.Mac和 ...
- shiro自定义拦截url
在实际项目上,我们针对不同的用户(guste,user,admin,mobile user)等等,需要进入不同的页面,比如,手机端用户需要进入Mobile/这个路径下的,这个时候,我们需要自定义拦截u ...