HBase常用操作-HBaseUtil
package com.zhen.hbase; import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.SubstringComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger; public class HBaseUtil { //@Value("${hbase.zookeeper.quorum}")
private String addr="HDP1,HDP2,HDP3"; //@Value("${hbase.zookeeper.property.clientPort}")
private String port="2181"; Logger logger = Logger.getLogger(getClass()); private static Connection connection; @SuppressWarnings("static-access")
public void getConnection(){
Configuration conf = HBaseConfiguration.create(); conf.set("hbase.zookeeper.quorum",addr);
conf.set("hbase.zookeeper.property.clientPort", port);
try {
this.connection = ConnectionFactory.createConnection(conf);
} catch (IOException e) {
e.printStackTrace();
}
} public HBaseUtil() {
getConnection();
} /*
* 判断表是否存在
*
*
* @tableName 表名
*/ public boolean isExist(String tableName) throws IOException {
TableName table_name = TableName.valueOf(tableName);
Admin admin = connection.getAdmin();
boolean exit = admin.tableExists(table_name);
admin.close();
return exit;
} /**
* 单行添加
* @param tableName
* @param rowKey
* @param family
* @param keyValue
* @throws IOException
*/
@SuppressWarnings("unused")
private static void addRow(String tableName, String rowKey, String family, Map<String, String> keyValue) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Put put = new Put(Bytes.toBytes(rowKey));
for (Entry<String, String> entry : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry.getKey()), Bytes.toBytes(entry.getValue()));
}
table.put(put);
table.close();
keyValue.clear();
} public static void addRows(String tableName, String rowFamilySeparator, Map<String, Map<String, String>> keyValues) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
List<Put> puts = new ArrayList<Put>();
for (Entry<String, Map<String, String>> entry : keyValues.entrySet()) {
String key = entry.getKey();
if (null == rowFamilySeparator || rowFamilySeparator.isEmpty()) {
rowFamilySeparator = "_";
}
String rowKey = key.split(rowFamilySeparator)[0];
String family = key.split(rowFamilySeparator)[1];
Map<String, String> keyValue = entry.getValue();
Put put = new Put(Bytes.toBytes(rowKey), System.currentTimeMillis());
for (Entry<String, String> entry2 : keyValue.entrySet()) {
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(entry2.getKey()), Bytes.toBytes(entry2.getValue()));
}
puts.add(put);
}
table.put(puts);
table.close();
keyValues.clear();
} /**
* 单行删除
* @param tableName
* @param rowKey
* @param family
* @throws IOException
*/
public static void deleteByRowKey(String tableName, String rowKey) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Delete delete = new Delete(Bytes.toBytes(rowKey));
table.delete(delete);
table.close();
} /**
* 查询所有
* @param tableName
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName, String family) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 查询所有rowKey
* @param tableName
* @return
* @throws IOException
*/
public static List<String> queryAllRowKeyForScan(String tableName) throws IOException {
List<String> result = new ArrayList<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
if (null == cells || cells.length <= 0) {
continue;
}
Cell cell = cells[0];
String rowKey = new String(CellUtil.cloneRow(cell));
result.add(rowKey);
}
} finally {
rs.close();
}
return result;
} /**
* 查询所有字段
* @param tableName
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForScan(String tableName) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据时间范围
* @param tableName
* @param family
* @param minStamp
* @param maxStamp
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForTimeRange(String tableName, String family, long minStamp, long maxStamp) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setTimeRange(minStamp, maxStamp);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据RowKey查询
* @param tableName
* @param rowKey
* @param family
* @return
* @throws IOException
*/
public static Map<String, String> queryForRowKey(String tableName, String rowKey, String family) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
Scan scan = new Scan(get);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
}
}
} finally {
rs.close();
}
return row;
} /**
* 根据多个RowKey查询
* @param tableName
* @param rowKeys
* @param family
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeys(String tableName, List<String> rowKeys, String family) throws IOException {
List<Map<String, String>> resultList = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
List<Get> getList = new ArrayList();
for (String rowKey : rowKeys) {
Get get = new Get(Bytes.toBytes(rowKey));
get.addFamily(Bytes.toBytes(family));
getList.add(get);
}
Result[] results = table.get(getList);
for (Result result : results){//对返回的结果集进行操作
Map<String, String> row = new HashMap<String, String>();
for (Cell kv : result.rawCells()) {
row.put(new String(CellUtil.cloneQualifier(kv)), new String(CellUtil.cloneValue(kv),"UTF-8"));
}
resultList.add(row);
}
return resultList;
} /**
* 根据RowKey范围查询
* @param tableName
* @param family
* @param startRow
* @param stopRow
* @return
* @throws IOException
*/
public static List<Map<String, String>> queryForRowKeyRange(String tableName, String family, String startRow, String stopRow) throws IOException {
List<Map<String, String>> rows = new ArrayList<Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(family));
scan.setStartRow(Bytes.toBytes(startRow));
scan.setStopRow(Bytes.toBytes(stopRow));
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
row = new HashMap<String, String>();
for (Cell cell : cells) {
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", new String(CellUtil.cloneRow(cell)));
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell)));
}
rows.add(row);
}
} finally {
rs.close();
}
return rows;
} /**
* 根据指定列名匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfier(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
PageFilter p = new PageFilter(5);
scan.setFilter(filter);
scan.setFilter(p);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
} /**
* 根据指定列名完全匹配列值
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static Collection<Map<String, String>> queryForQuilfierExactly(String tableName, String family, String column, String value) throws IOException {
Map<String, Map<String, String>> rows = new HashMap<String, Map<String, String>>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL,value.getBytes());
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
Map<String, String> row = null;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
if (null == row || !rows.containsKey(rowKey)) {
row = new HashMap<String, String>();
}
row.put("timestamp", cell.getTimestamp() + "");
row.put("rowKey", rowKey);
row.put("family", new String(CellUtil.cloneFamily(cell)));
row.put(new String(CellUtil.cloneQualifier(cell)), new String(CellUtil.cloneValue(cell),"UTF-8"));
rows.put(rowKey,row);
}
}
} finally {
rs.close();
}
return rows.values();
} /**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter filter = new QualifierFilter(CompareOp.EQUAL,new BinaryComparator(Bytes.toBytes(qualifier)));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList filterList = new FilterList();
filterList.addFilter(filter);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} /**
* 获取多个列值匹配的rowkey
* @param tableName
* @param family
* @param qualifierMap 列名:列值
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForMultiQuilfierName(String tableName, String family, Map<String, String> qualifierMap, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
for (Entry<String, String> entry : qualifierMap.entrySet()) {
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), entry.getKey().getBytes(), CompareOp.EQUAL,entry.getValue().getBytes());
filter.setFilterIfMissing(true);
filterList.addFilter(filter);
}
PageFilter pageFilter = new PageFilter(pageSize);
filterList.addFilter(pageFilter);
scan.setFilter(filterList);
scan.addFamily(Bytes.toBytes(family));
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} public static void qualifierFilter() throws IOException {
Table mTable = connection.getTable(TableName.valueOf("portrait"));
QualifierFilter columnsNameFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL, new BinaryComparator("basictag-14".getBytes()));
Scan scan = new Scan();
scan.setFilter(columnsNameFilter);
ResultScanner rs = mTable.getScanner(scan);
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
}
}
} /**
* 获取列名匹配的rowkey
* @param tableName
* @param qualifier 列名
* @param pageSize 数量
* @return
* @throws IOException
*/
public static List<String> queryForQuilfierName(String tableName, String family, String qualifier, long pageSize) throws IOException {
Set<String> rowKeys = new HashSet<String>();
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
Filter qualifierFilter = new QualifierFilter(CompareFilter.CompareOp.EQUAL,new BinaryComparator(qualifier.getBytes()));
PageFilter pageFilter = new PageFilter(pageSize);
FilterList list = new FilterList();
list.addFilter(pageFilter);
list.addFilter(qualifierFilter);
scan.addFamily(Bytes.toBytes(family));
scan.setFilter(list);
System.out.println(scan.toJSON());
ResultScanner rs = table.getScanner(scan);
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
Cell[] cells = r.rawCells();
for (Cell cell : cells) {
System.out.println("==== "+new String(CellUtil.cloneRow(cell))+"\t"+
new String(CellUtil.cloneFamily(cell))+"\t"+
new String(CellUtil.cloneQualifier(cell))+"\t"+
new String(CellUtil.cloneValue(cell)));
String rowKey = new String(CellUtil.cloneRow(cell));
rowKeys.add(rowKey);
}
}
} finally {
rs.close();
}
List<String> rows = new ArrayList<String>(rowKeys);
return rows;
} /**
* 根据指定列名匹配列值条数
* @param tableName
* @param family
* @param qualifier
* @param value
* @return
* @throws IOException
*/
public static long queryForQuilfierCount(String tableName, String family, String column, String value) throws IOException {
Table table = connection.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
SubstringComparator comp = new SubstringComparator(value);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family.getBytes(), column.getBytes(), CompareOp.EQUAL, comp);
filter.setFilterIfMissing(true);
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
long count = 0;
try {
for (Result r = rs.next(); r != null; r = rs.next()) {
count++;
}
} finally {
rs.close();
}
return count;
} /*
* 关闭连接
*
*/
public void close() {
/**
* close connection
**/
if (connection != null) {
try {
connection.close();
} catch (IOException e) {
e.printStackTrace();
}
}
} }
HBase常用操作-HBaseUtil的更多相关文章
- Hbase常用操作记录
Hbase常用操作记录 Hbase 创建表 查看表结构 修改表结构 删除表 创建表 语法:create <table>, {NAME => <family>, VERSI ...
- Hbase常用操作(增删改查)
Hbase常用操作(增删改查) [日期:2014-01-03] 来源:Linux社区 作者:net19880504 [字体:大 中 小] 运行Eclipse,创建一个新的Java工程“HBa ...
- HBase常用操作之namespace
1.介绍 在HBase中,namespace命名空间指对一组表的逻辑分组,类似RDBMS中的database,方便对表在业务上划分.Apache HBase从0.98.0, 0.95.2两个版本开始支 ...
- Hbase常用操作
下面我们看看HBase Shell的一些基本操作命令,我列出了几个常用的HBase Shell命令,如 名称 命令表达式 创建表 create '表名称', '列名称1','列名称2','列名称N' ...
- HBASE常用操作增删改查
http://javacrazyer.iteye.com/blog/1186881 http://www.cnblogs.com/invban/p/5667701.html
- Hbase到Solr同步常用操作
Hbase到Solr同步常用操作 1. 整体流程 2. 常用操作 Hbase常用操作 Solr常用操作 hbase-index常用操作 3. 其他资料 Lily HBase Indexer使用整理 h ...
- HBase Shell操作
Hbase 是一个分布式的.面向列的开源数据库,其实现是建立在google 的bigTable 理论之上,并基于hadoop HDFS文件系统. Hbase不同于一般的关系型数据库(RDBMS ...
- HBase常用操作命令
HBase常用操作命令 1.进入HBase脚本客户端 #hbase shell #进入HBase脚本客户端 > whoami #查看当前登录用户 > status ...
- HBase伪分布式安装(HDFS)+ZooKeeper安装+HBase数据操作+HBase架构体系
HBase1.2.2伪分布式安装(HDFS)+ZooKeeper-3.4.8安装配置+HBase表和数据操作+HBase的架构体系+单例安装,记录了在Ubuntu下对HBase1.2.2的实践操作,H ...
随机推荐
- is_callable — 检测参数是否为合法的可调用结构
说明 bool is_callable ( callable $name [, bool $syntax_only = false [, string &$callable_name ]] ) ...
- Lua基本函数库 【转】
转自:http://www.cnblogs.com/whiteyun/archive/2009/08/12/1543184.html 基本函数库为Lua内置的函数库,不需要额外装载 assert (v ...
- 使用新版MonoDevelop来进行unity工程调试
现在可以使用新版的MonoDeveloper来调试Unity工程了,新版的MonoDeveloper界面上漂亮很多,而且使用.调试上感觉也更舒服了.先上一张图: 安装方法如下: 登陆网址:http ...
- secureCrt Linux 文件传输
1.在secureCRT终端下输入rz命令,查看Linux是否安装rz文件传输服务 如果提示未安装则先安装rz服务:安装命令: apt-get install lrzsz 2.进入要上传的目的文件 ...
- python在windows下安装paramiko模块和安装pycrypto模块（3步搞定）（转）
Python中使用SSH需要用到OpenSSH,而OpenSSH依赖于paramiko模块,而paramiko模块又依赖于pycrypto模块,因此要在Python中使用SSH,我们需要先安装pycr ...
- [SCOI2009]生日礼物(尺取法)
Time Limit: 10 Sec Memory Limit: 162 MBSubmit: 2201 Solved: 1186[Submit][Status][Discuss] Descript ...
- 星球大战starwar(并查集)
1015: [JSOI2008]星球大战starwar Time Limit: 3 Sec Memory Limit: 162 MBSubmit: 5253 Solved: 2395[Submit ...
- EasyPlayer播放器浏览器ActiveX/OCX插件RTSP播放/抓拍/录像功能调用说明
EasyPlayerPro与EasyPlayer-RTSP新增ocx多窗口播放功能 这里以EasyPlayerPro为例,使用方法如下: 打开播放器文件夹,进入Bin/C++目录,可以看到reg.ba ...
- 九度OJ 1203：IP地址 （字符串处理）
时间限制:1 秒 内存限制:32 兆 特殊判题:否 提交:3038 解决:1496 题目描述: 输入一个ip地址串,判断是否合法. 输入: 输入的第一行包括一个整数n(1<=n<=500) ...
- 我的Android进阶之旅------>/storage/sdcard0， /sdcard， /mnt/sdcard ，/storage/emulated/legacy 的区别
转自:http://bbs.gfan.com/android-5382920-1-1.html 关于android的4.2的0文件夹的详解---- android 4.0 ----在galaxy ne ...