Hbase API 简单封装

>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>欢迎转载，转载请注明出处-VirgoArt，www.cnblogs.com

一、Web项目链接配置（这里可以优化成连接池）（关于构造，是为了满足JUnit测试需求）

package com.pj.util;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent;

public class HBaseConnectionUtil implements ApplicationListener<ContextRefreshedEvent> {
    public static Connection connection;
    private static boolean isCreate = false;

    public HBaseConnectionUtil() {
        if (!isCreate) {
            this.onApplicationEvent(null);
        }
    }

    /** 这里结合Spring配置，实现初始化完成后的消息事件触发 */
    @Override
    public void onApplicationEvent(ContextRefreshedEvent arg0) {
        if (!isCreate) {
            Configuration conf = HBaseConfiguration.create();
            conf.set(SystemConfigUtil.getInstance().getProValue("HBASE_ZK_PATH"),
                    SystemConfigUtil.getInstance().getProValue("HBASE_ZK_IP"));
            try {
                connection = ConnectionFactory.createConnection(conf);
            } catch (IOException e) {
                e.printStackTrace();
            }
            isCreate = true;
        }
    }
}

二、常用的表级、列级操作

package com.pj.util;

import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.LoggerFactory;

import com.pj.entity.ResultBean;

import ch.qos.logback.classic.Logger;

public class HBaseUtil {
    public final static Logger log = (Logger) LoggerFactory.getLogger(HBaseUtil.class);

    /** 封装Table
     *
     * @deprecated 建议使用getTablePlus */
    @Deprecated
    public static Table getTable(String tableName) {
        try {
            return HBaseConnectionUtil.connection.getTable(TableName.valueOf(tableName));
        } catch (IOException e) {
            log.info(tableName + "表对象创建失败.");
            e.printStackTrace();
            return null;
        }
    }

    public static ResultBean<Table> getTablePlus(String tableName) {
        ResultBean<Table> res = null;
        try {
            Table table = HBaseConnectionUtil.connection.getTable(TableName.valueOf(tableName));
            res = new ResultBean<Table>(table);
        } catch (IOException e) {
            log.info(tableName + "表对象创建失败.");
            res = new ResultBean<>(e);
        }
        return res;
    }

    /** 通过Get进行单条查询，返回为单条结果
     *
     * @param table
     * @param rowKey
     * @param clazz
     * @return */
    @SuppressWarnings({ "deprecation", "unchecked" })
    public static <T> ResultBean<T> getDataFBean(Table table, String rowKey, Class<?> clazz) {
        Get get = new Get(Bytes.toBytes(rowKey));
        Result result = null;
        try {
            result = table.get(get);
        } catch (IOException e) {
            return new ResultBean<T>(e);
        }
        T t = null;
        try {
            t = (T) clazz.newInstance();
        } catch (InstantiationException e) {
            return new ResultBean<T>(e);
        } catch (IllegalAccessException e) {
            return new ResultBean<T>(e);
        }
        Method[] methods = clazz.getMethods();
        try {
            if (null == result) {
                return new ResultBean<T>(true, "Get Null result");
            }
            for (KeyValue keyValue : result.list()) {
                String row = Bytes.toString(result.getRow());
                String k = Bytes.toString(keyValue.getQualifier());
                String v = Bytes.toString(keyValue.getValue());
                for (Method method : methods) {
                    // 如果Bean中存在rowKey字段，用Key进行赋值
                    if (method.getName().equalsIgnoreCase("setRowKey")) {
                        try {
                            method.invoke(t, row);
                        } catch (IllegalAccessException e) {
                            e.printStackTrace();
                        } catch (IllegalArgumentException e) {
                            e.printStackTrace();
                        } catch (InvocationTargetException e) {
                            e.printStackTrace();
                        }
                    }
                    if (method.getName().equalsIgnoreCase("set" + k)) {
                        try {
                            method.invoke(t, v);
                        } catch (IllegalAccessException e) {
                            e.printStackTrace();
                        } catch (IllegalArgumentException e) {
                            e.printStackTrace();
                        } catch (InvocationTargetException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
        } catch (Exception e) {
            return new ResultBean<T>(e);
        }
        return new ResultBean<T>(t);
    }

    /** 封装结果集 */
    @Deprecated
    public static ResultScanner getResultScanner(Table table, Scan scan) {
        try {
            return table.getScanner(scan);
        } catch (IOException e) {
            log.info("扫描" + table.getName() + "失败.");
            e.printStackTrace();
            return null;
        }
    }

    /** get方式查询数据
     *
     * @param table
     * @param getList
     * @param clazz
     * @return */
    public static List<?> getResultScannerPlus(Table table, List<Get> getList, Class<?> clazz) {
        List<Object> list = new ArrayList<Object>();
        Result[] results;
        try {
            results = table.get(getList);
            for (Result result : results) {
                Object instance = clazz.newInstance();
                for (Cell kv : result.rawCells()) {
                    Field field = clazz.getDeclaredField(new String(CellUtil.cloneQualifier(kv)));
                    field.setAccessible(true);
                    field.set(instance, new String(CellUtil.cloneValue(kv)));
                }
                list.add(instance);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return list;
    }

    public static ResultBean<ResultScanner> getResultScannerPlus(Table table, Scan scan) {
        try {
            //scan.setCaching(1000);
            return new ResultBean<ResultScanner>(table.getScanner(scan));
        } catch (IOException e) {
            log.info("扫描" + table.getName() + "失败.");
            e.printStackTrace();
            return new ResultBean<>(e);
        }
    }

    /** 封装结果数据Map */
    @SuppressWarnings({ "unchecked", "deprecation" })
    public static <T> Map<String, T> getResultObjectMap(ResultScanner resultScanner, Class<?> clazz) {
        Map<String, T> resultMap = new HashMap<String, T>();
        Method[] methods = clazz.getMethods();

        for (Result res : resultScanner) {
            String key = Bytes.toString(res.getRow());
            T t = null;
            try {
                t = (T) clazz.newInstance();
            } catch (InstantiationException e) {
                e.printStackTrace();
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            }

            for (KeyValue keyValue : res.list()) {
                String k = Bytes.toString(keyValue.getQualifier());
                String v = Bytes.toString(keyValue.getValue());
                for (Method method : methods) {
                    // 如果Bean中存在rowKey字段，用Key进行赋值
                    if (method.getName().equalsIgnoreCase("setRowKey")) {
                        try {
                            method.invoke(t, key);
                        } catch (IllegalAccessException e) {
                            e.printStackTrace();
                        } catch (IllegalArgumentException e) {
                            e.printStackTrace();
                        } catch (InvocationTargetException e) {
                            e.printStackTrace();
                        }
                    }
                    if (method.getName().equalsIgnoreCase("set" + k)) {
                        try {
                            method.invoke(t, v);
                        } catch (IllegalAccessException e) {
                            e.printStackTrace();
                        } catch (IllegalArgumentException e) {
                            e.printStackTrace();
                        } catch (InvocationTargetException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
            resultMap.put(key, t);
        }
        return resultMap;
    }

    /** 封装结果数据List */
    @SuppressWarnings({ "unchecked", "deprecation" })
    public static <T> List<T> getResultObjectList_bck(ResultScanner resultScanner, Class<?> clazz) {
        List<T> resultList = new ArrayList<>();
        Method[] methods = clazz.getMethods();

        for (Result res : resultScanner) {
            String key = Bytes.toString(res.getRow());
            T t = null;
            try {
                t = (T) clazz.newInstance();
            } catch (InstantiationException e) {
                e.printStackTrace();
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            }
            for (int i = 0; i < res.list().size(); i++) {
                KeyValue keyValue = res.list().get(i);
                String k = Bytes.toString(keyValue.getQualifier());
                String v = Bytes.toString(keyValue.getValue());
                for (int j = 0; j < methods.length; i++) {
                    Method method = methods[i];
                    if (method.getName().equalsIgnoreCase("setRowKey")) {
                        try {
                            method.invoke(t, key);
                        } catch (IllegalAccessException e) {
                            e.printStackTrace();
                        } catch (IllegalArgumentException e) {
                            e.printStackTrace();
                        } catch (InvocationTargetException e) {
                            e.printStackTrace();
                        }
                    }
                    if (method.getName().equalsIgnoreCase("set" + k)) {
                        try {
                            method.invoke(t, v);
                        } catch (IllegalAccessException e) {
                            e.printStackTrace();
                        } catch (IllegalArgumentException e) {
                            e.printStackTrace();
                        } catch (InvocationTargetException e) {
                            e.printStackTrace();
                        }
                    }
                }
            }
            /*
             * for (KeyValue keyValue : res.list()) { String k =
             * Bytes.toString(keyValue.getQualifier()); String v =
             * Bytes.toString(keyValue.getValue()); for (Method method : methods) { //
             * 如果Bean中存在rowKey字段，用Key进行赋值 if
             * (method.getName().equalsIgnoreCase("setRowKey")) { try { method.invoke(t,
             * key); } catch (IllegalAccessException e) { e.printStackTrace(); } catch
             * (IllegalArgumentException e) { e.printStackTrace(); } catch
             * (InvocationTargetException e) { e.printStackTrace(); } } if
             * (method.getName().equalsIgnoreCase("set" + k)) { try { method.invoke(t, v); }
             * catch (IllegalAccessException e) { e.printStackTrace(); } catch
             * (IllegalArgumentException e) { e.printStackTrace(); } catch
             * (InvocationTargetException e) { e.printStackTrace(); } } } }
             */
            resultList.add(t);
        }
        return resultList;
    }

    @SuppressWarnings({ "deprecation", "unchecked" })
    public static <T> List<T> getResultObjectList(ResultScanner resultScanner, Class<?> clazz) {
        List<T> resultList = new ArrayList<>();
        // Method[] methods = clazz.getMethods();
        for (Result res : resultScanner) {
            String key = Bytes.toString(res.getRow());
            T t = null;
            try {
                t = (T) clazz.newInstance();
            } catch (InstantiationException e) {
                e.printStackTrace();
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            }
            for (int i = 0; i < res.list().size(); i++) {
                KeyValue keyValue = res.list().get(i);
                String k = Bytes.toString(keyValue.getQualifier());
                String v = Bytes.toString(keyValue.getValue());
                Field field = null;
                try {
                    if (!k.equals("serialVersionUID")) {
                        if (i == 1) {
                            field = clazz.getDeclaredField("rowKey");
                            field.setAccessible(true);
                            field.set(t, key);
                        }
                        field = clazz.getDeclaredField(k);
                        field.setAccessible(true);
                        field.set(t, v);
                    }
                } catch (IllegalArgumentException | NoSuchFieldException | SecurityException
                        | IllegalAccessException e) {
                    e.printStackTrace();
                }
            }
            resultList.add(t);
        }
        return resultList;
    }

    /** 封装Put get */
    public static Put getPut(String rowKye) {
        Put put = new Put(Bytes.toBytes(rowKye));
        return put;
    }

    /** 封装Put set
     *
     * @throws IOException */
    @Deprecated
    public static void setPut(Table table, Put put) throws IOException {
        table.put(put);
    }

    public static ResultBean<String> setPutPlus(Table table, Put put) {
        try {
            table.put(put);
            return new ResultBean<String>("Put Success");
        } catch (IOException e) {
            e.printStackTrace();
            return new ResultBean<String>(e);
        }
    }

    /** 封装Put add */
    @SuppressWarnings("deprecation")
    public static Put putAdd(Put put, String colunmFamily, String colunmName, String value) {
        put.add(Bytes.toBytes(colunmFamily), Bytes.toBytes(colunmName), Bytes.toBytes(value));
        return put;
    }

    /** 封装Delete，ROWKEY delete */
    @Deprecated
    public static void deleteRowByRowKey(Table table, String rowKey) {
        Delete delete = new Delete(Bytes.toBytes(rowKey));
        try {
            table.delete(delete);
        } catch (IOException e) {
            log.info(rowKey + "行删除失败.");
            e.printStackTrace();
        }
    }

    public static ResultBean<String> deleteRowByRowKeyPlus(Table table, String rowKey) {
        Delete delete = new Delete(Bytes.toBytes(rowKey));
        try {
            table.delete(delete);
            return new ResultBean<String>("Delete Success");
        } catch (IOException e) {
            log.info(rowKey + "行删除失败.");
            return new ResultBean<String>(e);
        }
    }

    /** Close：需要调用时注意序列，从小到大进行排列 */
    public static void close(Object... objects) {
        for (Object o : objects) {
            if (null != o) {
                if (o instanceof ResultScanner) {
                    ((ResultScanner) o).close();
                } else if (o instanceof Scan) {

                } else if (o instanceof Table) {
                    try {
                        ((Table) o).close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                } else if (o instanceof Connection) {
                    try {
                        ((Connection) o).close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                } else {
                    throw new RuntimeException("未知对象:" + o.getClass().getName());
                }
            }
        }
    }

    /** 列过滤器：SingleColumnValueFilter<br>
     * 用于在进行条件查询时，通过过滤参数条件自动生成过滤器（单个）<br>
     * compareOp:
     * <code>LESS 小于，LESS_OR_EQUAL 小于等于，EQUAL 等于，NOT_EQUAL 不等于，GREATER_OR_EQUAL 大于等于，GREATER 大于</code> */
    public static Filter setSingleColumnValueFilter(String colunmFilterString, String splitString, String compareOp) {
        String[] sp = colunmFilterString.split(splitString);
        return new SingleColumnValueFilter(Bytes.toBytes(sp[0]), Bytes.toBytes(sp[1]), CompareOp.valueOf(compareOp),
                Bytes.toBytes(sp[2].toString()));
    }

    /** 对列过滤器设计过滤条件：SingleColumnValueFilter<br>
     * 用于在进行条件查询时，通过对象合成过滤条件 */
    public static List<String> setSingleColumnValueFilterParam(String colunmFamily, Object object, String splitString) {
        List<String> params = new ArrayList<>();
        for (Field field : object.getClass().getDeclaredFields()) {
            Object value = ReflectUtil.getFiledValueByName(field.getName(), object);
            if (null != value && !"".equals(value.toString())) {
                params.add(colunmFamily + splitString + field.getName() + splitString + value.toString());
            }
        }
        return params;
    }

    /** 对HTable求Count（在有查询条件时，表现有问题，初步判断是由于FirstKeyOnlyFilter与其他过滤器不兼容） */
    public static long getCountByFilter(String tableName, FilterList fl) {
        long count = 0;
        Scan scan = new Scan();
        scan.setFilter(fl);
        ResultScanner rs = null;
        try {
            rs = HBaseUtil.getTable(tableName).getScanner(scan);
            Iterator<Result> it = rs.iterator();
            while (it.hasNext()) {
                it.next();
                count += 1;
            }
        } catch (IOException e) {
            e.printStackTrace();
        }
        close(rs);
        return count;
    }

    /** 做全表扫描时，根据page对象进行分页（可以通过FilterList传入过滤条件） */
    public static <T> List<T> scanTableByPage(String tableName, Page page, Class<?> clazz, FilterList fl) {
        List<T> rsList = null;
        Scan scan = new Scan();// 全表扫描Scan
        Filter pageFilter = null;// 分页过滤器
        ResultScanner rs = null;
        page.setTotalRecord((int) getCountByFilter(tableName, fl));// 设置page对象中总记录数
        int pageNow = page.getPageNo();// 当前页
        int pageSize = page.getPageSize();// 页大小
        int recordSize = page.getTotalPage();// 总条数
        int begin = 0;// 内容显示时开始的行号
        if (pageNow == 1) {
            // 首页分页处理，直接返回页大小值
            pageFilter = new PageFilter(pageSize);
            fl.addFilter(pageFilter);
            scan.setFilter(fl);
            rs = HBaseUtil.getResultScanner(HBaseUtil.getTable(tableName), scan);
            rsList = HBaseUtil.getResultObjectList(rs, clazz);
        } else {
            begin = (pageNow - 1) * pageSize + 1;
            if (begin > recordSize) {
                // 如果是尾页，修改开始扫描行号
                begin = (pageNow - 1) * pageSize + 1;
            }
            pageFilter = new PageFilter(begin);// 一次性扫描前n页+1的数据
            fl.addFilter(pageFilter);
            scan.setFilter(fl);
            rs = HBaseUtil.getResultScanner(HBaseUtil.getTable(tableName), scan);
            Iterator<Result> it = rs.iterator();
            int b = 0;
            byte[] startRow = null;
            while (it.hasNext()) {
                b++;
                if (b == begin) {
                    startRow = it.next().getRow();// 通过迭代获取当前显示页首数据RowKey
                }
                it.next();
            }
            pageFilter = new PageFilter(pageSize);// 重置分页过滤器为页面数据大小
            fl.addFilter(pageFilter);
            scan.setFilter(fl);
            scan.setStartRow(startRow);// 设置起始扫描RowKey
            rs = HBaseUtil.getResultScanner(HBaseUtil.getTable(tableName), scan);
            rsList = HBaseUtil.getResultObjectList(rs, clazz);

        }
        // page.setTotalRecord(rsList.size());//
        // 由于getCountByFilter在条件环境下有问题，重新进行值的设定
        close(rs);
        return rsList;
    }

    /** 从KeyValue类型中，getKeyString方法获取真实的RowKey */
    @SuppressWarnings({ "deprecation", "static-access" })
    public static String getRealRowKey(KeyValue kv) {
        int rowLength = Bytes.toShort(kv.getBuffer(), kv.getOffset() + kv.ROW_OFFSET);
        String rowKey = Bytes.toStringBinary(kv.getBuffer(), kv.getOffset() + kv.ROW_OFFSET + Bytes.SIZEOF_SHORT,
                rowLength);
        return rowKey;
    }

    public static Put putAddByBean(Put put, Object obj) {
        for (Field field : obj.getClass().getDeclaredFields()) {
            String firstLetter = field.getName().substring(0, 1).toUpperCase();
            String getter = "get" + firstLetter + field.getName().substring(1);
            Object value = null;
            try {
                // 构造的GET方法，如果没有，抛异常NoSuchMethodException
                Method getMethod = obj.getClass().getMethod(getter, new Class[] {});
                // 代理obj的GET方法，获取返回值
                value = getMethod.invoke(obj, new Object[] {});
            } catch (NoSuchMethodException e) {
                e.printStackTrace();
            } catch (SecurityException e) {
                e.printStackTrace();
            } catch (IllegalAccessException e) {
                e.printStackTrace();
            } catch (IllegalArgumentException e) {
                e.printStackTrace();
            } catch (InvocationTargetException e) {
                e.printStackTrace();
            }
            if (null == value || value.equals("") || value.equals("null")) {

            } else {
                HBaseUtil.putAdd(put, "cf1", field.getName(), value.toString());
            }
        }
        return put;
    }
}

三、简单的HDFS文件存储（为了支持Hbase中的文件索引）

package com.pj.util;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

/** 操作分布式集群HDFS工具类
 *
 * @author ChenWen */
public class HDFSOptionUtil {

    /** 将本地文件上传到HDFS(不清楚为什么本地文件复制到HDFS，API会将本地文件删除掉):方法将true改成false
     *
     * @param localPathStr
     * @param hdfsPathStr
     * @throws IOException */
    public static void localfileUpload(String localPathStr, String hdfsPathStr) throws IOException {
        Configuration conf = new Configuration();
        conf.addResource(new Path("hdfs-site.xml"));
        conf.addResource(new Path("core-site.xml"));
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localPathStr);
        Path hdfsPath = new Path(hdfsPathStr);
        fs.copyFromLocalFile(true, localPath, hdfsPath);
    }

    /** HDFS文件下载
     *
     * @param hdfsPathStr
     * @param localPathStr
     * @throws IOException */
    public static void fsFileDownload(String hdfsPathStr, String localPathStr) throws IOException {
        Configuration conf = new Configuration();
        conf.addResource(new Path("hdfs-site.xml"));
        conf.addResource(new Path("core-site.xml"));
        FileSystem fs = FileSystem.get(conf);
        FSDataInputStream in = null;
        FileOutputStream out = null;
        try {
            File aimFilePath = new File(localPathStr);
            if (!aimFilePath.exists()) {
                aimFilePath.mkdirs();
            }
            Path srcPath = new Path(hdfsPathStr);

            in = fs.open(srcPath);
            File aimFile = new File(aimFilePath, srcPath.getName());
            if (!aimFile.exists()) {
                aimFile.createNewFile();
            }
            out = new FileOutputStream(aimFile);
            IOUtils.copyBytes(in, out, 4096, false);
        } finally {
            IOUtils.closeStream(in);
            IOUtils.closeStream(out);
        }
    }
}

四、补充

　　1.使用的SystemConfig是用来获取系统配置文件键值对。

　　2.ResultBean用来自定义返回值，具有异常栈存储、消息、值存储。