hbase数据导入

hbase数据导入：

参考http://blog.csdn.net/hua840812/article/details/7414875，在把代码copy下来后，发现运行总是报错：

java.io.IOException: Type mismatch in key from map: expected org.apache.hadoop.hbase.io.ImmutableBytesWritable, recieved org.apache.hadoop.io.LongWritable；

原因是map的输出必须按照现有的版本来写，也就是extends Mapper<LongWritable, Text, ImmutableBytesWritable, KeyValue>

要这样写，不能简单的写成extends Mapper,

代码还是贴出来：

生成hfile的代码：

package com.bonc.db;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class TestHFileToHBase {

    public static class TestHFileToHBaseMapper  extends Mapper<LongWritable, Text, ImmutableBytesWritable, KeyValue> {

        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String[] values = value.toString().split("\\|");
            ImmutableBytesWritable rowkey = new ImmutableBytesWritable(
                    values[].toString().trim().getBytes());
            KeyValue kvProtocol;
            if (values.length>){
             kvProtocol = new KeyValue(values[].toString().trim().getBytes(), "url_type".getBytes(), "type".getBytes(),System.currentTimeMillis(), values[].toString().trim()
                    .getBytes());
            }else{
                kvProtocol=new KeyValue(values[].toString().trim().getBytes(), "url_type".getBytes(), "type".getBytes(),System.currentTimeMillis(), "NULL".getBytes());
            }
            context.write(rowkey, kvProtocol);
            // KeyValue kvSrcip = new KeyValue(row, "SRCIP".getBytes(),
            // "SRCIP".getBytes(), values[1].getBytes());
            // context.write(k, kvSrcip);
//             HFileOutputFormat.getRecordWriter
        }

    }

    public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
        Configuration conf = HBaseConfiguration.create();
        Job job = new Job(conf, "TestHFileToHBase");
        job.setJarByClass(TestHFileToHBase.class);

        job.setOutputKeyClass(ImmutableBytesWritable.class);
        job.setOutputValueClass(KeyValue.class);

        job.setMapperClass(TestHFileToHBaseMapper.class);
        job.setReducerClass(KeyValueSortReducer.class);
//        job.setOutputFormatClass(org.apache.hadoop.hbase.mapreduce.HFileOutputFormat.class);
        job.setOutputFormatClass(HFileOutputFormat.class);
        // job.setNumReduceTasks(4);
        // job.setPartitionerClass(org.apache.hadoop.hbase.mapreduce.SimpleTotalOrderPartitioner.class);

        // HBaseAdmin admin = new HBaseAdmin(conf);
        HTable table = new HTable(conf, "url_rule");

        HFileOutputFormat.configureIncrementalLoad(job, table);

        FileInputFormat.addInputPath(job, new Path(args[]));
        FileOutputFormat.setOutputPath(job, new Path(args[]));

        System.exit(job.waitForCompletion(true) ?  : );
    }

}

hfile导入到表的代码：

 package com.bonc.db;
 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
 import org.apache.hadoop.hbase.util.Bytes;  

 public class TestLoadIncrementalHFileToHBase {  

     // private static final byte[] TABLE = Bytes.toBytes("hua");
     // private static final byte[] QUALIFIER = Bytes.toBytes("PROTOCOLID");
     // private static final byte[] FAMILY = Bytes.toBytes("PROTOCOLID");  

     public static void main(String[] args) throws IOException {
         Configuration conf = HBaseConfiguration.create();
 //      byte[] TABLE = Bytes.toBytes("hua");
         byte[] TABLE = Bytes.toBytes(args[]);
         HTable table = new HTable(TABLE);
         LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
         loader.doBulkLoad(new Path(args[]), table);
 //      loader.doBulkLoad(new Path("/hua/testHFileResult/"), table);
     }  

 }  

悲剧的是在从hfile导入到表的时候报错：

java.io.IOException: java.io.IOException: Failed rename of maprfs://133.0.76.41:7222/user/hbasetest/url_type/4447706551787973235 to maprfs://133.0.76.41:7222/hbase/url_rule/732e6d3d150caa8bd3d8d228e3d9c9a0/url_type/914168143008836217
        at org.apache.hadoop.hbase.regionserver.StoreFile.rename(StoreFile.java:512)

虽然解决办法在这里：

http://answers.mapr.com/questions/685/hbase-and-completebulkload

但是我实在是没看懂。so，我采用了最原始的方法：

split将文件分割成小文件，然后：

 package com.bonc.db;
 import java.io.BufferedReader;
 import java.io.FileReader;
 import java.io.IOException;

 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.HTablePool;
 import org.apache.hadoop.hbase.client.Put;

 import com.bonc.URLMatch.HBaseMain;
 public class URL_HBase {

         public static void main(String[] args) throws IOException{
             //文件绝对路径改成你自己的文件路径
             FileReader fr=new FileReader(args[]);
             //可以换成工程目录下的其他文本文件
             HTablePool pool = new HTablePool(HBaseMain.conf, );
             HTable table = (HTable) pool.getTable("url_rule");
             BufferedReader br=new BufferedReader(fr);
             while(br.readLine()!=null){
                 String[] s=br.readLine().toString().split("\\|");
                 if(s.length>){
                 Put put = new Put(s[].trim().getBytes());
                 put.add("url_type".getBytes(), "type".getBytes(), s[].trim().getBytes());
                 table.put(put);
                 }else{
                     System.out.println(s);
                 }
             }
             br.close();
         }
     }

终于成功了，希望有人能够帮我翻译一下，怎么解决是个什么意思。。唉。