Exception in thread "main" java.io.IOException: Mkdirs failed to create /user/centos/hbase-staging (exists=false, cwd=file:/home/centos)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:440)
at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:426)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:906)
at org.apache.hadoop.io.SequenceFile$Writer.<init>(SequenceFile.java:1071)
at org.apache.hadoop.io.SequenceFile$RecordCompressWriter.<init>(SequenceFile.java:1371)
at org.apache.hadoop.io.SequenceFile.createWriter(SequenceFile.java:272)
at org.apache.hadoop.io.SequenceFile.createWriter(SequenceFile.java:294)
at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.writePartitions(HFileOutputFormat2.java:335)
at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.configurePartitioner(HFileOutputFormat2.java:596)
at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.configureIncrementalLoad(HFileOutputFormat2.java:440)
at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.configureIncrementalLoad(HFileOutputFormat2.java:405)
at org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2.configureIncrementalLoad(HFileOutputFormat2.java:386)
at bulkloadExample.BulkloadData.main(BulkloadData.java:88)

  1. package bulkloadExample;
  2.  
  3. import org.apache.hadoop.conf.Configuration;
  4. import org.apache.hadoop.fs.FileSystem;
  5. import org.apache.hadoop.fs.Path;
  6. import org.apache.hadoop.hbase.HBaseConfiguration;
  7. import org.apache.hadoop.hbase.TableName;
  8. import org.apache.hadoop.hbase.client.Connection;
  9. import org.apache.hadoop.hbase.client.ConnectionFactory;
  10. import org.apache.hadoop.hbase.client.Put;
  11. import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
  12. import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2;
  13. import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
  14. import org.apache.hadoop.hbase.mapreduce.PutSortReducer;
  15. import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
  16. import org.apache.hadoop.hbase.util.Bytes;
  17. import org.apache.hadoop.io.LongWritable;
  18. import org.apache.hadoop.io.Text;
  19. import org.apache.hadoop.mapreduce.Job;
  20. import org.apache.hadoop.mapreduce.Mapper;
  21. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  22. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  23.  
  24. import java.io.IOException;
  25. import java.net.URI;
  26.  
  27. public class BulkloadData {
  28.  
  29. static final String JOBNAME = "BulkLoad";
  30. static final String TABLENAME = "wqbin:duowan_user_bulkLoad";
  31. static final String PATH_IN = "/datain/duowan_user.txt"; //输入路径
  32. static final String PATH_OUT = "/dataout"; //输入路径HFILE
  33.  
  34. static final String SEPARATOR = "\t";
  35.  
  36. static final byte[] ColumnFamily = "f".getBytes(); // 列簇
  37. // static final byte[] row_id = "id".getBytes(); // 列名
  38. static final byte[] name = "name".getBytes();
  39. static final byte[] pass = "pass".getBytes();
  40. static final byte[] mail = "mail".getBytes();
  41. static final byte[] nickname = "nickname".getBytes();
  42.  
  43. public static class Map extends Mapper<LongWritable, Text, ImmutableBytesWritable, Put> {
  44. protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
  45. String[] strArr = value.toString().split(SEPARATOR);
  46. String row = strArr[0];
  47. Put put = new Put(Bytes.toBytes(row.toString())); // rowkey
  48. put.add(ColumnFamily, name, Bytes.toBytes(strArr[1]));
  49. put.add(ColumnFamily, pass, Bytes.toBytes(strArr[2]));
  50. put.add(ColumnFamily, mail, Bytes.toBytes(strArr[3]));
  51. put.add(ColumnFamily, nickname, Bytes.toBytes(strArr[3]));
  52.  
  53. context.write(new ImmutableBytesWritable(value.getBytes()), put);
  54. }
  55. }
  56.  
  57. public static void main(String[] args) throws Exception {
  58.  
  59. Configuration conf = HBaseConfiguration.create();
  60. Connection conn = ConnectionFactory.createConnection(conf);
  61. // conf.set("hbase.zookeeper.quorum", "xx,xx,xx");
  62. conf.set("fs.defaultFS","file:///");
  63. System.setProperty("HADOOP_USER_NAME", "centos");
  64.  
  65. Job job = Job.getInstance(conf, JOBNAME);
  66. job.setJarByClass(BulkloadData.class);
  67.  
  68. job.setMapOutputKeyClass(ImmutableBytesWritable.class);
  69. job.setMapOutputValueClass(Put.class);
  70. TableMapReduceUtil.addDependencyJars(job);
  71. job.setMapperClass(Map.class);
  72. //这个 SorterReducer(KeyValueSortReducer或PutSortReducer) 可以不指定,
  73. //因为源码中已经做了判断
  74. job.setReducerClass(PutSortReducer.class);
  75. job.setOutputFormatClass(HFileOutputFormat2.class);
  76.  
  77. FileSystem fs = FileSystem.get(URI.create("/"),conf);
  78.  
  79. Path outPath = new Path(PATH_OUT);
  80. // if (fs.exists(outPath))fs.delete(outPath, true);
  81.  
  82. FileOutputFormat.setOutputPath(job, outPath);
  83. FileInputFormat.setInputPaths(job, new Path(PATH_IN));
  84.  
  85. // HTable table = new HTable(conf, TABLENAME);
  86.  
  87. HFileOutputFormat2.configureIncrementalLoad(job, conn.getTable(TableName.valueOf(TABLENAME)),
  88. conn.getRegionLocator(TableName.valueOf(TABLENAME)));
  89. System.out.println("load完毕");
  90.  
  91. if(job.waitForCompletion(true)){
  92. LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf);
  93. // loader.doBulkLoad( table);
  94.  
  95. }
  96.  
  97. System.exit(0);
  98. }
  99. }

"main" java.io.IOException: Mkdirs failed to create /user/centos/hbase-staging (exists=false, cwd=file:/home/centos)的更多相关文章

  1. Exception in thread "main" java.io.IOException: Mkdirs failed to create /var/folders/q0/1wg8sw1x0dg08cmm5m59sy8r0000gn/T/hadoop-unjar6090005653875084137/META-INF/license at org.apache.hadoop.util.Run

    在使用hadoop运行jar时出现. 解决方法 zip -d Test.jar LICENSE zip -d Test.jar META-INF/LICENSE 完美解决.

  2. nutch爬取时Exception in thread “main” java.io.IOException: Job failed!

    用cygwin运行nutch 1.2爬取提示IOException: $ bin/nutch crawl urls -dir crawl -depth 3 -topN 10 crawl started ...

  3. Exception in thread "main" java.io.IOException: Failed to set permissions of path

    在跑BuildForest的时候,编写了下面的程序: package test.breiman; import org.apache.mahout.classifier.df.mapreduce.Bu ...

  4. windows下eclipse远程连接hadoop错误“Exception in thread"main"java.io.IOException: Call to Master.Hadoop/172.20.145.22:9000 failed ”

    在VMware虚拟机下搭建了hadoop集群,ubuntu-12.04,一台master,三台slave.hadoop-0.20.2版本.在 master机器上利用eclipse-3.3连接hadoo ...

  5. github提交失败并报错java.io.IOException: Authentication failed:

    一.概述 我最近在写一个android的项目. 软件:android studio.Android studio VCS integration(插件) Android studio VCS inte ...

  6. 解决方案--java执行cmd命令ProcessBuilder--出错Exception in thread "main" java.io.IOException: Cannot run program "dir d:\": CreateProcess error=2(xjl456852原创)

    当我尝试在java中通过ProcessBuilder运行window的cmd命令时出现错误: public static void main(String [] args) throws IOExce ...

  7. android java.io.IOException: open failed: EBUSY (Device or resource busy)

    今天遇到一个奇怪的问题, 测试在程序的下载界面,下载一个文件第一次下载成功,删除后再下载结果下载报错, 程序:file.createNewFile(); 报错:java.io.IOException: ...

  8. java.io.IOException: open failed: EACCES (Permission denied)问题解决

    1.  问题描述:在Android中,用程序访问Sdcard时,有时出现“java.io.IOException: open failed: EACCES (Permission denied)&qu ...

  9. hadoop错误Ignoring exception during close for org.apache.hadoop.mapred.MapTask$NewOutputCollector@17bda0f2 java.io.IOException Spill failed

    1.错误    Ignoring exception during close for org.apache.hadoop.mapred.MapTask$NewOutputCollector@17bd ...

随机推荐

  1. OpenStack组件——Neutron网络服务(1)

    1.neutron 介绍: 1)Neutron 概述 传统的网络管理方式很大程度上依赖于管理员手工配置和维护各种网络硬件设备:而云环境下的网络已经变得非常复杂,特别是在多租户场景里,用户随时都可能需要 ...

  2. 强化学习应用于游戏Tic-Tac-Toe

    Tic-Tac-Toe游戏为3*3格子里轮流下棋,一方先有3子成直线的为赢家. 参考代码如下,我只删除了几个没用的地方: ####################################### ...

  3. IIS配置相关问题:Framework 4.5 在IIS 7.5中运行

    <system.webServer>    <validation validateIntegratedModeConfiguration="false" /&g ...

  4. nginx文件路径配置(root|alias)

    nginx指定文件路径主要有两种方式:root|alias. 那么他们究竟有什么区别呢? 指令的使用方法和作用域: [root] 语法:root path 默认值:root html 配置段:http ...

  5. 2019牛客暑期多校训练营(第一场)-E(DP)

    题目链接:https://ac.nowcoder.com/acm/contest/881/E 题意:求可分解成n个AB和m个BA的字符串的个数. 思路: 首先根据贪心思想,前n个A可作为AB的A,后m ...

  6. 如何使用JavaScript实现前端导入和导出excel文件

    一.SpreadJS 简介 SpreadJS 是一款基于 HTML5 的纯 JavaScript 电子表格和网格功能控件,以“高速低耗.纯前端.零依赖”为产品特色,可嵌入任何操作系统,同时满足 .NE ...

  7. SpringMVC基础教程

    1. 最简单的配置 首先是要有相应的配置文件: 文件内容: <context:component-scan base-package="com.imethsoft.server.*&q ...

  8. 使用Redis實現秒殺功能

    <?php $id = 1; $pdo=new PDO("mysql:host=127.0.0.1;dbname=test","root","r ...

  9. AC自动机练习题1:地图匹配

    AC自动机板子,学习之前要是忘记了就看一下 1465: [AC自动机]地图匹配 poj1204 时间限制: 1 Sec  内存限制: 256 MB提交: 78  解决: 46[提交] [状态] [讨论 ...

  10. python_0基础开始_day08

    第八节 1,文件操作 文件操作目的: 持久化,永久存储 (数据库之前 -- 文件操作就是代替数据库) 读 1,找到文件位 2,双击打开 3,进行一些操作 4,关闭文件 open() 打开,通过pyth ...