sqoop1.99.4 JAVA API操作
貌似天国还没有介绍1.99.4的java操作代码的，自己吃一次螃蟹吧
如果你是MAVEN项目
<dependency>
<groupId>org.apache.sqoop</groupId>
<artifactId>sqoop-client</artifactId>
<version>1.99.4</version>
</dependency>
如果你是java项目
导入sqoop1.99.4中shell目录下的lib里面全部jar包就行(不用server中的）
HDFS->MYSQL
package org.admln.sqoopOperate; import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
import org.apache.sqoop.validation.Status; public class HDFSToMysql {
public static void main(String[] args) {
sqoopTransfer();
}
public static void sqoopTransfer() {
//初始化
String url = "http://hadoop:12000/sqoop/";
SqoopClient client = new SqoopClient(url); //创建一个源链接 HDFS
long fromConnectorId = 1;
MLink fromLink = client.createLink(fromConnectorId);
fromLink.setName("HDFS connector");
fromLink.setCreationUser("admln");
MLinkConfig fromLinkConfig = fromLink.getConnectorLinkConfig();
fromLinkConfig.getStringInput("linkConfig.uri").setValue("hdfs://hadoop:8020/");
Status fromStatus = client.saveLink(fromLink);
if(fromStatus.canProceed()) {
System.out.println("创建HDFS Link成功，ID为: " + fromLink.getPersistenceId());
} else {
System.out.println("创建HDFS Link失败");
}
//创建一个目的地链接 JDBC
long toConnectorId = 2;
MLink toLink = client.createLink(toConnectorId);
toLink.setName("JDBC connector");
toLink.setCreationUser("admln");
MLinkConfig toLinkConfig = toLink.getConnectorLinkConfig();
toLinkConfig.getStringInput("linkConfig.connectionString").setValue("jdbc:mysql://hadoop:3306/hive");
toLinkConfig.getStringInput("linkConfig.jdbcDriver").setValue("com.mysql.jdbc.Driver");
toLinkConfig.getStringInput("linkConfig.username").setValue("hive");
toLinkConfig.getStringInput("linkConfig.password").setValue("hive");
Status toStatus = client.saveLink(toLink);
if(toStatus.canProceed()) {
System.out.println("创建JDBC Link成功，ID为: " + toLink.getPersistenceId());
} else {
System.out.println("创建JDBC Link失败");
} //创建一个任务
long fromLinkId = fromLink.getPersistenceId();
long toLinkId = toLink.getPersistenceId();
MJob job = client.createJob(fromLinkId, toLinkId);
job.setName("HDFS to MySQL job");
job.setCreationUser("admln");
//设置源链接任务配置信息
MFromConfig fromJobConfig = job.getFromJobConfig();
fromJobConfig.getStringInput("fromJobConfig.inputDirectory").setValue("/out/aboutyunLog/HiveExport/ipstatistical/data"); //创建目的地链接任务配置信息
MToConfig toJobConfig = job.getToJobConfig();
toJobConfig.getStringInput("toJobConfig.schemaName").setValue("aboutyunlog");
toJobConfig.getStringInput("toJobConfig.tableName").setValue("ipstatistical");
//toJobConfig.getStringInput("fromJobConfig.partitionColumn").setValue("id");
// set the driver config values
//MDriverConfig driverConfig = job.getDriverConfig();
//driverConfig.getStringInput("throttlingConfig.numExtractors").setValue("3");//这句还没弄明白
Status status = client.saveJob(job);
if(status.canProceed()) {
System.out.println("JOB创建成功，ID为: "+ job.getPersistenceId());
} else {
System.out.println("JOB创建失败。");
} //启动任务
long jobId = job.getPersistenceId();
MSubmission submission = client.startJob(jobId);
System.out.println("JOB提交状态为 : " + submission.getStatus());
while(submission.getStatus().isRunning() && submission.getProgress() != -1) {
System.out.println("进度 : " + String.format("%.2f %%", submission.getProgress() * 100));
//三秒报告一次进度
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
System.out.println("JOB执行结束... ...");
System.out.println("Hadoop任务ID为 :" + submission.getExternalId());
Counters counters = submission.getCounters();
if(counters != null) {
System.out.println("计数器:");
for(CounterGroup group : counters) {
System.out.print("\t");
System.out.println(group.getName());
for(Counter counter : group) {
System.out.print("\t\t");
System.out.print(counter.getName());
System.out.print(": ");
System.out.println(counter.getValue());
}
}
}
if(submission.getExceptionInfo() != null) {
System.out.println("JOB执行异常，异常信息为 : " +submission.getExceptionInfo());
}
System.out.println("HDFS通过sqoop传输数据到MySQL统计执行完毕");
}
}
MYSQL->HDFS
package org.admln.sqoopOperate; import org.apache.sqoop.client.SqoopClient;
import org.apache.sqoop.model.MDriverConfig;
import org.apache.sqoop.model.MFromConfig;
import org.apache.sqoop.model.MJob;
import org.apache.sqoop.model.MLink;
import org.apache.sqoop.model.MLinkConfig;
import org.apache.sqoop.model.MSubmission;
import org.apache.sqoop.model.MToConfig;
import org.apache.sqoop.submission.counter.Counter;
import org.apache.sqoop.submission.counter.CounterGroup;
import org.apache.sqoop.submission.counter.Counters;
import org.apache.sqoop.validation.Status; public class MysqlToHDFS {
public static void main(String[] args) {
sqoopTransfer();
}
public static void sqoopTransfer() {
//初始化
String url = "http://hadoop:12000/sqoop/";
SqoopClient client = new SqoopClient(url); //创建一个源链接 JDBC
long fromConnectorId = 2;
MLink fromLink = client.createLink(fromConnectorId);
fromLink.setName("JDBC connector");
fromLink.setCreationUser("admln");
MLinkConfig fromLinkConfig = fromLink.getConnectorLinkConfig();
fromLinkConfig.getStringInput("linkConfig.connectionString").setValue("jdbc:mysql://hadoop:3306/hive");
fromLinkConfig.getStringInput("linkConfig.jdbcDriver").setValue("com.mysql.jdbc.Driver");
fromLinkConfig.getStringInput("linkConfig.username").setValue("hive");
fromLinkConfig.getStringInput("linkConfig.password").setValue("hive");
Status fromStatus = client.saveLink(fromLink);
if(fromStatus.canProceed()) {
System.out.println("创建JDBC Link成功，ID为: " + fromLink.getPersistenceId());
} else {
System.out.println("创建JDBC Link失败");
}
//创建一个目的地链接HDFS
long toConnectorId = 1;
MLink toLink = client.createLink(toConnectorId);
toLink.setName("HDFS connector");
toLink.setCreationUser("admln");
MLinkConfig toLinkConfig = toLink.getConnectorLinkConfig();
toLinkConfig.getStringInput("linkConfig.uri").setValue("hdfs://hadoop:8020/");
Status toStatus = client.saveLink(toLink);
if(toStatus.canProceed()) {
System.out.println("创建HDFS Link成功，ID为: " + toLink.getPersistenceId());
} else {
System.out.println("创建HDFS Link失败");
} //创建一个任务
long fromLinkId = fromLink.getPersistenceId();
long toLinkId = toLink.getPersistenceId();
MJob job = client.createJob(fromLinkId, toLinkId);
job.setName("MySQL to HDFS job");
job.setCreationUser("admln");
//设置源链接任务配置信息
MFromConfig fromJobConfig = job.getFromJobConfig();
fromJobConfig.getStringInput("fromJobConfig.schemaName").setValue("sqoop");
fromJobConfig.getStringInput("fromJobConfig.tableName").setValue("sqoop");
fromJobConfig.getStringInput("fromJobConfig.partitionColumn").setValue("id");
MToConfig toJobConfig = job.getToJobConfig();
toJobConfig.getStringInput("toJobConfig.outputDirectory").setValue("/usr/tmp");
MDriverConfig driverConfig = job.getDriverConfig();
driverConfig.getStringInput("throttlingConfig.numExtractors").setValue("3"); Status status = client.saveJob(job);
if(status.canProceed()) {
System.out.println("JOB创建成功，ID为: "+ job.getPersistenceId());
} else {
System.out.println("JOB创建失败。");
} //启动任务
long jobId = job.getPersistenceId();
MSubmission submission = client.startJob(jobId);
System.out.println("JOB提交状态为 : " + submission.getStatus());
while(submission.getStatus().isRunning() && submission.getProgress() != -1) {
System.out.println("进度 : " + String.format("%.2f %%", submission.getProgress() * 100));
//三秒报告一次进度
try {
Thread.sleep(3000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
System.out.println("JOB执行结束... ...");
System.out.println("Hadoop任务ID为 :" + submission.getExternalId());
Counters counters = submission.getCounters();
if(counters != null) {
System.out.println("计数器:");
for(CounterGroup group : counters) {
System.out.print("\t");
System.out.println(group.getName());
for(Counter counter : group) {
System.out.print("\t\t");
System.out.print(counter.getName());
System.out.print(": ");
System.out.println(counter.getValue());
}
}
}
if(submission.getExceptionInfo() != null) {
System.out.println("JOB执行异常，异常信息为 : " +submission.getExceptionInfo());
}
System.out.println("MySQL通过sqoop传输数据到HDFS统计执行完毕");
}
}
别问为什么没有MYSQL和HBASE、HIVE互导的代码
20150102
sqoop1.99.4 JAVA API操作的更多相关文章
- MongoDB Java API操作很全的整理
MongoDB 是一个基于分布式文件存储的数据库.由 C++ 语言编写,一般生产上建议以共享分片的形式来部署. 但是MongoDB官方也提供了其它语言的客户端操作API.如下图所示: 提供了C.C++ ...
- hive-通过Java API操作
通过Java API操作hive,算是测试hive第三种对外接口 测试hive 服务启动 package org.admln.hive; import java.sql.SQLException; i ...
- hadoop2-HBase的Java API操作
Hbase提供了丰富的Java API,以及线程池操作,下面我用线程池来展示一下使用Java API操作Hbase. 项目结构如下: 我使用的Hbase的版本是 hbase-0.98.9-hadoop ...
- 使用Java API操作HDFS文件系统
使用Junit封装HFDS import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org ...
- Kafka系列三 java API操作
使用java API操作kafka 1.pom.xml <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xs ...
- Hadoop之HDFS（三）HDFS的JAVA API操作
HDFS的JAVA API操作 HDFS 在生产应用中主要是客户端的开发,其核心步骤是从 HDFS 提供的 api中构造一个 HDFS 的访问客户端对象,然后通过该客户端对象操作(增删改查)HDFS ...
- zookeeper的java api操作
zookeeper的java api操作 创建会话: Zookeeper(String connectString,int sessionTimeout,Watcher watcher) Zookee ...
- java api操作
java api操作 导入开发包 将hbase安装包中lib下包导入java项目 创建表 Configuration conf = HBaseConfiguration.create(); c ...
- HDFS 05 - HDFS 常用的 Java API 操作
目录 0 - 配置 Hadoop 环境(Windows系统) 1 - 导入 Maven 依赖 2 - 常用类介绍 3 - 常见 API 操作 3.1 获取文件系统(重要) 3.2 创建目录.写入文件 ...
随机推荐
- 实体框架 (EF) 入门 => 二、在全新的数据库中使用 Code First
学习资料:http://msdn.microsoft.com/zh-cn/data/jj193542 视频建立的控制台应用程序,我在这里使用MVC. 一.非常有磁性的,非常优雅 很喜欢看这个人的视频, ...
- Java Applet and ServiceLoader
http://stackoverflow.com/questions/14062813/java-applet-and-serviceloader
- Linux下的hostname命令详解
1.临时修改: #hostname ***** //*****为修改的hostname 2.永久修改: # vi /etc/sysconfig/networkNETWORKING=yesHOSTNAM ...
- C++11用于计算函数对象返回类型的统一方法
[C++11用于计算函数对象返回类型的统一方法] 模板 std::result_of 被TR1 引进且被 C++11 所采纳,可允许我们决定和使用一个仿函数其回返值的类别.底下,CalculusVer ...
- LDR指令的格式：
http://blog.csdn.net/tanyouliang/article/details/6767011 LDR指令的格式: LDR{条件} 目的寄存器 <存储器地址> ...
- Android中使用gzip传递数据
HTTP协议上的GZIP编码是一种用来改进WEB应用程序性能的技术.大流量的WEB站点常常使用GZIP压缩技术来减少文件大小,减少文件大小有两个明显的好处,一是可以减少存储空间,二是通过网络传输文件时 ...
- POJ3630Phone List(字典树)
经典的字典树的题目了,这次完全是按照自己的风格来写的,没有参考其他人的代码风格来写. 分析:如果采用常规的暴力枚举,那么复杂度就是O(n*n*str.length) = O(10^9),这明显是会超时 ...
- java使用org.apache.poi读取与保存EXCEL文件
一.读EXCEL文件 package com.ruijie.wis.cloud.utils; import java.io.FileInputStream; import java.io.FileNo ...
- linux TCP数据包重传过程----小结
于TCP/IP协议栈的TCP协议的重传功能是由在linux内核源码(net/ipv4/tcp_output.c)中的函数tcp_retransmit_skb()实现的 代码如下: /* This re ...
- android 动画属性（一）之Animation
Animation 在android 程序当中很多时候要用到动画效果,而动画效果主要是Animation来实现的,API给出的解释: 其中包含4种动画效果 AlphaAnimation 渐变透明度 R ...