thrift实现HDFS文件操作
thrift 文件如下
namespace java com.pera.file.transform
struct File{
1:string path ,
2:string content,
}
service FileTransform {
bool exists(1:string path),
void mkdir(1:string path ),
void store(1:File file),
set<string> ls(1:string path),
bool isDirectory(1:string path),
string getParentFile(1:string path),
void createNewFile(1:string path),
void deleteFile(1:string path),
void deleteOnExit(1:string path),
bool isFile(1:string path),
void rename(1:string srcPath ,2:string destPath),
string getName(1:string path),
i64 totalSpace(1:string path),
i64 usedSpace(1:string path),
string read(1:string path)
}
客户端：
public class FileTransFormPair {
private TTransport transport ;
private Client client ;
public FileTransFormPair(){}
public FileTransFormPair( TTransport transport ,Client client ){
this.transport=transport ;
this.client =client;
}
public TTransport getTransport() {
return transport;
}
public void setTransport(TTransport transport) {
this.transport = transport;
}
public Client getClient() {
return client;
}
public void setClient(Client client) {
this.client = client;
}
}
public class FileTransFormClient implements Iface {
public static String SERVER_IP = "localhost";
public static int SERVER_PORT = 8090;
public static int TIMEOUT = 30000;
public FileTransFormClient(String host,int port ,int timeout){
SERVER_IP=host;
SERVER_PORT=port;
TIMEOUT=timeout;
}
private static FileTransFormPair getClient() throws TTransportException{
TTransport transport = new TSocket(SERVER_IP, SERVER_PORT, TIMEOUT);
TProtocol protocol = new TBinaryProtocol(transport);
Client client =new Client(protocol);
transport.open();
return new FileTransFormPair(transport, client);
}
@Override
public boolean exists(String path) throws TException {
boolean result =false;
FileTransFormPair parameter =getClient();
result =parameter.getClient().exists(path);
parameter.getTransport().close();
return result;
}
@Override
public void mkdir(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().mkdir(path);
parameter.getTransport().close();
}
@Override
public void store(File file) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().store(file);
parameter.getTransport().close();
}
@Override
public Set<String> ls(String path) throws TException {
FileTransFormPair parameter =getClient();
Set<String> result =parameter.getClient().ls(path);
parameter.getTransport().close();
return result;
}
@Override
public boolean isDirectory(String path) throws TException {
boolean result =false ;
FileTransFormPair parameter =getClient();
result =parameter.getClient().isDirectory(path);
parameter.getTransport().close();
return result;
}
@Override
public String getParentFile(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getParentFile(path);
parameter.getTransport().close();
return result;
}
@Override
public void createNewFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().createNewFile(path);
parameter.getTransport().close();
}
@Override
public void deleteFile(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteFile(path);
parameter.getTransport().close();
}
@Override
public void deleteOnExit(String path) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().deleteOnExit(path);
parameter.getTransport().close();
}
@Override
public boolean isFile(String path) throws TException {
FileTransFormPair parameter =getClient();
boolean result =parameter.getClient().isFile(path);
parameter.getTransport().close();
return result ;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
FileTransFormPair parameter =getClient();
parameter.getClient().rename(srcPath, destPath);
parameter.getTransport().close();
}
@Override
public String getName(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().getName(path);
parameter.getTransport().close();
return result;
}
@Override
public long totalSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().totalSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public long usedSpace(String path) throws TException {
FileTransFormPair parameter =getClient();
long result =parameter.getClient().usedSpace(path);
parameter.getTransport().close();
return result;
}
@Override
public String read(String path) throws TException {
FileTransFormPair parameter =getClient();
String result =parameter.getClient().read(path);
parameter.getTransport().close();
return result;
}
}
服务端：
public class ThriftServer {
public static int SERVER_PORT ;
private static TServer server =null;
private static Log LOG =LogFactory.getLog(ThriftServer.class);
public void init(){
SERVER_PORT = 9090;
LOG.info("init the thriftServer");
}
public void start() throws TTransportException {
TProcessor tprocessor = new FileTransform.Processor<FileTransform.Iface>(
new FileStoreService());
TServerSocket serverTransport = new TServerSocket(SERVER_PORT);
TThreadPoolServer.Args ttpsArgs = new TThreadPoolServer.Args(
serverTransport);
ttpsArgs.processor(tprocessor);
ttpsArgs.protocolFactory(new TBinaryProtocol.Factory());
server = new TThreadPoolServer(ttpsArgs);
server.serve();
LOG.info("start the thriftServer");
}
public void stop() {
if(null!=server){
server.stop();
LOG.info("stop the thriftServer");
}
}
public static void main(String[] args) {
ThriftServer server =new ThriftServer();
server.init();
try {
server.start();
} catch (TTransportException e) {
e.printStackTrace();
}
}
}
public class FileStoreService implements Iface{
private Configuration getConfiguration(){
Configuration conf =new Configuration();
Properties pro = new Properties();
try {
pro.load(new FileInputStream(new File("conf/filetransform-site.properties")));
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
// conf.set("dfs.default.name", pro.getProperty("dfs.default.name"));
return conf;
}
@Override
public boolean exists(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
} catch (IOException e) {
throw new TException(e.getMessage());
}
boolean exists =false ;
try {
exists = fs.exists(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return exists;
}
@Override
public void mkdir(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs=null;
try {
fs = FileSystem.get(conf);
fs.mkdirs(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void store(com.pera.file.transform.gen.File file) throws TException {
String path =file.getPath();
String content =file.getContent();
Configuration conf =getConfiguration();
FileSystem fs =null;
FSDataOutputStream out=null;
try {
Path p =new Path(path);
fs =FileSystem.get(conf);
if(!fs.exists(p.getParent())){
throw new TException( path+" is not exists");
}
out =fs.create(p);
if(null!=content){
byte [] tmp =content.getBytes();
out.write(tmp);
out.flush();
out.close();
}
} catch (Exception e) {
throw new TException(e.getMessage());
}finally{
try {
if(null!=out){
out.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
try {
if(null!=fs){
fs.close();
}
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
@Override
public Set<String> ls(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
Set<String> result =new HashSet<String>();
try {
fs=FileSystem.get(conf);
FileStatus [] fstatus=fs.listStatus(new Path(path));
int length =fstatus==null?0:fstatus.length;
for (int i = 0; i < length; i++) {
String url =fstatus[i].getPath().toString();
result.add(url);
}
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
@SuppressWarnings("deprecation")
public boolean isDirectory(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
boolean result =false ;
try {
fs =FileSystem.get(conf);
result =fs.isDirectory(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
@Override
public String getParentFile(String path) throws TException {
Path parentPath =new Path(path).getParent();
return parentPath.toString();
}
@Override
public void createNewFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.createNewFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteFile(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public void deleteOnExit(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.deleteOnExit(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public boolean isFile(String path) throws TException {
boolean isFile =false ;
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
isFile =fs.isFile(new Path(path));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return isFile;
}
@Override
public void rename(String srcPath, String destPath) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
try {
fs =FileSystem.get(conf);
fs.rename(new Path(srcPath), new Path(destPath));
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
}
@Override
public String getName(String path) throws TException {
String name =new Path(path).getName();
return name;
}
@Override
public long totalSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long length =0 ;
try {
fs =FileSystem.get(conf);
length =fs.getFileStatus(new Path(path)).getLen();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return length;
}
@Override
public long usedSpace(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
long used =0 ;
try {
fs =FileSystem.get(conf);
used =fs.getUsed();
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return used;
}
@Override
public String read(String path) throws TException {
Configuration conf =getConfiguration();
FileSystem fs =null;
String result ="" ;
try {
fs =FileSystem.get(conf);
FSDataInputStream in =fs.open(new Path(path));
int length =in.available();
byte [] tmp = new byte [length] ;
in.readFully(tmp);
result =new String(tmp);
} catch (IOException e) {
throw new TException(e.getMessage());
}finally{
if(null!=fs){
try {
fs.close();
} catch (IOException e) {
throw new TException(e.getMessage());
}
}
}
return result;
}
}
thrift实现HDFS文件操作的更多相关文章
- Hadoop之HDFS文件操作常有两种方式（转载）
摘要:Hadoop之HDFS文件操作常有两种方式,命令行方式和JavaAPI方式.本文介绍如何利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- hadoop的hdfs文件操作实现上传文件到hdfs
这篇文章主要介绍了使用hadoop的API对HDFS上的文件访问,其中包括上传文件到HDFS上.从HDFS上下载文件和删除HDFS上的文件,需要的朋友可以参考下hdfs文件操作操作示例,包括上传文件到 ...
- HDFS文件操作(命令行)
HDFS是一种分布式文件系统,为MapReduce这种框架下的海量数据分布式处理而设计. Hadoop之HDFS文件操作常有两种方式,一种是命令行方式,即Hadoop提供了一套与Linux文件命令类似 ...
- Hadoop之HDFS文件操作
摘要:Hadoop之HDFS文件操作常有两种方式.命令行方式和JavaAPI方式.本文介绍怎样利用这两种方式对HDFS文件进行操作. 关键词:HDFS文件 命令行 Java API HD ...
- JAVA API 实现hdfs文件操作
java api 实现hdfs 文件操作会出现错误提示: Permission denied: user=hp, access=WRITE, inode="/":hdfs:supe ...
- HDFS文件操作
hadoop装好后,文件系统中没有任何目录与文件 1. 创建文件夹 hadoop fs -mkdir -p /hkx/learn 参数-p表示递归创建文件夹 2. 浏览文件 hadoop fs -ls ...
- eclipse 对 hadoop1.2.1 hdfs 文件操作
package com.hdfs; import java.io.FileInputStream;import java.io.FileNotFoundException;import java.io ...
- Hadoop HDFS文件操作
1.创建目录 import java.io.IOException; import org.apache.hadoop.conf.Configuration; import org.apache.ha ...
- [bigdata] 使用Flume hdfs sink， hdfs文件未关闭的问题
现象: 执行mapreduce任务时失败 通过hadoop fsck -openforwrite命令查看发现有文件没有关闭. [root@com ~]# hadoop fsck -openforwri ...
随机推荐
- Android 系统自动重启Bug(高通平台)
点击打开链接 最近客户反馈了一个Bug,我们的系统用着用着会自动重启,尤其是在拨号的时候极容易死机或者进入下载模式.根据老大和高通的支持得到了一个解决方案. 在Android系统中,有这么一个文件夹: ...
- 删除表中重复行SQL
delete from table_name a where rowid < (select max(rowid) from table_name b where a.col1 = b.col1 ...
- 一步步创建Qt Widget项目+TextFinder案例（摘自笔者2015年将出的《QT5权威指南》，本文为试读篇）
创建一个基于应用的QtWidget应用程序 这个手册描述了怎样使用QtCreater创建个一个小的Qt应用程序,Text Finder.它是Qt工具Text Finder例子的简写版本.这个应用 ...
- ajax post请求request.getParameter("")取值为null
今天在写提交一个json数据到后台,然后后台返回一个json数据类型.但是发现后台通过request.getParamter("")取到的值为null. 于是写一个简单的ajax ...
- Linux内核编译时会遇到的问题--缺少mkimage
由于各大平台所带的内核不同,所以编译方式也不同,但方法都是大同小异. 但是,编译内核有时候会提示缺少mkimage这个命令,如何解决? 供应商提供内核的同时也会提供其它的一些,比如Uboot,root ...
- 统计git代码提交量
以下是我写的一个脚本,可以统计在某个项目中,自己修改代码的行数,包括增加多少行,删除多少行. 可以统计当天,24小时内或全部时间内.使用时需要把代码中的author对应的值换成自己的名字. 代码如下: ...
- Docker教程：使用docker配置python开发环境
http://blog.csdn.net/pipisorry/article/details/50808034 Docker的安装和配置 [Docker教程:docker的安装] [Docker教程: ...
- [C++学习历程]Visual Studio 2010 的HelloWorld
大学时期曾学过C++的知识,那时候也没有使用VS这样高档的IDE工具,就是C++6.0那样来的.对于重新拾起C++来说,换了个IDE,那么就先从使用IDE学起吧~ 作者:苏生米沿 本文链接:http: ...
- UNIX环境高级编程——线程
线程包含了表示进程内执行环境必需的信息,其中包括进程中标示线程的线程ID.一组寄存器值.栈.调度优先级和策略.信号屏蔽字.errno变量以及线程私有数据. 进程的所有信息对该进程的所有线程都是共享的, ...
- HTML5 在<a>标签内放置块级元素
原文地址:HTML5: Wrap Block-Level Elements with A's 原文日期: 2010年06月26日 翻译日期: 2013年08月22日 对比起XHTML来说,HTML5通 ...