ceph的object storage 提供了和amazon s3兼容的接口以供客户访问。

在ceph的官网上,可以看到它提供了多种语言的访问范本,例如python的(http://docs.ceph.com/docs/master/radosgw/s3/python/):

import boto
import boto.s3.connection
access_key = 'abc'
secret_key = 'cba'
conn = boto.connect_s3(
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
host = '1.2.3.4', port=,
is_secure=False,
calling_format = boto.s3.connection.OrdinaryCallingFormat(),
)
#bucket = conn.create_bucket('my-first-s3-bucket')
bucket = conn.get_bucket('my-first-s3-bucket')
print "{name}".format(
name = bucket.name,
# created = bucket.creation_date,
)
for obj in bucket.get_all_keys():
print(obj.key)
key = bucket.new_key('auto')
key.set_contents_from_filename('/tmp/test.yaml')

但是找不到golang版本的,研究了很久,才在网上找到golang版本的example,链接:https://my.oschina.net/u/578775/blog/863370

package main

import (
"bufio"
"bytes"
"errors"
"fmt"
"github.com/astaxie/beego"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"io"
"os"
) type CephMgmt struct {
host string `ceph host`
DoName string `ceph doname`
bucket_id string `bucket_id`
PathStyle bool `ceph url style, true means you can use host directly, false means bucket_id.doname will be used`
AccessKey string `aws s3 aceessKey`
SecretKey string `aws s3 secretKey`
block_size int64 `block_size`
} var Ceph CephMgmt type MyProvider struct{} func (m *MyProvider) Retrieve() (credentials.Value, error) { return credentials.Value{
AccessKeyID: Ceph.AccessKey, //"9YGYCFH1V3QUT4B9KZD1",
SecretAccessKey: Ceph.SecretKey, //"9hdiwSjCfyrZPKGObH8Kctdur8PiBPJu3B4zGYaZ",
}, nil
}
func (m *MyProvider) IsExpired() bool { return false } func (this *CephMgmt) Init() error {
// "http://s3.devopscloud.com"
this.host = beego.AppConfig.String("ceph::host")
if len(this.host) <= {
return errors.New("ceph conf host is nil")
}
this.bucket_id = beego.AppConfig.String("ceph::bucket_id")
if len(this.bucket_id) <= {
return errors.New("ceph conf bucket_id is nil")
}
this.block_size, _ = beego.AppConfig.Int64("ceph::block_size")
if this.block_size <= ** {
this.block_size = * *
}
return nil
} func (this *CephMgmt) Init2(host string, bucket string, accesskey string, secretkey string) error { this.host = host
this.bucket_id = bucket
this.AccessKey = accesskey
this.SecretKey = secretkey
this.PathStyle = true
this.block_size = * * return nil
} func (this *CephMgmt) connect() (*s3.S3, error) {
///////////////创建连接
sess := session.Must(session.NewSessionWithOptions(session.Options{
Config: aws.Config{
Region: aws.String("default"), //Required 目前尚未分区,填写default即可
//EndpointResolver: endpoints.ResolverFunc(s3CustResolverFn),
Endpoint: &this.host,
S3ForcePathStyle: &this.PathStyle,
Credentials: credentials.NewCredentials(&MyProvider{}),
},
}))
// Create the S3 service client with the shared session. This will
// automatically use the S3 custom endpoint configured in the custom
// endpoint resolver wrapping the default endpoint resolver.
return s3.New(sess), nil
} func (this *CephMgmt) Download(src_name, dst_name string) error {
s3Svc, _ := this.connect()
// Operation calls will be made to the custom endpoint.
resp, err := s3Svc.GetObject(&s3.GetObjectInput{
Bucket: &this.bucket_id, //Required 可认为是cpid,也可认为是对于根目录而言的第一级目录.必须先创建,在存储。
Key: &src_name, //Required 文件名,中间可以带着路径,格式如:{path}/{filename}
//Range: aws.String("bytes=0-499"), //not must be Required 文件范围,如果没有则是全文件
})
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
fmt.Println("hello , i met error")
fmt.Println(err.Error())
return err
} fmt.Println("AcceptRanges :", *resp.AcceptRanges)
fmt.Println("ContentLength:", *resp.ContentLength)
//fmt.Println("ContentRange :", *resp.ContentRange)
//fmt.Println("PartsCount :", *resp.PartsCount)
// Pretty-print the response data.
//fmt.Println(resp) //数据都在resp.Body中,可以转储到文件,具体操作参考go的操作
out, err := os.OpenFile(dst_name, os.O_CREATE|os.O_RDWR, )
if out == nil {
fmt.Println("Open fail")
return err
}
num, err := io.Copy(out, resp.Body)
fmt.Printf("\n write %d err %v \n", num, err)
return nil
} func (this *CephMgmt) Upload(src_name, dst_name string) error {
s3Svc, _ := this.connect() dst_full_name := fmt.Sprintf("/%s/%s", this.bucket_id, dst_name) //Notice: 如果要讲桶ID拼到全路径下,一定要在最前面加'/' //////////////////////////////////////// 存入 /////////////////////////////
/////////////// 创建一个分片上传context
param_init := &s3.CreateMultipartUploadInput{
Bucket: aws.String(this.bucket_id), // Required {bucket}
Key: aws.String(dst_full_name), // Required /{bucket}/{path}/{filename}
/*
Metadata: map[string]*string{
"Key": aws.String("lasttime_if_need"), // Required 可以填充一些文件属性,ceph云存储不关心其中的内容
},
*/
} resp_init, err := s3Svc.CreateMultipartUpload(param_init)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
fmt.Println("I am create upload.", err.Error())
return err }
// Pretty-print the response data.
up_id := resp_init.UploadId
fmt.Println(resp_init) // 需要保存该resp , resp.UploadId 是对象网关创建的标识 ///////////////////////// 分片上传
f, err := os.Open(src_name)
if err != nil {
return err
}
defer f.Close() bfRd := bufio.NewReader(f)
buf := make([]byte, this.block_size) //TODO:一次读取多少个字节 var part_num int64 =
var completes []*s3.CompletedPart for {
n, err := bfRd.Read(buf)
if err == io.EOF {
fmt.Println("read data finished")
break
}
if int64(n) != this.block_size {
data := make([]byte, n)
data = buf[:n]
buf = data
}
//////////////////////// 每次上传一个分片,每次的PartNumber都要唯一
part_num++
param := &s3.UploadPartInput{
Bucket: aws.String(this.bucket_id), // Required bucket
Key: aws.String(dst_name), // Required {path}/{filename}
PartNumber: aws.Int64(part_num), // Required 每次的序号唯一且递增
UploadId: up_id, // Required 创建context时返回的值
Body: bytes.NewReader(buf), // Required 数据内容
ContentLength: aws.Int64(int64(n)), // Required 数据长度
} resp2, err := s3Svc.UploadPart(param)
if err != nil {
fmt.Printf("Hello ,i am wrong[%s][%d][%d]\n", dst_name, part_num, n)
return err
}
fmt.Println(resp2) // 需要保存该resp,因为resp.Etag 需要在通知完成上传时使用 var c s3.CompletedPart
c.PartNumber = aws.Int64(part_num) // Required Etag对应的PartNumber, 上一步返回的
c.ETag = resp2.ETag // Required 上传分片时返回的值 Etag
completes = append(completes, &c)
} /////////////////////////// 结束上传
params := &s3.CompleteMultipartUploadInput{
Bucket: aws.String(this.bucket_id), // Required {bucket}
Key: aws.String(dst_name), // Required {path}/{filename}
UploadId: up_id, // Required 创建context时返回的值
MultipartUpload: &s3.CompletedMultipartUpload{
Parts: completes,
},
//RequestPayer: aws.String("RequestPayer"),
}
resp_comp, err := s3Svc.CompleteMultipartUpload(params)
if err != nil {
// Print the error, cast err to awserr.Error to get the Code and
// Message from an error.
fmt.Println(err.Error())
return err
}
// Pretty-print the response data.
fmt.Println(resp_comp)
return nil
} func main() {
//既可以使用域名,也可以使用ip+端口的形式
Ceph.Init2( /*"http://172.16.20.15"*/ "http://s3.devopscloud.com:80", "new-bucket-5de4d39a", "9YGYCFH1V3QUT4B9KZD1", "9hdiwSjCfyrZPKGObH8Kctdur8PiBPJu3B4zGYaZ")
Ceph.Download("1.mp4", "./1.mp4")
Ceph.Upload("1.mp4", "api/rest/2017-3-16-10-38.mp4")
}

golang 操作ceph object storage的更多相关文章

  1. centos6.4 ceph安装部署之ceph object storage

    preface: ceph-deploy does not provide a rapid installation for Ceph Object Storage install Configura ...

  2. [译]OpenStack Object Storage Monitoring

    注:翻译的不完整,主要是有些地方翻译后反而妨碍理解,有些不知道怎么翻,anyway,需要时拿来用用也是可行的,顺便共享啦.欢迎提意见. 一个OpenStack Object Storage(OSOS) ...

  3. OpenStack Object Storage(Swift)概述

    概述 OpenStack Object Storage(Swift)是OpenStack开源云计算项目的子项目之一,被称为对象存储,提供了强大的扩展性.冗余和持久性. Swift并不是文件系统或者实时 ...

  4. Oracle Schema Objects(Schema Object Storage And Type)

    One characteristic of an RDBMS is the independence of physical data storage from logical data struct ...

  5. swift(Object Storage对象存储服务)(单节点)

    # 在部署对象存储服务(swift)之前,你的环境必须包含身份验证服务(keystone); # keystone需要MySQL数据库,Rabbitmq服务,Memcached服务; # 内存:4G ...

  6. 《Object Storage on CRAQ: High-throughput chain replication for read-mostly workloads》论文总结

    CRAQ 论文总结 说明:本文为论文 <Object Storage on CRAQ: High-throughput chain replication for read-mostly wor ...

  7. openStack 对象存储object storage swift

  8. go语言之行--golang操作redis、mysql大全

    一.redis 简介 redis(REmote DIctionary Server)是一个由Salvatore Sanfilippo写key-value存储系统,它由C语言编写.遵守BSD协议.支持网 ...

  9. [转帖]golang操作mysql使用总结

    golang操作mysql使用总结 https://www.cnblogs.com/hanyouchun/ 讲解的很详细~ 前言 Golang 提供了database/sql包用于对SQL数据库的访问 ...

随机推荐

  1. bat 提示窗口,带换行

    bat 提示窗口 各种窗口样式 mshta vbscript:msgbox("内容1",1,"标题1")(window.close) mshta vbscrip ...

  2. dstat命令详解

    dstat 如果系统没有些工具 yum -y install dstat安装下即妥,此软件小巧玲珑,软件包大小只有144k,安装大小660k. 此工具默认情况下会动态显示CPU,disk,net,pa ...

  3. bind支持mysql

    最近打算将bind的记录信息存入到数据库中去,网上找了下,原来早有老外写好了mysql-bind的补丁,重新编译bind即可实现bind支持mysql存储.(http://mysql-bind.sou ...

  4. 多项式乘法(FFT)模板 && 快速数论变换(NTT)

    具体步骤: 1.补0:在两个多项式最前面补0,得到两个 $2n$ 次多项式,设系数向量分别为 $v_1$ 和 $v_2$. 2.求值:用FFT计算 $f_1 = DFT(v_1)$ 和 $f_2=DF ...

  5. 【Android-数据库Sqlite】Sqlite数据库 增、删、改、查

    1.先创建一个Product类 Product.java 变量如下: int id; String code; String name; int qty; 2.创建一个DBHelper类 DBHelp ...

  6. 菜鸟刷面试题(五、Java容器篇)

    目录: java 容器都有哪些? Collection 和 Collections 有什么区别? List.Set.Map 之间的区别是什么? HashMap 和 Hashtable 有什么区别? 如 ...

  7. 排列组合C、A

    排列组合是组合学最基本的概念.所谓排列,就是指从给定个数的元素中取出指定个数的元素进行排序.组合则是指从给定个数的元素中仅仅取出指定个数的元素,不考虑排序. 排列组合定义及公式 排列的定义:从n个不同 ...

  8. python 学习资料 常用

    https://www.cnblogs.com/vamei/archive/2012/09/13/2682778.html 利用python 进行数据分析第二版 https://www.jianshu ...

  9. OSError: cannot open resource(pillow错误处理)

    https://www.jianshu.com/p/c64ae3e9b196 pillow使用备忘之OSError: cannot open resource错误处理 在使用pillow过程中,Pyt ...

  10. [Luogu] 选择客栈

    https://www.luogu.org/problemnew/show/P1311 思路就是,从1到n枚举,输入color和price的值,我们需要记录一个距离第二个客栈最近的咖啡厅价钱合理的客栈 ...