阿里云oss存储图片
单线程版-上传网络流
import pymysql
import oss2
import requests
import logging # 添加日志 logging.basicConfig(
level=logging.INFO, # 定义输出到文件的log级别，大于此级别的都被输出
format='%(asctime)s %(filename)s %(levelname)s : %(message)s', # 定义输出log的格式
datefmt='%Y-%m-%d %H:%M:%S', # 时间
filename='Error.log', # log文件名
filemode='a') # 写入模式“w”或“a” class osss(object): def __init__(self):
self.db = pymysql.connect(host='', port=3306, database='spider_yu', user='spider',
password='', charset='utf8')
# self.db = pymysql.connect(host='127.0.0.1', port=3306, database='shuo', user='root',
# password='root', charset='utf8')
self.cursor = self.db.cursor()
self.auth = oss2.Auth('LTAIjiyFNM8SukLq', 'fH83Q8o4JDMCpid7XxJSSDm4zPGxDW')
self.bucket = oss2.Bucket(self.auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yxb-dev')
self.numbre = 0
while True:
self.ssh_fun() def ssh_fun(self):
num = self.cursor.execute(
'select id, big, avatar from cqyy_product_info where id>{} limit 1000'.format(self.numbre))
if int(num) == int(0):
exit()
data_tuple = self.cursor.fetchall()
for data in data_tuple:
self.numbre = data[0]
big = data[1]
if "?" in big:
big = big.split('?')[0]
avatar = data[2]
if "?" in avatar:
avatar = avatar.split('?')[0]
# print('numbre:"{}", big:"{}", avatar:"{}"'.format(self.numbre, big, avatar))
logging.info('numbre:"{}", big:"{}", avatar:"{}"'.format(self.numbre, big, avatar))
try:
if ',' in big:
big = big.split(',')
big_list = []
for big_url in big:
if '/' in big_url:
big_name = big_url.split('/')[-1]
big_list.append(big_name)
num = self.cursor.execute('select id from cqyy_cqyy where url="{}"'.format(big_name))
if not num:
input = requests.get(big_url)
self.bucket.put_object('yxb-cqyy/drug_image/{}'.format(big_name), input)
self.cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(big_name))
self.db.commit() big_oss = ','.join(big_list)
self.cursor.execute(
'update cqyy_product_info set big_oss="{}" where id={}'.format(big_oss, self.numbre))
self.db.commit() else:
if '/' in big:
big_name = big.split('/')[-1]
self.cursor.execute(
'update cqyy_product_info set big_oss="{}" where id={}'.format(big_name, self.numbre))
self.db.commit()
num = self.cursor.execute('select id from cqyy_cqyy where url="{}"'.format(big_name))
if not num:
input = requests.get(big)
self.bucket.put_object('yxb-cqyy/drug_image/{}'.format(big_name), input)
self.cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(big_name))
self.db.commit() if ',' in avatar:
avatar = avatar.split(',')
avatar_list = []
for avatar_url in avatar:
if '/' in avatar_url:
avatar_name = avatar_url.split('/')[-1]
avatar_list.append(avatar_name)
num = self.cursor.execute('select id from cqyy_cqyy where url="{}"'.format(avatar_name))
if not num:
input = requests.get(avatar_url)
self.bucket.put_object('yxb-cqyy/head_portrait/{}'.format(avatar_name), input)
self.cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(avatar_name))
self.db.commit()
avatar_oss = ','.join(avatar_list)
self.cursor.execute(
'update cqyy_product_info set avatar_oss="{}" where id={}'.format(avatar_oss, self.numbre))
self.db.commit() else:
if '/' in avatar:
avatar_name = avatar.split('/')[-1]
self.cursor.execute(
'update cqyy_product_info set avatar_oss="{}" where id={}'.format(avatar_name,
self.numbre))
self.db.commit()
num = self.cursor.execute('select id from cqyy_cqyy where url="{}"'.format(avatar_name))
if not num:
input = requests.get(avatar)
self.bucket.put_object('yxb-cqyy/head_portrait/{}'.format(avatar_name), input)
self.cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(avatar_name))
self.db.commit()
except Exception as e:
# print('oss存入失败 id:"{}" 错误类型:"{}"'.format(self.numbre, e))
logging.info('oss存入失败 id:"{}" 错误类型:"{}"'.format(self.numbre, e)) if __name__ == '__main__':
osss = osss()
多线程版-上传网络流
import paramiko, threading
import queue
import pymysql
import oss2
import requests
import logging # 添加日志 logging.basicConfig(
level=logging.INFO, # 定义输出到文件的log级别，大于此级别的都被输出
format='%(asctime)s %(filename)s %(levelname)s : %(message)s', # 定义输出log的格式
datefmt='%Y-%m-%d %H:%M:%S', # 时间
filename='Error.log', # log文件名
filemode='a') # 写入模式“w”或“a” class ThreadPool(object):
def __init__(self, maxsize):
self.maxsize = maxsize
self._q = queue.Queue(self.maxsize)
for i in range(self.maxsize):
self._q.put(threading.Thread) def getThread(self):
return self._q.get() def addThread(self):
self._q.put(threading.Thread) dbs = pymysql.connect(host='', port=3306, database='spider_yu', user='spider',
password='', charset='utf8') def ssh_fun(numbre, big, avatar, pool, db, bucket):
print('numbre:"{}", big:"{}", avatar:"{}"'.format(numbre, big, avatar))
logging.info('numbre:"{}", big:"{}", avatar:"{}"'.format(numbre, big, avatar))
# dbs = pymysql.connect(host='127.0.0.1', port=3306, database='shuo', user='root',
# password='root', charset='utf8')
# dbs = pymysql.connect(host='115.238.111.198', port=3306, database='spider_yu', user='spider',
# password='Kangce@0608', charset='utf8')
cursor = dbs.cursor()
try:
if ',' in big:
big = big.split(',')
big_list = []
for big_url in big:
if '/' in big_url:
big_name = big_url.split('/')[-1]
big_list.append(big_name)
num = cursor.execute('select id from cqyy_cqyy where url="{}"'.format(big_name))
if not num:
# auth = oss2.Auth('LTAIjiyFNM8SukLq', 'fH83Q8o4JDMCpid7XxJSSDm4zPGxDW')
# bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yxb-dev')
input = requests.get(big_url)
bucket.put_object('yxb-cqyy/drug_image/{}'.format(big_name), input)
cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(big_name))
dbs.commit() big_oss = ','.join(big_list)
cursor.execute(
'update cqyy_product_info set big_oss="{}" where id={}'.format(big_oss, numbre))
dbs.commit() else:
if '/' in big:
big_name = big.split('/')[-1]
cursor.execute(
'update cqyy_product_info set big_oss="{}" where id={}'.format(big_name, numbre))
dbs.commit()
num = cursor.execute('select id from cqyy_cqyy where url="{}"'.format(big_name))
if not num:
# auth = oss2.Auth('LTAIjiyFNM8SukLq', 'fH83Q8o4JDMCpid7XxJSSDm4zPGxDW')
# bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yxb-dev')
input = requests.get(big)
bucket.put_object('yxb-cqyy/drug_image/{}'.format(big_name), input)
cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(big_name))
dbs.commit() if ',' in avatar:
avatar = avatar.split(',')
avatar_list = []
for avatar_url in avatar:
if '/' in avatar_url:
avatar_name = avatar_url.split('/')[-1]
avatar_list.append(avatar_name)
num = cursor.execute('select id from cqyy_cqyy where url="{}"'.format(avatar_name))
if not num:
# auth = oss2.Auth('LTAIjiyFNM8SukLq', 'fH83Q8o4JDMCpid7XxJSSDm4zPGxDW')
# bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yxb-dev')
input = requests.get(avatar_url)
bucket.put_object('yxb-cqyy/head_portrait/{}'.format(avatar_name), input)
cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(avatar_name))
dbs.commit()
avatar_oss = ','.join(avatar_list)
cursor.execute(
'update cqyy_product_info set avatar_oss="{}" where id={}'.format(avatar_oss, numbre))
dbs.commit() else:
if '/' in avatar:
avatar_name = avatar.split('/')[-1]
cursor.execute(
'update cqyy_product_info set avatar_oss="{}" where id={}'.format(avatar_name, numbre))
dbs.commit()
num = cursor.execute('select id from cqyy_cqyy where url="{}"'.format(avatar_name))
if not num:
# auth = oss2.Auth('LTAIjiyFNM8SukLq', 'fH83Q8o4JDMCpid7XxJSSDm4zPGxDW')
# bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yxb-dev')
input = requests.get(avatar)
bucket.put_object('yxb-cqyy/head_portrait/{}'.format(avatar_name), input)
cursor.execute(
'insert into cqyy_cqyy(url) values ("{}")'.format(avatar_name))
dbs.commit()
except Exception as e:
# print('oss存入失败 id:"{}" 错误类型:"{}"'.format(numbre, e))
logging.info('oss存入失败 id:"{}" 错误类型:"{}"'.format(numbre, e)) finally:
# dbs.close()
pool.addThread() if __name__ == '__main__':
t_list = []
pool = ThreadPool(3)
numbre = 0
# db = pymysql.connect(host='127.0.0.1', port=3306, database='shuo', user='root',
# password='root', charset='utf8')
db = pymysql.connect(host='', port=3306, database='spider_yu', user='spider',
password='', charset='utf8')
cursor = db.cursor()
auth = oss2.Auth('LTAIjiyFNM8SukLq', 'fH83Q8o4JDMCpid7XxJSSDm4zPGxDW')
bucket = oss2.Bucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'yxb-dev')
while True:
num = cursor.execute('select id, big, avatar from cqyy_product_info where id>{} limit 1000'.format(numbre))
if int(num) == int(0):
exit()
data_tuple = cursor.fetchall()
for data in data_tuple:
numbre = data[0]
big = data[1]
if "?" in big:
big = big.split('?')[0]
avatar = data[2]
if "?" in avatar:
avatar = avatar.split('?')[0]
# if int(numbre) == int(20):
# exit()
th = pool.getThread()
t = th(target=ssh_fun, args=(numbre, big, avatar, pool, db, bucket))
t.start()
t_list.append(t)
for i in t_list:
i.join()
db.close()
阿里云oss存储图片的更多相关文章
- php将图片存储在阿里云oss存储上
创建两个方法 1.上传方法 use OSS\OssClient; use think\Config; use OSS\Core\OssException; /** * 存储文件 * * @param ...
- 备份MySQL数据库并上传到阿里云OSS存储
1. 环境配置 要将本地文件上传到阿里云oss中, 必须使用阿里云提供的工具 ossutil, 有32位,也有64位的, Linux和Windows都有.具体可以到阿里云官网下载 官网及文档: htt ...
- 前端（react）上传到阿里云OSS存储 实例
需求背景 由于现有的后台管理系统,上传的视频越来越大,加上上传视频较慢,后端小哥提出直接从前端上传视频或者其他文件到阿里云OSS存储. 阿里云OSS 阿里云OSS文档介绍,这里不做过多赘述 安装 原本 ...
- 谷粒 | 10 | 阿里云OSS存储对象服务
阿里云OSS对象存储服务 准备工作 1.在service模块新建子模块service_oss 2.引入pom.xml文件中引入oss服务依赖 <dependencies> <!--a ...
- Laravel 使用阿里云 oss 存储对象
一.下载安装 composer require jacobcyl/ali-oss-storage 二.注册服务提供者 在config/app.php的providers下添加: //阿里云OSS对象存 ...
- 阿里云OSS存储开发（一）
Step 1. 初始化一个OSSClient OSSClient是与OSS服务交互的客户端,SDK的OSS操作都是通过OSSClient完成的. 下面代码新建了一个OSSClient: using A ...
- 阿里云OSS存储
1.accessKeyId 与 accessKeySecret 是由系统分配给用户的,称为ID对,用于标识用户,为访问OSS做签名验证. 2.Bucket是OSS上的命名空间,相当于数据的容器,可以存 ...
- Java项目接入阿里云OSS存储
需求背景 目前公司内部项目所支持的文件云存储方式还是公司内部项目组提供的方案,但在时间的考验之下,弊端显现,尤其是灾备切换过程中需要切换访问地址,这种操作不方便,更可能因为中间过程的失误导致资源不可用 ...
- 阿里云OSS存储前端API上传（签名上传）
一.创建用户 在阿里云创建用户https://ram.console.aliyun.com/users,并勾选Open API 保存好信息,很重要,返回后就再也找不到了 新增授权(这里视个人情况,需要 ...
随机推荐
- Windows Server 2016 启用完整版任务管理器
众所周知 Windows Server 2012以上的任务管理器是被阉割过的 那么如何启用呢?首先把你的任务管理器复制一份出来位置:系统盘\Windows\System32\Taskmgr.exe和系 ...
- Vue中 等待DOM或者数据完成 在执行 --this.$nextTick()
虽然 Vue.js 通常鼓励开发人员沿着“数据驱动”的方式思考,避免直接接触 DOM,但是有时我们确实要这么做.比如一个新闻滚动的列表项.如果在这里需要操作dom, 应该是等待 Vue 完成更新 DO ...
- linux 下的常用操作命令
一,Linux的介绍 略. 二,文件类的常用命令 三,权限类的常用命令
- <创新思维与实践>总结梳理
2017年12月3-4号 培训了两天的创新思维与实践,有时间要补充总结一下. ---todo
- Java实现图像对比类
package com.function; import java.awt.image.BufferedImage; import java.io.BufferedWriter; import jav ...
- eclipse 添加svn资源库卡死。长时间等待
使用https://localhost:8443/svn/xx方式打入浏览器判断其服务器是否正常 如果正常通过,而eclipse新建库卡死时.可以等待一点时间看是否卡 问题依旧,考虑更改地址 主机名 ...
- 文件查重工具 ultraCompare 和 UltraFinder 用法
UltraCompare 是一款文件内容比较工具,它可以对于文本.文件夹.二进制进行比较.可进行文本模式,文件夹模式以及二进制模式的比较,可对比较的文件.文件夹等进行合并,同步等操作.是进行比较操作的 ...
- python pyqt面板切换
- tkinter widget
tkinter messagebox
- js分钟数转天-时-分
//js格式化分钟转为天.时.分 function formatMinutes(minutes) { )); ? Math.floor((minutes - day * ) / ) : Math.fl ...