celery 4.1下报kombu.exceptions.EncodeError: Object of type 'bytes' is not JSON serializable 处理方式
#python代码如下
from celery import Celery
import subprocess
app = Celery('tasks',
broker='redis://localhost',
backend='redis://localhost')
@app.task
def add(x,y):
print("running...",x,y)
return x+y
@app.task
def run_cmd(cmd):
cmd_obj=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
return cmd_obj.stdout.read()
[python@qinhan file]$ python
Python 3.6. (default, Jan , ::)
[GCC 4.8. (Red Hat 4.8.-)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import task
>>> task.run_cmd('df') #此条命令不会发送到celery后台
b'Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/mapper/centos-root 5232640 2481244 2751396 48% /\ndevtmpfs 490308 0 490308 0% /dev\ntmpfs 500680 0 500680 0% /dev/shm\ntmpfs 500680 6968 493712 2% /run\ntmpfs 500680 0 500680 0% /sys/fs/cgroup\n/dev/mapper/centos-home 2086912 33076 2053836 2% /home\n/dev/mapper/centos-var 975532 633120 342412 65% /var\n/dev/sda1 508588 126972 381616 25% /boot\ntmpfs 100136 0 100136 0% /run/user/0\n'
>>> task.run_cmd.delay('df') #此条命令会发送到celery后台,celery后台报错not JSON serializable
<AsyncResult: 2c671a6b-be73-4f46-8b55-c58f0a05ed2f>
>>> #celery后台报错信息如下
Traceback (most recent call last):
File "/usr/local/python3/lib/python3.6/site-packages/celery/app/trace.py", line 434, in trace_task
uuid, retval, task_request, publish_result,
File "/usr/local/python3/lib/python3.6/site-packages/celery/backends/base.py", line 152, in mark_as_done
self.store_result(task_id, result, state, request=request)
File "/usr/local/python3/lib/python3.6/site-packages/celery/backends/base.py", line 309, in store_result
request=request, **kwargs)
File "/usr/local/python3/lib/python3.6/site-packages/celery/backends/base.py", line 652, in _store_result
self.set(self.get_key_for_task(task_id), self.encode(meta))
File "/usr/local/python3/lib/python3.6/site-packages/celery/backends/base.py", line 259, in encode
_, _, payload = self._encode(data)
File "/usr/local/python3/lib/python3.6/site-packages/celery/backends/base.py", line 263, in _encode
return dumps(data, serializer=self.serializer)
File "/usr/local/python3/lib/python3.6/site-packages/kombu/serialization.py", line 221, in dumps
payload = encoder(data)
File "/usr/local/python3/lib/python3.6/contextlib.py", line 99, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/python3/lib/python3.6/site-packages/kombu/serialization.py", line 54, in _reraise_errors
reraise(wrapper, wrapper(exc), sys.exc_info()[2])
File "/usr/local/python3/lib/python3.6/site-packages/vine/five.py", line 178, in reraise
raise value.with_traceback(tb)
File "/usr/local/python3/lib/python3.6/site-packages/kombu/serialization.py", line 50, in _reraise_errors
yield
File "/usr/local/python3/lib/python3.6/site-packages/kombu/serialization.py", line 221, in dumps
payload = encoder(data)
File "/usr/local/python3/lib/python3.6/site-packages/kombu/utils/json.py", line 72, in dumps
**dict(default_kwargs, **kwargs))
File "/usr/local/python3/lib/python3.6/json/__init__.py", line 238, in dumps
**kw).encode(obj)
File "/usr/local/python3/lib/python3.6/json/encoder.py", line 199, in encode
chunks = self.iterencode(o, _one_shot=True)
File "/usr/local/python3/lib/python3.6/json/encoder.py", line 257, in iterencode
return _iterencode(o, 0)
File "/usr/local/python3/lib/python3.6/site-packages/kombu/utils/json.py", line 62, in default
return super(JSONEncoder, self).default(o)
File "/usr/local/python3/lib/python3.6/json/encoder.py", line 180, in default
o.__class__.__name__)
kombu.exceptions.EncodeError: Object of type 'bytes' is not JSON serializable
间接处理方法,直接把数据转换为json格式(只是做测试用,可以忽略)
>>> r=b'Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/mapper/centos-root 5232640 2481244 2751396 48% /\ndevtmpfs 490308 0 490308 0% /dev\ntmpfs 500680 0 500680 0% /dev/shm\ntmpfs 500680 6968 493712 2% /run\ntmpfs 500680 0 500680 0% /sys/fs/cgroup\n/dev/mapper/centos-home 2086912 33056 2053856 2% /home\n/dev/mapper/centos-var 975532 633208 342324 65% /var\n/dev/sda1 508588 126972 381616 25% /boot\ntmpfs 100136 0 100136 0% /run/user/0\n'
>>> import json
>>> r.decode()
'Filesystem 1K-blocks Used Available Use% Mounted on\n/dev/mapper/centos-root 5232640 2481244 2751396 48% /\ndevtmpfs 490308 0 490308 0% /dev\ntmpfs 500680 0 500680 0% /dev/shm\ntmpfs 500680 6968 493712 2% /run\ntmpfs 500680 0 500680 0% /sys/fs/cgroup\n/dev/mapper/centos-home 2086912 33056 2053856 2% /home\n/dev/mapper/centos-var 975532 633208 342324 65% /var\n/dev/sda1 508588 126972 381616 25% /boot\ntmpfs 100136 0 100136 0% /run/user/0\n'
>>> json.dumps(r.decode())
'"Filesystem 1K-blocks Used Available Use% Mounted on\\n/dev/mapper/centos-root 5232640 2481244 2751396 48% /\\ndevtmpfs 490308 0 490308 0% /dev\\ntmpfs 500680 0 500680 0% /dev/shm\\ntmpfs 500680 6968 493712 2% /run\\ntmpfs 500680 0 500680 0% /sys/fs/cgroup\\n/dev/mapper/centos-home 2086912 33056 2053856 2% /home\\n/dev/mapper/centos-var 975532 633208 342324 65% /var\\n/dev/sda1 508588 126972 381616 25% /boot\\ntmpfs 100136 0 100136 0% /run/user/0\\n"'
最终解决方法
#修改编码格式为utf-8
from celery import Celery
import subprocess app = Celery('tasks',
broker='redis://localhost',
backend='redis://localhost') @app.task
def add(x,y):
print("running...",x,y)
return x+y @app.task
def run_cmd(cmd):
cmd_obj=subprocess.Popen(cmd,shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
return cmd_obj.stdout.read().decode("utf-8") 重启celery
celery -A task worker --loglevel=debug
#前端执行python脚本
[python@qinhan file]$ python
Python 3.6.2 (default, Jan 10 2018, 16:15:47)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-16)] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import task
>>> task.run_cmd.delay('df')
<AsyncResult: b0d9d480-e51d-4304-9128-1b676d70d3fe>
>>> a=task.run_cmd.delay("df -h")
>>> a.get()
'Filesystem Size Used Avail Use% Mounted on\n/dev/mapper/centos-root 5.0G 2.4G 2.7G 48% /\ndevtmpfs 479M 0 479M 0% /dev\ntmpfs 489M 0 489M 0% /dev/shm\ntmpfs 489M 6.9M 483M 2% /run\ntmpfs 489M 0 489M 0% /sys/fs/cgroup\n/dev/mapper/centos-home 2.0G 33M 2.0G 2% /home\n/dev/mapper/centos-var 953M 619M 335M 65% /var\n/dev/sda1 497M 124M 373M 25% /boot\ntmpfs 98M 0 98M 0% /run/user/0\n'
>>>
#后台celery执行成功,不在报错not JSON serializable
[2018-01-12 10:26:27,236: DEBUG/MainProcess] | Worker: Hub.register Pool...
[2018-01-12 10:26:27,237: INFO/MainProcess] celery@qinhan ready.
[2018-01-12 10:26:27,238: DEBUG/MainProcess] basic.qos: prefetch_count->16
[2018-01-12 10:27:06,818: INFO/MainProcess] Received task: task.run_cmd[b0d9d480-e51d-4304-9128-1b676d70d3fe]
[2018-01-12 10:27:06,819: DEBUG/MainProcess] TaskPool: Apply <function _fast_trace_task at 0x7f02b051cd08> (args:('task.run_cmd', 'b0d9d480-e51d-4304-9128-1b676d70d3fe', {'lang': 'py', 'task': 'task.run_cmd', 'id': 'b0d9d480-e51d-4304-9128-1b676d70d3fe', 'eta': None, 'expires': None, 'group': None, 'retries': 0, 'timelimit': [None, None], 'root_id': 'b0d9d480-e51d-4304-9128-1b676d70d3fe', 'parent_id': None, 'argsrepr': "('df',)", 'kwargsrepr': '{}', 'origin': 'gen71158@qinhan', 'reply_to': '61d4cf29-4204-397e-854d-446c648c8b9e', 'correlation_id': 'b0d9d480-e51d-4304-9128-1b676d70d3fe', 'delivery_info': {'exchange': '', 'routing_key': 'celery', 'priority': 0, 'redelivered': None}}, b'[["df"], {}, {"callbacks": null, "errbacks": null, "chain": null, "chord": null}]', 'application/json', 'utf-8') kwargs:{})
[2018-01-12 10:27:06,825: DEBUG/MainProcess] Task accepted: task.run_cmd[b0d9d480-e51d-4304-9128-1b676d70d3fe] pid:71152
[2018-01-12 10:27:06,842: INFO/ForkPoolWorker-1] Task task.run_cmd[b0d9d480-e51d-4304-9128-1b676d70d3fe] succeeded in 0.020323514996562153s: 'Filesystem 1K-blocks Used Available Use% Mounted on
/dev/mapper/centos-root 5232640 2481252 2751388 48% /
devtmpfs 490308 0 490308 0% /dev
tmpfs 500680 0 500680 0% /dev/shm
tmpfs 500680 6968 493712 2% /run
tmpfs 500680 0 500680 0% /sys/fs/cgroup
/dev/mapper/centos-home 2086912 33056 2053856 2% /home
/dev/mapper/centos-var 975532 633120 342412 65% /var
/dev/sda1 508588 126972 381616 25% /boot
tmpfs 100136 0 100136 0% /run/user/0
'
celery 4.1下报kombu.exceptions.EncodeError: Object of type 'bytes' is not JSON serializable 处理方式的更多相关文章
- Django+Celery+redis kombu.exceptions.EncodeError:Object of type is not JSON serializable报错
在本文中例子中遇到问题的各种开发版本如下: Python3.6.8 Django==2.2 celery==4.4.0 kombu==4.6.7 redis==3.3.0 大概的报错如下截图: 是在开 ...
- pymysql下报错:numpy.float64 object has no attribute 'translate' 可能是pandas版本的问题
pymysql下报错:numpy.float64 object has no attribute 'translate'.定位到db.merge函数中,dataframe中浮点型元素的类型为numpy ...
- python3运行报错:TypeError: Object of type 'type' is not JSON serializable解决方法(详细)
返回结果先转成str 字符创
- bootstrap-datetimepicker在火狐下报错的问题
问题:使用bootstrap-datetimepicker这个日期插件来显示日期,但在火狐下报如下错误:TypeError: (intermediate value).toString(...).sp ...
- jquery.validate.js在IE8下报错不运行
因为项目要表单验证,所以特意去找了jquery.validate这个应该是jquery官方团队出品的表单验证插件. 实际应用时,却在IE8调试时碰到了问题. 从jquery.validate官网下载了 ...
- IE8、IE9浏览器下报:JSON未定义 解决方法
IE8.IE9浏览器下报:JSON未定义的问题 解决方法: 在jsp中引入如下代码 <!-- 解决 IE8.IE9 下显示混乱的问题--><% String browserStrin ...
- Flask 与 Celery 在 windows 下的集成问题
Flask 与 Celery 在 windows 下的集成问题 所有的 Web 框架内部的视图中不适合执行需要长时间运行的任务,包括 Flask .Django 等.这类型的任务会阻塞 Web 的响应 ...
- JavaScript在IE6,IE7下报错'expected identifier, string or number'
问题: 代码在Forefox和IE8下工作正常,但是在IE6下报错: expected identifier, string or number 假如变量options有多个选项,那么我们可以用逗号分 ...
- ASP.NET服务器控件在IE10浏览器(非兼容模式)下报脚本错误的可能解决办法
关于IE10出现LinkButton点击无效的情况: 一般高配置的系统如Win7旗舰版SP1系统不会出现这种情况,针对家庭普通版和专业版的用户通过测试都有这种情况,对于开发人员要解决不同 ...
随机推荐
- 解决 MySQL 比如我要拉取一个消息表中用户id为1的前10条最新数据
我们都知道,各种主流的社交应用或者阅读应用,基本都有列表类视图,并且都有滑到底部加载更多这一功能, 对应后端就是分页拉取数据.好处不言而喻,一般来说,这些数据项都是按时间倒序排列的,用户只关心最新的动 ...
- Rabbitmq基本使用 SpringBoot整合Rabbit SpringCloud Stream+Rabbit
https://blog.csdn.net/m0_37867405/article/details/80793601 四.docker中使用rabbitmq 1. 搭建和启动 使用地址:rabbitm ...
- swift - 本地通知2 - 啰嗦版
1. import UIKit @UIApplicationMain class AppDelegate: UIResponder, UIApplicationDelegate { var windo ...
- swift - scrollview 判断左右移动, 以及上下两个view联动
核心代码 1. 2. 3. 界面代码VFL /* 浏览作品view*/ import UIKit /** * 图片浏览器(大图和缩略图) */ class JYBrowseWorksView: UIV ...
- 大话listview之设置item监听器无效大坑之一:item设了属性clickable
今天一个listview设置item监听器居然没有作用: 看了半天,怀疑是item设置了这个属性, 于是删了,果然就可以了. 大坑 ...
- make ;makefile; cmake; qmake的区分
1. make 是用来执行Makefile的.2. Makefile是类unix环境下(比如Linux)的类似于批处理的"脚本"文件.其基本语法是: 目标+依赖+命令,只有在目标文 ...
- MVC中Ajax post 和Ajax Get——提交对象
HTTP 请求:GET vs. POST两种在客户端和服务器端进行请求-响应的常用方法是:GET 和 POST.GET - 从指定的资源请求数据POST - 向指定的资源提交要处理的数据GET 基本上 ...
- kubadm创建k8s v1.10集群
kubadm创建k8s集群 1:服务器信息以及节点介绍 主机名 ip 备注 k8s-master 192.168.0.104 master etcd keepalived k8s-client1 19 ...
- 多进程copy文件
from multiprocessing import Pool,Manager import os,time def copyFileTask(fileName,oldFolderName,newF ...
- cesium 中地图发生了平移,放缩,旋转等动作所要执行的动作
1.在canvas上得到鼠标点击的是那个键 <html><head><title>js判断鼠标左.中.右键哪个被点击-柯乐义</title><sc ...