python实现RabbitMQ同步跟异步消费模型

1，消息推送类

 import pika
 
 # 同步消息推送类
 class RabbitPublisher(object):
 
     # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
     def __init__(self, host, user, password):
         self.host = host
         self.user = user
         self.password = password
         self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
         self.channel = self.connection.channel()
 
     # 发送消息在队列中
     def send(self, queue_name, body):
         self.channel.queue_declare(queue=queue_name, durable=True)  # 声明一个持久化队列
         self.channel.basic_publish(exchange='',
                                    routing_key=queue_name,  # 队列名字
                                    body=body,  # 消息内容
                                    properties=pika.BasicProperties(
                                        delivery_mode=2,  # 消息持久化
                                    ))
 
     # 清除指定队列的所有的消息
     def purge(self, queue_name):
         self.channel.queue_purge(queue_name)
 
     # 删除指定队列
     def delete(self, queue_name, if_unused=False, if_empty=False):
         self.channel.queue_delete(queue_name, if_unused=if_unused, if_empty=if_empty)
 
     # 断开连接
     def stop(self):
         self.connection.close()

2.消息消费类

（1）同步消息消费

在同步消息消费的时候可能会出现pika库断开的情况，原因是因为pika客户端没有及时发送心跳，连接就被server端断开了。解决方案就是做一个心跳线程来维护连接。

心跳线程类

 class Heartbeat(threading.Thread):
 
     def __init__(self, connection):
         super(Heartbeat, self).__init__()
         self.lock = threading.Lock()  # 线程锁
         self.connection = connection  # rabbit连接
         self.quitflag = False  # 退出标志
         self.stopflag = True  # 暂停标志
         self.setDaemon(True)  # 设置为守护线程，当消息处理完，自动清除
 
     # 间隔10s发送心跳
     def run(self):
         while not self.quitflag:
             time.sleep(10)  # 睡10s发一次心跳
             self.lock.acquire()  # 加线程锁
             if self.stopflag:
                 self.lock.release()
                 continue
             try:
                 self.connection.process_data_events()  # 一直等待服务段发来的消息
             except Exception as e:
                 print "Error format: %s" % (str(e))
                 self.lock.release()
                 return
             self.lock.release()
 
     # 开启心跳保护
     def startheartbeat(self):
         self.lock.acquire()
         if self.quitflag:
             self.lock.release()
             return
         self.stopflag = False
         self.lock.release()

消息消费类

 # 同步消息消费类
 class RabbitConsumer(object):
 
     # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
     def __init__(self, host, user, password):
         self.host = host
         self.user = user
         self.password = password
         self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
         self.channel = self.connection.channel()
 
     # 进行消费
     def receive(self, queue_name, callback_worker, prefetch_count=1):  # callback_worker为消费的回调函数
         self.channel.queue_declare(queue=queue_name, durable=True)
         self.channel.basic_qos(prefetch_count=prefetch_count)  # 设置预取的数量，如果为0则不预取，消费者处理越快，可以将这个这设置的越高
         self.channel.basic_consume(callback_worker, queue=queue_name)  # callback_worker为消费的回调函数
         heartbeat = Heartbeat(self.connection)  # 实例化一个心跳类
         heartbeat.start()  # 开启一个心跳线程，不传target的值默认运行run函数
         heartbeat.startheartbeat()  # 开启心跳保护
         self.channel.start_consuming()  # 开始消费

调用方法

# 消费回调函数
def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)
    ch.basic_ack(delivery_tag=method.delivery_tag)  # 告诉生产者处理完成
 
consumer = RabbitConsumer(host="12.12.12.12", user="test", password="")
consumer.receive(queue_name="queue1", callback_worker=callback)  

（2）异步消息消费（推荐）

pika提供了支持异步发送模式的selectconnection方法支持异步发送接收（通过回调的方式）

在连接的时候stop_ioloop_on_close=False需要低版本的pika，比如0.13.1，安装方式 pip install pika==0.13.1

connectioon建立时回调建立channel， channel建立时一次回调各种declare方法，declare建立时依次回调publish。

同使用blockconnection方法相比，通过wireshark抓包来看，使用 异步的方式会对发包进行一些优化，会将几个包合并成一个大包，然后做一次ack应答从而提高效率，与之相反使用blockconnection时将会做至少两次ack，head一次content一次等

因此再试用异步的方式时会获得一定的优化

异步消息消费类

 # 异步消息消费类
 class RabbitConsumerAsync(object):
     EXCHANGE = 'amq.direct'
     EXCHANGE_TYPE = 'direct'
 
     def __init__(self, host, user, password, queue_name="fish_test", callback_worker=None, prefetch_count=1):
         self.host = host
         self.user = user
         self.password = password
         self._connection = None
         self._channel = None
         self._closing = False
         self._consumer_tag = None
         self.QUEUE = queue_name
         self.callbackworker = callback_worker
         self.prefetch_count = prefetch_count
 
     def connect(self):
         return pika.SelectConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)), self.on_connection_open,
                                      stop_ioloop_on_close=False)
 
     def on_connection_open(self, unused_connection):
         self.add_on_connection_close_callback()
         self.open_channel()
 
     def add_on_connection_close_callback(self):
         self._connection.add_on_close_callback(self.on_connection_closed)
 
     def on_connection_closed(self, connection, reply_code, reply_text):
         self._channel = None
         if self._closing:
             self._connection.ioloop.stop()
         else:
             self._connection.add_timeout(5, self.reconnect)
 
     def reconnect(self):
         self._connection.ioloop.stop()
         if not self._closing:
             self._connection = self.connect()
             self._connection.ioloop.start()
 
     def open_channel(self):
         self._connection.channel(on_open_callback=self.on_channel_open)
 
     def on_channel_open(self, channel):
         self._channel = channel
         self._channel.basic_qos(prefetch_count=self.prefetch_count)
         self.add_on_channel_close_callback()
         self.setup_exchange(self.EXCHANGE)
 
     def add_on_channel_close_callback(self):
         self._channel.add_on_close_callback(self.on_channel_closed)
 
     def on_channel_closed(self, channel, reply_code, reply_text):
         print reply_text
         self._connection.close()
 
     def setup_exchange(self, exchange_name):
         self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self.EXCHANGE_TYPE, durable=True)
 
     def on_exchange_declareok(self, unused_frame):
         self.setup_queue()
 
     def setup_queue(self):
         self._channel.queue_declare(self.on_queue_declareok, self.QUEUE, durable=True)
 
     def on_queue_declareok(self, method_frame):
         self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, self.QUEUE)
 
     def on_bindok(self, unused_frame):
         self.start_consuming()
 
     def start_consuming(self):
         self.add_on_cancel_callback()
         self._consumer_tag = self._channel.basic_consume(self.on_message, self.QUEUE)
 
     def add_on_cancel_callback(self):
         self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
 
     def on_consumer_cancelled(self, method_frame):
         if self._channel:
             self._channel.close()
 
     def on_message(self, unused_channel, basic_deliver, properties, body):
         self.callbackworker(body)
         self.acknowledge_message(basic_deliver.delivery_tag)
 
     def acknowledge_message(self, delivery_tag):
         self._channel.basic_ack(delivery_tag)
 
     def stop_consuming(self):
         if self._channel:
             self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
 
     def on_cancelok(self, unused_frame):
         self.close_channel()
 
     def close_channel(self):
         self._channel.close()
 
     def run(self):
         self._connection = self.connect()
         self._connection.ioloop.start()
 
     def stop(self):
         self._closing = True
         self.stop_consuming()
         self._connection.ioloop.start()
 
     def close_connection(self):
         self._connection.close()

调用方法

# 消费回调函数
def callback(body):
    print(" [x] Received %r" % body)
 
consumer = RabbitConsumerAsync(host="12.12.12.12", user="test", password="", queue_name="fish_test", callback_worker=callback, prefetch_count=2)
consumer.run()

（后面这两个可不加入）守护进程类（保证消费运行）

class CDaemon(object):
    """
    a generic daemon class.
    usage: subclass the CDaemon class and override the run() method
    stderr  表示错误日志文件绝对路径, 收集启动过程中的错误日志
    verbose 表示将启动运行过程中的异常错误信息打印到终端,便于调试,建议非调试模式下关闭, 默认为1, 表示开启
    save_path 表示守护进程pid文件的绝对路径
    """
 
    def __init__(self, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
        self.stdin = stdin
        self.stdout = stdout
        self.stderr = stderr
        self.pidfile = save_path  # pid文件绝对路径
        self.home_dir = home_dir
        self.verbose = verbose  # 调试开关
        self.umask = umask
        self.daemon_alive = True
 
    def daemonize(self):
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write('fork #1 failed: %d (%s)\n' % (e.errno, e.strerror))
            sys.exit(1)
 
        os.chdir(self.home_dir)
        os.setsid()
        os.umask(self.umask)
 
        try:
            pid = os.fork()
            if pid > 0:
                sys.exit(0)
        except OSError, e:
            sys.stderr.write('fork #2 failed: %d (%s)\n' % (e.errno, e.strerror))
            sys.exit(1)
 
        sys.stdout.flush()
        sys.stderr.flush()
 
        si = file(self.stdin, 'r')
        so = file(self.stdout, 'a+')
        if self.stderr:
            se = file(self.stderr, 'a+', 0)
        else:
            se = so
 
        os.dup2(si.fileno(), sys.stdin.fileno())
        os.dup2(so.fileno(), sys.stdout.fileno())
        os.dup2(se.fileno(), sys.stderr.fileno())
 
        def sig_handler(signum, frame):
            self.daemon_alive = False
 
        signal.signal(signal.SIGTERM, sig_handler)
        signal.signal(signal.SIGINT, sig_handler)
 
        if self.verbose >= 1:
            print 'daemon process started ...'
 
        atexit.register(self.del_pid)
        pid = str(os.getpid())
        file(self.pidfile, 'w+').write('%s\n' % pid)
 
    def get_pid(self):
        try:
            pf = file(self.pidfile, 'r')
            pid = int(pf.read().strip())
            pf.close()
        except IOError:
            pid = None
        except SystemExit:
            pid = None
        return pid
 
    def del_pid(self):
        if os.path.exists(self.pidfile):
            os.remove(self.pidfile)
 
    def start(self, *args, **kwargs):
        if self.verbose >= 1:
            print 'ready to starting ......'
        # check for a pid file to see if the daemon already runs
        pid = self.get_pid()
        if pid:
            msg = 'pid file %s already exists, is it already running?\n'
            sys.stderr.write(msg % self.pidfile)
            sys.exit(0)
        # start the daemon
        self.daemonize()
        self.run(*args, **kwargs)
 
    def stop(self):
        if self.verbose >= 1:
            print 'stopping ...'
        pid = self.get_pid()
        if not pid:
            msg = 'pid file [%s] does not exist. Not running?\n' % self.pidfile
            sys.stderr.write(msg)
            if os.path.exists(self.pidfile):
                os.remove(self.pidfile)
            return
        # try to kill the daemon process
        try:
            i = 0
            while 1:
                os.kill(pid, signal.SIGTERM)
                time.sleep(0.1)
                i = i + 1
                if i % 10 == 0:
                    os.kill(pid, signal.SIGHUP)
        except OSError, err:
            err = str(err)
            if err.find('No such process') > 0:
                if os.path.exists(self.pidfile):
                    os.remove(self.pidfile)
            else:
                print str(err)
                sys.exit(1)
            if self.verbose >= 1:
                print 'Stopped!'
 
    def restart(self, *args, **kwargs):
        self.stop()
        self.start(*args, **kwargs)
 
    def is_running(self):
        pid = self.get_pid()
        # print(pid)
        return pid and os.path.exists('/proc/%d' % pid)
 
    def run(self, *args, **kwargs):
        # NOTE: override the method in subclass
        print 'base class run()'

调用

class RabbitDaemon(CDaemon):
    def __init__(self, name, save_path, stdin=os.devnull, stdout=os.devnull, stderr=os.devnull, home_dir='.', umask=022, verbose=1):
        CDaemon.__init__(self, save_path, stdin, stdout, stderr, home_dir, umask, verbose)
        self.name = name  # 派生守护进程类的名称
 
    def run(self, **kwargs):
        # 新建一个队列链接
        rab_con = RabbitConsumerAysnc(queuename="test", callbackworker=liando_sf_consumer, prefetch_count=2)
        # 开启消费者进程
        rab_con.run()
 
p_name = 'test'  # 守护进程名称
pid_fn = '/www/rabbit/test.pid'  # 守护进程pid文件的绝对路径
err_fn = '/www/rabbit/test_err.log'  # 守护进程启动过程中的错误日志,内部出错能从这里看到
cD = RabbitDaemon(p_name, pid_fn, stderr=err_fn, verbose=1)

参考链接

https://pika.readthedocs.io/en/0.10.0/examples.html

https://www.jianshu.com/p/a4671c59351a

源码下载地址：https://github.com/sy159/RabbiyMQ.git