协程-遇到I/O自动切换

参考博客：http://www.cnblogs.com/alex3714/articles/5248247.html

一、前言

　　Gevent 是一个第三方库，可以轻松通过gevent实现并发同步或异步编程，在gevent中用到的主要模式是Greenlet, 它是以C扩展模块形式接入Python的轻量级协程。 Greenlet全部运行在主程序操作系统进程的内部，但它们被协作式地调度。

二、Gevent

import gevent
 
def func1():
    print('running in func1...')
    gevent.sleep(2)    # I/O中断2秒
    print('Explicit context switch to func1 again...')
 
def func2():
    print('Explicit context func2...')
    gevent.sleep(1)
    print('Implicit context switch to func2 again...')
 
gevent.joinall([
    gevent.spawn(func1),
    gevent.spawn(func2),
])

　　

import gevent
 
def func1():
    print('running in func1...')
    gevent.sleep(2)
    print('Explicit context switch to func1 again...')
 
def func2():
    print('Explicit context func2...')
    gevent.sleep(1)
    print('Implicit context switch to func1 again...')
 
def func3():
    print('running in func3...')
    gevent.sleep(0)    # 表示中断，没有时间
    print('running func3 again...')
 
gevent.joinall([
    gevent.spawn(func1),
    gevent.spawn(func2),
    gevent.spawn(func3),
])

　　

三、遇到I/O阻塞时自动切换任务

　　简单的gevent并发爬网页　　

# -*- coding: UTF-8 -*-
 
from urllib import request
import time
from gevent import monkey
import gevent
 
monkey.patch_all()
# gevent并不能识别urllib 的I/O中断，monkey.patch_all会给所有I/O中断做标签
 
def f(url):
    print('Get: %s' % url)
    resp = request.urlopen(url)
    data = resp.read()
    print('%d bytes received from %s.' % (len(data), url))
 
# 同步处理
urls = [
    'https://www.python.org/',
    'https://www.yahoo.com/',
    'https://github.com/'
]
 
time_start = time.time()
for url in urls:
    f(url)
print('同步take %s seconds' % (time.time() - time_start))
 
# 异步处理
async_time = time.time()
gevent.joinall([
    gevent.spawn(f, 'https://www.python.org/'),
    gevent.spawn(f, 'https://www.yahoo.com/'),
    gevent.spawn(f, 'https://github.com/')
])
print('异步take %s seconds' % (time.time() - async_time))

　　结果：

Get: https://www.python.org/
48956 bytes received from https://www.python.org/.
Get: https://www.yahoo.com/
517389 bytes received from https://www.yahoo.com/.
Get: https://github.com/
51473 bytes received from https://github.com/.
同步take 4.6533966064453125 seconds
Get: https://www.python.org/
Get: https://www.yahoo.com/
Get: https://github.com/
48956 bytes received from https://www.python.org/.
521617 bytes received from https://www.yahoo.com/.
51473 bytes received from https://github.com/.
异步take 1.916191816329956 seconds
 
# 可以看到两者之间时间差别还是很大的

　　如果没有monkey.patch_all(),程序也能正常运行，但是同步和异步时间基本没有差别，因为gevent并不知道urllib的I/O中断，所以也不会自动切换，所以哪怕使用了gevent，程序也是同步执行（串行）。而monkey.patch_all()就可以为所有urllib中的I/O中断做标记，gevent遇到这些中断就会自动切换，执行其他没有I/O操作的程序。

四、gevent实现单线程下的多socket并发

　　server side

import sys
import socket
import time
import gevent
 
from gevent import socket,monkey
monkey.patch_all()
 
def server(port):
    s = socket.socket()
    s.bind(('0.0.0.0', port))
    s.listen(500)
    while True:
        cli, addr = s.accept()
        gevent.spawn(handle_request, cli)
 
def handle_request(conn):
    try:
        while True:
            data = conn.recv(1024)
            print("recv:", data)
            conn.send(data)
            if not data:
                conn.shutdown(socket.SHUT_WR)
 
    except Exception as  ex:
        print(ex)
    finally:
        conn.close()
if __name__ == '__main__':
    server(8001)

　　client side

import socket
 
HOST = 'localhost'    # The remote host
PORT = 8001           # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
while True:
    msg = bytes(input(">>:"),encoding="utf8")
    s.sendall(msg)
    data = s.recv(1024)
    #print(data)
 
    print('Received', repr(data))
s.close()

　　

# 并发100 个socket连接
 
import socket
import threading
 
def sock_conn():
 
    client = socket.socket()
 
    client.connect(("localhost",8001))
    count = 0
    while True:
        #msg = input(">>:").strip()
        #if len(msg) == 0:continue
        client.send( ("hello %s" %count).encode("utf-8"))
 
        data = client.recv(1024)
 
        print("[%s]recv from server:" % threading.get_ident(),data.decode()) #结果
        count +=1
    client.close()
 
for i in range(100):
    t = threading.Thread(target=sock_conn)
    t.start()