Python 多进程 + 协程 (gevent) 实例

利用多进程（multiprocessing）+多协程（gevent）实现同时请求12个网站url

import gevent
from gevent import monkey;monkey.patch_socket();monkey.patch_ssl()

from multiprocessing import Pool
import requests
import time
from collections import deque
from functools import wraps

list = [
	"http://www.baidu.com",
	"http://www.qq.com",
	"http://www.weibo.com",
	"http://www.tencent.com",
	"http://www.jd.com",
	"http://www.meituan.com",
	"http://www.douban.com",
	"http://www.hao123.com",
	"http://www.vip.com",
	"http://www.sohu.com",
	"http://www.alibaba.com",
	"http://www.jumei.com",
]


def getUrlContent(url):
    '''
    获取url内容
    '''
    content = requests.post(url)
    print(content)
    return content

def coroutine(*url_args):
    '''
    创建协程
    '''
	spawnList = []
	for url in url_args:
		spawn = gevent.spawn(getUrlContent, url)
		spawnList.append(spawn)

	gevent.joinall(spawnList)

def printTime(func):
    '''
    包装函数
    '''
	@wraps(func)
	def wrapper(*args):
		startTime = time.time()
		r = func(*args)
		endTime = time.time()
		print("time %s" % str(endTime - startTime))
		return r
	return wrapper

@printTime
def createMultiProcess(processNumber, coroutineNumber):
    '''
    创建进程
    '''
	urlList = deque(list)
	p = Pool()
	for i in range(processNumber):
		urlBlock = []
		for process in range(coroutineNumber):
			url = urlList.pop()
			urlBlock.append(url)
		p.apply_async(coroutine, args=urlBlock)
	p.close()
	p.join()
	

一、使用6个进程 + 2个协程

http://www.jd.com

http://www.meituan.com

http://www.sohu.com

http://www.vip.com

http://www.tencent.com

http://www.weibo.com

http://www.douban.com

http://www.hao123.com

http://www.baidu.com

http://www.qq.com

http://www.jumei.com

http://www.alibaba.com


time 1.625

二、使用2个进程 + 6个协程

http://www.baidu.com

http://www.tencent.com

http://www.jd.com

http://www.qq.com

http://www.meituan.com

http://www.weibo.com

http://www.sohu.com

http://www.vip.com

http://www.jumei.com

http://www.douban.com

http://www.hao123.com

http://www.alibaba.com


time 1.505000114440918