网络爬虫 | 猫眼电影最受期待榜信息抓取 —— 基于multiprocessing / re（多进程/正则表达式）

import re
import requests
from requests.exceptions import RequestException
import multiprocessing
import random


headers = {'User-Agent':  "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.151 Safari/534.16"}
def get_one_page(url):
    """
    获取当前页的html文本信息
    :param url:
    :return:
    """
    try:
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            # 此处要进行重新编码
            return response.text
        return None
    except RequestException as e:
        return None


def parse_one_page(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?(.*?).*?
(.*?)
.*?'
                         '(.*?)
', re.S)
    items = re.findall(pattern, html)
    for i in items:
        yield {
            "index": items[0],
            "title": items[1],
            "star": items[2],
            "releasetime": items[3]
        }


def main(offset):
    url = "https://maoyan.com/board/6?offset=%s"
    html = get_one_page(url % offset)
    print([item for item in parse_one_page(html)])


if __name__ == "__main__":
    pool = multiprocessing.Pool(4)
    for i in range(10):
        pool.apply_async(main, (i*10, ))

    pool.close()
    pool.join()