爬取天猫top100

# encoding:utf-8
import requests

import re

import json

from multiprocessing import Pool  # 多线程模块


# 获取网页源代码

def get_one_page(url):
    # 添加头信息

    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.3964.2 Safari/537.36'}

    html = requests.get(url, headers=headers)

    if html.status_code == 200:
        return html.text

    return None


# 根据正则表达式提取信息

def parse_one_page(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?title="(.*?)".*?data-src="(.*?)".*?star">(.*?)
.*?'

                         + 'releasetime">(.*?)
.*?integer">(.*?).*?fraction">(.*?).*?
', re.S)

    data = re.findall(pattern,html)

    for item in data:
        yield {

            'top': item[0],

            'name': item[1],

            'image_src': item[2],

            'actor': item[3].strip()[3:],

            'releasetime': item[4].strip()[5:],

            'score': item[5] + item[6]

        }


# 将提取得数据写人文件

def write_of_file(content):
    # 'a'保留原先的内容，在尾部写入，因为用的是for循环写入，如果用'w'写入只会保留最后一组的数据，或者早打开文件，写入完在关闭，encoding='utf-8'以utf-8编码

    with open('result.txt', 'a', encoding='utf-8') as f:
        # ensure_ascii = False使中文正常输出

        f.write(json.dumps(content, ensure_ascii=False) + '\n')

        f.close()


# 主函数

def main(offset):
    # 要爬取的网址

    url = "http://maoyan.com/board/4?offset=" + str(offset)

    html = get_one_page(url)

    for item in parse_one_page(html):
        print(item)

        write_of_file(item)


if __name__ == '__main__':
    # 定义多线程

    pool = Pool()

    pool.map(main, [i * 10 for i in range(10)])