Python爬虫学习笔记(实例：爬取猫眼电影排行前100)

#抓取猫眼电影排行，以文件的形式保存结果
import json
import requests
from requests.exceptions import RequestException
import re
import time

#抓取第一页内容
def get_one_page(url):
    try:   #此处的cookies,headers,params需要根据自己浏览器登陆猫眼电影之后后台生成并经过相应网        
           #站处理，具体见之前的博客——爬取淘宝商品信息实例
        cookies = {
            '__mta': '208109101.1580868743910.1580869334708.1580869391871.6',
            'uuid_n_v': 'v1',
            'uuid': 'F1A445C047BC11EAB2946104D9850EAF183D937C41A04091ADBCEF33315FF34C',
            '_csrf': '29dfa3ddd059c3a9db12143e29026730092990239ef75898af0b1751e3f8f4ff',
            '_lx_utm': 'utm_source%3DBaidu%26utm_medium%3Dorganic',
            '_lxsdk_cuid': '170131eb0cac8-06dda5778e29a9-b383f66-100200-170131eb0ca1',
            '_lxsdk': 'F1A445C047BC11EAB2946104D9850EAF183D937C41A04091ADBCEF33315FF34C',
            'mojo-uuid': 'b9015c74d0f312cee2a094515d087a8a',
            'mojo-session-id': '{"id":"71271fe111c7821d0b4446a38b2b7dcc","time":1580868744683}',
            'SL_GWPT_Show_Hide_tmp': '1',
            'SL_wptGlobTipTmp': '1',
            'lt': 'Yty7hi4TdDYZxEZhvpeYtaajBpoAAAAA-QkAAJKPH-4NCbTaSVLzCzf_TNVZrtzxSTBgGfPP4-CJmai8YCLwDdVe-7I0NZ-BGQdd1g',
            'lt.sig': 'WXkP89KWgZ0hwZC1_IufMi11VrA',
            'mojo-trace-id': '11',
            'Hm_lvt_703e94591e87be68cc8da0da7cbd0be2': '1580868743,1580869392',
            'Hm_lpvt_703e94591e87be68cc8da0da7cbd0be2': '1580869392',
            '_lxsdk_s': '170131eb0cc-ef-30a-66b%7C%7C12',
        }

        headers = {
            'Connection': 'keep-alive',
            'Pragma': 'no-cache',
            'Cache-Control': 'no-cache',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36',
            'Sec-Fetch-User': '?1',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Sec-Fetch-Site': 'cross-site',
            'Sec-Fetch-Mode': 'navigate',
            'Referer': 'https://passport.meituan.com/account/secondverify?response_code=ff1857e9b5eb422eb5da21966a3dd53d&request_code=da67f3c519af4f57a311dab06eb7cced',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'zh-CN,zh;q=0.9',
        }

        params = (
            ('offset', '0'),
        )

        response = requests.get(url, headers=headers, params=params, cookies=cookies)
        response.encoding = response.apparent_encoding #设置编码方式与正文相同

        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        return None

#从网站控制台的Network中查看源码，不能从Element中查看，因为经过JS处理
#通过正则表达式解析页面
def parse_one_page(html):
    pattern = re.compile('
.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name">(.*?).*?star">(.*?)
.*?releasetime">(.*?)
'
                         + '.*?integer">(.*?).*?fraction">(.*?).*?
', re.S)
    items = re.findall(pattern, html) #匹配HTML中的正则对象pattern
    #数据按照类别整理
    for item in items:
        yield {
            'index': item[0],
            'image': item[1],
            'title': item[2],
            'actor': item[3].strip()[3:],#删除头尾空格换行符
            'time': item[4].strip()[5:],
            'score': item[5] + item[6]
        }

#写入文件并存入指定目录
def write_to_file(content):
    with open('D:/result.txt', 'a', encoding='utf-8') as f:
        f.write(json.dumps(content, ensure_ascii=False) + '\n') #JSON库的dumps()方法实现字典的序列化，ensure_ascii=false保证中文输出


def main(offset):
    url = 'http://maoyan.com/board/4?offset=' + str(offset) #拼接出所有页面的URL
    html = get_one_page(url) #得到各个页面的HTML
    for item in parse_one_page(html): #解析各个页面并将结果序列化输出并写入文件
        print(item) #输出
        write_to_file(item) #写入文件

#main函数调用这里实现URL的更改，得到所有页面的URL
if __name__ == '__main__':
    for i in range(10):
        main(offset=i * 10)
        time.sleep(1)