爬虫感悟2

import requests
from requests.exceptions import RequestException
import re
import json
def getHTMLText(url):
    try:
        response = requests.get(url, headers={'user-agent': 'Mozilla/5.0'})
        if response.status_code == 200:
            return response.text
    except RequestException:
        return None

def parseHTMLText(html):
    pattern = re.compile('

.*?board-index.*?>(\d+).*?data-src="(.*?)".*?name"> (.*?).*?star">(.*?).*?releasetime">(.*?)' + '.*?integer">(.*?).*?fraction">(.*?).*?', re.S) items = re.findall(pattern,html) for item in items: yield{ "index": item[0], "image": item[1], "title": item[2], "stars": item[3].strip()[3:], "time": item[4].strip()[5:], "score": item[5]+item[6] } def writeHTMLText(content): with open('result.text','a',encoding='utf-8') as f: f.write(json.dumps(content,ensure_ascii=False) + '\n') f.close() def main(page): url = 'http://maoyan.com/board/4?' + 'offset=' + str(page*10) html = getHTMLText(url) for item in parseHTMLText(html): print(item) writeHTMLText(item) '''item = parseHTMLText(html) print(item)''' if __name__ == '__main__': for i in range(10): main(i)

一开始equests的访问被拒绝，然后重新改变了headers

引入requests库的requests.exceptions,使问题更具体化

第二个函数使用了生成器，减少内存的使用

读写文件时加上了 encoding='utf-8’使生成的文件显示中文