Python3.6爬虫爬取豆瓣电影Top250信息

此爬虫程序基于requests-BeautifulSoup技术路线，对豆瓣电影Top250的电影进行信息的爬取，整个程序解析点击打开链接，查看此网页源代码，逐一匹配电影名，评价人数，分数，链接，一句话影评并输出。

获取网页Html链接：

def getHTML(url):#获取链接Html
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ''

打印爬取的所有信息：

def print_info(lis):
    count = 0
    for movie in lis:
        count += 1
        flist = 'Top{}\n电影名:{}\n评价人数:{}\n分数:{}\n链接:{}\n一句话影评:{}\n'
        print(flist.format(count, movie[0], movie[1], movie[2], movie[3], movie[4]))

解析Html页面并保存下来：

parasepage(html, lis):
    soup = BeautifulSoup(html, 'html.parser')
    movies = soup.find_all('div', class_ = 'info')
    for info in movies:
        name_info = info.find('span', class_ = 'title').string                #取影片的名字
        comment = info.find('span', class_ = "rating_num")                    #标记此标签
        score_info = comment.string                                           #取影片的评分
        people_info = comment.find_next_sibling().find_next_sibling().string  #平行遍历取出下一标签的内容（人数）
        link_info = info.find('a').get('href')                                #获取影片链接
        try:
            review_info = info.find('span', class_ = 'inq').string            #获取一句话影评
        except:
            review_info = '暂无信息'
        lis.append([name_info, people_info, score_info, link_info, review_info])

主函数：

main():
    depth = 11
    lis = []
    start_url = 'https://movie.douban.com/top250?start='
    for i in range(depth):
        url = start_url + str(i * 25) + '&filter='  #i*25表示翻页操作
        html = getHTML(url)
        parasepage(html, lis)

以下是完整代码：

import requests
from bs4 import BeautifulSoup


def getHTML(url):#获取链接Html
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ''

def print_info(lis):
    count = 0
    for movie in lis:
        count += 1
        flist = 'Top{}\n电影名:{}\n评价人数:{}\n分数:{}\n链接:{}\n一句话影评:{}\n'
        print(flist.format(count, movie[0], movie[1], movie[2], movie[3], movie[4]))

def parasepage(html, lis):
    soup = BeautifulSoup(html, 'html.parser')
    movies = soup.find_all('div', class_ = 'info')
    for info in movies:
        name_info = info.find('span', class_ = 'title').string                #取影片的名字
        comment = info.find('span', class_ = "rating_num")                    #标记此标签
        score_info = comment.string                                           #取影片的评分
        people_info = comment.find_next_sibling().find_next_sibling().string  #平行遍历取出下一标签的内容（人数）
        link_info = info.find('a').get('href')                                #获取影片链接
        try:
            review_info = info.find('span', class_ = 'inq').string            #获取一句话影评
        except:
            review_info = '暂无信息'
        lis.append([name_info, people_info, score_info, link_info, review_info])
    print_info(lis)


def main():
    depth = 11
    lis = []
    start_url = 'https://movie.douban.com/top250?start='
    for i in range(depth):
        url = start_url + str(i * 25) + '&filter='  #i*25表示翻页操作
        html = getHTML(url)
        parasepage(html, lis)


if __name__ == '__main__':
    main()