Pythoon3.x对豆瓣TOP250电影爬虫

一直想要学习爬虫,看了两天Python书籍,把基础数据类型和基础语法了解了一下.

在网上找了学习资料和BeautifulSoup4中文说明,链接在下方给出.

主要使用三个模块requests/BeautifulSoup4/pymysql

要抓取数据的网页为 https://movie.douban.com/top250

代码和解释都挺清晰了

#! encoding = utf-8
import requests
from bs4 import BeautifulSoup
import pymysql
import traceback

base_url = "https://movie.douban.com/top250"
conn = pymysql.connect("localhost", "root", "123456", "test", use_unicode=True, charset="utf8")
cursor = conn.cursor()

def save_to_database(movie_id, movie_title, movie_image, movie_des, movie_quote):
    sql = 'insert into douban(movie_id,movie_title,movie_image,movie_description,movie_quote) VALUES ("%s","%s","%s","%s","%s")' % \
          (movie_id, movie_title, movie_image, movie_des, movie_quote)

    try:
        cursor.execute(sql)
        conn.commit()
        print("success")
    except:
        conn.rollback()
        traceback.print_exc()

def get_list(url):
    global base_url
    req = requests.get(url).text
    soup = BeautifulSoup(req, "html.parser")
    movie_list = soup.find_all("ol", "grid_view")
    for item in movie_list:
        for detail in item.find_all('li'):
            movie_id = str(detail.find('em').text)
            image_url = str(detail.img.attrs['src'])
            title = str(detail.find('span', 'title').text)
            description = str(detail.find('div', 'bd').p.text)
            try:
                quote = str(detail.find('p', 'quote').text)
            except:
                quote = '暂无引用'
            # 测试打印的数据
            # print(movie_id, title, image_url, description, quote)
            # 进行数据库插入
            save_to_database(movie_id, title, image_url, description, quote)

    # 下一页地址
    next_page = soup.find('span', attrs={'class': 'next'}).find('a')
    if next_page:
        new_url = base_url + next_page['href']
        # 使用递归,重复调用此函数,直到没有新的数据页
        get_list(new_url)
        return base_url
    else:
        base_url = None
    return None


get_list(base_url)

结果如图:

豆瓣爬虫数据库.png

参考链接:
BeautifulSoup4中文文档
学习的文章