Python爬取豆瓣读书Top250（正则表达式）

#  -*- coding = utf-8 -*-
#导入所需库
from bs4 import BeautifulSoup
import re
import xlwt
import urllib.request, urllib.error
import sqlite3

爬取步骤
#第一步 爬取数据
#第二步 解析数据
#第三步 保存数据

def main():
    baseurl = "https://book.douban.com/top250?start="
    datalist = getData(baseurl)
    savepath = "C:\\Users\\J\\Desktop\\读书2.xls"#保存路径
    # askURL("https://book.douban.com/top250?start=")
    saveData(datalist, savepath)
    
#使用正则表达式进行内容提取
#规则设定
findLink = re.compile(r'(.*?)')#外文名
findInq = re.compile(r'
(.*?)
',re.S)#相关信息
findScore = re.compile(r'span class="rating_nums">(.*?)')#得分
findCount = re.compile(r'(.*?)', re.S)#评价人数
findIntro = re.compile(r'(.*)',re.S)#简介

def getData(baseurl):
    datalist = []
    for i in range(0,10):
        url = baseurl + str(i *25)
        html = askURL(url)
        #将得到的数据进行逐一解析
        soup = BeautifulSoup(html, "html.parser")
        for item in soup.find_all("tr", class_ = "item"):
            # print(item)
            data = []
            item = str(item)

            link = re.findall(findLink, item)[0]
            data.append(link)

            img = re.findall(findImgSrc, item)[0]
            data.append(img)

            title = re.findall(findTitle, item)[0]
            data.append(title)

            Ftitle = re.findall(findFT,item)
            if (len(Ftitle)==0):
                data.append(' ')
            else:
                data.append(Ftitle[0])


            Inq = re.findall(findInq, item)[0]
            Inq = Inq.replace("[","")
            Inq = Inq.replace("]","")
            data.append(Inq)

            score = re.findall(findScore, item)[0]
            data.append(score)

            count = re.findall(findCount, item)[0]
            count = re.sub('((\s+)?)(\s+)?',"",count)
            count = count.replace("(","")
            count = count.replace("人评价)","")
            data.append(count)

            introduction = re.findall(findIntro, item)
            if len(introduction) != 0 :
                introduction = introduction[0]
                data.append(introduction)
            else:
                data.append("暂无介绍")

            datalist.append(data)

    return datalist

def askURL(url):
    head ={
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36"#模拟浏览器访问网页
    }
    request =  urllib.request.Request(url,headers=head)
    html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode("utf-8")
    except urllib.error.URLError as e:
        if hasattr(e,"code"):
            print(e.code)
        if hasattr(e,"reason"):
            print(e.reason)
    return html

#保存xls数据
def saveData(datalist, savepath):
    print("Save....")
    book = xlwt.Workbook(encoding="utf-8", style_compression=0)
    sheet = book.add_sheet('豆瓣读书Top250', cell_overwrite_ok=True)
    col = ("书链接", "图片链接", "书名", "副标题", "相关信息", "豆瓣评分", "评价人数", "简介")
    for i in range(0,8):
        sheet.write(0, i, col[i])
    for i in range(0,250):
        print("第%d条"%(i+1))
        data = datalist[i]
        for j in range(0,8):
            sheet.write(i+1, j, data[j])

    book.save(savepath)

if __name__ == '__main__':
    main()
    print("完成")