爬虫小案例之爬取京东商品链接

观察URL翻页的变化

爬取页面URL如下

    base_url='https://search.jd.com/Search?keyword='+keyword
    for x in range(1,num+ 1):
        url=base_url+'&page=%d'% x

爬取结果如下：

 

import requests
import re
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36'}
def parse_page(results,text):
    titles_ret = re.findall(r'
", "",title)
        titles.append(temp.strip())
    print('title', titles)
    prices = re.findall(r'
.*?(.*?).*?
', text,flags=re.DOTALL)
    hrefs=re.findall(r'
.*?
', text, flags=re.DOTALL)
    for value in zip(titles, prices, hrefs):
        title, price,href = value
        results.append([title, price,href])
    return results


def getHTMLText(url):
    '''从网络上获取网页内容'''
    try:
        r = requests.get(url, headers=headers,timeout=40)
        # #如果状态不是200，就会引发HTTPError异常
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""

def printGoodsList(ilt):
    tplt = "{:4}\t{:8}\t{:40}\t{:60}"
    print(tplt.format("序号", "价格", "链接","商品名称"))
    count = 0
    for g in ilt:
        count = count + 1
        print(tplt.format(count, g[1], g[2],g[0]))

def main():
    keyword ='女装套装'
    base_url='https://search.jd.com/Search?keyword='+keyword
    num=5
    list=[]
    for x in range(1,num+ 1):
        url=base_url+'&page=%d'% x
        print(url)
        text = getHTMLText(url)
        list = parse_page(list,text)
    printGoodsList(list)
if __name__ == '__main__':
    main()

股票爬取