python(点点点工程师自我代码修养)--小案例--爬取静态网页

小案例–爬取静态网页(猫咪交易网)

import requests
import parsel
import csv


# 创建一个用于存储数据的csv文件
f = open('猫咪.csv', mode='a', encoding='utf-8', newline='')
csv_write = csv.DictWriter(f, fieldnames=['标题', '商店', '价格', '浏览量', '卖家承诺', '连接', '地区'])
csv_write.writeheader()

for page in range(1, 11):
	# 发送请求
	print('==========正在爬取第{0}页内容=========='.format(page))
	url = 'http://www.maomijiaoyi.com/index.php?/chanpinliebiao_c_2_{0}--24.html'.format(page)
	headers = {
	'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36'
	}

	# 获取数据
	response = requests.get(url, headers=headers)

	# 解析数据
	selector = parsel.Selector(response.text)
	href = selector.css('#content > div.breeds_floor > div > div > a::attr(href)').getall()
    area = selector.css('#content > div.breeds_floor > div > div > a > div.price_area > div.area > span.color_333::text').getall()
    area = [i.strip() for i in area]
    for cat_url, city in zip(href, area):
    	cat_url = "http://www.maomijiaoyi.com" + cat_url
    	# 发送请求
        html_data = requests.get(url=cat_url, headers=headers).text
        # 获取数据
        # 解析数据
        selector1 = parsel.Selector(html_data)
        title = selector1.css('#content > div.center_1220.detail > div.left.top > div.detail_text > div.title::text').get().strip()
        shop = selector1.css('#content > div.center_1220.detail > div.right > div.dinming::text').get().strip()
        price = selector1.css('.red.size_24::text').get().strip()
        views = selector1.css('#content > div.center_1220.detail > div.left.top > div.detail_text > div.info1 > div:nth-child(1) > span:nth-child(4)::text').get()
        promise = selector1.css('#content > div.center_1220.detail > div.left.top > div.detail_text > div.info1 > div:nth-child(2) > span::text').get().replace('卖家承诺', '')
        dict = {
            '标题': title,
            '商店': shop,
            '价格': price,
            '浏览量': views,
            '卖家承诺': promise,
            '连接': cat_url,
            '地区': city
        }
        # 保存数据
        csv_write.writerow(dict)

爬取结果(部分)