scrapy爬虫框架测试某个功能函数测试文件

import sys
sys.path.insert(0, '../')
sys.path.insert(0, '../../')
from scrapy.utils.log import configure_logging
from news.abc.spiders.abc_spider import NewsSpider
import requests

headers = {
    'user-agent': 'Mozilla/5.0 (Linux; Android 13; Pixel 7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Mobile Safari/537.36'
}
configure_logging({'LOG_LEVEL':'INFO'})
news_url = "https://www.xxxx.com/news/xxxx.html"

spider = NewsSpider()

response = requests.get(news_url, headers=headers)
response.meta = {}
response.meta['classify'] = ''
response.meta['is_hot'] = ''
gen = spider.news_info(response)
for value in gen:
    print(value)