爬虫模块
# -*- coding: utf-8 -*-
'''深度爬虫：采集工作信息及分页深度采集'''
# 引入爬虫模块
from scrapy.spiderimport CrawlSpider
# 引入链接提取模块
from scrapy.linkextractorsimport LinkExtractor
# 引入规则模块
from scrapy.spiderimport Rule
#导入item模块
from ..itemsimport MyspiderItem
class ZhilianJobSpider(CrawlSpider):
name ="zhilian1"
allowed_domains = ["zhaopin.com"]
start_urls = ["http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E6%97%A0%E9%94%A1&kw=python&isadv=0&isfilter=1&p=1&pd=30",]
# 提取信息1：提取工作链接
link_job = LinkExtractor(restrict_xpaths=("//td[@class='zwmc']/div/a[1]"))
# 提起信息2：提取下一页链接
link_next_page = LinkExtractor(restrict_xpaths=("//a[@class='next-page']"))
# 定义规则和指定的处理函数
# 注意：千万千万千万不要重写parse函数
rules = [
Rule(link_job,callback="parse_job"),
Rule(link_next_page,follow=True),
]
def parse_job(self, response):
'''response中包含的就是详细工作信息'''
print("---------------------job_name---------------------")
item =MyspiderItem()
#职位
job_name = response.xpath("//div[@class='top-fixed-box'][1]/div[@class='fixed-inner-box']/div[@class='inner-left fl']/h1/text()").extract()[0]
if len(job_name) >0:
item['name'] = job_name
# 薪资 .strip去图片
job_salary = response.xpath("//ul[@class='terminal-ul clearfix']/li[1]/strong/text()").extract()[0].strip()
if len(job_salary) >0:
item['salary'] = job_salary
#公司
job_company = response.xpath("//div[@class='top-fixed-box'][1]/div[@class='fixed-inner-box']/div[@class='inner-left fl']/h2/a/text()").extract()[0]
if len(job_company) >0:
item['company'] = job_company
#招聘日期
job_day = response.xpath("//strong/span[@id='span4freshdate']/text()").extract()[0]
if len(job_day) >0:
item['day'] = job_day
#工作经验
job_experience= response.xpath("//ul[@class='terminal-ul clearfix']/li[5]/strong/text()").extract()[0]
if len(job_experience) >0:
item['experience'] = job_experience
#工作地点
job_area = response.xpath("//ul[@class='terminal-ul clearfix']/li[2]/strong/a/text()").extract()[0]
if len(job_area) >0:
item['area'] = job_area
#招聘人数
job_number = response.xpath("//ul[@class='terminal-ul clearfix']/li[7]/strong/text()").extract()[0]
if len(job_number) >0:
item['number'] = job_number
#工作性质
job_nature = response.xpath("//ul[@class='terminal-ul clearfix']/li[4]/strong/text()").extract()[0]
if len(job_nature) >0:
item['nature'] = job_nature
#学历
education_background = response.xpath("//ul[@class='terminal-ul clearfix']/li[6]/strong/text()").extract()[0]
if len(education_background) >0 :
item['education'] = education_background
#职位描述
# job_description = response.xpath("//div[@class='terminalpage-main clearfix']/div[@class='tab-cont-box']/div[@class='tab-inner-cont'][1]")
# if len(job_description) > 0:
# item['description'] = job_description.xpath("string(.)").extract()[0].strip()
job_description = response.xpath("//div[@class='terminalpage-main clearfix']/div[@class='tab-cont-box']/div[@class='tab-inner-cont'][1]")
if len(job_description) >0:
item['description'] = job_description.xpath("string(.)").extract()[0].strip()
# 交给 pipelines管道模块进行处理
yield item
# print (job_name)
# print (job_salary)
# print (job_company)
# print (job_day)
# print (job_experience)
# print (job_area)
# print (job_number)
# print (job_nature)
# print (education_background)
# print (item['description'])