Python爬虫-selenium模块

selenium模块

基本使用

from selenium.webdriver import Chrome
# 导入自定义配置模块
from selenium.webdriver.chrome.options import Options

# 创建配置对象
chrome_options = Options()
# 谷歌文档提到需要加上这个属性来规避bug
chrome_options.add_argument('--disable-gpu')
# 不加载图片，可以提升速度
chrome_options.add_argument('blink-settings=imagesEnabled=false')
# 浏览器不提供可视化页面，Linux下如果系统如果无界面不加这条会启动失败
chrome_options.add_argument('--headless')
# 取消浏览器驱动提示
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])

# 如果驱动路径设置到环境遍历中，可以不用传参数
# options 表示配置项
driver = Chrome(options=chrome_options)
# 访问百度
driver.get("https://www.baidu.com")
# 获取页面源码
text = driver.page_source
print(text)

解析数据

from selenium.webdriver import Chrome

driver = Chrome()
driver.get("https://www.baidu.com")

# selenium提供了很多用于解析数据的函数
# tag = driver.find_element_by_class_name("index-logo-src")
# tag = driver.find_element_by_css_selector(".index-logo-src")
# print(tag.text)
# 得到的是driver对象，不是父标签
# print(tag.parent)
# print(tag.get_attribute("src"))
# print(tag.tag_name)

# 一些特别的查找方式
# 根据链接的文本查找，文本完全匹配
# tag = driver.find_element_by_link_text("学术")
# 根据链接的文本查找，文本部分匹配
# tag = driver.find_element_by_partial_link_text("术")
# print(tag)

# name属性等于tj_trhao123
# tag = driver.find_element_by_name("tj_trhao123")
# 标签名称等于body
# tag = driver.find_element_by_tag_name("body")
# print(tag)
# print(tag.tag_name,tag.get_attribute("href"),tag.text)

#所有方法都有带s的版本 用来查找所有匹配的 返回一个列表
# res = driver.find_elements_by_class_name("mnav")
# print(res)

# 嵌套查找
# tag = driver.find_element_by_name("tj_trhao123")
# tag2 = tag.find_element_by_tag_name("a")


# 与显示相关的属性
tag = driver.find_element_by_class_name("index-logo-src")
print(tag.location)
print(tag.size)

等待元素加载

from selenium.webdriver import Chrome
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

driver = Chrome()
driver.get("https://www.baidu.com")

# 隐式等待
# 当要查找的某个元素不存在时，会过一会儿在查找一次(轮询)直到找到，找到超过10就报错
driver.implicitly_wait(10)

# 找到输入框
key_input = driver.find_element_by_id("kw")
key_input.send_keys("python")
key_input.send_keys(Keys.ENTER)

# 等到页面上出现了一个id为content_left的元素位为止，最长等10秒
WebDriverWait(driver,10).until(EC.presence_of_element_located((By.ID,"content_left")))

动作链

动作链指的是一系列动作的集合
例如：滑动验证
1.点击并按住
2.移动鼠标
3.移到指定位置松手

from selenium.webdriver import Chrome
from selenium.webdriver import ActionChains

driver = Chrome()
driver.get("http://www.runoob.com/try/try.php?filename=jqueryui-api-droppable")
driver.implicitly_wait(5)

# 切换fream
driver.switch_to.frame("iframeResult")

# 获取请拖拽我这个标签
tag = driver.find_element_by_id("draggable")
# 获取目标位置的标签
tag2 = driver.find_element_by_id("droppable")

# 线性移动
# 创建一个动作对象
asc = ActionChains(driver)
# perform()表示执行这个动作 点击并按住
asc.click_and_hold(tag).perform()

# 循环逐渐移动
while tag.location["x"] < tag2.location["x"]:
    ActionChains(driver).move_by_offset(1, 0).perform()
# 松手
asc.release().perform()

执行js

from selenium.webdriver import Chrome

driver = Chrome()
driver.get("https://www.baidu.com")
# driver.execute_script("alert('python')")

# driver.get("https://www.baidu.com")
# driver.get("https://www.qq.com")
# driver.get("https://www.sina.com")
# 后退
# driver.back()
# 前进
# driver.forward()

# 切换选项卡
driver.execute_script("window.open()")
driver.switch_to.window(driver.window_handles[1])
driver.get("https://www.qq.com")
driver.switch_to.window(driver.window_handles[0])

xpath解析数据

from lxml.html import etree
doc = """


    
        

        
          
          29.99
        

        11111111111111111111
          
          39.95
        

        
    
    

"""

html = etree.HTML(doc)
# print(html.xpath("/bookstore"))
# print(html.xpath("//bookstore"))

# 通配符
# print(html.xpath("//book"))
# print(html.xpath("//*"))

# 获取属性
# print(html.xpath("//bookstore/@id"))
# print(html.xpath("//bookstore/@*"))

# 嵌套
# print(html.xpath("//bookstore/book/title/text()"))

# 加上谓语(条件)
# 指定要获取的索引
# print(html.xpath("//bookstore/book[1]/title/text()")) # 获取第一个
# print(html.xpath("//bookstore/book[last()-1]/title/text()")) # 获取最后一个
# print(html.xpath("//bookstore/book[position()>1]/title/text()")) # 索引大于1的

# 查找price的值大于30的book标签
# e = html.xpath("//book[price > 30]")[0]
# 访问文本
# print(e.text)
# 访问属性
# print(e.attrib)

# 只要存在lang属性即可
# print(html.xpath("//*[@lang]"))
# 找具备lang并且值为abc的标签
# print(html.xpath("//*[@lang='abc']")[0].attrib)
# 只要有属性即可
# print(html.xpath("//*[@*]"))
# 多个匹配条件
# print(html.xpath("//title|//price"))

# 轴匹配(先拿到一个标签,在相对这个标签找其他标签)
# 所有先辈
# print(html.xpath("//bookstore/ancestor::*"))
# 所有叫body的先辈
# print(html.xpath("//bookstore/ancestor::body"))
# 所有叫body的先辈
# print(html.xpath("//bookstore/ancestor-or-self::*"))

# 获取属性
print(html.xpath("//bookstore/attribute::id"))
# 所有子级标签
print(html.xpath("//bookstore/child::*"))
# 所有后代标签
print(html.xpath("//bookstore/descendant::*"))

# 在这个标签后面的所有标签,与层级无关
print(html.xpath("//book[1]/following::*"))
# 它弟弟们
print(html.xpath("//book[1]/following-sibling::*"))
# 它哥哥们
print(html.xpath("//book[1]/preceding-sibling::*"))

# 获取父级
print(html.xpath("//book[1]/parent::*"))
# 获取既有id属性,又有class属性的标签
print(html.xpath("//*[@id and @class]"))

案例

import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys

# 创建配置对象
chrome_options = Options()
# 谷歌文档提到需要加上这个属性来规避bug
chrome_options.add_argument('--disable-gpu')
# 不加载图片, 可以提升速度
chrome_options.add_argument('blink-settings=imagesEnabled=false')
driver = webdriver.Chrome()
driver.get("https://pages.tmall.com/wow/car/act/zhizu")

# 执行js获取总高度
height = driver.execute_script("return document.body.clientHeight")
# 已经滑动的距离
dis = 0
while dis < height:
    driver.execute_script("""
    window.scrollTo({
        top: %s,
        behavior: "smooth"
    });""" % dis)
    dis += 200
    time.sleep(0.2)
driver.implicitly_wait(5)

# 获取页头 四个推荐数据
big_div = driver.find_element_by_class_name("zebra-car-goods-1x1")
item_div = big_div.find_element_by_css_selector("div")
print(item_div.get_attribute("class"))
res = item_div.find_elements_by_css_selector(".item")
print(res)
for i in res:
    print(i.find_element_by_tag_name("a").get_attribute("href"))
    print(i.find_element_by_css_selector(".img").get_attribute("src"))
    print(i.find_element_by_css_selector(".info .title").text)
    infos = i.find_elements_by_css_selector(".interest")
    print([k.text for k in infos])


# 获取汽车列表
sections = driver.find_elements_by_class_name("zebra-car-goods-1x2")
print(len(sections))
for s in sections:
    bloks = s.find_elements_by_css_selector(".block")
    for b in bloks:
        print(b.find_element_by_css_selector("a").get_attribute("href"))
        print(b.find_element_by_css_selector(".main").get_attribute("src"))
        print(b.find_element_by_css_selector(".title").text)
        infos = b.find_elements_by_css_selector(".interest")
        print([k.text for k in infos])
driver.close()