day04-selenium语法

1. selenium语法

"""__author__= 雍新有"""
import time

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

# # 打开浏览器
# # 默认Chrome()中需要传入chromedriver.exe,只需配置chromedriver环境变量就行了
# browser = webdriver.Chrome()
# # get(url): 表示打开某个地址
# browser.get('https://www.baidu.com')
# # 关闭浏览器
# browser.close()


# 1.获取豆瓣电影信息
# browser = webdriver.Chrome()
# browser.get('https://movie.douban.com/explore#!type=movie&tag=%E7%83%AD%E9%97%A8&sort=recommend&page_limit=20&page_start=0')
# # 获取‘加载更多按钮’, 实现点击事件
# # 模拟向下拉进度条
# js1 = 'window.scrollTo(0, 1000)'
# # browser.execute_script(js1)
# # 获取加载按钮
# # 按钮的xpath代码  -- //*[@id="content"]/div/div[1]/div/div[4]/a
# more_button = browser.find_element_by_xpath('//*[@id="content"]/div/div[1]/div/div[4]/a')
# # 实现点击
# more_button.click()
# # 由于点击事件刚执行，页面还没有被ajax获取到的数据所渲染，所以要睡眠几秒
# time.sleep(3)
#
# # page_source获取源码,经过ajax渲染后的源码
# a1 = browser.page_source
# # a1为源码，解析可采用正则、lxml.etree、beactifulsoup4
# print(a1)
# browser.close()

# 2. 京东 - 查询元素
# browser = webdriver.Chrome()
# browser.get('https://www.jd.com')
# # 输入框
# input = browser.find_element_by_xpath('//*[@id="key"]')
# print(input)
# input = browser.find_element_by_id('key')
# print(input)
# # 选择器，copy里面有
# input = browser.find_element_by_css_selector('#key')
#
# # 通用写法
# # input = browser.find_element(By.XPATH, '//*[@id="key"]')
# # input = browser.find_element(By.ID, 'ID')
# # input = browser.find_element(By.CSS_SELECTOR, '#key')
#
# # 输入内容
# input.send_keys('Mac')

# 3. 前进后退
# 依次在一个窗口打开下面3个网站
# browser = webdriver.Chrome()
# browser.get('https://www.baidu.com')
# browser.get('https://www.jd.com')
# browser.get('http://blog.vincent-whf.top')
#
# # 回退
# browser.back()
# # 前进
# browser.forward()


# 4.获取京东，输入内容，点击搜索
# 等待(隐式等待、显式等待)
# browser = webdriver.Chrome()
# browser.get('https://www.jd.com')
# # 不睡眠，可能网页加载不出来，后面的xpath就找不到元素
# # time.sleep(3)
#
# # 隐式等待，如果获取元素找不到，则等待规定时长(默认为0)，等待时长自定义
# browser.implicitly_wait(10)
#
# input = browser.find_element(By.XPATH, '//*[@id="key"]')
# # 将输入框的内容清空
# input.clear()
# # 输入信息
# input.send_keys('零食')
# # 获取搜索点击按钮
# button = browser.find_element(By.XPATH, '//*[@id="search"]/div/div[2]/button')
# # 搜索点击
# button.click()
# browser.implicitly_wait(10)
#
# # 获取页码
# total = browser.find_element(By.XPATH, '//*[@id="J_bottomPage"]/span[2]/input')

# 显式等待
browser = webdriver.Chrome()
browser.get('https://www.jd.com')

# 获取等待对象,等待10秒
wait = WebDriverWait(browser, 10)
# presence_of_all_elements_located: 当元素被加载出来了才获取信息
input = wait.until(
    EC.presence_of_element_located((By.XPATH, '//*[@id="key"]'))
)

# 将输入框的内容清空
input.clear()
# 输入信息
input.send_keys('水果')

# element_to_be_clickable: 等待元素可被点击
button = wait.until(
    EC.element_to_be_clickable((By.XPATH, '//*[@id="search"]/div/div[2]/button'))
)
# 搜索点击
button.click()

# 获取页码
total = wait.until(
    EC.presence_of_element_located((By.XPATH, '//*[@id="J_bottomPage"]/span[2]/em[1]/text()'))
)
print(total)

2. 京东selenium爬虫

"""__author__= 雍新有"""
import re
import time

import pymongo
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait


chrome_options = webdriver.ChromeOptions()
# 配置不用加载图片
prefs = {"profile.managed_default_content_settings.images":2}
chrome_options.add_experimental_option("prefs", prefs)

# 获取浏览器
browser = webdriver.Chrome(chrome_options=chrome_options)
# 显式等待
wait = WebDriverWait(browser, 20)


def search():
    browser.get('https://www.jd.com')
    # 获取输入框，输入内容
    input = wait.until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="key"]'))
    )
    input.clear()
    input.send_keys('水果')
    # 获取按钮，实现点击
    button = wait.until(
        EC.element_to_be_clickable((By.XPATH, '//*[@id="search"]/div/div[2]/button'))
    )
    button.click()

    for i in range(1, 17):
        js = f'window.scrollTo(0, {i} * document.body.scrollHeight / 16)'
        browser.execute_script(js)
        time.sleep(5)

    total = wait.until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="J_bottomPage"]/span[2]/em[1]'))
    )
    # 获取第一页源码
    html = browser.page_source
    result = parse_html(html)
    save_mongo(result, 1)

    return total.text


def next_page(page):
    # 获取从第二页开始的每一页信息z
    # 翻页: 1. 可点击下一页  2. 可输入页面，实现点击
    # 下一页xpath =  // *[ @ id = "J_bottomPage"] / span[1] / a[9]

    # 执行滚动
    for i in range(1, 33):
        js = f'window.scrollTo(0, {i} * document.body.scrollHeight / 32)'
        browser.execute_script(js)
        time.sleep(5)

    # js = 'window.scrollTo(0, 2*document.body.scrollHeight / 4)'
    # browser.execute_script(js)
    # time.sleep(1)
    #
    # js = 'window.scrollTo(0, 3*document.body.scrollHeight / 4)'
    # browser.execute_script(js)
    # time.sleep(1)
    #
    # js = 'window.scrollTo(0, 4*document.body.scrollHeight / 4)'
    # browser.execute_script(js)
    # time.sleep(1)

    # print(f'----------点击{page}页--------')
    # next_input = wait.until(
    #     EC.element_to_be_clickable((By.XPATH, '//*[@id="J_bottomPage"]/span[1]/a[9]'))
    # )
    # next_input.click()

    page_input = wait.until(
        EC.presence_of_element_located((By.XPATH, '// *[ @ id = "J_bottomPage"] / span[2] / input'))
    )
    page_input.clear()
    print(f'---------第{page}页----')
    page_input.send_keys(str(page))
    button = wait.until(
        EC.element_to_be_clickable((By.XPATH, '//*[@id="J_bottomPage"]/span[2]/a'))
    )
    button.click()

    # 点击过后，页面将重新加载,主动等待3秒后再获取源码
    time.sleep(10)
    html = browser.page_source
    return html
    # 判断当前页是否调转过来了


def parse_html(html):
    tree = etree.HTML(html)
    goods_list = tree.xpath('//*[@id="J_goodsList"]/ul/li')
    result = []
    for item in goods_list:
        # item就是每一个li
        data = {
            'img': item.xpath('./div/div[1]/a/img/@src'),
            'goods_name': item.xpath('./div/div[3]/a/@title'),
            'goods_price': item.xpath('./div/div[2]/strong/i/text()')
        }
        result.append(data)
    return result


def save_mongo(data, page):
    db = pymongo.MongoClient(host='127.0.0.1', port=27017)
    print(len(data))
    try:
        print(f'第{page}页数据插入成功')
        for item in data:
            print(item)
            db['spider']['jd1'].insert_one(item)
    except:
        print(f'第{page}页数据插入失败')


if __name__ == '__main__':
    # search(): 打开浏览器输入京东，搜索某商品，返回总页码
    total = search()
    # 正则search匹配字符串内的内容，fallmatch要匹配' -- 字符串标识符,
    all_page = int(re.search('(\d+)', total).group())
    for page in range(2, 4):
        html = next_page(page)
        result = parse_html(html)
        save_mongo(result, page)