Python爬虫练习（二）爬取NCBI pubmed搜索结果doi号并进行文献下载（requests+xpath+正则）

继上文的requests+正则解析https://www.jianshu.com/p/c4c041459f58，接下来使用requests+re+xpath解析。用pubmed的高级搜索不知道为什么，比如用关键字1 关键字2搜索，总是会有缺少一个关键字的情况，就算用AND连接也是这样，给文献的搜索造成了很大的麻烦，所以就用爬虫提取关键字都出现的结果的doi号然后调用sci-hub地址下载。
用法：运行代码。然后输入关键字，如"human evolution" "mammalia“，这是两个关键字，用双引号圈起来，如果你输入human evolution mammlia不加双引号，那么human evolution将不会紧密连在一起搜索，这是三个关键字。接下来确认下载的文件夹，输入y或者n确认是否直接下载pdf还是将下载地址写入download_url.txt这个文件。后续也许还会写爬取Google学术的类似类，也会放到这个module里面，会同时更新到我的github上https://github.com/ijustwanthaveaname/web-crawler/blob/main/%E7%AC%AC%E4%BA%8C%E7%AB%A0%EF%BC%9Arequest%E6%A8%A1%E5%9D%97%E5%9F%BA%E7%A1%80/searchpaper.py

import requests
import re
import os
from lxml import etree


class Searchpubmed:
    def __init__(self, term):
        self.__term = term
        self.__headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
              (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36"
        }
        self.url = "https://pubmed.ncbi.nlm.nih.gov/"
        self.__size, self.__page = 200, 1
        self.__param = {}
        self.param = {
            "term": self.__term,
            "size": self.__size,
            "page": self.__page
        }
        self.__doi, self.__doi_list, self.__down_url = [], [], []
        self.__response, self.__page_text, self.__step_num, self.__results_amount, \
        self.__tree, self.__snippet, self.__citation, self.__title = [None]*8

    def get_text(self, url, param):
        self.__response = requests.get(url=url, params=param, headers=self.__headers)
        self.__page_text = self.__response.text
        return self.__page_text

    @staticmethod
    def __format_text(xpathelement):
        xpathelement = "".join(xpathelement.xpath(".//text()")).replace("", "").replace("", "").lower()
        return xpathelement

    def get_doi(self, page_text):
        self.__tree = etree.HTML(page_text)
        self.__title = self.__tree.xpath("//div[@class='docsum-content']/a")
        self.__snippet = self.__tree.xpath("//div[@class='full-view-snippet']")
        self.__citation = self.__tree.xpath("//span[@class='docsum-journal-citation full-journal-citation']")
        for title, snippet, citation in zip(self.__title, self.__snippet, self.__citation):
            title = self.__format_text(title)
            snipper = self.__format_text(snippet)
            searchtext = title + snipper
            self.__doi = re.search(r"""(doi: (10\..*?)\. )|(doi: (10\.\S+)\.$)""", citation.xpath("./text()")[0])
            self.__doi = [self.__doi.group(2) if self.__doi.group(2) else self.__doi.group(4)] if self.__doi else []
            for kw in self.__term:
                if kw.lower() in searchtext:
                    continue
                else:
                    self.__doi = []
                    break
            self.__doi_list += self.__doi
        return self.__doi_list

    def get_allpagedoi(self, page_text):
        print("Processing Page1")
        self.__results_amount = int(re.search(r"""(\d+(?:,?\d+)?).*?results""", page_text,
                                            re.DOTALL).group(1).replace(",", ""))
        self.get_doi(page_text)
        if self.__results_amount % 200 == 0:
            self.__step_num = self.__results_amount / 200 - 1
        else:
            self.__step_num = self.__results_amount // 200
        if self.__step_num:
            for page in range(2, self.__step_num + 2):
                print(f"Processing Page{page}")
                self.__size = 200
                self.__page = page
                self.__param = {
                    "term": self.__term,
                    "size": self.__size,
                    "page": self.__page
                }
                self.__page_text = self.get_text(url=self.url, param=self.__param)
                self.get_doi(self.__page_text)
        return self.__doi_list

    def scihuburl(self, doi_list):
        for doi in doi_list:
            self.__down_url.append(r"https://sci.bban.top/pdf/"+doi+".pdf")
        return self.__down_url

    @staticmethod
    def getpdf(down_url, path="./", direct=False):
        downloadpath = os.path.join(path, "download_url.txt")
        if os.path.isfile(downloadpath) and not direct:
            os.remove(downloadpath)
        for doiurl in down_url:
            if direct:
                r = requests.get(url=down_url)
                with open(os.path.join(path, os.path.basename(doiurl)), "wb") as f:
                     f.write(r.content)
            else:
                with open(downloadpath, "a") as u:
                    u.write(doiurl + "\n")


if __name__ == "__main__":
    term = input("Please input your keywords: ")
    if '"' in term:
        term = [i for i in term.split('"') if i not in ("", " ")]
    else:
        term = term.split(" ")
    downloaddir= input("Please specify the directory for downloading: ")
    direct = input("Do you want to download the pdf directly?[y/n]")
    while direct not in ["y", "n"]:
        print("You only can input y or n!")
        direct = input("Do you want to download the pdf directly?[y/n]")
    direct = True if direct == "y" else False
    getpub = Searchpubmed(term)
    page_text = getpub.get_text(getpub.url, getpub.param)
    doi_list = getpub.get_allpagedoi(page_text)
    down_url = getpub.scihuburl(doi_list)
    getpub.getpdf(down_url, downloaddir, direct)