nodejs 爬虫支持代理IP 原创雕虫小技
欢迎一起交流学习，废话不说直接上代码
const request = require("request")
const iconv = require('iconv-lite')
const cheerio = require("cheerio")
const _ = require("lodash")
const crypto = require('crypto')
class spider {
constructor(config) {
/**
* @param {boolen} debug
* @param {boolen} proxy
* @param {string} proxy_ip
* @param {array} temp_proxy
* @param {array} headers
*/
this.config = config
this.doc_type = config.doc_type || 'html'
this.headers = {
'Host': this.config.Host
}
this.result_obj = {
id: Number,
title: String,
link: String,
desc: String,
from: String
}
this.result_ext = {}
this.debug = false
this.proxy = false
this.proxy_ip = null
this.temp_proxy = []
this.headers = []
this.result_list = []
}
/**
* @param {string} url
* @param {string} method
* @return {string}
*/
async so(url = "baidu.com", method = 'get') {
let options = {
method: method,
url: url,
timeout: 8000,
headers: this.headers
}
console.log('开始爬取', url)
if (this.temp_proxy.length && this.proxy) {
if (!this.proxy_ip) {
this.say('找出可用的代理ip')
this.proxy_ip = await this.checkIp()
console.log('获取到了才到下一步')
}
if (!this.proxy_ip) {
console.log('还没有的话。估计是不行了')
return false
}
options.proxy = this.proxy_ip
return new Promise((resolve, reject) => {
request(options, (error, response, body) => {
try {
if (error) throw error;
if (/meta.*charset=gb2312/.test(body)) {
body = iconv.decode(body, 'gbk');
}
if (this.proxy_ip) {
this.say('这个IP果然牛逼!!!!', this.proxy_ip)
}
this.say('爬取完成了，丢出去html给下一个兄弟处理\n')
if (this.doc_type == 'json') {
resolve(this.handle(body[this.res_data]))
} else {
resolve(this.handle(body))
}
} catch (e) {
this.say(options.proxy + '爬取失败了，因为这原因代理ip换一下吧' + options.proxy, e.response)
this.tempProxy = []
this.proxyIp = null
resolve(false)
}
});
}).catch(e => {
this.say(e);
return false
})
} else {
return new Promise((resolve, reject) => {
request(options, (error, response, body) => {
try {
if (error) throw error;
if (/meta.*charset=gb2312/.test(body)) {
body = iconv.decode(body, 'gbk');
}
let result
this.say('爬取完成了，丢出去html给下一个兄弟处理\n')
if (this.doc_type == 'json') {
result = JSON.parse(body)
// console.log(result)
return resolve(this.handle(result[this.res_data]))
} else {
console.log(this.config.doc_type)
resolve(this.handle(body))
}
} catch (e) {
this.say('爬取失败了，因为这原因', e.response)
return reject(e);
}
});
}).catch(e => {
console.log(e)
})
}
}
/**
* 获取代理IP
* @returns
*/
get_proxy_list() {
let api_url = 'http://www.66ip.cn/mo.php?sxb=&tqsl=100&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=http%3A%2F%2Fwww.66ip.cn%2F%3Fsxb%3D%26tqsl%3D100%26ports%255B%255D2%3D%26ktip%3D%26sxa%3D%26radio%3Dradio%26submit%3D%25CC%25E1%2B%2B%25C8%25A1';
return new Promise((resolve, reject) => {
let options = {
method: 'GET',
url: api_url,
gzip: true,
encoding: null,
headers: {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36',
'referer': 'http://www.66ip.cn/'
}
}
request(options, (error, response, body) => {
try {
if (error) throw error;
if (/meta.*charset=gb2312/.test(body)) {
body = iconv.decode(body, 'gbk');
}
let ret = body.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,4}/g);
resolve(ret);
} catch (e) {
return reject(e);
}
})
})
}
/**
* 设置代理ip
*/
async set_proxy() {
// this.say(this.tempProxy)
this.proxy = true
this.say('好吧请稍等，代理IP稍后奉上....')
if (!this.tempProxy.length) {
this.temp_proxy = await this.get_proxy_list()
this.say('这次一用力获取到了' + this.temp_proxy.length + '个IP，请测试！')
} else {
this.say('原来的代理IP还有慢慢慢用')
//其实该换新的了
this.temp_proxy = []
}
}
/**
* 处理数据
* @returns
*/
handle(data) {
if (this.doc_type == 'json') {
let item = {}
for (let i in data) {
item = {}
for (let it in this.result_obj) {
//如果是数读取到最后一个字符串
if (_.isArray(this.result_obj[it])) {
if (this.result_obj[it].length == 2) {
item[it] = data[i][this.result_obj[it][0]][this.result_obj[it][1]]
}
if (this.result_obj[it].length == 3) {
item[it] = data[i][this.result_obj[it][0]][this.result_obj[it][1]][this.result_obj[it][2]]
}
} else {
item[it] = data[i][this.result_obj[it]]
}
}
let new_item = Object.assign(item, this.result_ext)
new_item['key'] = this.set_key(new_item.title)
this.result_list[i] = new_item
}
if (this.debug) {
this.say(data[0])
this.say(this.result_list[0])
}
}
if (this.doc_type == 'html') {
let $ = cheerio.load(data)
this.say('这是一个html文档，具体是否可用要问一下下面的兄弟才知道')
return $
}
return this.result_list
}
async test_proxy(proxy_ip) {
if (proxyIp == undefined) {
this.say('ip 已经没了，重新获取')
this.temp_proxy = []
await this.set_proxy()
}
//测试这个代理IP 可用就设置
return new Promise((resolve, reject) => {
let target_options = {
method: 'GET',
url: 'http://ip.chinaz.com/getip.aspx',
timeout: 8000
};
//这里修改一下，变成你要访问的目标网站
this.say(`开始测试这个IP ${proxyIp}`);
target_options.proxy = 'http://' + proxy_ip;
request(target_options, (error, response, body) => {
try {
if (error) throw error;
body = body.toString();
// this.say(body);
if (body.length < 100) {
this.say(`兄弟这个可以拿去用==>> ${proxy_ip}`);
resolve('http://' + proxy_ip)
} else {
this.say(`这个IP无效==>> ${proxy_ip}`);
resolve(false)
}
} catch (e) {
return reject(false);
}
});
}).catch(e => {
return e
})
}
say(lang, lang1 = '') {
if (this.debug) {
console.log(lang, lang1)
}
}
async check_ip() {
//从第一个开始找找到合适的返回
if (this.proxy) {
for (let i in this.temp_proxy) {
console.log(this.temp_proxy[i])
let true_ip = await this.testProxy(this.temp_proxy[i])
if (true_ip) {
//记住这个索引
return true_ip
}
}
}
return false
}
set_key(str) {
if (!str) {
return 'no str'
}
let rand = Math.ceil(Math.random() * 100)
const md5 = crypto.createHash('md5')
let password = md5.update(str + rand).digest('hex')
return password
}
}
/**
* api 例子
*/
const api_spider = new spider({
Host: 'juejin.com',
doc_type: 'json'
})
api_spider.res_data = ['d']
api_spider.result_obj = {
id: 'objectId',
title: 'title',
link: 'originalUrl',
desc: 'content',
like: 'collectionCount',
comments: 'commentsCount',
createdAt: 'createdAt'
}
api_spider.result_ext = {
from: '掘金'
}
api_spider.so('https://search-merger-ms.juejin.im/v1/search?query=node&page=0&raw_result=false&src=web')
.then((data) => {
console.log(data[0])
})
/**
* html 例子
*/
const so = async function () {
let start = new spider({
Host: 'zzk.cnblogs.com',
doc_type: 'html'
})
start.headers['Cookie'] = 'GA1.2.182722864.1520590357; UM_distinctid=162105121121049-0b14491452f4e4-32667b04-1aeaa0-162105121136f2; _gid=GA1.2.1004931403.1527083606; __utma=59123430.182722864.1520590357.1527094058.1527094058.1; __utmc=59123430; __utmz=59123430.1527094058.1.1.utmcsr=cnblogs.com|utmccn=(referral)|utmcmd=referral|utmcct=/; __utmt=1; __utmb=59123430.3.10.1527094058'
let $ = await start.so('http://zzk.cnblogs.com/s?t=b&w=node')
$('div.searchItem').each(function (i, elem) {
let item = $(this).children('.searchItemTitle')
start.result_list[i] = {
id: i + 1,
title: item.text(),
link: item.children('a').attr('href'),
desc: $(this).children('.searchCon').text().substr(0, 80),
from: '博客园',
createdAt: $(this).children('.searchItemInfo').children('.searchItemInfo-publishDate').text(),
comments: $(this).children('.searchItemInfo').children('.searchItemInfo-comments').text(),
views: $(this).children('.searchItemInfo').children('.searchItemInfo-views').text(),
key: start.set_key(item.text())
}
})
// console.log(start.resultList)
return start.result_list
}
so().then((data)=>{
console.log(data[0])
})
module.exports = spider
``