python 爬虫 CSDN博客下载-改进版

主要是对代码结构进行了调整

新分出了用户类

加入了多线程，实现多个用户同时下载

# -*- coding: utf-8 -*-
import os
import sys
import urllib
import urllib2
import re
import thread
import time
import codecs
import threading
class USER:
	def __init__(self,userName,path):
		self.name=userName
		self.articleList=[]
		self.articleNum=0
		self.pageIndex=1
		self.rootURL='http://blog.csdn.net'
		self.contentURL='http://blog.csdn.net/'+self.name+'/article/list/'+str(self.pageIndex)
		self.rootPath=path+'/BlogCSDN/'+self.name
	def openURL(self,url):
		headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64)'}
		try:
			request = urllib2.Request(url, headers = headers)
			response = urllib2.urlopen(request)
			pageCode = response.read().decode('utf-8')
			return pageCode
		except urllib2.URLError, e:
			if hasattr(e, "reason"):
				print u"连接CSDN，错误原因", e.reason
				return None
	def getList(self):
		try:
			url = self.rootURL+'/' +self.name+'/article/list/'+ str(self.pageIndex)
			if self.articleNum==0:
				pageCode=self.openURL(url)
				if pageCode!=None:
					#正则表达式当有多个括号时，返回为字符串元祖列表[(),(),...()]
					#每一个元组元素包含的字符串数与表达式中的括号一致
					#比如下式返回值可能为[('20','18','3')]
					#正则表达式有1个或0个括号时，返回值为字符串列表['aaaa','ddd']
					pattern=re.compile('blog_statistics">.*?<span>(\d+).*?</span>.*?<span>(\d+).*?</span>.*?<span>(\d+).*?</span>',re.S)
					items=re.findall(pattern,pageCode)
					for m in items[0]:
						self.articleNum+=int(m)
			num=0
			articleList=[]
			while num<self.articleNum:
				url = self.rootURL+'/' +self.name+'/article/list/'+ str(self.pageIndex)
				pageCode=self.openURL(url)
				if not pageCode:
					return None
				pattern=re.compile('link_title"><a href="(.*?)">(.*?)</a>',re.S)
				items=re.findall(pattern,pageCode)
				for item in items:
					articleList.append([self.rootURL+item[0].strip(),item[1].strip()])
					#print self.rootURL+item[0]+'-----'+item[1].strip()
					num+=1
					if num>=self.articleNum:
						break
				self.pageIndex+=1
			self.articleList=articleList
		except urllib2.URLError, e:
			if hasattr(e, "reason"):
				print u"连接CSDN，错误原因", e.reason
				return None
	def getArticle(self):
		for item in self.articleList:
			pageCode=self.openURL(item[0])
			if not pageCode:
				print u'链接无效'
				return None
			pattern=re.compile('class="article_content">(.*?)</div>',re.S)
			content=re.findall(pattern,pageCode)
			articleID=(item[0].split('/'))[6]
			title=item[1]
			fileName=self.rootPath+'/'+title+'---'+articleID+'.txt'
			#这个地方刚开始少写了'[0]'，造成编码对不上，花费了许多时间
			text=content[0]
			text=self.repairCode(text)
			self.saveArticle(fileName,text)
			print self.name+':'+articleID+'.txt has saved!'
	def saveArticle(self,fileName,text):
		if not os.path.exists(self.rootPath):
			os.makedirs(self.rootPath)
		try:
			if os.path.exists(fileName):
				print u'%s 文件已经存在' % (fileName)
				return None
			fobj =codecs.open(fileName, 'w+','utf-8')
			fobj.write(text)
			fobj.close()
		except IOError:
			print 'IO error!'
			return None
	def repairCode(self,text):
		#去除标签
		replace = re.compile('</\w+>')
		text = re.sub(replace,"\r\n",text)
		replace = re.compile('<.*?>')
		text = re.sub(replace,"",text)
		rTermsDict={'<':'<','>':'>','&':'&','"':'"',' ':' ',''':'\'','+':'+'}
		for item in rTermsDict.keys():
			replace = re.compile(item)
			text = re.sub(replace,rTermsDict[item],text)
		#if codeType!='cpp':
			#扩展不同的代码
			#return text
		#对齐代码块
		return text
	def start(self):
		print 'reading %s blog...' % (self.name)
		self.getList()
		self.getArticle()
		print "%s  blog %d has saved!" % (self.name,self.articleNum)

class CSDNSPIDER:

	def __init__(self):
		# 存放程序是否继续运行的变量
		self.rootPATH='E:/Python_Spider'
		self.enable = False
	def start(self):
		#想继续扩展成多线程
		userList=['waytoaccept','xiaosebi1111','zhiyuan_2007']
		thrList=[]
		for name in userList:
			user=USER(name,self.rootPATH)
			t=threading.Thread(target=user.start,name=name)
			thrList.append(t)
		for thr in thrList:
			thr.start()
		for thr in thrList:
			thr.join()
		self.enable = True
		print u'欢迎下次使用CSDN爬虫！'
spider = CSDNSPIDER()
spider.start()

感觉爬虫越来越有意思了