爬虫 蜘蛛 信息采集
HttpWebRequest
System.Net.HttpWebRequest request = (System.Net.HttpWebRequest)System.Net.WebRequest.Create("");
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
System.Net.WebResponse response = request.GetResponse();
System.IO.Stream resStream = response.GetResponseStream();
System.IO.StreamReader sr = new System.IO.StreamReader(resStream, encoding);
string content=sr.ReadToEnd();
resStream.Close();
sr.Close();
webrequest,WebClient
System.Net.WebClient wc = new System.Net.WebClient();
wc.Credentials = System.Net.CredentialCache.DefaultCredentials;
Byte[] pageData = wc.DownloadData("");
string content= System.Text.Encoding.Default.GetString(pageData);
///
<summary>
///
实现登录
///
</summary>
///
<param name="targetURL">
请求的路径，必须是实现登录的路径(*)
</param>
///
<param name="cc">
用于维持cookies Or Session
</param>
///
<param name="param">
Post提交的信息（用户名，密码）
</param>
///
<returns>
html page
</returns>
public
static
CookieContainer cc
=
new
CookieContainer();
//
维持cookie或Session
public
static
string
PostAndGetHTML(
string
targetURL, Hashtable param) {
//
formData用于保存提交的信息
string
formData
=
""
;
foreach
(DictionaryEntry de
in
param) { formData
+=
de.Key.ToString()
+
"
=
"
+
de.Value.ToString()
+
"
&
"
; }
if
(formData.Length
>
0
) formData
=
formData.Substring(
0
, formData.Length
-
1
);
//
去除最后一个 '&'
//
把提交的信息转码（post提交必须转码）
ASCIIEncoding encoding
=
new
ASCIIEncoding();
byte
[] data
=
encoding.GetBytes(formData);
//
开始创建请求
HttpWebRequest request
=
(HttpWebRequest)WebRequest.Create(targetURL); request.Method
=
"
POST
"
;
//
提交方式：post
request.ContentType
=
"
application/x-www-form-urlencoded
"
; request.ContentLength
=
data.Length; request.UserAgent
=
"
Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; SV1; .NET CLR 2.0.1124)
"
; request.AllowAutoRedirect
=
true
; request.KeepAlive
=
true
; Stream newStream
=
request.GetRequestStream(); newStream.Write(data,
0
, data.Length);
//
将请求的信息写入request
newStream.Close(); request.CookieContainer
=
cc;
//
向服务器发送请求
HttpWebResponse response
=
(HttpWebResponse)request.GetResponse();
//
获得Cookie 保存到Appliction中
string
cookieHeader
=
request.CookieContainer.GetCookieHeader(
new
Uri(
"
http://login.xiaonei.com/Login.do
"
)); HttpContext.Current.Application.Lock(); HttpContext.Current.Application[
"
cookieHeader
"
]
=
cookieHeader; HttpContext.Current.Application.UnLock();
return
"
OK
"
; }
///
<summary>
///
访问其他页面
///
</summary>
///
<param name="strUrl"></param>
///
<returns></returns>
public
static
string
ReGetHtml(
string
strUrl) {
//
第二次请求
HttpWebRequest request1
=
(HttpWebRequest)WebRequest.Create(strUrl);
string
cookhead
=
HttpContext.Current.Application[
"
cookieHeader
"
].ToString(); request1.Method
=
"
GET
"
; request1.Headers.Add(
"
cookie:
"
+
cookhead); request1.KeepAlive
=
true
; request1.AllowAutoRedirect
=
true
; HttpWebResponse response1
=
(HttpWebResponse)request1.GetResponse(); Stream stream2
=
response1.GetResponseStream();
//
获得回应的数据流
//
将数据流转成 String
string
result1
=
new
StreamReader(stream2, System.Text.Encoding.UTF8).ReadToEnd();
return
result1; }