数据采集之-页面读取，图片采集 类

1. namespace DBUtility
2. {
3.  public    class common
4.     {
5.      public static string GetPageHTML(string url)
6.         {
7.             HttpWebRequest wr = WebRequest.Create(url) as HttpWebRequest;
8.             try
9.             {
10.                 wr.Method = "get";
11.                 wr.Accept = "*/*";
12.                 wr.Headers.Add("Accept-Language:   zh-cn");
13.                 wr.Headers.Add("UA-CPU:   x86");
14.                 wr.Headers.Add("Accept-Encoding:   gzip, deflate");
15.                 wr.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1;  Embedded Web Browser from: http://bsalsa.com/; InfoPath.2; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022)";
16.                 wr.KeepAlive = true;
17.                 wr.ServicePoint.Expect100Continue = false;
18.                 wr.AllowAutoRedirect = false;
19.                 wr.Timeout = 30000;
21.                 HttpWebResponse wre = wr.GetResponse() as HttpWebResponse;
22.                 StreamReader sreader = new StreamReader(wre.GetResponseStream(), Encoding.Default);
23.                 string sHtml = sreader.ReadToEnd();
24.                 wre.Close();
25.                 return sHtml;
26.             }
27.             catch
28.             {
29.                 return "";
30.             }
31.         }
32.      public static  int SaveImageFromWeb(string imgUrl, string path,string imgname)
33.      {
34.           path = "e://test";
35.           string imgName = imgname+imgUrl.ToString().Substring(imgUrl.ToString().LastIndexOf(".") );
36.          path = path + "//" + imgName;
37.          string defaultType = ".jpg";
38.          string[] imgTypes = new string[] { ".jpg", ".jpeg", ".png", ".gif", ".bmp" };
39.          string imgType = imgUrl.ToString().Substring(imgUrl.ToString().LastIndexOf("."));
40.          foreach (string it in imgTypes)
41.          {
42.              if (imgType.ToLower().Equals(it))
43.                  break;
44.              if (it.Equals(".bmp"))
45.                  imgType = defaultType;
46.          }
47.          //try
48.          //{
49.              HttpWebRequest request = (HttpWebRequest)WebRequest.Create(imgUrl);
50.              request.UserAgent = "Mozilla/6.0 (MSIE 6.0; Windows NT 5.1; Natas.Robot)";
51.              request.Timeout = 10000;
52.              WebResponse response = request.GetResponse();
53.              Stream stream = response.GetResponseStream();
54.              if (response.ContentType.ToLower().StartsWith("image/"))
55.              {
56.                  byte[] arrayByte = new byte[1024];
57.                  int imgLong = (int)response.ContentLength;
58.                  int l = 0;
59.                  // CreateDirectory(path);
60.                  FileStream fso = new FileStream(path, FileMode.Create);
61.                  while (l < imgLong)
62.                  {
63.                      int i = stream.Read(arrayByte, 0, 1024);
64.                      fso.Write(arrayByte, 0, i);
65.                      l += i;
66.                  }
67.                  fso.Close();
68.                  stream.Close();
69.                  response.Close();
70.                  return 1;
71.              }
72.              else
73.              {
74.                  return 0;
75.              }
76.          //}
77.          //catch 
78.          //{
79.          //    return 0;
80.          //}
81.         
82.      }
83.     } 
84.    
85. }