//此方法获取远程地址的源代码
public static string GetHtml( string url)
{
try
{
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream stream = webResponse.GetResponseStream();
System.IO.StreamReader streamReader = new StreamReader(stream, System.Text.Encoding.GetEncoding( "GB2312"));
string content = streamReader.ReadToEnd();
streamReader.Close();
webResponse.Close();
return content.Trim();
}
catch
{
return "-1";
}
}
//按钮
protected void btnRead_Click( object sender, EventArgs e)
{
string rssURL = txtUrl.Text.Trim(); //列表页面url 地址
string resultlist = GetHtml(rssURL); //列表页面数据
Return(resultlist, int.Parse(TextBox7.Text.ToString())); //调用采集主要方法
}
//主体方法
private string Return( string result, int j)
{
string strReg = TextBox1.Text.Trim() + @"[(\s\S)]*?" + TextBox2.Text.Trim(); //输入新闻列表开始结尾-- /news/2008 ---.shtml
Regex re = new Regex(strReg, RegexOptions.IgnoreCase);
Match m = re.Match(result);
int mn = 0;
if (i <= j)
{
string NewsString = GetHtml( this.TextBox8.Text.Trim() + m.Value); //查询新闻内容数据
string rg_title = TextBox5.Text.Trim() + @"([\s\S]*?)" + TextBox6.Text.Trim(); //新闻标题标签
Regex re_title = new Regex(rg_title, RegexOptions.IgnoreCase);
Match m_title = re_title.Match(NewsString);
string n_title = System.Text.RegularExpressions.Regex.Replace(m_title.Value.ToString().Replace( "'", ""), "<[^>]+> ", ""); //新闻标题
string rg_note = TextBox3.Text.Trim() + @"([\s\S]*?)" + TextBox4.Text.Trim(); //新闻内容标签
Regex re_note = new Regex(rg_note, RegexOptions.IgnoreCase);
Match m_note = re_note.Match(NewsString);
string n_note = m_note.Value.ToString().Replace( "'", ""); //新闻内容
string selecttitle = "select n_title from zy_News where n_title='" + FunctionLib.StrLib.cutStr(n_title, 48) + "'";
DataSet dstitle = SqlHelp.ExecuteDataSet(SqlHelp.sqlCon, CommandType.Text, selecttitle, null);
if (dstitle.Tables[0].Rows.Count == 0)
{
if (n_note != "" && n_note != null)
{
string insertsql = "insert into zy_News values(58,0,'" + n_title + "','','','" + n_note + "','','',1,'" + n_title + "','',1,'','',0,0,getdate(),getdate(),0,0,0,0,getdate())";
SqlHelp.ExecuteNonQuery(SqlHelp.sqlCon, CommandType.Text, insertsql, null);
mn++;
i++;
}
}
result1 = result.Replace(m.Value, " ");
Return(result1, j); //递归
}
}
