C#实现web信息自动抓取


背景随着Internet的普及，网络信息正以极高的速度增长，在这么多数据中找到自己需要的信息是一件很繁琐的事情，找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。所谓Internet信息抓取程序，就是程序会按照用户的关键词或关键网站来收集相应的信息，并提供给用户想要的信息格式。信息量的增加会带来信息网站发布人员工作量的剧增，为实现信息发布系统实现信息自动发布、减少工作人员工作量、即时跟踪最新信息，就需要自动信息提供程序，因此Internet信息抓取程序应运而生。目标实现自定义网站信息分类抓取，存入本地数据库、生成静态页面或其它用户定义的信息结构，并下载与信息相关的多媒体文件。开发l 目标站点结构分析本步骤是准确抓取信息个关键。首先要选择更新频率高的页面做为抓取地址，然后分析要抓取内容页面url特点。然后分析要抓取信息页面的元素特性，比如标题位置，内容位置 等，得到定位标记点。将以上信息写成自己的配置文件或存到数据库中。每个网站都需要分析，写出单独的配置文件，供抓取程序使用。l 信息提取根据配置文件取得要抓取页面url，使用HttpWebRequest类获取内容：//获取http页面函数public string Get_Http(string a_strUrl,int timeout){string strResult ; try{HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;myReq.Timeout = timeout;HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse();Stream myStream = HttpWResp.GetResponseStream () ;StreamReader sr = new StreamReader(myStream , Encoding.Default);StringBuilder strBuilder = new StringBuilder();while (-1 != sr.Peek()){strBuilder.Append(sr.ReadLine() "\r\n");}strResult = strBuilder.ToString();}catch(Exception exp){strResult = "错误："  exp.Message ;}return strResult ;}获取页面内容后，分析页面中连接地址取到要抓取的url：//处理页面标题和链接public string SniffWebUrl( string urlStr,string blockB,string blockE ){ string urlch1 = "";string urlch2 = ""; int end_n1 = 0;int end_nums = 0;int end_nums1 = 0;int end_nums2 = 0;int end_nums3 = 0; string reUTStr = "";string reTitle = "";string ret = ""; try{int pos01 = urlStr.IndexOf( "." );int pos02 = urlStr.LastIndexOf( "/" );if( pos01 < 0 ) {return "";}if( pos02 < 0 ) {return "";}int pos03 = urlStr.IndexOf( "/",pos01 );if ( pos03 < 0 ){urlch1 = urlStr;urlch2 = urlStr;}else{urlch1 = urlStr.Substring( 0,pos03 );urlch2 = urlStr.Substring( 0,pos02 );}string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);int pos1 = tmpAllStr.IndexOf( blockB );int pos2 = tmpAllStr.IndexOf( blockE,pos1  blockB.Length );if ( pos1>0 &;amp;&;amp; pos2>0 &;amp;&;amp; pos2>pos1 ){ret = tmpAllStr.Substring( pos1  blockB.Length,pos2 - pos1 - blockB.Length );ret = ret.Substring( ret.IndexOf( "<" ));while( ret.IndexOf( "<A" ) >= 0 ){ret = ret.Substring( 0,ret.IndexOf( "<A" ) )  "<a"  ret.Substring( ret.IndexOf( "<A" )  2 );}while( ret.IndexOf( "</A" ) >=0 ){ret = ret.Substring( 0,ret.IndexOf( "</A" ) )  "</a"  ret.Substring( ret.IndexOf( "</A" )  3 );}while( ret.IndexOf( "Href=" ) >=0 ){ret = ret.Substring( 0,ret.IndexOf( "Href=" ))  "href="  ret.Substring( ret.IndexOf( "Href=" )  5 );}while( ret.IndexOf( "HREF=" ) >=0 ){ret = ret.Substring( 0,ret.IndexOf( "HREF=" ))  "href="  ret.Substring( ret.IndexOf( "HREF=" )  5 );}while( ret.IndexOf( "href= " ) >=0 ){ret = ret.Substring( 0,ret.IndexOf( "href= " ))  "href=\""  ret.Substring( ret.IndexOf( "href= " )  6 );}} tmpAllStr = ret; int begin_nums = tmpAllStr.IndexOf( "href=" );while ( begin_nums >= 0 ){ string tmpStrA = "";string tmpStrB = tmpAllStr.Substring( begin_nums  5,1 );if ( tmpStrB == "\"" ){end_n1 = begin_nums  6;if ( ( end_n1  1 ) > tmpAllStr.Length ){return "";}tmpStrA = tmpAllStr.Substring( begin_nums 6,1 );}else{end_n1 = begin_nums  5;tmpStrA = tmpStrB;}if ( tmpStrA == "#" ){tmpAllStr = tmpAllStr.Substring( end_n1 );begin_nums = tmpAllStr.IndexOf( "href=" );}else{ end_nums1 = tmpAllStr.IndexOf( " ",end_n1 );end_nums2 = tmpAllStr.IndexOf( ">",end_n1 );end_nums3 = tmpAllStr.IndexOf( "</a",end_nums2 );if ( ( end_nums3 >= 0 ) &;amp;&;amp; ( end_nums2 >= 0 ) ){reTitle = tmpAllStr.Substring( end_nums2  1,end_nums3 - end_nums2 - 1 );if ( end_nums1 > end_nums2 ){end_nums = end_nums2;}else{if ( end_nums1 < 0 ){end_nums = end_nums2;}else{end_nums = end_nums1;}}string str4 = tmpAllStr.Substring( end_nums-1, end_nums - end_nums  1 );if ( str4 =="\"" || str4 == " " ){end_nums = end_nums - 1;}string sTotalOne = tmpAllStr.Substring( end_n1,end_nums - end_n1 );if ( sTotalOne.IndexOf( "http://" ) <0 ){if ( sTotalOne.IndexOf( "/" ) == 0 ){sTotalOne = urlch1  sTotalOne;}else{ int linshiIntNum = 0;int flags = 0;string urlChange = urlStr;;while( sTotalOne.IndexOf( "../" ) >= 0 ){sTotalOne = sTotalOne.Substring( sTotalOne.IndexOf( "../" )  3 );linshiIntNum = linshiIntNum  1;flags = flags  1;}while( ( urlChange.LastIndexOf( "/" ) >= 0 ) &;amp;&;amp; ( linshiIntNum >= 0 ) ){urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );linshiIntNum = linshiIntNum - 1;}if ( flags == 0 ){sTotalOne = urlch2  "/"  sTotalOne;}else{sTotalOne = urlChange  "/"  sTotalOne;}}}reUTStr = reUTStr  new PublicFun().RemoveHtmlCode( reTitle )  sTotalOne;tmpAllStr = tmpAllStr.Substring( end_nums3  4 );begin_nums = tmpAllStr.IndexOf( "href=" );}else{begin_nums = -1;} }}return reUTStr;}catch( Exception e){return "";}}得到要抓取内容的url后，处理该页面：//获取链接内容并分类处理public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root ){string tmpAllStr = ""; string dfStrB = "";string dfStrE = ""; string rePicStr = "";//图片返回路径 string reContentStr = "";string picHtml = "images"; //本地图片路径string urlch1 ="";string urlch2 ="";int pos1 = gatherUrl.IndexOf( "." );int pos2 = gatherUrl.LastIndexOf( "/" );if( pos1 < 0 ){return "";}if( pos2 < 0 ){ return "";}int pos3 = gatherUrl.IndexOf( "/",pos1 );if ( pos3 < 0 ){urlch1 = gatherUrl;urlch2 = gatherUrl;}else{urlch1 = gatherUrl.Substring( 0,pos3 );urlch2 = gatherUrl.Substring( 0,pos2 );} tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );//取稿源string docFromStr = "";if ( dfStrB != "" &;amp;&;amp; dfStrE != "" ){if ( tmpAllStr != "" ){int b_docF = tmpAllStr.IndexOf( dfStrB );if ( b_docF > 0 ){int e_docF = tmpAllStr.IndexOf( dfStrE,b_docF  dfStrB.Length );if ( e_docF > 0 &;amp;&;amp; e_docF > b_docF &;amp;&;amp; e_docF - b_docF < 20 ){docFromStr = tmpAllStr.Substring( b_docF  dfStrB.Length, e_docF - b_docF - dfStrB.Length );}}}}//取内容if ( tmpAllStr != "" ){ int begin_strnum = tmpAllStr.IndexOf( b_Content );if ( begin_strnum < 0 ){ return "";}int end_strnum = tmpAllStr.IndexOf( e_Content,begin_strnum  b_Content.Length );if ( end_strnum < 0 ){ return "";}string sTotalSubM = "";if ( end_strnum > begin_strnum ){sTotalSubM = tmpAllStr.Substring ( begin_strnum,end_strnum - begin_strnum );}if ( sTotalSubM == "" ){ return "";} //过滤无用信息int bfnum = sTotalSubM.IndexOf( b_Filter );if ( bfnum > -1 ){int efnum = sTotalSubM.IndexOf( e_Filter,bfnum );if ( efnum > -1 ){if ( efnum > bfnum ){sTotalSubM = sTotalSubM.Substring( 0,bfnum )  sTotalSubM.Substring( efnum  e_Filter.Length );}}}//格式化图片标记while( sTotalSubM.IndexOf( "Src=" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "Src=" ) )  "src="  sTotalSubM.Substring( sTotalSubM.IndexOf( "Src=" )  4 );}while( sTotalSubM.IndexOf( "SRC=" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "SRC=" ) )  "src="  sTotalSubM.Substring( sTotalSubM.IndexOf( "SRC=" )  4 );}while( sTotalSubM.IndexOf( "src= " ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "src= " ) )  "src=\""  sTotalSubM.Substring( sTotalSubM.IndexOf( "src= " )  5 );}//取图片地址int end_n12 = 0;int end_nums2 = 0;int begin_nums2 = sTotalSubM.IndexOf( "src=" );while( begin_nums2 >= 0 ){String tmpStr = sTotalSubM.Substring( begin_nums2  4,1 );if ( tmpStr == "\"" ){end_n12 = begin_nums2  5;}else{end_n12 = begin_nums2  4;}int end_nums2a = sTotalSubM.IndexOf( " ",end_n12 );int end_nums2b = sTotalSubM.IndexOf( ">",end_n12 );if ( end_nums2b < 0 ){break;}if ( end_nums2a > end_nums2b ){end_nums2 = end_nums2b;}else{if (end_nums2a<0){end_nums2 = end_nums2b;}else{end_nums2 = end_nums2a;}}tmpStr = sTotalSubM.Substring( end_nums2-1,1 );if ( tmpStr == "\"" || tmpStr == " " ){end_nums2 = end_nums2 - 1;}string tmpPicStr = sTotalSubM.Substring( end_n12,end_nums2 - end_n12 );if ( tmpPicStr.IndexOf( "http://" ) < 0 ){if ( tmpPicStr.IndexOf( "/" ) == 0 ){tmpPicStr = urlch1  tmpPicStr;}else{ int linshiIntNum = 0;int flags = 0;string urlChange = subUrl;while( tmpPicStr.IndexOf( "../" ) >= 0 ){tmpPicStr = tmpPicStr.Substring( tmpPicStr.IndexOf("../")  3 );linshiIntNum = linshiIntNum  1;flags = flags  1;}while( ( urlChange.LastIndexOf( "/" ) >= 0 ) &;amp;&;amp; ( linshiIntNum >= 0 ) ){urlChange = urlChange.Substring( 0,urlChange.LastIndexOf( "/" ) );linshiIntNum = linshiIntNum - 1;}if ( flags == 0 ){tmpPicStr = urlch2  "/"  tmpPicStr;}else{tmpPicStr = urlChange  "/"  tmpPicStr;}}}//tmpPicStr = tmpPicStr.ToLower();string tmpPicStrTmp = tmpPicStr.ToLower();//if ( tmpPicStr.IndexOf( ".jpg" ) > 0 || tmpPicStr.IndexOf( ".gif" ) > 0 || tmpPicStr.IndexOf( ".bmp" ) > 0 )if ( tmpPicStrTmp.IndexOf( ".jpg" ) > 0 || tmpPicStrTmp.IndexOf( ".gif" ) > 0 || tmpPicStrTmp.IndexOf( ".bmp" ) > 0 ){rePicStr = rePicStr  "||"  tmpPicStr ;int flagN2 = tmpPicStr.LastIndexOf( "/" );string fileN2 = picHtml  tmpPicStr.Substring( flagN2 );sTotalSubM = sTotalSubM.Substring( 0,end_nums2 )  ">******"  fileN2  "******<"  sTotalSubM.Substring( end_nums2 );begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2  fileN2.Length  22 );}else{begin_nums2 = sTotalSubM.IndexOf( "src=", end_nums2  4 ); } }if ( rePicStr.Length > 2 ) rePicStr = rePicStr.Substring(2); //内容处理 格式化关键标记while( sTotalSubM.IndexOf( "<P" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<P" ) )  "|****|<"  sTotalSubM.Substring( sTotalSubM.IndexOf( "<P" )  2 );}while( sTotalSubM.IndexOf( "<p" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<p" ) )  "|****|<"  sTotalSubM.Substring( sTotalSubM.IndexOf( "<p" )  2 );}while( sTotalSubM.IndexOf( "</P" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</P" ) )  "|****|<"  sTotalSubM.Substring( sTotalSubM.IndexOf( "</P" )  3 );}while( sTotalSubM.IndexOf( "</p" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "</p" ) )  "|****|<"  sTotalSubM.Substring( sTotalSubM.IndexOf( "</p" )  3 );}while( sTotalSubM.IndexOf( "<br" ) >=0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<br" ) )  " **** <"  sTotalSubM.Substring( sTotalSubM.IndexOf( "<br" )  3 );}while( sTotalSubM.IndexOf( "<BR" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<BR" ) )  " **** <"  sTotalSubM.Substring( sTotalSubM.IndexOf( "<BR" )  3 );}while( sTotalSubM.IndexOf( "<Br" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<Br" ) )  " **** <"  sTotalSubM.Substring( sTotalSubM.IndexOf( "<Br" )  3 );}while( sTotalSubM.IndexOf( "<bR" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "<bR" ) )  " **** <"  sTotalSubM.Substring( sTotalSubM.IndexOf( "<bR" )  3 );}//去除html标记int linshiInt1 = sTotalSubM.IndexOf( "<" );int linshiInt2 = sTotalSubM.IndexOf( ">" ); if ( linshiInt2 < linshiInt1 ){sTotalSubM = sTotalSubM.Substring( linshiInt2  1 );}int linshiInt11 = sTotalSubM.LastIndexOf( "<" );int linshiInt12 = sTotalSubM.LastIndexOf( ">" );if ( linshiInt12 < linshiInt11 ){sTotalSubM = sTotalSubM.Substring( 0,linshiInt12  1 );}linshiInt1 = sTotalSubM.IndexOf( "<" );while ( linshiInt1 >= 0 ){linshiInt2 = sTotalSubM.IndexOf( ">",linshiInt1 );if ( linshiInt2 >= 0 ){ sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 )  sTotalSubM.Substring( linshiInt2  1 );}else{sTotalSubM = sTotalSubM.Substring( 0,linshiInt1 );}linshiInt1 = sTotalSubM.IndexOf("<");}//还原关键标记int linshiInt3 = 0;int linshiInt4 = 0;while( sTotalSubM.IndexOf( " **** " ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( " **** " ) )  "<br>\n"  sTotalSubM.Substring( sTotalSubM.IndexOf( " **** " )  9 );}while( sTotalSubM.IndexOf( "|****|" ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( "|****|" ) )  "<br>\n"  sTotalSubM.Substring( sTotalSubM.IndexOf( "|****|" )  9 );}while( sTotalSubM.IndexOf( "******" ) >= 0 ){linshiInt3 = sTotalSubM.IndexOf( "******" )  9;linshiInt4 = sTotalSubM.IndexOf( "******",linshiInt3 );if ( linshiInt4 >= 0 ){int tmpPos = sTotalSubM.IndexOf( "******" );string tmpStr1 = sTotalSubM.Substring( 0,tmpPos ); string tmpStr2 = sTotalSubM.Substring( linshiInt3,linshiInt4 - linshiInt3 );string tmpStr3 = sTotalSubM.Substring( linshiInt4  9 );sTotalSubM = tmpStr1  "<img src=" http://www.aspxboy.com/private/363/  tmpStr2  ">"  tmpStr3;}else{break;}}//去除内容中的标题if ( sTotalSubM.IndexOf( subTitle ) >= 0 ){sTotalSubM = sTotalSubM.Substring( 0,sTotalSubM.IndexOf( subTitle ) )  sTotalSubM.Substring( sTotalSubM.IndexOf( subTitle )  subTitle.Length );}reContentStr = sTotalSubM;//调用下载图片功能//下载图片到指定目录string[] img_Url = new PublicFun().split( rePicStr,"||" );for ( int i=0;i<img_Url.Length;i  ){if ( img_Url != "" ){new PublicFun().Get_Img( img_Url,10000,root  "\\images\\"  img_Url.Substring( img_Url.LastIndexOf("/") 1 ) );}}}return reContentStr;}以上方法返回要取得的信息，包括标题内容，图片地址等。下载页面中图片：//下载图片public void Get_Img(string a_strUrl,int timeout,string filepath){try{HttpWebRequest myReq = (HttpWebRequest)HttpWebRequest.Create(a_strUrl) ;myReq.Timeout = timeout;HttpWebResponse HttpWResp = (HttpWebResponse)myReq.GetResponse(); Stream myStream = HttpWResp.GetResponseStream () ; Bitmap map = new Bitmap( myStream );PictureBox picB = new PictureBox();picB.Image = (Image)map;string path = filepath.Substring( 0,filepath.LastIndexOf( "\\" ) );if (!Directory.Exists(path)){CreateDir( path );} picB.Image.Save(filepath); }catch(Exception exp){string ss = exp.Message;WriteLog( filepath.Substring(0,filepath.LastIndexOf("\\"))  "\\error.log",a_strUrl  "--"  ss  "\r\n"); }}l 保存文件或入库上面取得的信息可以按自己的要求保存。****设计的时候没有使用url按层次循环抓取，这样定义抓取url效率更高，速度更快。测试程序下载：
http://bjfile.focus.cn/file/15379/NetBugV102.rar
如有建议请发送EMail：xiancai@qianlong.com 或msn yutao728@hotmail.com注：此版本只提供静态文件存储功能，不提供数据库接口，不提供自定义网站功能。本程序运行需要先安装.net 框架1.1。  (2005-6-14:03:05)

感谢原创者的辛勤劳动，希望对您有所帮助，转载请注明原出处。	警告：持续变种木马正在发起农历新年攻势！
您可能对 [C#] 的这些文章也感兴趣： C#实现所有经典排序算法 使用C#的BitmapData编程实例 用C#编写网络电话 C# 实现 RDLC 自定义报表 C#实现Web代理服务程序 简易C#入门教程 C#中的“Squiggles”特性 C#数字图像探索系列(1)--伪渐变效果原理及应用初级篇 C#数字图像探索系列(2)--渐变二 如何在MFC中调用C#编写的DLL C# wm6通过udp协议和pc通讯 异步网络编程之图片批量抓取下载（C#）