HtmlAgilityPack教程

解析html教程(重点) http://www.cnblogs.com/kissdodog/archive/2013/02/28/2936950.html

完整的教程 http://www.cnblogs.com/kissdodog/category/453229.html

1 解析html

路径
//div 属于平行路径
/html/body/div/ul 属于xml类型的路径
//table/tr 平行路径+xml类型路径，混合使用
//*[@id='div1'] 可以根据id选择，也可以根据其它的属性
*代表匹配所有类型的标签，也可以换成其它的标签，如div等
如果要选择多个使用：var nodes = doc.DocumentNode.SelectNodes("//*[@class='a']");
按节点的ChildNodes选择
divInfo.ChildNodes[0].ChildNodes[0].Attributes["src"].Value

1 选择网页中的所有的div
doc.DocumentNode.SelectNodes("//div")

2 选择doc.DocumentNode.SelectSingleNode("/html/body/div/ul")

3 根据属性id选择节点
HtmlNode node8 = doc.DocumentNode.SelectSingleNode("//*[@id='div1']");
Response.Write(node8.Id);
Response.Write(node8.InnerText);

属性
Name
InnerHtml
InnerText
OuterHtml
ParentNode
XPath

2 Get/Post请求网页

 using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Web;
 using System.Net;
 using System.Configuration;
 using System.IO;
 using System.Text;

 namespace MyLibrary.Common
 {
     public class BaseParser
     {
         private string _encode = "utf-8"; //默认编码格式

         #region 1.0 下载指定URL的HTML代码(默认编码格式) + string GetHtml(string strUrl)
         /// <summary>
         /// 下载指定URL的HTML代码
         /// </summary>
         /// <param name="strUrl">目标页URL</param>
         /// <returns>目标URL的HTML代码</returns>
         public string GetHtml(string strUrl)
         {
             HttpWebRequest httpReq;
             HttpWebResponse httpResp;

             httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             CookieContainer cc = new CookieContainer();
             httpReq.CookieContainer = cc;

             httpResp = (HttpWebResponse)httpReq.GetResponse();
             Stream respStream = httpResp.GetResponseStream();
             StreamReader respStreamReader = new StreamReader(respStream, Encoding.GetEncoding(_encode));
             string html = respStreamReader.ReadToEnd();
             respStream.Close();
             respStreamReader.Close();

             return html;
         }
         #endregion

         #region 1.1 下载指定URL的HTML代码(默认编码格式，并加了try catch) + string GetHtml2(string strUrl)
         /// <summary>
         /// 下载指定URL的HTML代码
         /// </summary>
         /// <param name="strUrl">目标页URL</param>
         /// <returns>目标URL的HTML代码,如果报错，则返回error</returns>
         public string GetHtml2(string strUrl)
         {
             HttpWebRequest httpReq;
             HttpWebResponse httpResp;

             httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             CookieContainer cc = new CookieContainer();
             httpReq.CookieContainer = cc;
             try
             {
                 httpResp = (HttpWebResponse)httpReq.GetResponse();
                 Stream respStream = httpResp.GetResponseStream();
                 StreamReader respStreamReader = new StreamReader(respStream, Encoding.GetEncoding(_encode));
                 string html = respStreamReader.ReadToEnd();
                 respStream.Close();
                 respStreamReader.Close();

                 return html;
             }
             catch
             {
                 return "error";
             }

         }
         #endregion

         #region 2.0 下载指定URL的HTML代码 + string GetHtml(string strUrl, Encoding encode)
         /// <summary>
         /// 下载指定URL的HTML代码
         /// </summary>
         /// <param name="strUrl">目标页URL</param>
         ///<param name="encode">编码格式</param>
         /// <returns>目标URL的HTML代码</returns>
         public string GetHtml(string strUrl, Encoding encode)
         {
             HttpWebRequest httpReq;
             HttpWebResponse httpResp;

             httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             CookieContainer cc = new CookieContainer();
             httpReq.CookieContainer = cc;

             httpResp = (HttpWebResponse)httpReq.GetResponse();
             Stream respStream = httpResp.GetResponseStream();
             StreamReader respStreamReader = new StreamReader(respStream, encode);
             string html = respStreamReader.ReadToEnd();
             respStream.Close();
             respStreamReader.Close();

             return html;
         }
         #endregion

         #region 3.0 带Cookie凭据下载有登录限制URL的HTML代码(默认编码格式) + string GetHtml(string strUrl, CookieContainer cc)
         /// <summary>
         /// 带Cookie凭据下载有登录限制URL的HTML代码
         /// </summary>
         /// <param name="strUrl">目标URL</param>
         /// <param name="cc">Cookie凭据</param>
         /// <returns>目标URL的HTML代码</returns>
         public string GetHtml(string strUrl, CookieContainer cc)
         {
             HttpWebRequest httpReq;
             HttpWebResponse httpResp;

             httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             httpReq.CookieContainer = cc;

             httpResp = (HttpWebResponse)httpReq.GetResponse();
             Stream respStream = httpResp.GetResponseStream();
             StreamReader respStreamReader = new StreamReader(respStream, Encoding.GetEncoding(_encode));
             string html = respStreamReader.ReadToEnd();
             respStream.Close();
             respStreamReader.Close();

             return html;
         }
         #endregion

         #region 4.0 带Cookie凭据下载有登录限制URL的HTML代码 + string GetHtml(string strUrl, CookieContainer cc, Encoding encode)
         /// <summary>
         /// 带Cookie凭据下载有登录限制URL的HTML代码
         /// </summary>
         /// <param name="strUrl">目标URL</param>
         /// <param name="cc">Cookie凭据</param>
         /// <param name="encode">编码格式</param>
         /// <returns>目标URL的HTML代码</returns>
         public string GetHtml(string strUrl, CookieContainer cc, Encoding encode)
         {
             HttpWebRequest httpReq;
             HttpWebResponse httpResp;

             httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             httpReq.CookieContainer = cc;

             httpResp = (HttpWebResponse)httpReq.GetResponse();
             Stream respStream = httpResp.GetResponseStream();
             StreamReader respStreamReader = new StreamReader(respStream, encode);
             string html = respStreamReader.ReadToEnd();
             respStream.Close();
             respStreamReader.Close();

             return html;
         }
         #endregion

         #region 5.0 带Cookie凭据模拟发送POST请求(默认编码格式) + string PostWebRequest(string strUrl, IDictionary<string, string> dicParams, ref CookieContainer container)
         /// <summary>
         /// 带Cookie凭据模拟发送POST请求
         /// </summary>
         /// <param name="strUrl">目标URL</param>
         /// <param name="dicParams">参数列表</param>
         /// <param name="container">Cookie凭据</param>
         /// <param name="encode">编码格式</param>
         /// <returns>请求成功返回目标URL的HTML代码，失败则返回error</returns>
         public string PostWebRequest(string strUrl, IDictionary<string, string> dicParams, ref CookieContainer container)
         {
             string postData = string.Empty;
             if (dicParams != null)
             {
                 foreach (string key in dicParams.Keys)
                 {
                     postData += string.Format("{0}={1}&", key, dicParams[key]);
                 }
                 if (postData != string.Empty) postData = postData.Substring(, postData.Length - );
             }
             byte[] byteArray = Encoding.GetEncoding(_encode).GetBytes(postData);
             HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             //httpReq.Credentials = CredentialCache.DefaultCredentials;
             httpReq.KeepAlive = true;
             httpReq.Method = "POST";
             httpReq.ContentType = "application/x-www-form-urlencoded";
             httpReq.ContentLength = byteArray.Length;

             if (container != null) httpReq.CookieContainer = container;
             else httpReq.CookieContainer = new CookieContainer();

             Stream reqStream = httpReq.GetRequestStream();
             reqStream.Write(byteArray, , byteArray.Length);    //写入参数
             reqStream.Close();

             HttpWebResponse httpResp = (HttpWebResponse)httpReq.GetResponse();
             httpResp.Cookies = httpReq.CookieContainer.GetCookies(httpReq.RequestUri);
             int cookies = httpResp.Cookies.Count;
             if (container == null) container = httpReq.CookieContainer;

             StreamReader respStream = new StreamReader(httpResp.GetResponseStream(), Encoding.GetEncoding(_encode));
             string html = respStream.ReadToEnd();

             respStream.Close();
             httpReq.Abort();
             httpResp.Close();

             if (cookies > ) return html;
             else return "error";
         }
         #endregion

         #region 5.1 带Cookie凭据模拟发送POST请求(默认编码格式,即使报错也返回HTML代码) + string PostWebRequest(string strUrl, IDictionary<string, string> dicParams, ref CookieContainer container)
         /// <summary>
         /// 带Cookie凭据模拟发送POST请求(即使报错也返回HTML代码)
         /// </summary>
         /// <param name="strUrl">目标URL</param>
         /// <param name="dicParams">参数列表</param>
         /// <param name="container">Cookie凭据</param>
         /// <param name="encode">编码格式</param>
         /// <returns>请求成功返回目标URL的HTML代码，失败则返回error和HTML代码(格式：error|HTML代码)</returns>
         public string PostWebRequest2(string strUrl, IDictionary<string, string> dicParams, ref CookieContainer container)
         {
             string postData = string.Empty;
             if (dicParams != null)
             {
                 foreach (string key in dicParams.Keys)
                 {
                     postData += string.Format("{0}={1}&", key, dicParams[key]);
                 }
                 if (postData != string.Empty) postData = postData.Substring(, postData.Length - );
             }
             byte[] byteArray = Encoding.GetEncoding(_encode).GetBytes(postData);
             HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             //httpReq.Credentials = CredentialCache.DefaultCredentials;
             httpReq.KeepAlive = true;
             httpReq.Method = "POST";
             httpReq.ContentType = "application/x-www-form-urlencoded";
             httpReq.ContentLength = byteArray.Length;

             if (container != null) httpReq.CookieContainer = container;
             else httpReq.CookieContainer = new CookieContainer();

             Stream reqStream = httpReq.GetRequestStream();
             reqStream.Write(byteArray, , byteArray.Length);    //写入参数
             reqStream.Close();

             HttpWebResponse httpResp = (HttpWebResponse)httpReq.GetResponse();
             httpResp.Cookies = httpReq.CookieContainer.GetCookies(httpReq.RequestUri);
             int cookies = httpResp.Cookies.Count;
             if (container == null) container = httpReq.CookieContainer;

             StreamReader respStream = new StreamReader(httpResp.GetResponseStream(), Encoding.GetEncoding(_encode));
             string html = respStream.ReadToEnd();

             respStream.Close();
             httpReq.Abort();
             httpResp.Close();

             if (cookies > ) return html;
             else return "error|"+html;
         }
         #endregion

         #region 6.0 带Cookie凭据模拟发送POST请求 + string PostWebRequest(string strUrl, IDictionary<string, string> dicParams, ref CookieContainer container, Encoding encode)
         /// <summary>
         /// 带Cookie凭据模拟发送POST请求
         /// </summary>
         /// <param name="strUrl">目标URL</param>
         /// <param name="dicParams">参数列表</param>
         /// <param name="container">Cookie凭据</param>
         /// <param name="encode">编码格式</param>
         /// <returns>请求成功返回目标URL的HTML代码，失败则返回error</returns>
         public string PostWebRequest(string strUrl, IDictionary<string, string> dicParams, ref CookieContainer container, Encoding encode)
         {
             string postData = string.Empty;
             if (dicParams != null)
             {
                 foreach (string key in dicParams.Keys)
                 {
                     postData += string.Format("{0}={1}&", key, dicParams[key]);
                 }
                 if (postData != string.Empty) postData = postData.Substring(, postData.Length - );
             }
             byte[] byteArray = encode.GetBytes(postData);
             HttpWebRequest httpReq = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
             httpReq.AllowAutoRedirect = true;
             //httpReq.Credentials = CredentialCache.DefaultCredentials;
             httpReq.KeepAlive = true;
             httpReq.Method = "POST";
             httpReq.ContentType = "application/x-www-form-urlencoded";
             httpReq.ContentLength = byteArray.Length;

             if (container != null) httpReq.CookieContainer = container;
             else httpReq.CookieContainer = new CookieContainer();

             Stream reqStream = httpReq.GetRequestStream();
             reqStream.Write(byteArray, , byteArray.Length);    //写入参数
             reqStream.Close();

             HttpWebResponse httpResp = (HttpWebResponse)httpReq.GetResponse();
             httpResp.Cookies = httpReq.CookieContainer.GetCookies(httpReq.RequestUri);
             int cookies = httpResp.Cookies.Count;
             if (container == null) container = httpReq.CookieContainer;

             StreamReader respStream = new StreamReader(httpResp.GetResponseStream(), encode);
             string html = respStream.ReadToEnd();

             respStream.Close();
             httpReq.Abort();
             httpResp.Close();

             if (cookies > ) return html;
             else return "error";
         }
         #endregion

     }
 }