HttpWebRequest提高效率之连接数,代理，自动跳转,gzip请求等设置问题

先设置4个：
 
[csharp]
webrequest.ServicePoint.Expect100Continue = false;
               //是否使用 Nagle 不使用 提高效率
               webrequest.ServicePoint.UseNagleAlgorithm = false;
               //最大连接数
               webrequest.ServicePoint.ConnectionLimit = 65500;
               //数据是否缓冲 false 提高效率
               webrequest.AllowWriteStreamBuffering = false; 
 
代理的使用：我用的代理都是不要账号和密码的，因为要不是免费的，要是是我们自己的，对来访IP 有限制
 
[csharp]
if (proxy!=null)
               {
                   model.proxy = proxy;
                   WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
                   myProxy.Credentials = new NetworkCredential("", "");
                   webrequest.Proxy = myProxy;
               }
               else
               {
                   webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
               } 
 
webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
 
带上gzip，如果对方支持gzip，流量上可以节省五分之四，原来250KB，用gzip压缩有 大概在50KB，减少带宽压力，增加速度
 
值得注意的是，如果是gzip，返回值必须要gzip解压的。
 
[csharp]
if (response.ContentEncoding.ToLower().Contains("gzip"))
                   {
                       using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                       {
                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                           { 
 
                               model.html = reader.ReadToEnd();
                           }
                       }
                   } 
 
还有一点：gzip解压的时候回耗费CPU，所以这个需要注意下
 
附完成方法
 
[csharp]
/// <summary>
       /// Get 方式 获取数据
       /// </summary>
       /// <param name="webUrl"></param>
       /// <returns></returns>
       public Model.WebRequestReturnModel WebRequestGetHtmlByGet(string webUrl, CookieContainer cookieContainer, Encoding encoding, string refer,Model.Proxy proxy)
       {
           Model.WebRequestReturnModel model = new Model.WebRequestReturnModel();
           HttpWebRequest webrequest = null;
           try
           {
               webrequest = WebRequest.Create(webUrl) as HttpWebRequest; 
 
               webrequest.ServicePoint.Expect100Continue = false;
                webrequest.ServicePoint.UseNagleAlgorithm = false;
                webrequest.ServicePoint.ConnectionLimit = 65500;
               webrequest.AllowWriteStreamBuffering = false;
               if (proxy!=null)
               {
                   model.proxy = proxy;
                   WebProxy myProxy = new WebProxy(proxy.ProxyInfo, false);
                   myProxy.Credentials = new NetworkCredential("", "");
                   webrequest.Proxy = myProxy;
               }
               else
               {
                   webrequest.Proxy = GlobalProxySelection.GetEmptyWebProxy();
               }
               //设置其他的herader
               webrequest =WebRequestSetHeaders(webrequest);
               if (!String.IsNullOrEmpty(refer))
               {
                   webrequest.Referer = refer;
               }
               else
               {
                   webrequest.Referer = webUrl;
               }
               webrequest.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate");
               webrequest.CookieContainer = cookieContainer;
               webrequest.Timeout = 5000;
               webrequest.UserAgent = "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)";
               webrequest.Accept = "*/*";
               webrequest.KeepAlive = true;
               webrequest.Headers.Add("Accept-Language", "zh-cn,en-us;q=0.5"); 
 
               using (HttpWebResponse response = (HttpWebResponse)webrequest.GetResponse())
               { 
 
                   if (response.ContentEncoding.ToLower().Contains("gzip"))
                   {
                       using (GZipStream stream = new GZipStream(response.GetResponseStream(), CompressionMode.Decompress))
                       {
                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                           { 
 
                               model.html = reader.ReadToEnd();
                           }
                       }
                   }
                   else if (response.ContentEncoding.ToLower().Contains("deflate"))
                   {
                       using (DeflateStream stream = new DeflateStream(response.GetResponseStream(), CompressionMode.Decompress))
                       {
                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                           { 
 
                               model.html = reader.ReadToEnd();
                           } 
 
                       }
                   }
                   else
                   {
                       using (Stream stream = response.GetResponseStream())
                       {
                           using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
                           { 
 
                               model.html = reader.ReadToEnd();
                           }
                       }
                   }
               }
           }
           catch (Exception ex)
           {
               model.exception = ex;
               model.html = string.Empty; 
 
           }
           model.cookieContainer = cookieContainer;
           return model;
       }