C#小爬虫，通过URL进行模拟发送接收数据

 public async Task<string> SendDataAsync(HttpMethod httpMethod, string requestUrl, HttpContent postContent = null, string cookies = "")
         {
             HttpClientHandler httpHandler = new HttpClientHandler()
             {
                 AllowAutoRedirect = true,
                 //CookieContainer = new CookieContainer(),
                 UseCookies = false,
                 AutomaticDecompression = DecompressionMethods.GZip
             };

             using (HttpClient httpClient = new HttpClient(httpHandler))
             {
                 httpClient.DefaultRequestHeaders.ExpectContinue = false;
                 httpClient.DefaultRequestHeaders.Add("Accept", "*/*");
                 httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
                 httpClient.DefaultRequestHeaders.Add("Accept-Language", " zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
                 httpClient.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:49.0) Gecko/20100101 Firefox/49.0");
                 //var response = await httpClient.GetAsync(sUrl);
                 //return (await response.Content.ReadAsAsync<AvResult>());
                 string sResult;
                 try
                 {
                     HttpRequestMessage request = new HttpRequestMessage(httpMethod, requestUrl);
                     bool isNullCookie = string.IsNullOrEmpty(cookies);
                     if (!isNullCookie)
                     {
                         //httpHandler.CookieContainer.SetCookies(new Uri(requestUrl), cookies);
                         request.Headers.Add("Cookie", cookies);
                     }
                     if (httpMethod.Method == "POST")
                         request.Content = postContent;
                     var response = await httpClient.SendAsync(request);
                     sResult = await response.Content.ReadAsStringAsync();
                 }
                 catch (Exception ex)
                 {
                     sResult = ex.Message;
                 }
                 return sResult;
             }
         }

次上面的代码只是爬虫的模拟数据发送的部分代码，即自定义模拟浏览器客户端的，设置请求头，没有用Cookies进行登陆保存设置