HttpClient&Jsoup爬虫的简单应用

　　详细的介绍已经有很多前辈总结，引用一下该篇文章:https://blog.csdn.net/zhuwukai/article/details/78644484

　　下面是一个代码的示例：

package com.http.client;

import java.io.IOException;

import org.apache.http.HttpHost;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.params.ConnRouteParams;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.params.CoreConnectionPNames;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
/**
 *
 * @author oo
 * @date 2018-04-04
 */
public class MyHttpClient {
    private static Logger logger = Logger.getLogger(MyHttpClient.class);

    /**
     * 需求：使用httpclient 爬取 网站数据
     *
     * @param args
     */
    public static void main(String[] args) {
        // 创建ＨttpClient 对象
        HttpClient hclient = new DefaultHttpClient();

        // 设置响应时间 传输源码时间 代理服务器（设置代理服务器的目的是：防止爬数据被封ip）
        hclient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000)
                .setParameter(CoreConnectionPNames.SO_TIMEOUT, 20000)
                .setParameter(ConnRouteParams.DEFAULT_PROXY, new HttpHost("111.155.116.237", 8123));

        HttpGet hGet = new HttpGet("http://www.itcast.cn/");
        String content = "";
        try {
            // 向网站发送请求，获取网页源码
            HttpResponse execute = hclient.execute(hGet);
            // EntityUtils工具类把网页实体转换成字符串
            content = EntityUtils.toString(execute.getEntity(), "utf-8");
        } catch (ClientProtocolException e) {
            e.printStackTrace();
            logger.error("********ClientProtocolException" + e);
        } catch (IOException e) {
            e.printStackTrace();
            logger.error("********IOException" + e);
        }

        System.out.println(content);
    }

}

　　使用Jsoup进行请求：

package com.http.client;

import java.io.IOException;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class ＭyJsoup {
    private static Logger logger = Logger.getLogger(ＭyJsoup.class);

    public static void main(String[] args) {
        try {
            // 使用jsoup 发送请求
            Document document = Jsoup.connect("http://www.itcast.cn").get();
//            System.out.println(document);
            Elements elements = document.getElementsByTag("a");
            String val = elements.text();
            System.out.println(val);

            for (Element element : elements) {
                System.out.println(element.text()+":"+element.attr("href"));
            }
        } catch (IOException e) {
            e.printStackTrace();
            logger.error("***********IOException:　连接失败" + e);
        }
    }

}

　　HttpClient 结合Jsoup：

 package com.http.client;

 import java.io.IOException;

 import org.apache.http.HttpResponse;
 import org.apache.http.client.ClientProtocolException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.impl.client.DefaultHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;

 public class HttpCLientAndJsoup {

     public static void main(String[] args) throws ClientProtocolException, IOException {
         // 创建HttpClient对象
         HttpClient hClient = new DefaultHttpClient();
         // 爬虫URL大部分都是get请求，创建get请求对象
         HttpGet hget = new HttpGet("http://www.itcast.cn/");
         // 向网站发送请求，获取网页源码
         HttpResponse response = hClient.execute(hget);
         // EntityUtils工具类把网页实体转换成字符串
         String content = EntityUtils.toString(response.getEntity(), "utf-8");
         // Jsoup负责解析网页
         Document doc = Jsoup.parse(content);
         // 使用元素选择器选择网页内容
         Elements elements = doc.select("div.salary_con li");
         // System.out.println(elements.text());
         for (Element element : elements) {
             String text = element.text();
             System.out.println(text);
         }

     }

 }