java编写网站数据抓取

来公司已经俩月了，每天加班平均工时11个小时的我又想起了老东家温馨舒适安逸的生活。已经有好久没时间读博客写博客了，我觉得我退步了，嗯嗯，我很不开心

今天记录下抓数据的一些东西吧。

数据抓取现在是很普遍的事情，有用Python的，当然我还是很弱，我只能用java搞，以下就是正经话了。

以下需要注意的：

1.首先有个目标，抓取的目标页面

2.目标页面的数据结构

3.目标网站是否有反爬虫机制（就是会封你ip）

4.数据解析之 存库

获取httpClient

package com.app.utils;

import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.springframework.stereotype.Component;

import javax.annotation.PostConstruct;
import javax.net.ssl.SSLContext;
import java.security.NoSuchAlgorithmException;

@Component
public class HttpConnectionManager {
    PoolingHttpClientConnectionManager cm = null;

    @PostConstruct
    public void init() {
        LayeredConnectionSocketFactory sslsf = null;
        try {
            sslsf = new SSLConnectionSocketFactory(SSLContext.getDefault());
        } catch (NoSuchAlgorithmException e) {
            e.printStackTrace();
        }

        Registry<ConnectionSocketFactory> socketFactoryRegistry = RegistryBuilder.<ConnectionSocketFactory> create()
                .register("https", sslsf)
                .register("http", new PlainConnectionSocketFactory())
                .build();
        cm =new PoolingHttpClientConnectionManager(socketFactoryRegistry);
        cm.setMaxTotal(200);
        cm.setDefaultMaxPerRoute(20);
    }

    public CloseableHttpClient getHttpClient() {
        CloseableHttpClient httpClient = HttpClients.custom()
                .setConnectionManager(cm)
                .build();
        return httpClient;
    }
}

抓取页面工具类 因为很多网站都做了反爬虫机制，它会在单位时间内限制你的访问次数，如果你爬去的数据不停在变化，即时性要求很高，这就需要使用代理IP

package com.zyt.creenshot.util;

import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.util.EntityUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;

/**
 * @ClassName:DocumentHelper
 * @Description:<抓取页面工具类>
 * @Author:zhaiyutao
 * @Data:2019/7/1 11:15
 * @Vesion: v1.0
 */
@Component
public class DocumentHelper {

    @Autowired
    HttpConnectionManager connManager;

    public String getProxyHttp(String url, String address, int port, String charset) {
        CloseableHttpResponse response = null;
        CloseableHttpClient httpClient = connManager.getHttpClient();
        try {
            //发送get请求
            HttpGet httpGet = new HttpGet(url);
            //构建代理IP请求
            httpGet = buildProxy(httpGet, address, port);
            Map<String, String> headerMap = new HashMap<String, String>();
            headerMap.put("Referer", "http://*********.com/");
            headerMap.put("Content-Type", "text/html; charset=utf-8");
            headerMap.put("User-Agent", UserAgentUtil.getRandomUserAgent());
            headerMap.put("Accept-Language", "zh-CN,zh;q=0.9");
            headerMap.put("Accept-Encoding", "gzip, deflate");
            //构建请求header头
            httpGet = buildRequestHeader(headerMap, httpGet);
            response = httpClient.execute(httpGet);
            response.addHeader("Content-Type", "text/html; charset=utf-8");
            //获取响应实体
            HttpEntity entity = response.getEntity();
            if (entity != null) {
                String content = EntityUtils.toString(entity);
                if(null != charset && !"".equals(charset)) {
                    content = new String(content.getBytes(StandardCharsets.ISO_8859_1), charset);
                }
                return content;
            }
        } catch (Exception e){
            //出现问题不处理异常 再次请求 最后统一处理
            //log.error("代理解析内部出现问题 url {} address {} port {}",url,address,port);
            return "";
        }finally {
            try {
                response.close();
            } catch (Exception e) {
            }
        }
        return "";
    }

    private static HttpGet buildProxy(HttpGet httpGet,String address,int port) {
        RequestConfig requestConfig = null;
        if(StringUtils.isNotEmpty(address)){
            HttpHost proxy = new HttpHost(address, port);
            requestConfig = RequestConfig.custom()
                    .setProxy(proxy)
                    .setConnectTimeout(4000)
                    .setSocketTimeout(8000)
                    .setConnectionRequestTimeout(4000)
                    .build();
        }else{
            requestConfig = RequestConfig.custom()
                    .setConnectTimeout(4000)
                    .setSocketTimeout(8000)
                    .setConnectionRequestTimeout(4000)
                    .build();
        }
        httpGet.setConfig(requestConfig);
        return httpGet;
    }

    /**
     * 构建header头
     * @param headerMap
     * @param httpGet
     * @return
     */
    private static HttpGet buildRequestHeader(Map<String,String> headerMap,HttpGet httpGet) {
        if (MapUtils.isNotEmpty(headerMap)) {
            for (Map.Entry<String,String> kv :headerMap.entrySet()) {
                httpGet.setHeader(kv.getKey(),kv.getValue());
            }
        }
        return httpGet;
    }

}

请求头伪装工具类

package com.zyt.creenshot.util;

import java.util.Random;

public class UserAgentUtil {

    private static final String[] USER_AGENT = {
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36", //google
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 OPR/60.0.3255.109", //opera
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36 QIHU 360SE", //
        "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.7.3000", //遨游
        "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0", //firefox
        "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50" //safari
        //"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/18.17763" //IE
    };

    /**
     * 随机获取user-agent
     * @return
     */
    public static String getRandomUserAgent() {
        Random random = new Random();
        int i = random.nextInt(USER_AGENT.length);
        String userAgent = USER_AGENT[i];
        return userAgent;
    }

}

如下以爬取某个汽车网站数据为例 当然表的创建需要自己根据需要设计 我就不贴具体表结构了

package com.zyt.creenshot.service.crawlerData.impl;

import com.zyt.creenshot.entity.CarBaseData;
import com.zyt.creenshot.mapper.CarBaseDataMapper;
import com.zyt.creenshot.service.crawlerData.ICrawlerData;
import com.zyt.creenshot.util.DocumentHelper;
import com.zyt.creenshot.util.HttpConnectionManager;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.ArrayList;
import java.util.List;

/**
 * @ClassName:CrawlerDataImpl
 * @Description:<TODO>
 * @Author:zhaiyutao
 * @Data:2019/7/8 17:48
 * @Vesion: v1.0
 */
@Component
@Slf4j
public class CrawlerDataImpl implements ICrawlerData {

    @Autowired
    private HttpConnectionManager connectionManager;

    @Autowired(required = false)
    private CarBaseDataMapper carBaseDataMapper;

    @Override
    public void crawlerCarBaseData() {

        String url = "***********要爬取的网址*************";
        String resultHtml = DocumentHelper.getProxyHttp(url, null, 0, "GBK", connectionManager);
        if(StringUtils.isEmpty(resultHtml)){
            log.error("没有爬到网站数据");
        }
        Document html = Jsoup.parse(resultHtml);
        //解析品牌
        Elements brandList = html.select("div[class=braRow]");
        if(null != brandList && brandList.size() > 0 ){
            List<CarBaseData> listCar = new ArrayList<>();
            // 获取车的大品牌
            for(Element brand : brandList){
                Elements brandBig = brand.select("div[class=braRow-icon]");
                //大品牌名称 和 车标
                String brandName = brandBig.select("p").text().replace("?","·");
                String brandPic = brandBig.select("img[src]").attr("#src");

                Elements smallBrandList = brand.select("div[class=modA noBorder]");
                for( Element sb : smallBrandList){
                    Elements brandItem = sb.select("div[class=thA]");
                    // 细分品牌
                    String brandSmallName = brandItem.select("a[href]").text();

                    Elements sbInner = sb.select("div[class=tbA ]");
                    for(Element in : sbInner){
                        dealCarData(listCar, brandName, brandPic, brandSmallName, in);
                    }
                    Elements sbInnerNother = sb.select("div[class=tbA mt10 noBorder]");
                    for(Element inner : sbInnerNother){
                        dealCarData(listCar, brandName, brandPic, brandSmallName, inner);
                    }
                }
            }
            if(CollectionUtils.isNotEmpty(listCar)){
                carBaseDataMapper.insertBatch(listCar);
            }
        }
    }

    private void dealCarData(List<CarBaseData> listCar, String brandName, String brandPic, String brandSmallName, Element in) {
        String carTypeName = in.select("p[class=stit]").text().split("（")[0];
        Elements li = in.select("li");
        for(Element element : li){
            Element tit = element.select("p[class=tit]").get(0);
            Element price = element.select("p[class=price]").get(0);
            Elements carHref = tit.select("a[href]");
            String priceStr = price.text();
            if(null != carHref){
                String href = carHref.attr("href");
                if(StringUtils.isEmpty(href)){
                    continue;
                }
                String carName = carHref.attr("title");
                String carId = StringUtils.substring(href, 1, href.length() - 1);
                CarBaseData carBaseData = new CarBaseData();
                carBaseData.setCarId(carId);
                carBaseData.setCarName(carName);
                carBaseData.setBrandName(brandName);
                carBaseData.setBrandPic(brandPic);
                carBaseData.setSubBrandName(brandSmallName);
                carBaseData.setCarType(carTypeName);
                carBaseData.setCarPrice(priceStr);
                listCar.add(carBaseData);
            }
            if(listCar.size()>=500){
                carBaseDataMapper.insertBatch(listCar);
                listCar.clear();
            }
        }
    }
}

爬取的数据：