【JAVA】我的爬虫

简介：不是很智能的爬虫，效率慢，仅用作自娱自乐，需要观察目标网站的页面然后修改相关正则去获取自己想要的数据

环境：需要Http-client相关jar包，如下，可以去我的下载链接下载：

https://download.csdn.net/download/the_fool_/10046597

重新整理jar包：

<dependency>
    <groupId>org.apache.httpcomponents</groupId>
    <artifactId>httpclient</artifactId>
    <version>4.5</version>
</dependency>

工具类1，获取某个页面HTML代码

package com.zzt.spider;

import java.io.IOException;
import java.util.Random;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
/**
 * 用于获取整张页面的字符串表现形式
 * @author Administrator
 *
 */
public class SpiderChild {
	public static void main(String[] args) {
		Random r =new Random();
		int nextInt = r.nextInt(8)+1;
		System.out.println(nextInt);

//		String stringHtml = getStringHtml("http://www.163.com");
//		System.out.println(stringHtml);
//		String[] contents = stringHtml.split("<a href=\"");
//		for(String s :contents){
//			System.out.println(s);
//		}
	}

	//获取�?��张页面的字符串表现形�?
	public static String getStringHtml(String url){
		//实例化客户端
		HttpClient client = new DefaultHttpClient();
		HttpGet getHttp = new HttpGet(url);
		//整张页面
		String content = null;
		HttpResponse response;

		try {
			response = client.execute(getHttp);
			//获取到实�?
			HttpEntity entity = response.getEntity();
			if(entity!=null){
				content = EntityUtils.toString(entity);
				//System.out.println(content);
			}
		} catch (ClientProtocolException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}finally{
			client.getConnectionManager().shutdown();
		}
		return content;
	}

}

主类：

package com.zzt.spider;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
 * ZX 2017.7.26
 * 针对于不同网站定制化爬虫，并不全面
 * @author Administrator

 *
 */
public class BigBugSpiderSu {
	  public static void main(String[] args) throws Exception {
		  File file = new File("E:\\htmlfile");
		  if(!file.exists()){
			  boolean createNewFile = file.mkdir();
			  System.out.println(createNewFile);
		  }
		  crawl();

	  }
	  public static void crawl() {
		  System.out.println("begin+++++++++++++++++++++++++++++++++++++++++++");

			  String url = "http://www.xxxx.cn/html/";
			  for(int i=0; i<=30000; i++){
					String URLS = url + i+".html";
					System.out.println("共30000页，当前="+i+".html");
					try {
						spider(URLS);
						for(int j=0;j<urlUeue.size();j++){

							String u = urlUeue.get(j);System.out.println("页面"+u);
							String stringHtml = SpiderChild.getStringHtml(u);

							String fileName=(u.substring(u.indexOf("="))+".html").replace("=", "a");
							System.out.println(u);
							try {
								Thread.sleep(1000);
								writetoFile(stringHtml,fileName);
							} catch (Exception e) {
								e.printStackTrace();
								continue;
							}
						}
						urlUeue.clear();
					} catch (Exception e) {
						e.printStackTrace();
						continue;
					}
					try {
						Thread.sleep(0);
					} catch (InterruptedException e) {
						// TODO Auto-generated catch block
						e.printStackTrace();
					}
				}

				System.out.println("end+++++++++++++++++++++++++++++++++++++++++++");
				System.out.println("end+++++++++++++++++++++++++++++++++++++++++++");
				System.out.println("end+++++++++++++++++++++++++++++++++++++++++++");

		}

	    //链接容器
	    static List<String> urlUeue = new ArrayList<String>();
	    //获取�?��页面的所有连�?
		private static void spider(String URLS) throws Exception{
			 URL url = null;
		        URLConnection urlconn = null;
		        BufferedReader br = null;
		        PrintWriter pw = null;
		        //http://www.ajxxgk.jcy.cn/html/[0-9_/]+.html
		        //href="/html/20170904/2/7002870.html">
		        String regex = "//html//[0-9]+//2//[0-9_]+.html";
		        Pattern p = Pattern.compile(regex);
		        try {
		            url = new URL(URLS);
		            urlconn = url.openConnection();  //X-Forward-For
		            urlconn.setRequestProperty("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
		            urlconn.setRequestProperty("Accept-Encoding","gzip, deflate");
		            urlconn.setRequestProperty("Accept-Language","zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");
		            urlconn.setRequestProperty("Connection","keep-alive");
		            urlconn.setRequestProperty("Cookie", "__jsluid=053c2fe045bbf28d2215a0aa0aa713e5; Hm_lvt_2e64cf4f6ff9f8ccbe097650c83d719e=1502258037,1504571969; Hm_lpvt_2e64cf4f6ff9f8ccbe097650c83d719e=1504574596; sYQDUGqqzHpid=page_0; sYQDUGqqzHtid=tab_0; PHPSESSID=7ktaqicdremii959o4d0p2rgm6; __jsl_clearance=1504575799.118|0|cwzSt6rKCXJZrf5ZOVGhco1TpWw%3");
		            urlconn.setRequestProperty("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0");
		            urlconn.setRequestProperty("Host","www.ajxxgk.jcy.cn");
		            urlconn.setRequestProperty("Content-Type","text/html; charset=UTF-8");
		            urlconn.setRequestProperty("Referer","http://www.ajxxgk.jcy.cn/html/zjxflws/2.html");
		            pw = new PrintWriter(new FileWriter("e:/url.txt"), true);
		            br = new BufferedReader(new InputStreamReader(
		                    urlconn.getInputStream()));
		            String buf = null;
		            while ((buf = br.readLine()) != null) {
		            	System.out.println(buf);
//		            	String string = new String(buf.getBytes(), "utf-8");
//		            	System.out.println(string);
		                Matcher buf_m = p.matcher(buf);
		                while (buf_m.find()) {
		                    urlUeue.add("http://www.xxxx.cn/"+buf_m.group());
		                }
		            }
		            //System.out.println("获取成功");
		        } catch (MalformedURLException e) {
		            e.printStackTrace();
		        } catch (IOException e) {
		            e.printStackTrace();
		        } finally {
		            try {
		                br.close();
		            } catch (IOException e) {
		                e.printStackTrace();
		            }
		            pw.close();
		        }
		}

		 public static void writetoFile(String context,String fileName)throws Exception{
		        // 构建指定文件
		        File file = new File("E:" + File.separator + "htmlfile"+File.separator+fileName);
		        OutputStream out = null;
		        try {
		            // 根据文件创建文件的输出流
		            out = new FileOutputStream(file);
		            // 把内容转换成字节数组
		            byte[] data = context.getBytes();
		            // 向文件写入内�?
		            out.write(data);
		        } catch (Exception e) {
		            e.printStackTrace();
		        } finally {
		            try {
		                // 关闭输出�?
		                out.close();
		            } catch (Exception e) {
		                e.printStackTrace();
		            }
		        }
		    }
}

//class Dog implements Runnable{
//	public  List<String> urlUeue;
//	@Override
//	public void run() {
//		for(String u:urlUeue){
//
//			String stringHtml = SpiderChild.getStringHtml(u);
//			String fileName=u.substring(u.indexOf("xiangqing-"));
//			System.out.println("fileName"+fileName);
//			writetoFile(stringHtml,fileName);
//		}
//
//	}
//	 public  void writetoFile(String context,String fileName){
//	        // 构建指定文件
//	        File file = new File("E:" + File.separator + "htmlfile"+File.separator+fileName);
//	        OutputStream out = null;
//	        try {
//	            // 根据文件创建文件的输出流
//	            out = new FileOutputStream(file);
//	            // 把内容转换成字节数组
//	            byte[] data = context.getBytes();
//	            // 向文件写入内�?
//	            out.write(data);
//	        } catch (Exception e) {
//	            e.printStackTrace();
//	        } finally {
//	            try {
//	                // 关闭输出�?
//	                out.close();
//	            } catch (Exception e) {
//	                e.printStackTrace();
//	            }
//	        }
//	    }
//	public Dog(List<String> urlUeue) {
//		this.urlUeue = urlUeue;
//	}
//	public Dog() {
//		super();
//	}
//
//}