Miner.java 爬虫启动类

Miner.java 爬虫启动类

package com.iteye.injavawetrust.miner;

import java.util.concurrent.ThreadPoolExecutor;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * 爬虫启动类
 * @author InJavaWeTrust
 *
 */
public class Miner {

	private static final Log LOG = LogFactory.getLog(Miner.class);

	private MinerUrl minerUrl = null;

	private MinerConfig config = null;

	public Miner(MinerUrl minerUrl, MinerConfig config){
		this.minerUrl = minerUrl;
		this.config = config;
	}

	public void start(){
		//启动前 先校验配置
		if(!MinerUtil.checkBeforeStart(config)){
			LOG.info("配置出现问题，程序不能启动！！！");
			return;
		}
		MinerUtil.starTime = System.currentTimeMillis();
		//程序启动，将第一个起始页面放入待访问队列。
		MinerQueue.addUnVisited(minerUrl);
		//将URL 添加到URL队列 保证每个URL只访问一次
		MinerQueue.addUrlSet(minerUrl.getUrl());
		ThreadPoolExecutor executor = MinerThreadPool.getInstance();
		//download
		for(int i = 0; i < config.getMinerHtmlThreadNum(); i++){
			MinerHtmlThread minerHtml = new MinerHtmlThread(config);
			executor.execute(minerHtml);
		}
		//misering
		for(int i = 0; i < config.getMiseringThreadNum(); i++){
			MiseringThread misering = new MiseringThread(config);
			executor.execute(misering);
		}
		//store
		for(int i = 0; i < config.getMinserStoreThreadNum(); i++){
			MinerStoreThread minerStoreThread = new MinerStoreThread(config);
			executor.execute(minerStoreThread);
		}
		//monitor
		MinerMonitorThread minerMonitor = new MinerMonitorThread();
		executor.execute(minerMonitor);
	}

}

返回列表