go并发版爬虫
并发版爬虫
代码实现
/crawler/main.go
package main
import (
"learn/crawler/engine"
"learn/crawler/scheduler"
"learn/crawler/zhenai/parser"
)
func main() {
e := engine.ConcurrentEngine{
Scheduler: &scheduler.QueuedScheduler{},
WorkerCount: 20,
}
e.Run(engine.Request{
Url: "http://www.zhenai.com/zhenghun",
ParseFunc: parser.ParseCityList,
})
//测试上海单个城市
//e.Run(engine.Request{
// Url: "http://www.zhenai.com/zhenghun/shanghai",
// ParseFunc: parser.ParseCity,
//})
}
/crawler/engine/simple.go
package engine
import (
"learn/crawler/fetcher"
"log"
)
type SimpleEngine struct {
}
func (e SimpleEngine) Run(seeds ...Request) {
var requests []Request
for _, r := range seeds {
requests = append(requests, r)
}
for len(requests) > 0 {
r := requests[0]
requests = requests[1:]
parseResult, err := worker(r)
if err != nil {
continue
}
requests = append(requests, parseResult.Requests...)
for _, item := range parseResult.Items{
log.Printf("Got item %v", item)
}
}
}
func worker(r Request) (ParseResult, error) {
log.Printf("Fetching %s", r.Url)
body, err := fetcher.Fetch(r.Url)
if err != nil {
log.Printf("Fetcher: error" + "fetching url %s: %v", r.Url, err)
return ParseResult{}, err
}
return r.ParseFunc(body), nil
}
/crawler/engine/concurrent.go
package engine
import (
"log"
)
type ConcurrentEngine struct {
Scheduler Scheduler
WorkerCount int
}
type Scheduler interface {
ReadyNotifier
Submit(Request)
WorkerChan() chan Request
Run()
}
type ReadyNotifier interface {
WorkerReady(chan Request)
}
func (e *ConcurrentEngine) Run(seeds ...Request) {
out := make(chan ParseResult)
e.Scheduler.Run()
for i := 0; i < e.WorkerCount; i++ {
createWork(e.Scheduler.WorkerChan(), out, e.Scheduler)
}
for _, r := range seeds {
e.Scheduler.Submit(r)
}
itemCount := 0
for {
result := <- out
for _, item := range result.Items {
log.Printf("Got item #%d: %v", itemCount, item)
itemCount++
}
for _, request := range result.Requests {
e.Scheduler.Submit(request)
}
}
}
func createWork(in chan Request, out chan ParseResult, ready ReadyNotifier) {
go func() {
for {
ready.WorkerReady(in)
request := <- in
result, err := worker(request)
if err != nil {
continue
}
out <- result
}
}()
}
/crawler/engine/typers.go
package engine
type Request struct {
Url string
ParseFunc func([]byte) ParseResult
}
type ParseResult struct {
Requests []Request
Items []interface{}
}
func NilParser([]byte) ParseResult{
return ParseResult{}
}
/crawler/fetcher/fetcher.go
package fetcher
import (
"bufio"
"fmt"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
"io/ioutil"
"log"
"net/http"
"time"
)
var rateLimiter = time.Tick(100 * time.Millisecond)
func Fetch(url string) ([]byte, error) {
<- rateLimiter
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Wrong status code: %d", resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determineEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
return ioutil.ReadAll(utf8Reader)
}
func determineEncoding(r *bufio.Reader) encoding.Encoding {
bytes, err := r.Peek(1024)
if err != nil {
log.Printf("Fetcher error: %v", err)
return unicode.UTF8
}
e, _, _ := charset.DetermineEncoding(bytes, "")
return e
}
/crawler/zhenai/parser/citylist.go
package parser
import (
"learn/crawler/engine"
"regexp"
)
const cityListRe = `<a href="(http://www.zhenai.com/zhenghun/[0-9a-z]+)" [^>]*>([^<]+)</a>`
func ParseCityList(contents []byte) engine.ParseResult {
re := regexp.MustCompile(cityListRe)
matches := re.FindAllSubmatch(contents, -1)
result := engine.ParseResult{}
for _, m := range matches {
result.Items = append(result.Items, "City: "+string(m[2]))
result.Requests = append(result.Requests, engine.Request{
Url: string(m[1]),
ParseFunc: ParseCity,
})
}
return result
}
/crawler/zhenai/parser/city.go
package parser
import (
"learn/crawler/engine"
"regexp"
)
var (
profileRe = regexp.MustCompile(`<a href="(http://album.zhenai.com/u/[0-9]+)" [^>]*>([^<]+)</a>`)
cityUrlRe = regexp.MustCompile(`href="(http://www.zhenai.com/zhenghun/[^"]+)"`)
)
func ParseCity(contents []byte) engine.ParseResult {
matches := profileRe.FindAllSubmatch(contents, -1)
result := engine.ParseResult{}
for _, m := range matches {
name := string(m[2])
result.Items = append(result.Items, "User "+name)
result.Requests = append(result.Requests, engine.Request{
Url: string(m[1]),
ParseFunc: func(c []byte) engine.ParseResult {
return ParseProfile(c, "name:"+name)
},
})
}
matches = cityUrlRe.FindAllSubmatch(contents, -1)
for _, m := range matches {
result.Requests = append(result.Requests, engine.Request{
Url: string(m[1]),
ParseFunc: ParseCity,
})
}
return result
}
/crawler/zhenai/parser/profile.go
package parser
import (
"learn/crawler/engine"
"learn/crawler/model"
"regexp"
)
const all = `<div class="m-btn purple" data-v-8b1eac0c>([^<]+)</div>`
func ParseProfile(contents []byte, name string) engine.ParseResult {
profile := model.Profile{}
profile.User = append(profile.User, name)
re := regexp.MustCompile(all)
match := re.FindAllSubmatch(contents,-1)
if match != nil {
for _, m := range match {
profile.User = append(profile.User, string(m[1]))
}
}
result := engine.ParseResult{
Items: []interface{}{profile},
}
return result
}
/crawler/model/profile.go
package model
type Profile struct {
User []string
}
/crawler/scheduler/queued.go
package scheduler
import "learn/crawler/engine"
type QueuedScheduler struct {
requestChan chan engine.Request
workChan chan chan engine.Request
}
func (s *QueuedScheduler) WorkerChan() chan engine.Request {
return make(chan engine.Request)
}
func (s *QueuedScheduler) Submit(r engine.Request) {
s.requestChan <- r
}
func (s *QueuedScheduler) WorkerReady(w chan engine.Request){
s.workChan <- w
}
func (s *QueuedScheduler) Run(){
s.workChan = make(chan chan engine.Request)
s.requestChan = make(chan engine.Request)
go func() {
var requestQ []engine.Request
var workerQ []chan engine.Request
for {
var activeRequest engine.Request
var activeWorker chan engine.Request
if len(requestQ) > 0 && len(workerQ) > 0 {
activeRequest = requestQ[0]
activeWorker = workerQ[0]
}
select {
case r := <-s.requestChan:
requestQ = append(requestQ, r)
case w := <-s.workChan:
workerQ = append(workerQ, w)
case activeWorker <- activeRequest:
workerQ = workerQ[1:]
requestQ = requestQ[1:]
}
}
}()
}
/crawler/scheduler/simple.go
package scheduler
import "learn/crawler/engine"
type SimpleScheduler struct {
workerChan chan engine.Request
}
func (s *SimpleScheduler) WorkerChan() chan engine.Request {
return s.workerChan
}
func (s *SimpleScheduler) WorkerReady(chan engine.Request) {
}
func (s *SimpleScheduler) Run() {
s.workerChan = make(chan engine.Request)
}
func (s *SimpleScheduler) Submit(r engine.Request) {
go func() { s.workerChan <- r }()
}
完整项目
https://gitee.com/FenYiYuan/golang-cpdcrawler.git
go并发版爬虫的更多相关文章
- Go语言之进阶篇爬百度贴吧并发版
1.爬百度贴吧并发版 示例: package main import ( "fmt" "net/http" "os" "strco ...
- Go HelloWorld 网络版和并发版
网络版 package main import ( "net/http" "fmt" ) func main() { http.HandleFunc(" ...
- go-爬虫-百度贴吧(并发版)
爬取百度贴吧的网页 非并发版 package main import ( "fmt" "io" "net/http" "os&qu ...
- go单任务版爬虫
go单任务版爬虫(爬取珍爱网) 爬虫总体算法 单任务版爬虫架构 任务 获取并打印所在城市第一页用户的详细信息 代码实现 /crawler/main.go package main import ( & ...
- 区划代码 node 版爬虫尝试
前言 对于区划代码数据,很多人都不会陌生,大多公司数据库都会维护一份区划代码,包含省市区等数据.区划信息跟用户信息息息相关,往往由于历史原因很多数据都是比较老的数据,且不会轻易更改.网上也有很多人提供 ...
- python链家网高并发异步爬虫asyncio+aiohttp+aiomysql异步存入数据
python链家网二手房异步IO爬虫,使用asyncio.aiohttp和aiomysql 很多小伙伴初学python时都会学习到爬虫,刚入门时会使用requests.urllib这些同步的库进行单线 ...
- 最新IP地址数据库Dat格式-高性能高并发版(2019年3月)
最新IP地址数据库->Dat 二进制文件 高性能高并发-qqzeng-ip.dat 格式 全球IP数据库-20190301-Dat 版 国内IP数据库-20190 ...
- python链家网高并发异步爬虫and异步存入数据
python链家网二手房异步IO爬虫,使用asyncio.aiohttp和aiomysql 很多小伙伴初学python时都会学习到爬虫,刚入门时会使用requests.urllib这些同步的库进行单线 ...
- python学习_新闻联播文字版爬虫(V 1.0版)
python3的爬虫练习,爬取的是新闻联播文字版网站 #!/usr/bin/env python # -*- coding: utf-8 -*- ''' __author__ = 'wyf349' _ ...
随机推荐
- django1.11版本在python3.7中运行还有点兼容性问题.
django1.11版本在python3.7中运行还有点兼容性问题. 出现SyntaxError: Generator expression must be parenthesized这个报错 找到这 ...
- echats 的使用
第一步在我们的电脑上百度搜索echarts,点击进去,如下图所示: 2 第二步进去之后,点击下载,选择要下载的echarts版本,一般选择源代码,如下图所示: 3 第三步下载完成之后,我们也可以来使用 ...
- 养成编程思维,可以从python开始,今天说说python注释
先看思维导图!对内容简单了解一下. 提高代码的可读性,需要提供对代码的注释.python注释,主要作用在代码中,对代码功能进行解释,是一种标注性文字.一般情况下分成三类,单行注释.多行注释.中文声明注 ...
- vscode 调试 react 项目
主要分为以下三个步骤 安装 debug for chrome 配置 launch.json 文件 配置内容如下 { "version": "0.2.0", &q ...
- Java并发读书笔记:JMM与重排序
目录 Java内存模型(JMM) JMM抽象结构 重排序 源码->最终指令序列 编译器重排序 处理器重排序 数据依赖性 as-if-serial happens-before happens-b ...
- 初探 Node.js 框架:eggjs (环境搭配篇)
eggjs 是一个优秀的 Node.js 框架 概述:为什么标题上说 eggjs 是一个优秀的 Node.js 框架(可跳过)? 换言之,我们为什么选择 eggjs 进行开发而不是之前初学时使用的 E ...
- Webpack 一,打包JS
创建入口文件 app.js // es6 module 规范 import sum_d from './sum.js' import {sum_e} from './sum.js' // commco ...
- angularJS 格式化日期 转载
angularjs中的filter(过滤器)——格式化日期的date 标签: filterangularjsdate http://blog.csdn.net/smile_panda/article/ ...
- Django ORM各种查询
正向和反向查询 正向 ----> 关联字段在当前表中,从当前表向外查叫正向 反向 —> 关联字段不在当前表中,当当前表向外查叫反向 正向通过字段,反向通过表名查 表结构 from djan ...
- 【C++】C++程序加载lib静态库
使用Visual Studio 编写C++程序有几种配置lib的方法,以下是在代码中加载lib文件的方法: 在项目所在目录下创建文件夹lib,将lib文件此路径下,包括Debug和Release两种模 ...