并发版爬虫

代码实现

/crawler/main.go
package main

import (
"learn/crawler/engine"
"learn/crawler/scheduler"
"learn/crawler/zhenai/parser"
) func main() {
e := engine.ConcurrentEngine{
Scheduler: &scheduler.QueuedScheduler{},
WorkerCount: 20,
}
e.Run(engine.Request{
Url: "http://www.zhenai.com/zhenghun",
ParseFunc: parser.ParseCityList,
})
//测试上海单个城市
//e.Run(engine.Request{
// Url: "http://www.zhenai.com/zhenghun/shanghai",
// ParseFunc: parser.ParseCity,
//})
}
/crawler/engine/simple.go
package engine

import (
"learn/crawler/fetcher"
"log"
) type SimpleEngine struct { } func (e SimpleEngine) Run(seeds ...Request) {
var requests []Request
for _, r := range seeds {
requests = append(requests, r)
}
for len(requests) > 0 {
r := requests[0]
requests = requests[1:] parseResult, err := worker(r)
if err != nil {
continue
}
requests = append(requests, parseResult.Requests...)
for _, item := range parseResult.Items{
log.Printf("Got item %v", item)
}
}
}
func worker(r Request) (ParseResult, error) {
log.Printf("Fetching %s", r.Url)
body, err := fetcher.Fetch(r.Url)
if err != nil {
log.Printf("Fetcher: error" + "fetching url %s: %v", r.Url, err)
return ParseResult{}, err
}
return r.ParseFunc(body), nil
}
/crawler/engine/concurrent.go
package engine

import (
"log"
) type ConcurrentEngine struct {
Scheduler Scheduler
WorkerCount int
}
type Scheduler interface {
ReadyNotifier
Submit(Request)
WorkerChan() chan Request
Run()
}
type ReadyNotifier interface {
WorkerReady(chan Request)
}
func (e *ConcurrentEngine) Run(seeds ...Request) {
out := make(chan ParseResult)
e.Scheduler.Run() for i := 0; i < e.WorkerCount; i++ {
createWork(e.Scheduler.WorkerChan(), out, e.Scheduler)
}
for _, r := range seeds {
e.Scheduler.Submit(r)
}
itemCount := 0
for {
result := <- out
for _, item := range result.Items {
log.Printf("Got item #%d: %v", itemCount, item)
itemCount++
}
for _, request := range result.Requests {
e.Scheduler.Submit(request)
}
}
}
func createWork(in chan Request, out chan ParseResult, ready ReadyNotifier) {
go func() {
for {
ready.WorkerReady(in)
request := <- in
result, err := worker(request)
if err != nil {
continue
}
out <- result
}
}()
}
/crawler/engine/typers.go
package engine

type Request struct {
Url string
ParseFunc func([]byte) ParseResult
}
type ParseResult struct {
Requests []Request
Items []interface{}
}
func NilParser([]byte) ParseResult{
return ParseResult{}
}
/crawler/fetcher/fetcher.go
package fetcher

import (
"bufio"
"fmt"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
"io/ioutil"
"log"
"net/http"
"time"
) var rateLimiter = time.Tick(100 * time.Millisecond)
func Fetch(url string) ([]byte, error) {
<- rateLimiter
client := &http.Client{}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("Wrong status code: %d", resp.StatusCode)
}
bodyReader := bufio.NewReader(resp.Body)
e := determineEncoding(bodyReader)
utf8Reader := transform.NewReader(bodyReader, e.NewDecoder())
return ioutil.ReadAll(utf8Reader)
}
func determineEncoding(r *bufio.Reader) encoding.Encoding {
bytes, err := r.Peek(1024)
if err != nil {
log.Printf("Fetcher error: %v", err)
return unicode.UTF8
}
e, _, _ := charset.DetermineEncoding(bytes, "")
return e
}
/crawler/zhenai/parser/citylist.go
package parser

import (
"learn/crawler/engine"
"regexp"
) const cityListRe = `<a href="(http://www.zhenai.com/zhenghun/[0-9a-z]+)" [^>]*>([^<]+)</a>`
func ParseCityList(contents []byte) engine.ParseResult {
re := regexp.MustCompile(cityListRe)
matches := re.FindAllSubmatch(contents, -1)
result := engine.ParseResult{}
for _, m := range matches {
result.Items = append(result.Items, "City: "+string(m[2]))
result.Requests = append(result.Requests, engine.Request{
Url: string(m[1]),
ParseFunc: ParseCity,
})
}
return result
}
/crawler/zhenai/parser/city.go
package parser

import (
"learn/crawler/engine"
"regexp"
)
var (
profileRe = regexp.MustCompile(`<a href="(http://album.zhenai.com/u/[0-9]+)" [^>]*>([^<]+)</a>`)
cityUrlRe = regexp.MustCompile(`href="(http://www.zhenai.com/zhenghun/[^"]+)"`)
)
func ParseCity(contents []byte) engine.ParseResult {
matches := profileRe.FindAllSubmatch(contents, -1)
result := engine.ParseResult{}
for _, m := range matches {
name := string(m[2])
result.Items = append(result.Items, "User "+name)
result.Requests = append(result.Requests, engine.Request{
Url: string(m[1]),
ParseFunc: func(c []byte) engine.ParseResult {
return ParseProfile(c, "name:"+name)
},
})
}
matches = cityUrlRe.FindAllSubmatch(contents, -1)
for _, m := range matches {
result.Requests = append(result.Requests, engine.Request{
Url: string(m[1]),
ParseFunc: ParseCity,
})
}
return result
}
/crawler/zhenai/parser/profile.go
package parser

import (
"learn/crawler/engine"
"learn/crawler/model"
"regexp"
) const all = `<div class="m-btn purple" data-v-8b1eac0c>([^<]+)</div>`
func ParseProfile(contents []byte, name string) engine.ParseResult {
profile := model.Profile{}
profile.User = append(profile.User, name)
re := regexp.MustCompile(all)
match := re.FindAllSubmatch(contents,-1)
if match != nil {
for _, m := range match {
profile.User = append(profile.User, string(m[1]))
}
} result := engine.ParseResult{
Items: []interface{}{profile},
}
return result
}
/crawler/model/profile.go
package model

type Profile struct {
User []string
}
/crawler/scheduler/queued.go
package scheduler

import "learn/crawler/engine"

type QueuedScheduler struct {
requestChan chan engine.Request
workChan chan chan engine.Request
} func (s *QueuedScheduler) WorkerChan() chan engine.Request {
return make(chan engine.Request)
} func (s *QueuedScheduler) Submit(r engine.Request) {
s.requestChan <- r
}
func (s *QueuedScheduler) WorkerReady(w chan engine.Request){
s.workChan <- w
}
func (s *QueuedScheduler) Run(){
s.workChan = make(chan chan engine.Request)
s.requestChan = make(chan engine.Request)
go func() {
var requestQ []engine.Request
var workerQ []chan engine.Request
for {
var activeRequest engine.Request
var activeWorker chan engine.Request
if len(requestQ) > 0 && len(workerQ) > 0 {
activeRequest = requestQ[0]
activeWorker = workerQ[0]
}
select {
case r := <-s.requestChan:
requestQ = append(requestQ, r)
case w := <-s.workChan:
workerQ = append(workerQ, w)
case activeWorker <- activeRequest:
workerQ = workerQ[1:]
requestQ = requestQ[1:]
}
}
}()
}
/crawler/scheduler/simple.go
package scheduler

import "learn/crawler/engine"

type SimpleScheduler struct {
workerChan chan engine.Request
} func (s *SimpleScheduler) WorkerChan() chan engine.Request {
return s.workerChan
} func (s *SimpleScheduler) WorkerReady(chan engine.Request) {
} func (s *SimpleScheduler) Run() {
s.workerChan = make(chan engine.Request)
} func (s *SimpleScheduler) Submit(r engine.Request) {
go func() { s.workerChan <- r }()
}

完整项目

https://gitee.com/FenYiYuan/golang-cpdcrawler.git

go并发版爬虫的更多相关文章

  1. Go语言之进阶篇爬百度贴吧并发版

    1.爬百度贴吧并发版 示例: package main import ( "fmt" "net/http" "os" "strco ...

  2. Go HelloWorld 网络版和并发版

    网络版 package main import ( "net/http" "fmt" ) func main() { http.HandleFunc(" ...

  3. go-爬虫-百度贴吧(并发版)

    爬取百度贴吧的网页 非并发版 package main import ( "fmt" "io" "net/http" "os&qu ...

  4. go单任务版爬虫

    go单任务版爬虫(爬取珍爱网) 爬虫总体算法 单任务版爬虫架构 任务 获取并打印所在城市第一页用户的详细信息 代码实现 /crawler/main.go package main import ( & ...

  5. 区划代码 node 版爬虫尝试

    前言 对于区划代码数据,很多人都不会陌生,大多公司数据库都会维护一份区划代码,包含省市区等数据.区划信息跟用户信息息息相关,往往由于历史原因很多数据都是比较老的数据,且不会轻易更改.网上也有很多人提供 ...

  6. python链家网高并发异步爬虫asyncio+aiohttp+aiomysql异步存入数据

    python链家网二手房异步IO爬虫,使用asyncio.aiohttp和aiomysql 很多小伙伴初学python时都会学习到爬虫,刚入门时会使用requests.urllib这些同步的库进行单线 ...

  7. 最新IP地址数据库Dat格式-高性能高并发版(2019年3月)

    最新IP地址数据库->Dat  二进制文件 高性能高并发-qqzeng-ip.dat 格式 全球IP数据库-20190301-Dat 版                国内IP数据库-20190 ...

  8. python链家网高并发异步爬虫and异步存入数据

    python链家网二手房异步IO爬虫,使用asyncio.aiohttp和aiomysql 很多小伙伴初学python时都会学习到爬虫,刚入门时会使用requests.urllib这些同步的库进行单线 ...

  9. python学习_新闻联播文字版爬虫(V 1.0版)

    python3的爬虫练习,爬取的是新闻联播文字版网站 #!/usr/bin/env python # -*- coding: utf-8 -*- ''' __author__ = 'wyf349' _ ...

随机推荐

  1. 前端 network

    控制台 :https://blog.csdn.net/m0_37724356/article/details/79884006 原文链接:https://segmentfault.com/a/1190 ...

  2. 工具之grep

    转自:http://www.cnblogs.com/dong008259/archive/2011/12/07/2279897.html grep (global search regular exp ...

  3. jdk8中CompletableFuture的各个API用法,极大扩展了Future

    就不介绍了,直接贴代码,建议在代码中使用,真的很方便 package cn.hou.completablefuture; import org.junit.Test; import java.util ...

  4. maven jar 包问题

    1. Failure to transfer... 这种错误基本是所需要的 jar 包不存在,或者下载不完整,可去本地仓库查看相关 jar 文件的完整性 解决方案: 删除对应 jar 包,重新下载(删 ...

  5. Spring的aop思想

    1.AOP思想 (1)在解决中文乱码的应用: 一般情况下如果在Servlet中要解决中文乱码问题需要在每一个Servlet中书写解决乱码的代码,但是,在运用了过滤器之后,就不再需要每一个Servlet ...

  6. 死磕java(3)

    流程控制 if else // do while// switch case// while do// break// continue// 相关查询百度

  7. CSS 对于grid布局的理解,举例代码及解释

    网格布局介绍: CSS Grid(网格) 布局(又称为 “Grid(网格)” ),是一个二维的基于网格的布局系统它的目标是完全改变我们基于网格的用户界面的布局方式.CSS 一直用来布局我们的网页,但一 ...

  8. 闲谈一下,ES3、ES4、ES5、ES6 分别是什么

    上图按照时间顺序说明了JavaScript.JScript和ECMAScript的发展. 显示在网景工作的Brendan Eich临危受命,用十天时间设计出LiveScript的第一个版本.临时发布前 ...

  9. Codeforces 1062B Math(质因数分解)

    题意: 给一个数n,可以将它乘任意数,或者开方,问你能得到的最小数是多少,并给出最小操作次数 思路: 能将这个数变小的操作只能是开方,所以构成的最小数一定是 $n = p_1*p_2*p_3*\dot ...

  10. HDU 6186 CS Course (连续位运算)

    CS Course Time Limit: 4000/2000 MS (Java/Others)    Memory Limit: 32768/32768 K (Java/Others)Total S ...