基于SpringBoot WebMagic爬虫爬取大乐透双色球
注：程序仅用于个人兴趣爱好，不得用于商业行为，本人一概不负责
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.example</groupId>
<artifactId>dubbo-ball</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>dubbo-ball</name>
<description>dubbo-ball</description> <properties>
<java.version>1.17</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.7.3</spring-boot.version>
</properties> <dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-undertow</artifactId>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.7.5</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.7.5</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies> <dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement> <build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.example.ball.DubboBallApplication</mainClass>
</configuration>
<executions>
<execution>
<id>repackage</id>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build> </project>
Magic.java
package com.example.ball.conf; import com.example.ball.handle.DLTProcessor;
import com.example.ball.handle.DoubleProcessor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider; import javax.annotation.PostConstruct; @Component
@Slf4j
public class Magic { @Autowired
@Qualifier("threadPoolTaskExecutor")
private ThreadPoolTaskExecutor threadPoolTaskExecutor; @PostConstruct
public void processor() throws InterruptedException {
System.out.println("================================================大乐透==========================================================");
threadPoolTaskExecutor.execute(() -> {
Spider spiderDLTBall = new Spider(new DLTProcessor()).addRequest(buildRequestDLT()).thread(1);
spiderDLTBall.runAsync();
});
Thread.sleep(3000);
System.out.println("\r\n\n\n\n");
System.out.println("================================================双色球==========================================================");
threadPoolTaskExecutor.execute(() -> {
Spider spiderDoubleBall = new Spider(new DoubleProcessor()).addRequest(buildRequestDoubleBall()).thread(1);
spiderDoubleBall.runAsync();
});
} private Request buildRequestDLT() {
return new Request("https://kjh.55128.cn/dlt-history-360.htm");
} private Request buildRequestDoubleBall() {
return new Request("https://kjh.55128.cn/ssq-history-360.htm");
} }
ThreadExecutorsConf.java
package com.example.ball.conf; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import java.util.concurrent.ThreadPoolExecutor; @Configuration
public class ThreadExecutorsConf { /**
* 核心线程数（默认线程数）
*/
private static final int corePoolSize = 60;
/**
* 最大线程数
*/
private static final int maxPoolSize = 1000;
/**
* 允许线程空闲时间（单位：默认为秒）
*/
private static final int keepAliveTime = 100;
/**
* 缓冲队列数
*/
private static final int queueCapacity = 1000;
/**
* 线程池名前缀
*/
private static final String threadNamePrefix = "Thread-"; @Bean("threadPoolTaskExecutor")
public ThreadPoolTaskExecutor executor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(corePoolSize);
executor.setKeepAliveSeconds(keepAliveTime);
executor.setMaxPoolSize(maxPoolSize);
executor.setQueueCapacity(queueCapacity);
executor.setThreadNamePrefix(threadNamePrefix);
//设置拒绝策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
return executor;
}
}
处理算子
DLTProcessor.java
大乐透算子处理
package com.example.ball.handle; import com.alibaba.fastjson.util.TypeUtils;
import com.example.ball.entity.DLTBall;
import com.example.ball.entity.StatisticalDLTBall;
import com.example.ball.entity.StatisticalDLTResult;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html; import java.util.ArrayList;
import java.util.List; @Slf4j
public class DLTProcessor implements PageProcessor { private Site site; @Override
public void process(Page page) {
Html pageHtml = page.getHtml();
String html = pageHtml.toString();
Document document = Jsoup.parse(html);
Elements body = document.getElementById("table").getElementsByTag("tbody").get(0).getElementsByTag("tr");
List<DLTBall> DLTBallHistory = new ArrayList<>();
bodyHandle(body, DLTBallHistory);
List<DLTBall> DLTBalls3 = DLTBallHistory.subList(0, 3);
List<DLTBall> DLTBalls5 = DLTBallHistory.subList(0, 5);
List<DLTBall> DLTBalls8 = DLTBallHistory.subList(0, 8);
List<DLTBall> DLTBalls10 = DLTBallHistory.subList(0, 10);
List<DLTBall> DLTBalls20 = DLTBallHistory.subList(0, 20);
List<DLTBall> DLTBalls30 = DLTBallHistory.subList(0, 30);
List<DLTBall> DLTBalls45 = DLTBallHistory.subList(0, 45);
List<DLTBall> DLTBalls60 = DLTBallHistory.subList(0, 60);
List<DLTBall> DLTBalls90 = DLTBallHistory.subList(0, 90);
List<DLTBall> DLTBalls120 = DLTBallHistory.subList(0, 120);
List<DLTBall> DLTBalls360 = DLTBallHistory.subList(0, 360); StatisticalDLTResult statisticalDLTResult3 = statisticalResult(DLTBalls3, 3);
StatisticalDLTResult statisticalDLTResult5 = statisticalResult(DLTBalls5, 5);
StatisticalDLTResult statisticalDLTResult8 = statisticalResult(DLTBalls8, 8);
StatisticalDLTResult statisticalDLTResult10 = statisticalResult(DLTBalls10, 10);
StatisticalDLTResult statisticalDLTResult20 = statisticalResult(DLTBalls20, 20);
StatisticalDLTResult statisticalDLTResult30 = statisticalResult(DLTBalls30, 30);
StatisticalDLTResult statisticalDLTResult45 = statisticalResult(DLTBalls45, 45);
StatisticalDLTResult statisticalDLTResult60 = statisticalResult(DLTBalls60, 60);
StatisticalDLTResult statisticalDLTResult90 = statisticalResult(DLTBalls90, 90);
StatisticalDLTResult statisticalDLTResult120 = statisticalResult(DLTBalls120, 120);
StatisticalDLTResult statisticalDLTResult360 = statisticalResult(DLTBalls360, 360); StatisticalDLTBall statisticalResult = new StatisticalDLTBall(statisticalDLTResult3, statisticalDLTResult5, statisticalDLTResult8, statisticalDLTResult10, statisticalDLTResult20, statisticalDLTResult30, statisticalDLTResult45, statisticalDLTResult60, statisticalDLTResult90, statisticalDLTResult120, statisticalDLTResult360);
} private static StatisticalDLTResult statisticalResult(List<DLTBall> DLTBalls, int group) {
long redDNum = DLTBalls.stream().filter(ball -> "单".equals(ball.getRedDS())).count();
long redSNum = DLTBalls.stream().filter(ball -> "双".equals(ball.getRedDS())).count();
long blueDNum = DLTBalls.stream().filter(ball -> "单".equals(ball.getBlueDS())).count();
long blueSNum = DLTBalls.stream().filter(ball -> "双".equals(ball.getBlueDS())).count();
int redSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedSum())).sum();
int blueSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueSum())).sum();
int redKDSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedKD())).sum();
int blueKDSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueKD())).sum();
int redBallMinNum = DLTBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int redBallMaxNum = DLTBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0);
int blueBallMinNum = DLTBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int blueBallMaxNum = DLTBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0); StatisticalDLTResult statisticalDLTResult = new StatisticalDLTResult();
statisticalDLTResult.setRedContrastDS((int) redDNum + ":" + redSNum);
statisticalDLTResult.setBlueContrastDS((int) blueDNum + ":" + blueSNum);
statisticalDLTResult.setRedBallMiddleNum(String.valueOf(redSum / group));
statisticalDLTResult.setBlueBallMiddleNum(String.valueOf(blueSum / group));
statisticalDLTResult.setRedBallKDMiddleNum(String.valueOf(redKDSum / group));
statisticalDLTResult.setBlueBallKDMiddleNum(String.valueOf(blueKDSum / group));
statisticalDLTResult.setRedBallMinNum(String.valueOf(redBallMinNum));
statisticalDLTResult.setRedBallMaxNum(String.valueOf(redBallMaxNum));
statisticalDLTResult.setBlueBallMinNum(String.valueOf(blueBallMinNum));
statisticalDLTResult.setBlueBallMaxNum(String.valueOf(blueBallMaxNum));
System.out.println("\r\n");
System.out.println("===========================================================大乐透近" + group + "天出球信息统计===========================================================");
System.out.println("红球单双比:" + statisticalDLTResult.getRedContrastDS() + " 蓝球单双比:" + statisticalDLTResult.getBlueContrastDS() + " 红球中位数:" + statisticalDLTResult.getRedBallMiddleNum() + " 蓝球中位数:" + statisticalDLTResult.getBlueBallMiddleNum() + " 红球跨度中位数:" + statisticalDLTResult.getRedBallKDMiddleNum() + " 蓝球跨度中位数:" + statisticalDLTResult.getBlueBallKDMiddleNum() + " 红球最小值:" + statisticalDLTResult.getRedBallMinNum() + " 红球最大值:" + statisticalDLTResult.getRedBallMaxNum() + " 蓝球最小值:" + statisticalDLTResult.getBlueBallMinNum() + " 蓝球最大值:" + statisticalDLTResult.getBlueBallMaxNum());
return statisticalDLTResult;
} private static void bodyHandle(Elements body, List<DLTBall> DLTBallHistory) {
body.forEach(tr -> {
List<String> redList = new ArrayList<>();
List<String> blueList = new ArrayList<>();
Integer blueSum = 0;
String blueDS;
Elements td = tr.getElementsByTag("td");
DLTBall.DLTBallBuilder builder = DLTBall.builder().date(td.get(0).text()).redSum(td.get(3).text()).redDS(td.get(4).text()).redJOB(td.get(5).text()).redKD(td.get(7).text());
Elements span = td.get(2).getElementsByTag("span");
int blueKD;
for (int i = 0; i < 7; i++) {
String text = span.get(i).text();
if (i < 5) {
redList.add(text);
} else {
Integer num = TypeUtils.castToInt(text);
blueSum += num;
blueList.add(text);
}
}
if (blueSum % 2 == 0) {
blueDS = "双";
} else {
blueDS = "单";
}
long ouNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 0).count();
long jiNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 1).count();
blueKD = TypeUtils.castToInt(blueList.get(1)) - TypeUtils.castToInt(blueList.get(0));
DLTBall build = builder.redBalls(redList).blueBalls(blueList).blueSum(TypeUtils.castToString(blueSum)).blueDS(blueDS).blueJOB(jiNum + ":" + ouNum).blueKD(TypeUtils.castToString(blueKD)).build();
System.out.println("开奖时间:" + build.getDate() + " 红:" + build.getRedBalls() + " 蓝:" + build.getBlueBalls() + " 红球sum值:" + build.getRedSum() + " 红球sum单双:" + build.getRedDS() + " 红球奇偶比:" + build.getRedJOB() + " 红球跨度:" + build.getRedKD() + " 蓝球sum值:" + build.getBlueSum() + " 蓝球sum单双:" + build.getBlueDS() + " 蓝球奇偶比:" + build.getBlueJOB() + " 蓝球跨度:" + build.getBlueKD());
DLTBallHistory.add(build);
});
} public static void main(String[] args) {
List<String> ad = new ArrayList<>();
ad.add("a");
ad.add("b");
ad.add("c");
System.out.println(ad);
} @Override
public Site getSite() {
if (site == null) {
site = Site.me().setDomain(".55128.cn").setSleepTime(1000);
}
return site;
} }
DoubleProcessor.java
双色球处理算子
package com.example.ball.handle; import com.alibaba.fastjson.util.TypeUtils;
import com.example.ball.entity.SSQBall;
import com.example.ball.entity.StatisticalSSQBall;
import com.example.ball.entity.StatisticalSSQResult;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html; import java.util.ArrayList;
import java.util.List; @Slf4j
public class DoubleProcessor implements PageProcessor { private Site site; @Override
public void process(Page page) {
Html pageHtml = page.getHtml();
String html = pageHtml.toString();
Document document = Jsoup.parse(html);
Elements body = document.getElementById("table").getElementsByTag("tbody").get(0).getElementsByTag("tr");
List<SSQBall> SSQBallHistory = new ArrayList<>();
bodyHandle(body, SSQBallHistory);
List<SSQBall> SSQBalls3 = SSQBallHistory.subList(0, 3);
List<SSQBall> SSQBalls5 = SSQBallHistory.subList(0, 5);
List<SSQBall> SSQBalls8 = SSQBallHistory.subList(0, 8);
List<SSQBall> SSQBalls10 = SSQBallHistory.subList(0, 10);
List<SSQBall> SSQBalls20 = SSQBallHistory.subList(0, 20);
List<SSQBall> SSQBalls30 = SSQBallHistory.subList(0, 30);
List<SSQBall> SSQBalls45 = SSQBallHistory.subList(0, 45);
List<SSQBall> SSQBalls60 = SSQBallHistory.subList(0, 60);
List<SSQBall> SSQBalls90 = SSQBallHistory.subList(0, 90);
List<SSQBall> SSQBalls120 = SSQBallHistory.subList(0, 120);
List<SSQBall> SSQBalls360 = SSQBallHistory.subList(0, 360); StatisticalSSQResult statisticalSSQResult3 = statisticalResult(SSQBalls3, 3);
StatisticalSSQResult statisticalSSQResult5 = statisticalResult(SSQBalls5, 5);
StatisticalSSQResult statisticalSSQResult8 = statisticalResult(SSQBalls8, 8);
StatisticalSSQResult statisticalSSQResult10 = statisticalResult(SSQBalls10, 10);
StatisticalSSQResult statisticalSSQResult20 = statisticalResult(SSQBalls20, 20);
StatisticalSSQResult statisticalSSQResult30 = statisticalResult(SSQBalls30, 30);
StatisticalSSQResult statisticalSSQResult45 = statisticalResult(SSQBalls45, 45);
StatisticalSSQResult statisticalSSQResult60 = statisticalResult(SSQBalls60, 60);
StatisticalSSQResult statisticalSSQResult90 = statisticalResult(SSQBalls90, 90);
StatisticalSSQResult statisticalSSQResult120 = statisticalResult(SSQBalls120, 120);
StatisticalSSQResult statisticalSSQResult360 = statisticalResult(SSQBalls360, 360); StatisticalSSQBall statisticalResult = new StatisticalSSQBall(statisticalSSQResult3, statisticalSSQResult5, statisticalSSQResult8, statisticalSSQResult10, statisticalSSQResult20, statisticalSSQResult30, statisticalSSQResult45, statisticalSSQResult60, statisticalSSQResult90, statisticalSSQResult120, statisticalSSQResult360);
} private static StatisticalSSQResult statisticalResult(List<SSQBall> SSQBalls, int group) {
long redDNum = SSQBalls.stream().filter(ball -> "单".equals(ball.getRedDS())).count();
long redSNum = SSQBalls.stream().filter(ball -> "双".equals(ball.getRedDS())).count();
long blueDNum = SSQBalls.stream().filter(ball -> "单".equals(ball.getBlueDS())).count();
long blueSNum = SSQBalls.stream().filter(ball -> "双".equals(ball.getBlueDS())).count();
int redSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedSum())).sum();
int blueSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueSum())).sum();
int redKDSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedKD())).sum();
int blueKDSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueKD())).sum();
int redBallMinNum = SSQBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int redBallMaxNum = SSQBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0);
int blueBallMinNum = SSQBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int blueBallMaxNum = SSQBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0); StatisticalSSQResult statisticalSSQResult = new StatisticalSSQResult();
statisticalSSQResult.setRedContrastDS((int) redDNum + ":" + redSNum);
statisticalSSQResult.setBlueContrastDS((int) blueDNum + ":" + blueSNum);
statisticalSSQResult.setRedBallMiddleNum(String.valueOf(redSum / group));
statisticalSSQResult.setBlueBallMiddleNum(String.valueOf(blueSum / group));
statisticalSSQResult.setRedBallKDMiddleNum(String.valueOf(redKDSum / group));
statisticalSSQResult.setBlueBallKDMiddleNum(String.valueOf(blueKDSum / group));
statisticalSSQResult.setRedBallMinNum(String.valueOf(redBallMinNum));
statisticalSSQResult.setRedBallMaxNum(String.valueOf(redBallMaxNum));
statisticalSSQResult.setBlueBallMinNum(String.valueOf(blueBallMinNum));
statisticalSSQResult.setBlueBallMaxNum(String.valueOf(blueBallMaxNum));
System.out.println("\r\n");
System.out.println("===========================================================双色球近" + group + "天出球信息统计===========================================================");
System.out.println("红球单双比:" + statisticalSSQResult.getRedContrastDS() + " 蓝球单双比:" + statisticalSSQResult.getBlueContrastDS() + " 红球中位数:" + statisticalSSQResult.getRedBallMiddleNum() + " 蓝球中位数:" + statisticalSSQResult.getBlueBallMiddleNum() + " 红球跨度中位数:" + statisticalSSQResult.getRedBallKDMiddleNum() + " 蓝球跨度中位数:" + statisticalSSQResult.getBlueBallKDMiddleNum() + " 红球最小值:" + statisticalSSQResult.getRedBallMinNum() + " 红球最大值:" + statisticalSSQResult.getRedBallMaxNum() + " 蓝球最小值:" + statisticalSSQResult.getBlueBallMinNum() + " 蓝球最大值:" + statisticalSSQResult.getBlueBallMaxNum());
return statisticalSSQResult;
} private static void bodyHandle(Elements body, List<SSQBall> SSQBallHistory) {
body.forEach(tr -> {
List<String> redList = new ArrayList<>();
List<String> blueList = new ArrayList<>();
Integer blueSum = 0;
String blueDS;
Elements td = tr.getElementsByTag("td");
SSQBall.SSQBallBuilder builder = SSQBall.builder().date(td.get(0).text()).redSum(td.get(4).text()).redDS(td.get(5).text()).redJOB(td.get(6).text()).redKD(td.get(9).text());
Elements span = td.get(2).getElementsByTag("span");
int blueKD;
for (int i = 0; i < 7; i++) {
String text = span.get(i).text();
if (i < 6) {
redList.add(text);
} else {
Integer num = TypeUtils.castToInt(text);
blueSum += num;
blueList.add(text);
}
}
if (blueSum % 2 == 0) {
blueDS = "双";
} else {
blueDS = "单";
}
long ouNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 0).count();
long jiNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 1).count();
SSQBall build = builder.redBalls(redList).blueBalls(blueList).blueSum(TypeUtils.castToString(blueSum)).blueDS(blueDS).blueJOB(jiNum + ":" + ouNum).blueKD(TypeUtils.castToString(blueList.get(0))).build();
System.out.println("开奖时间:" + build.getDate() + " 红:" + build.getRedBalls() + " 蓝:" + build.getBlueBalls() + " 红球sum值:" + build.getRedSum() + " 红球sum单双:" + build.getRedDS() + " 红球奇偶比:" + build.getRedJOB() + " 红球跨度:" + build.getRedKD() + " 蓝球sum值:" + build.getBlueSum() + " 蓝球sum单双:" + build.getBlueDS() + " 蓝球奇偶比:" + build.getBlueJOB() + " 蓝球跨度:" + build.getBlueKD());
SSQBallHistory.add(build);
});
} @Override
public Site getSite() {
if (site == null) {
site = Site.me().setDomain(".55128.cn").setSleepTime(1000);
}
return site;
} }
实体类
DLTBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable;
import java.util.List; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class DLTBall implements Serializable { private String date; private List<String> redBalls; private List<String> blueBalls; //红色sum
private String redSum; //红色sum值单双
private String redDS; //红色奇偶比
private String redJOB; //红色跨度
private String redKD; //蓝色sum
private String blueSum; //蓝色sum值单双
private String blueDS; //蓝色奇偶比
private String blueJOB; //蓝色跨度
private String blueKD; }
SSQBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable;
import java.util.List; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class SSQBall implements Serializable { private String date; private List<String> redBalls; private List<String> blueBalls; //红色sum
private String redSum; //红色sum值单双
private String redDS; //红色奇偶比
private String redJOB; //红色跨度
private String redKD; //蓝色sum
private String blueSum; //蓝色sum值单双
private String blueDS; //蓝色奇偶比
private String blueJOB; //蓝色跨度
private String blueKD; }
StatisticalDLTBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalDLTBall implements Serializable { private StatisticalDLTResult statisticalDLTResult3;
private StatisticalDLTResult statisticalDLTResult5;
private StatisticalDLTResult statisticalDLTResult8;
private StatisticalDLTResult statisticalDLTResult10;
private StatisticalDLTResult statisticalDLTResult20;
private StatisticalDLTResult statisticalDLTResult30;
private StatisticalDLTResult statisticalDLTResult45;
private StatisticalDLTResult statisticalDLTResult60;
private StatisticalDLTResult statisticalDLTResult90;
private StatisticalDLTResult statisticalDLTResult120;
private StatisticalDLTResult statisticalDLTResult360; }
StatisticalDLTResult.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalDLTResult implements Serializable { //红球单双比
private String redContrastDS; //蓝球单双比
private String blueContrastDS; //红球中位数
private String redBallMiddleNum; //蓝球中位数
private String BlueBallMiddleNum; //红球跨度中位数
private String redBallKDMiddleNum; //蓝球跨度中位数
private String blueBallKDMiddleNum; //红球最小值
private String redBallMinNum; //红球最大值
private String redBallMaxNum; //蓝球最小值
private String blueBallMinNum; //蓝球最大值
private String blueBallMaxNum; }
StatisticalSSQBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalSSQBall implements Serializable { private StatisticalSSQResult statisticalSSQResult3;
private StatisticalSSQResult statisticalSSQResult5;
private StatisticalSSQResult statisticalSSQResult8;
private StatisticalSSQResult statisticalSSQResult10;
private StatisticalSSQResult statisticalSSQResult20;
private StatisticalSSQResult statisticalSSQResult30;
private StatisticalSSQResult statisticalSSQResult45;
private StatisticalSSQResult statisticalSSQResult60;
private StatisticalSSQResult statisticalSSQResult90;
private StatisticalSSQResult statisticalSSQResult120;
private StatisticalSSQResult statisticalSSQResult360; }
StatisticalSSQResult.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalSSQResult implements Serializable { //红球单双比
private String redContrastDS; //蓝球单双比
private String blueContrastDS; //红球中位数
private String redBallMiddleNum; //蓝球中位数
private String BlueBallMiddleNum; //红球跨度中位数
private String redBallKDMiddleNum; //蓝球跨度中位数
private String blueBallKDMiddleNum; //红球最小值
private String redBallMinNum; //红球最大值
private String redBallMaxNum; //蓝球最小值
private String blueBallMinNum; //蓝球最大值
private String blueBallMaxNum; }
基于SpringBoot WebMagic爬虫爬取大乐透双色球的更多相关文章
- 如何使用robots禁止各大搜索引擎爬虫爬取网站
ps:由于公司网站配置的测试环境被百度爬虫抓取,干扰了线上正常环境的使用,刚好看到每次搜索淘宝时,都会有一句由于robots.txt文件存在限制指令无法提供内容描述,于是便去学习了一波 1.原来一般来 ...
- python爬虫—爬取英文名以及正则表达式的介绍
python爬虫—爬取英文名以及正则表达式的介绍 爬取英文名: 一. 爬虫模块详细设计 (1)整体思路 对于本次爬取英文名数据的爬虫实现,我的思路是先将A-Z所有英文名的连接爬取出来,保存在一个cs ...
- MATLAB爬虫爬取股票数据
近年来,大数据盛行,有关爬虫的教程层次不穷.那么,爬虫到底是什么呢? 什么是爬虫? 百度百科是这样定义的: 网络爬虫(又被称为网页蜘蛛,网络机器人,在FOAF社区中间,更经常的称为网页追逐者),是一种 ...
- Python爬虫 - 爬取百度html代码前200行
Python爬虫 - 爬取百度html代码前200行 - 改进版, 增加了对字符串的.strip()处理 源代码如下: # 改进版, 增加了 .strip()方法的使用 # coding=utf-8 ...
- 如何利用Python网络爬虫爬取微信朋友圈动态--附代码（下）
前天给大家分享了如何利用Python网络爬虫爬取微信朋友圈数据的上篇(理论篇),今天给大家分享一下代码实现(实战篇),接着上篇往下继续深入. 一.代码实现 1.修改Scrapy项目中的items.py ...
- 使用scrapy爬虫,爬取17k小说网的案例-方法一
无意间看到17小说网里面有一些小说小故事,于是决定用爬虫爬取下来自己看着玩,下图这个页面就是要爬取的来源. a 这个页面一共有125个标题,每个标题里面对应一个内容,如下图所示 下面直接看最核心spi ...
- 基于 PHP 的数据爬取（QueryList）
基于PHP的数据爬取 官方网站站点 简单. 灵活.强大的PHP采集工具,让采集更简单一点. 简介: QueryList使用jQuery选择器来做采集,让你告别复杂的正则表达式:QueryList具有j ...
- python爬虫爬取京东、淘宝、苏宁上华为P20购买评论
爬虫爬取京东.淘宝.苏宁上华为P20购买评论 1.使用软件 Anaconda3 2.代码截图 三个网站代码大同小异,因此只展示一个 3.结果(部分) 京东 淘宝 苏宁 4.分析 这三个网站上的评论数据 ...
- 利用Python网络爬虫爬取学校官网十条标题
利用Python网络爬虫爬取学校官网十条标题 案例代码: # __author : "J" # date : 2018-03-06 # 导入需要用到的库文件 import urll ...
- node：爬虫爬取网页图片
代码地址如下:http://www.demodashi.com/demo/13845.html 前言 周末自己在家闲着没事,刷着微信,玩着手机,发现自己的微信头像该换了,就去网上找了一下头像,看着图片 ...
随机推荐
- cadence报错：Class must be one of IC, IO, DISCRETE, MECHANICAL, PLATING_BAR or DRIVER_CELL.
在原理图文件上右键选择Edit Object Properties, 然后在class一栏中修改class为IC, IO, DISCRETE, MECHANICAL, PLATING_BAR or D ...
- Application.HandleMessage与Application.ProcessMessage
HandleMessage: HandleMessage中断应用程序的执行,以便Windows可以在将控制权返回给应用程序之前处理来自Windows消息队列的单个消息. 如果消息队列为空,则Handl ...
- VS2012下没有ADO.NET实体数据模型
在C盘下搜"EFTools.msi"然后退出VS,点击修复在打开VS,数据下就有了
- C++ 函数与数组
C++ 函数与数组 #include<iostream> using namespace std; const int ArSize = 8; int sum_arr(int arr[], ...
- Gabor滤波（个人学习）
Gabor滤波 1.优点 Gabor小波与人类视觉系统中简单细胞的视觉刺激响应非常相似.在提取目标的局部空间和频率与信息方面具有良好的特性. 对于图像的边缘敏感,能够提供良好的方向选择和尺度选择.因此 ...
- @media移动PC端fontSize响应设置
@media only screen and (min-width: 320px) and (max-width: 359px) { html, body { font-size: 8.51px; } ...
- Spring Boot 中的异步调用
通常我们开发的程序都是同步调用的,即程序按照代码的顺序一行一行的逐步往下执行,每一行代码都必须等待上一行代码执行完毕才能开始执行.而异步编程则没有这个限制,代码的调用不再是阻塞的.所以在一些情景下,通 ...
- Java mysql查询数据库重复数据(单个或多个字段)
查询表重复数据: SELECT * FROM 表1 f WHERE (f.字段1,f.字段2) in (SELECT 字段1,字段2 FROM 表1 GROUP BY 字段1,字段2 HAVING c ...
- cf823div2C
cf823div2C 题目链接 题目 给你两个字符串\(s_1,s_2\).每次操作可以让\(s_1\)的前k个和\(s_2\)的后k个交换.询问是否可以通过多次上述操作,使得\(s_1=s_2\). ...
- 一起学JAVA：做一个简单的API项目吧（一）
由于一些特殊原因,最近想要去学习JAVA,了解JAVA ,然后在此之前对JAVA只停留在能读懂代码,但是写不懂的状态,那么最近又很闲,所以打算重新进行一波自己的学习计划,又因为是三分钟的热情,所以特意 ...