基于SpringBoot WebMagic爬虫爬取大乐透双色球
注：程序仅用于个人兴趣爱好，不得用于商业行为，本人一概不负责
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.example</groupId>
<artifactId>dubbo-ball</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>dubbo-ball</name>
<description>dubbo-ball</description> <properties>
<java.version>1.17</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.7.3</spring-boot.version>
</properties> <dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-undertow</artifactId>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.7.5</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.7.5</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
</dependencies> <dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement> <build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<mainClass>com.example.ball.DubboBallApplication</mainClass>
</configuration>
<executions>
<execution>
<id>repackage</id>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build> </project>
Magic.java
package com.example.ball.conf; import com.example.ball.handle.DLTProcessor;
import com.example.ball.handle.DoubleProcessor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider; import javax.annotation.PostConstruct; @Component
@Slf4j
public class Magic { @Autowired
@Qualifier("threadPoolTaskExecutor")
private ThreadPoolTaskExecutor threadPoolTaskExecutor; @PostConstruct
public void processor() throws InterruptedException {
System.out.println("================================================大乐透==========================================================");
threadPoolTaskExecutor.execute(() -> {
Spider spiderDLTBall = new Spider(new DLTProcessor()).addRequest(buildRequestDLT()).thread(1);
spiderDLTBall.runAsync();
});
Thread.sleep(3000);
System.out.println("\r\n\n\n\n");
System.out.println("================================================双色球==========================================================");
threadPoolTaskExecutor.execute(() -> {
Spider spiderDoubleBall = new Spider(new DoubleProcessor()).addRequest(buildRequestDoubleBall()).thread(1);
spiderDoubleBall.runAsync();
});
} private Request buildRequestDLT() {
return new Request("https://kjh.55128.cn/dlt-history-360.htm");
} private Request buildRequestDoubleBall() {
return new Request("https://kjh.55128.cn/ssq-history-360.htm");
} }
ThreadExecutorsConf.java
package com.example.ball.conf; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import java.util.concurrent.ThreadPoolExecutor; @Configuration
public class ThreadExecutorsConf { /**
* 核心线程数（默认线程数）
*/
private static final int corePoolSize = 60;
/**
* 最大线程数
*/
private static final int maxPoolSize = 1000;
/**
* 允许线程空闲时间（单位：默认为秒）
*/
private static final int keepAliveTime = 100;
/**
* 缓冲队列数
*/
private static final int queueCapacity = 1000;
/**
* 线程池名前缀
*/
private static final String threadNamePrefix = "Thread-"; @Bean("threadPoolTaskExecutor")
public ThreadPoolTaskExecutor executor() {
ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
executor.setCorePoolSize(corePoolSize);
executor.setKeepAliveSeconds(keepAliveTime);
executor.setMaxPoolSize(maxPoolSize);
executor.setQueueCapacity(queueCapacity);
executor.setThreadNamePrefix(threadNamePrefix);
//设置拒绝策略
executor.setRejectedExecutionHandler(new ThreadPoolExecutor.AbortPolicy());
return executor;
}
}
处理算子
DLTProcessor.java
大乐透算子处理
package com.example.ball.handle; import com.alibaba.fastjson.util.TypeUtils;
import com.example.ball.entity.DLTBall;
import com.example.ball.entity.StatisticalDLTBall;
import com.example.ball.entity.StatisticalDLTResult;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html; import java.util.ArrayList;
import java.util.List; @Slf4j
public class DLTProcessor implements PageProcessor { private Site site; @Override
public void process(Page page) {
Html pageHtml = page.getHtml();
String html = pageHtml.toString();
Document document = Jsoup.parse(html);
Elements body = document.getElementById("table").getElementsByTag("tbody").get(0).getElementsByTag("tr");
List<DLTBall> DLTBallHistory = new ArrayList<>();
bodyHandle(body, DLTBallHistory);
List<DLTBall> DLTBalls3 = DLTBallHistory.subList(0, 3);
List<DLTBall> DLTBalls5 = DLTBallHistory.subList(0, 5);
List<DLTBall> DLTBalls8 = DLTBallHistory.subList(0, 8);
List<DLTBall> DLTBalls10 = DLTBallHistory.subList(0, 10);
List<DLTBall> DLTBalls20 = DLTBallHistory.subList(0, 20);
List<DLTBall> DLTBalls30 = DLTBallHistory.subList(0, 30);
List<DLTBall> DLTBalls45 = DLTBallHistory.subList(0, 45);
List<DLTBall> DLTBalls60 = DLTBallHistory.subList(0, 60);
List<DLTBall> DLTBalls90 = DLTBallHistory.subList(0, 90);
List<DLTBall> DLTBalls120 = DLTBallHistory.subList(0, 120);
List<DLTBall> DLTBalls360 = DLTBallHistory.subList(0, 360); StatisticalDLTResult statisticalDLTResult3 = statisticalResult(DLTBalls3, 3);
StatisticalDLTResult statisticalDLTResult5 = statisticalResult(DLTBalls5, 5);
StatisticalDLTResult statisticalDLTResult8 = statisticalResult(DLTBalls8, 8);
StatisticalDLTResult statisticalDLTResult10 = statisticalResult(DLTBalls10, 10);
StatisticalDLTResult statisticalDLTResult20 = statisticalResult(DLTBalls20, 20);
StatisticalDLTResult statisticalDLTResult30 = statisticalResult(DLTBalls30, 30);
StatisticalDLTResult statisticalDLTResult45 = statisticalResult(DLTBalls45, 45);
StatisticalDLTResult statisticalDLTResult60 = statisticalResult(DLTBalls60, 60);
StatisticalDLTResult statisticalDLTResult90 = statisticalResult(DLTBalls90, 90);
StatisticalDLTResult statisticalDLTResult120 = statisticalResult(DLTBalls120, 120);
StatisticalDLTResult statisticalDLTResult360 = statisticalResult(DLTBalls360, 360); StatisticalDLTBall statisticalResult = new StatisticalDLTBall(statisticalDLTResult3, statisticalDLTResult5, statisticalDLTResult8, statisticalDLTResult10, statisticalDLTResult20, statisticalDLTResult30, statisticalDLTResult45, statisticalDLTResult60, statisticalDLTResult90, statisticalDLTResult120, statisticalDLTResult360);
} private static StatisticalDLTResult statisticalResult(List<DLTBall> DLTBalls, int group) {
long redDNum = DLTBalls.stream().filter(ball -> "单".equals(ball.getRedDS())).count();
long redSNum = DLTBalls.stream().filter(ball -> "双".equals(ball.getRedDS())).count();
long blueDNum = DLTBalls.stream().filter(ball -> "单".equals(ball.getBlueDS())).count();
long blueSNum = DLTBalls.stream().filter(ball -> "双".equals(ball.getBlueDS())).count();
int redSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedSum())).sum();
int blueSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueSum())).sum();
int redKDSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedKD())).sum();
int blueKDSum = DLTBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueKD())).sum();
int redBallMinNum = DLTBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int redBallMaxNum = DLTBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0);
int blueBallMinNum = DLTBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int blueBallMaxNum = DLTBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0); StatisticalDLTResult statisticalDLTResult = new StatisticalDLTResult();
statisticalDLTResult.setRedContrastDS((int) redDNum + ":" + redSNum);
statisticalDLTResult.setBlueContrastDS((int) blueDNum + ":" + blueSNum);
statisticalDLTResult.setRedBallMiddleNum(String.valueOf(redSum / group));
statisticalDLTResult.setBlueBallMiddleNum(String.valueOf(blueSum / group));
statisticalDLTResult.setRedBallKDMiddleNum(String.valueOf(redKDSum / group));
statisticalDLTResult.setBlueBallKDMiddleNum(String.valueOf(blueKDSum / group));
statisticalDLTResult.setRedBallMinNum(String.valueOf(redBallMinNum));
statisticalDLTResult.setRedBallMaxNum(String.valueOf(redBallMaxNum));
statisticalDLTResult.setBlueBallMinNum(String.valueOf(blueBallMinNum));
statisticalDLTResult.setBlueBallMaxNum(String.valueOf(blueBallMaxNum));
System.out.println("\r\n");
System.out.println("===========================================================大乐透近" + group + "天出球信息统计===========================================================");
System.out.println("红球单双比:" + statisticalDLTResult.getRedContrastDS() + " 蓝球单双比:" + statisticalDLTResult.getBlueContrastDS() + " 红球中位数:" + statisticalDLTResult.getRedBallMiddleNum() + " 蓝球中位数:" + statisticalDLTResult.getBlueBallMiddleNum() + " 红球跨度中位数:" + statisticalDLTResult.getRedBallKDMiddleNum() + " 蓝球跨度中位数:" + statisticalDLTResult.getBlueBallKDMiddleNum() + " 红球最小值:" + statisticalDLTResult.getRedBallMinNum() + " 红球最大值:" + statisticalDLTResult.getRedBallMaxNum() + " 蓝球最小值:" + statisticalDLTResult.getBlueBallMinNum() + " 蓝球最大值:" + statisticalDLTResult.getBlueBallMaxNum());
return statisticalDLTResult;
} private static void bodyHandle(Elements body, List<DLTBall> DLTBallHistory) {
body.forEach(tr -> {
List<String> redList = new ArrayList<>();
List<String> blueList = new ArrayList<>();
Integer blueSum = 0;
String blueDS;
Elements td = tr.getElementsByTag("td");
DLTBall.DLTBallBuilder builder = DLTBall.builder().date(td.get(0).text()).redSum(td.get(3).text()).redDS(td.get(4).text()).redJOB(td.get(5).text()).redKD(td.get(7).text());
Elements span = td.get(2).getElementsByTag("span");
int blueKD;
for (int i = 0; i < 7; i++) {
String text = span.get(i).text();
if (i < 5) {
redList.add(text);
} else {
Integer num = TypeUtils.castToInt(text);
blueSum += num;
blueList.add(text);
}
}
if (blueSum % 2 == 0) {
blueDS = "双";
} else {
blueDS = "单";
}
long ouNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 0).count();
long jiNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 1).count();
blueKD = TypeUtils.castToInt(blueList.get(1)) - TypeUtils.castToInt(blueList.get(0));
DLTBall build = builder.redBalls(redList).blueBalls(blueList).blueSum(TypeUtils.castToString(blueSum)).blueDS(blueDS).blueJOB(jiNum + ":" + ouNum).blueKD(TypeUtils.castToString(blueKD)).build();
System.out.println("开奖时间:" + build.getDate() + " 红:" + build.getRedBalls() + " 蓝:" + build.getBlueBalls() + " 红球sum值:" + build.getRedSum() + " 红球sum单双:" + build.getRedDS() + " 红球奇偶比:" + build.getRedJOB() + " 红球跨度:" + build.getRedKD() + " 蓝球sum值:" + build.getBlueSum() + " 蓝球sum单双:" + build.getBlueDS() + " 蓝球奇偶比:" + build.getBlueJOB() + " 蓝球跨度:" + build.getBlueKD());
DLTBallHistory.add(build);
});
} public static void main(String[] args) {
List<String> ad = new ArrayList<>();
ad.add("a");
ad.add("b");
ad.add("c");
System.out.println(ad);
} @Override
public Site getSite() {
if (site == null) {
site = Site.me().setDomain(".55128.cn").setSleepTime(1000);
}
return site;
} }
DoubleProcessor.java
双色球处理算子
package com.example.ball.handle; import com.alibaba.fastjson.util.TypeUtils;
import com.example.ball.entity.SSQBall;
import com.example.ball.entity.StatisticalSSQBall;
import com.example.ball.entity.StatisticalSSQResult;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html; import java.util.ArrayList;
import java.util.List; @Slf4j
public class DoubleProcessor implements PageProcessor { private Site site; @Override
public void process(Page page) {
Html pageHtml = page.getHtml();
String html = pageHtml.toString();
Document document = Jsoup.parse(html);
Elements body = document.getElementById("table").getElementsByTag("tbody").get(0).getElementsByTag("tr");
List<SSQBall> SSQBallHistory = new ArrayList<>();
bodyHandle(body, SSQBallHistory);
List<SSQBall> SSQBalls3 = SSQBallHistory.subList(0, 3);
List<SSQBall> SSQBalls5 = SSQBallHistory.subList(0, 5);
List<SSQBall> SSQBalls8 = SSQBallHistory.subList(0, 8);
List<SSQBall> SSQBalls10 = SSQBallHistory.subList(0, 10);
List<SSQBall> SSQBalls20 = SSQBallHistory.subList(0, 20);
List<SSQBall> SSQBalls30 = SSQBallHistory.subList(0, 30);
List<SSQBall> SSQBalls45 = SSQBallHistory.subList(0, 45);
List<SSQBall> SSQBalls60 = SSQBallHistory.subList(0, 60);
List<SSQBall> SSQBalls90 = SSQBallHistory.subList(0, 90);
List<SSQBall> SSQBalls120 = SSQBallHistory.subList(0, 120);
List<SSQBall> SSQBalls360 = SSQBallHistory.subList(0, 360); StatisticalSSQResult statisticalSSQResult3 = statisticalResult(SSQBalls3, 3);
StatisticalSSQResult statisticalSSQResult5 = statisticalResult(SSQBalls5, 5);
StatisticalSSQResult statisticalSSQResult8 = statisticalResult(SSQBalls8, 8);
StatisticalSSQResult statisticalSSQResult10 = statisticalResult(SSQBalls10, 10);
StatisticalSSQResult statisticalSSQResult20 = statisticalResult(SSQBalls20, 20);
StatisticalSSQResult statisticalSSQResult30 = statisticalResult(SSQBalls30, 30);
StatisticalSSQResult statisticalSSQResult45 = statisticalResult(SSQBalls45, 45);
StatisticalSSQResult statisticalSSQResult60 = statisticalResult(SSQBalls60, 60);
StatisticalSSQResult statisticalSSQResult90 = statisticalResult(SSQBalls90, 90);
StatisticalSSQResult statisticalSSQResult120 = statisticalResult(SSQBalls120, 120);
StatisticalSSQResult statisticalSSQResult360 = statisticalResult(SSQBalls360, 360); StatisticalSSQBall statisticalResult = new StatisticalSSQBall(statisticalSSQResult3, statisticalSSQResult5, statisticalSSQResult8, statisticalSSQResult10, statisticalSSQResult20, statisticalSSQResult30, statisticalSSQResult45, statisticalSSQResult60, statisticalSSQResult90, statisticalSSQResult120, statisticalSSQResult360);
} private static StatisticalSSQResult statisticalResult(List<SSQBall> SSQBalls, int group) {
long redDNum = SSQBalls.stream().filter(ball -> "单".equals(ball.getRedDS())).count();
long redSNum = SSQBalls.stream().filter(ball -> "双".equals(ball.getRedDS())).count();
long blueDNum = SSQBalls.stream().filter(ball -> "单".equals(ball.getBlueDS())).count();
long blueSNum = SSQBalls.stream().filter(ball -> "双".equals(ball.getBlueDS())).count();
int redSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedSum())).sum();
int blueSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueSum())).sum();
int redKDSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getRedKD())).sum();
int blueKDSum = SSQBalls.stream().mapToInt(ball -> TypeUtils.castToInt(ball.getBlueKD())).sum();
int redBallMinNum = SSQBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int redBallMaxNum = SSQBalls.stream().mapToInt(ball -> ball.getRedBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0);
int blueBallMinNum = SSQBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).min().orElse(0)).min().orElse(0);
int blueBallMaxNum = SSQBalls.stream().mapToInt(ball -> ball.getBlueBalls().stream().mapToInt(TypeUtils::castToInt).max().orElse(0)).max().orElse(0); StatisticalSSQResult statisticalSSQResult = new StatisticalSSQResult();
statisticalSSQResult.setRedContrastDS((int) redDNum + ":" + redSNum);
statisticalSSQResult.setBlueContrastDS((int) blueDNum + ":" + blueSNum);
statisticalSSQResult.setRedBallMiddleNum(String.valueOf(redSum / group));
statisticalSSQResult.setBlueBallMiddleNum(String.valueOf(blueSum / group));
statisticalSSQResult.setRedBallKDMiddleNum(String.valueOf(redKDSum / group));
statisticalSSQResult.setBlueBallKDMiddleNum(String.valueOf(blueKDSum / group));
statisticalSSQResult.setRedBallMinNum(String.valueOf(redBallMinNum));
statisticalSSQResult.setRedBallMaxNum(String.valueOf(redBallMaxNum));
statisticalSSQResult.setBlueBallMinNum(String.valueOf(blueBallMinNum));
statisticalSSQResult.setBlueBallMaxNum(String.valueOf(blueBallMaxNum));
System.out.println("\r\n");
System.out.println("===========================================================双色球近" + group + "天出球信息统计===========================================================");
System.out.println("红球单双比:" + statisticalSSQResult.getRedContrastDS() + " 蓝球单双比:" + statisticalSSQResult.getBlueContrastDS() + " 红球中位数:" + statisticalSSQResult.getRedBallMiddleNum() + " 蓝球中位数:" + statisticalSSQResult.getBlueBallMiddleNum() + " 红球跨度中位数:" + statisticalSSQResult.getRedBallKDMiddleNum() + " 蓝球跨度中位数:" + statisticalSSQResult.getBlueBallKDMiddleNum() + " 红球最小值:" + statisticalSSQResult.getRedBallMinNum() + " 红球最大值:" + statisticalSSQResult.getRedBallMaxNum() + " 蓝球最小值:" + statisticalSSQResult.getBlueBallMinNum() + " 蓝球最大值:" + statisticalSSQResult.getBlueBallMaxNum());
return statisticalSSQResult;
} private static void bodyHandle(Elements body, List<SSQBall> SSQBallHistory) {
body.forEach(tr -> {
List<String> redList = new ArrayList<>();
List<String> blueList = new ArrayList<>();
Integer blueSum = 0;
String blueDS;
Elements td = tr.getElementsByTag("td");
SSQBall.SSQBallBuilder builder = SSQBall.builder().date(td.get(0).text()).redSum(td.get(4).text()).redDS(td.get(5).text()).redJOB(td.get(6).text()).redKD(td.get(9).text());
Elements span = td.get(2).getElementsByTag("span");
int blueKD;
for (int i = 0; i < 7; i++) {
String text = span.get(i).text();
if (i < 6) {
redList.add(text);
} else {
Integer num = TypeUtils.castToInt(text);
blueSum += num;
blueList.add(text);
}
}
if (blueSum % 2 == 0) {
blueDS = "双";
} else {
blueDS = "单";
}
long ouNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 0).count();
long jiNum = blueList.stream().map(TypeUtils::castToInt).filter(integer -> integer % 2 == 1).count();
SSQBall build = builder.redBalls(redList).blueBalls(blueList).blueSum(TypeUtils.castToString(blueSum)).blueDS(blueDS).blueJOB(jiNum + ":" + ouNum).blueKD(TypeUtils.castToString(blueList.get(0))).build();
System.out.println("开奖时间:" + build.getDate() + " 红:" + build.getRedBalls() + " 蓝:" + build.getBlueBalls() + " 红球sum值:" + build.getRedSum() + " 红球sum单双:" + build.getRedDS() + " 红球奇偶比:" + build.getRedJOB() + " 红球跨度:" + build.getRedKD() + " 蓝球sum值:" + build.getBlueSum() + " 蓝球sum单双:" + build.getBlueDS() + " 蓝球奇偶比:" + build.getBlueJOB() + " 蓝球跨度:" + build.getBlueKD());
SSQBallHistory.add(build);
});
} @Override
public Site getSite() {
if (site == null) {
site = Site.me().setDomain(".55128.cn").setSleepTime(1000);
}
return site;
} }
实体类
DLTBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable;
import java.util.List; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class DLTBall implements Serializable { private String date; private List<String> redBalls; private List<String> blueBalls; //红色sum
private String redSum; //红色sum值单双
private String redDS; //红色奇偶比
private String redJOB; //红色跨度
private String redKD; //蓝色sum
private String blueSum; //蓝色sum值单双
private String blueDS; //蓝色奇偶比
private String blueJOB; //蓝色跨度
private String blueKD; }
SSQBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable;
import java.util.List; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class SSQBall implements Serializable { private String date; private List<String> redBalls; private List<String> blueBalls; //红色sum
private String redSum; //红色sum值单双
private String redDS; //红色奇偶比
private String redJOB; //红色跨度
private String redKD; //蓝色sum
private String blueSum; //蓝色sum值单双
private String blueDS; //蓝色奇偶比
private String blueJOB; //蓝色跨度
private String blueKD; }
StatisticalDLTBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalDLTBall implements Serializable { private StatisticalDLTResult statisticalDLTResult3;
private StatisticalDLTResult statisticalDLTResult5;
private StatisticalDLTResult statisticalDLTResult8;
private StatisticalDLTResult statisticalDLTResult10;
private StatisticalDLTResult statisticalDLTResult20;
private StatisticalDLTResult statisticalDLTResult30;
private StatisticalDLTResult statisticalDLTResult45;
private StatisticalDLTResult statisticalDLTResult60;
private StatisticalDLTResult statisticalDLTResult90;
private StatisticalDLTResult statisticalDLTResult120;
private StatisticalDLTResult statisticalDLTResult360; }
StatisticalDLTResult.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalDLTResult implements Serializable { //红球单双比
private String redContrastDS; //蓝球单双比
private String blueContrastDS; //红球中位数
private String redBallMiddleNum; //蓝球中位数
private String BlueBallMiddleNum; //红球跨度中位数
private String redBallKDMiddleNum; //蓝球跨度中位数
private String blueBallKDMiddleNum; //红球最小值
private String redBallMinNum; //红球最大值
private String redBallMaxNum; //蓝球最小值
private String blueBallMinNum; //蓝球最大值
private String blueBallMaxNum; }
StatisticalSSQBall.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalSSQBall implements Serializable { private StatisticalSSQResult statisticalSSQResult3;
private StatisticalSSQResult statisticalSSQResult5;
private StatisticalSSQResult statisticalSSQResult8;
private StatisticalSSQResult statisticalSSQResult10;
private StatisticalSSQResult statisticalSSQResult20;
private StatisticalSSQResult statisticalSSQResult30;
private StatisticalSSQResult statisticalSSQResult45;
private StatisticalSSQResult statisticalSSQResult60;
private StatisticalSSQResult statisticalSSQResult90;
private StatisticalSSQResult statisticalSSQResult120;
private StatisticalSSQResult statisticalSSQResult360; }
StatisticalSSQResult.java
package com.example.ball.entity; import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor; import java.io.Serializable; @Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class StatisticalSSQResult implements Serializable { //红球单双比
private String redContrastDS; //蓝球单双比
private String blueContrastDS; //红球中位数
private String redBallMiddleNum; //蓝球中位数
private String BlueBallMiddleNum; //红球跨度中位数
private String redBallKDMiddleNum; //蓝球跨度中位数
private String blueBallKDMiddleNum; //红球最小值
private String redBallMinNum; //红球最大值
private String redBallMaxNum; //蓝球最小值
private String blueBallMinNum; //蓝球最大值
private String blueBallMaxNum; }
基于SpringBoot WebMagic爬虫爬取大乐透双色球的更多相关文章
- 如何使用robots禁止各大搜索引擎爬虫爬取网站
ps:由于公司网站配置的测试环境被百度爬虫抓取,干扰了线上正常环境的使用,刚好看到每次搜索淘宝时,都会有一句由于robots.txt文件存在限制指令无法提供内容描述,于是便去学习了一波 1.原来一般来 ...
- python爬虫—爬取英文名以及正则表达式的介绍
python爬虫—爬取英文名以及正则表达式的介绍 爬取英文名: 一. 爬虫模块详细设计 (1)整体思路 对于本次爬取英文名数据的爬虫实现,我的思路是先将A-Z所有英文名的连接爬取出来,保存在一个cs ...
- MATLAB爬虫爬取股票数据
近年来,大数据盛行,有关爬虫的教程层次不穷.那么,爬虫到底是什么呢? 什么是爬虫? 百度百科是这样定义的: 网络爬虫(又被称为网页蜘蛛,网络机器人,在FOAF社区中间,更经常的称为网页追逐者),是一种 ...
- Python爬虫 - 爬取百度html代码前200行
Python爬虫 - 爬取百度html代码前200行 - 改进版, 增加了对字符串的.strip()处理 源代码如下: # 改进版, 增加了 .strip()方法的使用 # coding=utf-8 ...
- 如何利用Python网络爬虫爬取微信朋友圈动态--附代码（下）
前天给大家分享了如何利用Python网络爬虫爬取微信朋友圈数据的上篇(理论篇),今天给大家分享一下代码实现(实战篇),接着上篇往下继续深入. 一.代码实现 1.修改Scrapy项目中的items.py ...
- 使用scrapy爬虫,爬取17k小说网的案例-方法一
无意间看到17小说网里面有一些小说小故事,于是决定用爬虫爬取下来自己看着玩,下图这个页面就是要爬取的来源. a 这个页面一共有125个标题,每个标题里面对应一个内容,如下图所示 下面直接看最核心spi ...
- 基于 PHP 的数据爬取（QueryList）
基于PHP的数据爬取 官方网站站点 简单. 灵活.强大的PHP采集工具,让采集更简单一点. 简介: QueryList使用jQuery选择器来做采集,让你告别复杂的正则表达式:QueryList具有j ...
- python爬虫爬取京东、淘宝、苏宁上华为P20购买评论
爬虫爬取京东.淘宝.苏宁上华为P20购买评论 1.使用软件 Anaconda3 2.代码截图 三个网站代码大同小异,因此只展示一个 3.结果(部分) 京东 淘宝 苏宁 4.分析 这三个网站上的评论数据 ...
- 利用Python网络爬虫爬取学校官网十条标题
利用Python网络爬虫爬取学校官网十条标题 案例代码: # __author : "J" # date : 2018-03-06 # 导入需要用到的库文件 import urll ...
- node：爬虫爬取网页图片
代码地址如下:http://www.demodashi.com/demo/13845.html 前言 周末自己在家闲着没事,刷着微信,玩着手机,发现自己的微信头像该换了,就去网上找了一下头像,看着图片 ...
随机推荐
- 2022-4-7内部群每日三题-清辉PMP
1.公司聘用一名项目经理来协调一个期限紧迫的敏捷项目,项目经理和敏捷团队都由一位项目组合经理管理,该项目组合经理倾向于根据需要将开发人员重新分配给其他紧急事项,当项目经理与其接洽时,项目组合经理坚持认 ...
- IDEA2022 搭建SpringMVC
https://blog.csdn.net/LiuNengJing/article/details/125888494
- 实时中文语音克隆——开源项目MockingBird体验
[引子] 在今年大型网络攻防演练前不久,笔者接到一个公司的座机号码来电,上来就问防守准备得怎么样了,哪里还有不足等.等等,这声音不认识,笔者第一反应就是蓝军(Red Team)来进行社会工程攻击,于是 ...
- nmon 采坑
1.安装 wget http://sourceforge.net/projects/nmon/files/nmon16g_x86.tar.gz 2.解压 tar -zxvf nmon16g_x86.t ...
- 使用Latex错误集
1.写公式的函数--align(最怕空行) (1)align用法示例 \begin{align} & \left\{ \begin{array}{ll} \sup\limits_{\tilde ...
- 解决ESP8266反复启动问题
ESP8266刷不同固件后,有时会出现反复启动的问题,可能是FLASH里有未擦除干净的区域. 使用乐鑫 Flash 下载工具擦除 选好串口后点击erase擦除,即可正常使用
- oculus按键大全
// OVRInput.Update(); if (OVRInput.GetUp(OVRInput.Button.Three)) { Debug.Log("remote click" ...
- mockjs 加上 json-server 快速生成前端数据
const mock = require('mockjs'); // 引入mockjs const data = mock.mock({ "data|20": [{ "i ...
- axios使用总结
一.请求配置 // 引入import axios from 'axios';import qs from 'qs';this.$axios({ method:"get", // g ...
- 二，使用axios
1,下载https://unpkg.com/axios@1.3.2/dist/axios.min.js保存在js目录下,命名为axios.js 2,http.js let baseUrl = &quo ...