java实现网页结构分析列表发现
原文出处：http://www.yund.tech/zdetail.html?type=1&id=ee06002e2b83e7677c30aedc52d3429e
作者：jstarseven
现在的网站千奇百怪，什么样格式的都有，需要提取网页中的列表数据，有时候挨个分析处理很头疼，本文是一个页面结构分析的程序，可以分析处理页面大致列表结构。
废话不多说，我也不会说，show me code，code is terrible，so what hahaha。-------jstarseven
1.抽取元素dom结构框架
/**
* 分析元素dom结构框架
*
* @param node
* @return
*/
public String filterHtml(Element node) {
//去除节点的属性值
Document new_node = Jsoup.parse(node.outerHtml());
Elements elements = new_node.getAllElements();
for (Element item : elements) {
Attributes attributes = item.attributes();
for (Attribute a : attributes) {
if (a.getKey().equals(KeysEnum.attr_scroce)) {
item.removeAttr(a.getKey());
continue;
}
a.setValue(StringUtils.EMPTY);
}
}
//去除注释节点,节点文本内容
String str_new = new_node.outerHtml().replaceAll("<!--?(.*?)-->", "");
str_new = str_new.replaceAll("\\s*", "");
str_new = str_new.replaceAll(">(.*?)<", "><");
return str_new;
}
2.采用动态规划处理两个字符串相似度
public class SimilarDegree {
public static final double degree = 0.8;
/**
* 采用动态规划的方法解决
*
* @param source
* @param target
* @return
*/
public static int EditDistance(String source, String target) {
char[] sources = source.toCharArray();
char[] targets = target.toCharArray();
int sourceLen = sources.length;
int targetLen = targets.length;
int[][] d = new int[sourceLen + 1][targetLen + 1];
for (int i = 0; i <= sourceLen; i++) {
d[i][0] = i;
}
for (int i = 0; i <= targetLen; i++) {
d[0][i] = i;
}
for (int i = 1; i <= sourceLen; i++) {
for (int j = 1; j <= targetLen; j++) {
if (sources[i - 1] == targets[j - 1]) {
d[i][j] = d[i - 1][j - 1];
} else {
//插入
int insert = d[i][j - 1] + 1;
//删除
int delete = d[i - 1][j] + 1;
//替换
int replace = d[i - 1][j - 1] + 1;
d[i][j] = Math.min(insert, delete) > Math.min(delete, replace) ? Math.min(delete, replace) :
Math.min(insert, delete);
}
}
}
return d[sourceLen][targetLen];
}
public static void main(String[] args) {
System.out.println(EditDistance("html > body > ul > li.proiect_item:nth-child(1) > div.item_row.item_row_title > div:nth-child(1) > a",
"html > body > ul > li.proiect_item:nth-child(2) > div.item_row.item_row_title > div:nth-child(1) > a"));
}
}
3.对网页中每个节点的一级孩子节点分类
/**
* 统计列表下各个一级节点类型及个数
*
* @param node
* @return
*/
private Map<String, Integer> getGroupNode(Element node) {
Map<String, Integer> map = new HashMap<String, Integer>();
Elements children = node.children();
for (Element item : children) {
if (KeysEnum.input.equalsIgnoreCase(item.tagName()) || KeysEnum.br.equalsIgnoreCase(item.tagName())
|| KeysEnum.script.equalsIgnoreCase(item.tagName()) || KeysEnum.link.equalsIgnoreCase(item.tagName())
|| KeysEnum.style.equalsIgnoreCase(item.tagName()) || KeysEnum.meta.equalsIgnoreCase(item.tagName())
|| KeysEnum.select.equalsIgnoreCase(item.tagName()) || KeysEnum.option.equalsIgnoreCase(item.tagName())
|| KeysEnum.video.equals(item.tagName()) || KeysEnum.audio.equals(item.tagName())
|| KeysEnum.textarea.equals(item.tagName())) continue;
String key = filterHtml(item);
if (map.containsKey(key)) {
map.put(key, (Integer) map.get(key) + 1);
} else {
boolean is_like = false;
for (String map_key : map.keySet()) {
int dis = SimilarDegree.EditDistance(key, (String) map_key);
float v = (float) (key.length() - dis) / key.length();
if (v > SimilarDegree.degree) {
map.put(map_key, (Integer) map.get(map_key) + 1);
is_like = true;
break;
}
}
if (!is_like) map.put(key, 1);
}
}
return map;
}
4.处理网页中每个元素的叶子节点
/**
* 获取叶子节点选择器
*
* @param node
*/
public static List<String> getYeziNodeSel(Element node) {
List<String> list = new ArrayList<String>();
Elements all = node.getAllElements();
for (Element item : all) {
if (item.children().isEmpty()) list.add(item.cssSelector());
}
return list;
}
5.时间提取工具类
/**
* jstarseven
* 通用时间处理类 return Date
* */
public class DateParser {
private static int timezone = 0;
private static final Pattern[] DPTN = { Pattern.compile(
"(\\d{1,2})[\\s\\-\\/](\\d{1,2})[\\s\\-\\/](20\\d{2})\\s{0,2}((\\d{1,2})[:\\s](\\d{1,2})[:\\s]?(\\d{1,2})?)?"), Pattern.compile(
"((20)?\\d{2}) {0,2}[\\.\\-/年] {0,2}(\\d{1,2}) {0,2}[\\.\\-/月] {0,2}(\\d{1,2}) {0,2}[日 \\s]{0,2}((上午)|(下午))?\\s{0,2}((\\d{1,2})[:\\s时](\\d{1,2})[:\\s分]?(\\d{1,2})?)?"), Pattern.compile("((20)?\\d{2})/(\\d{2})(\\d{2})"), Pattern.compile(
"(\\d{1,2})[\\.\\-\\s/月](\\d{1,2})[日\\s]{0,2}((上午)|(下午))?\\s{0,2}((\\d{1,2})[:\\s](\\d{1,2})[:\\s]?(\\d{1,2})?)?"), Pattern.compile("([今前昨]天)?\\s{0,4}(\\d{1,2})[:\\s]{1,3}(\\d{1,2})[:\\s]?(\\d{1,2})?"), Pattern.compile("[今前昨]天"), Pattern.compile("((\\d{1,2})|(半))\\s*个?([天秒小时分钟周月年]{1,2})前"), Pattern.compile("(\\d{1,2})小?时(\\d{1,2})分钟?前"), Pattern.compile("(20\\d{2})[01]?(\\d{2})[012]?(\\d{2})") }; public static Date parse(Object obj) {
if (obj == null) {
return null;
}
if ((obj instanceof Date)) {
return (Date) obj;
}
if ((obj instanceof Number)) {
return new Date(((Number) obj).longValue());
}
String str = ((String) obj).trim();
if ((str.length() == 0) || ("null".equalsIgnoreCase(str))) {
return null;
}
str = transZH(str);
Calendar c = Calendar.getInstance();
c.setTimeInMillis(System.currentTimeMillis()); Matcher mt = DPTN[0].matcher(str);
if (mt.find()) {
int date = Integer.parseInt(mt.group(2));
if ((date == 0) || (date > 31)) {
return null;
}
int month = Integer.parseInt(mt.group(1));
if (month <= 0) {
return null;
}
if (month > 12) {
if ((date > 0) && (date <= 12) && (month < 32)) {
int tmp = month;
month = date;
date = tmp;
} else {
return null;
}
}
String sy = mt.group(3);
int year = Integer.parseInt(sy);
if ((year < 2000) || (year > 2099)) {
return null;
}
String hms = mt.group(4);
if ((hms == null) || (hms.length() == 0)) {
c.set(year, month - 1, date, timezone > 0 ? timezone : 0, 0, 0);
return c.getTime();
}
int hour = Integer.parseInt(mt.group(5));
if (hour >= 24) {
return null;
}
int min = Integer.parseInt(mt.group(6));
if (min >= 60) {
return null;
}
String ssec = mt.group(7);
int sec = (ssec == null) || (ssec.length() == 0) ? 0 : Integer.parseInt(ssec);
c.set(year, month - 1, date, hour, min, sec);
return c.getTime();
}
mt = DPTN[1].matcher(str);
if (mt.find()) {
String sy = mt.group(1);
if (sy.length() == 2) {
sy = "20" + sy;
}
int year = Integer.parseInt(sy);
if ((year < 2000) || (year > 2099)) {
return null;
}
int month = Integer.parseInt(mt.group(3)) - 1;
if ((month < 0) || (month > 11)) {
return null;
}
int date = Integer.parseInt(mt.group(4));
if (date > 31) {
return null;
}
String ss = mt.group(8);
if ((ss == null) || (ss.length() == 0)) {
c.set(year, month, date, timezone > 0 ? timezone : 0, 0, 0);
return c.getTime();
}
int hour = Integer.parseInt(mt.group(9));
if (hour >= 24) {
return null;
}
int min = Integer.parseInt(mt.group(10));
if (min >= 60) {
return null;
}
String ssec = mt.group(11);
int sec = (ssec == null) || (ssec.length() == 0) ? 0 : Integer.parseInt(ssec);
if (("下午".equals(mt.group(5))) && (hour < 12)) {
hour += 12;
}
c.set(year, month, date, hour, min, sec);
return c.getTime();
}
mt = DPTN[2].matcher(str);
if (mt.find()) {
String strYear = mt.group(1);
if (!strYear.startsWith("20")) {
strYear = "20" + strYear;
}
int year = Integer.parseInt(strYear);
int month = Integer.parseInt(mt.group(3)) - 1;
int day = Integer.parseInt(mt.group(4));
c.set(year, month, day, 0, 0, 0);
return c.getTime();
}
mt = DPTN[3].matcher(str);
if (mt.find()) {
int year = c.get(1);
int month = Integer.parseInt(mt.group(1)) - 1;
if (month < 0) {
return null;
}
if (month > c.get(2)) {
year--;
}
int date = Integer.parseInt(mt.group(2));
if (date > 31) {
return null;
}
String p = mt.group(6);
if ((p == null) || (p.length() == 0)) {
c.set(year, month, date, timezone > 0 ? timezone : 0, 0, 0);
return c.getTime();
}
int hour = Integer.parseInt(mt.group(7));
if (hour >= 24) {
return null;
}
int min = Integer.parseInt(mt.group(8));
if (min >= 60) {
return null;
}
String ssec = mt.group(9);
int sec = (ssec == null) || (ssec.length() == 0) ? 0 : Integer.parseInt(ssec);
if (("下午".equals(mt.group(3))) && (hour < 12)) {
hour += 12;
}
c.set(year, month, date, hour, min, sec);
return c.getTime();
}
mt = DPTN[4].matcher(str);
if (mt.find()) {
int hour = Integer.parseInt(mt.group(2));
if (hour >= 24) {
return null;
}
int min = Integer.parseInt(mt.group(3));
if (min >= 60) {
return null;
}
String day = mt.group(1);
if ("昨天".equals(day)) {
c.add(5, -1);
} else if ("前天".equals(day)) {
c.add(5, -2);
}
c.set(11, hour);
c.set(12, min);
return c.getTime();
}
mt = DPTN[5].matcher(str);
if (mt.find()) {
String day = mt.group(0);
if ("昨天".equals(day)) {
c.add(5, -1);
} else if ("前天".equals(day)) {
c.add(5, -2);
}
return c.getTime();
}
mt = DPTN[6].matcher(str);
if (mt.find()) {
String s = mt.group(4);
long t;
if ("年".equals(s)) {
t = 31536000000L;
} else {
if ("月".equals(s)) {
t = 2592000000L;
} else {
if ("周".equals(s)) {
t = 604800000L;
} else {
if ("天".equals(s)) {
t = 86400000L;
} else {
if ("小时".equals(s)) {
t = 3600000L;
} else {
if ("时".equals(s)) {
t = 3600000L;
} else {
if ("分钟".equals(s)) {
t = 60000L;
} else {
if ("分".equals(s)) {
t = 60000L;
} else {
if ("秒".equals(s)) {
t = 1000L;
} else {
return null;
}
}
}
}
}
}
}
}
}
String vs = mt.group(1);
if ("半".equals(vs)) {
t = System.currentTimeMillis() - t / 2L;
} else {
t = System.currentTimeMillis() - Integer.parseInt(vs) * t;
}
return new Date(t);
}
mt = DPTN[7].matcher(str);
if (mt.find()) {
int hh = Integer.parseInt(mt.group(1));
int nn = Integer.parseInt(mt.group(2));
long t = 3600000 * hh + 60000 * nn;
return new Date(System.currentTimeMillis() - t);
}
mt = DPTN[8].matcher(str);
if (mt.find()) {
String sy = mt.group(1);
int year = Integer.parseInt(sy);
if ((year < 2000) || (year > 2099)) {
return null;
}
int month = Integer.parseInt(mt.group(2)) - 1;
if ((month < 0) || (month > 11)) {
return null;
}
int date = Integer.parseInt(mt.group(3));
if (date > 31) {
return null;
}
c.set(year, month, date, timezone > 0 ? timezone : 0, 0, 0);
return c.getTime();
}
return null;
} private static String transZH(String string) {
String zh = "〇一二三四五六七八九";
string = string.replace("整", "0分").replaceAll("[上下]午", "");
StringBuffer buffer = new StringBuffer();
for (Character Char : string.toCharArray()) {
int index = zh.indexOf(Char);
if (index >= 0) {
buffer.append(index);
} else {
buffer.append(Char);
}
}
String str = buffer.toString();
int index = str.indexOf("十");
if (index == -1) {
return str;
} else {
if (!Character.isDigit(str.charAt(index-1)) && !Character.isDigit(str.charAt(index+1))) {
str=str.replace("十", "10");
}else if (Character.isDigit(str.charAt(index-1)) && !Character.isDigit(str.charAt(index+1))) {
str=str.replace("十", "0");
}else if(!Character.isDigit(str.charAt(index-1)) && Character.isDigit(str.charAt(index+1))){
str=str.replace("十", "1");
}else if(Character.isDigit(str.charAt(index-1)) && Character.isDigit(str.charAt(index+1))){
str=str.replace("十", "");
}
return str;
} } public static void main(String[] args) {
System.out.println(parse("1982-01-01 00:00:00"));
System.out.println(transZH("二〇一七年九月十日 上午十时整"));
System.out.println(transZH("二〇一七年九月二十日 上午九时整"));
System.out.println(transZH("二〇一七年九月十九日 上午九时整"));
System.out.println(transZH("二〇一七年九月二十三日 上午九时整"));
System.out.println("timezone=" + timezone);
String[] testdata = { "1982-01-01 00:00:00","11-13 15:24", "2009-8-30 16:42:10", "8-23 15:24", "2周前", "3 天前", "12 分钟前", "3天前",
"前天 09:36", "昨天 09:21 ", "2010-12-17 00:23 ", "2010-12-17 ", "昨天 12:37 ", "2011-8-15 08:42",
"25-7-2011 11:43:57", "1-9-2011", "06-03", "半小时前", "今天发表", "昨天发表", "前天发表", "06-03-2010",
"02-01-2010 00:39", "3小时26分钟前", "2010-8-24 上午 01:17:32", "2010-8-24 下午 01:17:32", "7小时前 »",
"4/29/2010 1:31:00", "2012 年 1 月 31 日", "17时20分前", "2017年10月12日 14时30分", "二〇一七年九月十九日 上午九时整" }; DateFormat df = DateFormat.getDateTimeInstance(2, 2);
for (String s : testdata) {
Date d = parse(s);
System.out.println(s + "\t\t" + (d == null ? d : df.format(d)));
}
} }
6.自定义比较器对网页所有元素排序，发现结果靠前的基本都是列表元素
比较器：按照疑似列表的可能性
/**
* 排序子节点
* 1.最大相同dom结构长度
* 2.最大相同dom结构元素数量
*
* @param nodes
* @return
*/
private Elements sortBy(Elements nodes, String base_url) {
// System.setProperty("java.util.Arrays.useLegacyMergeSort", "true");
nodes.sort(new Comparator<Element>() {
@Override
public int compare(Element o1, Element o2) {
double o1_rate = reckonRate(o1);
double o2_rate = reckonRate(o2);
return (o2_rate > o1_rate) ? 1 : ((o2_rate == o1_rate) ? 0 : -1);
} private double reckonRate(Element o) {
if (StringUtils.isNotBlank(base_url) && KeysEnum.a.equalsIgnoreCase(o.tagName()) && base_url.equalsIgnoreCase(o.attr(KeysEnum.attr_href)))
o.attr(KeysEnum.attr_list_tag_name, o.text());
if (null == o || o.children().size() < 2
|| KeysEnum.html.equalsIgnoreCase(o.tagName()) || KeysEnum.body.equalsIgnoreCase(o.tagName()) || KeysEnum.link.equalsIgnoreCase(o.tagName())
|| KeysEnum.head.equalsIgnoreCase(o.tagName()) || KeysEnum.title.equalsIgnoreCase(o.tagName()) || KeysEnum.meta.equalsIgnoreCase(o.tagName())
|| KeysEnum.script.equalsIgnoreCase(o.tagName()) || KeysEnum.style.equalsIgnoreCase(o.tagName())) {
o.attr(KeysEnum.attr_scroce, "0");
return 0;
}
String style = o.attr(KeysEnum.style);
if (StringUtils.isNotBlank(style) && style.contains(KeysEnum.display_none)) {
o.attr(KeysEnum.attr_scroce, "0");
return 0;
}
Map<String, Object> maxKeyDom = getMaxKeyDom(o);
String key = (String) maxKeyDom.get(KeysEnum.max_key);
int num = (int) maxKeyDom.get(KeysEnum.max_num);
if (num < 2) {
o.attr(KeysEnum.attr_scroce, "0");
return 0;
}
int scroce = num * key.length();
Elements tags = o.children();
for (Element a : tags) {
if (KeysEnum.div.equalsIgnoreCase(a.tagName())) scroce += 5;
if (KeysEnum.ul.equalsIgnoreCase(a.tagName())) scroce += 10;
if (KeysEnum.li.equalsIgnoreCase(a.tagName())) scroce += 10;
if (KeysEnum.tbody.equalsIgnoreCase(a.tagName())) scroce += 5;
if (KeysEnum.table.equalsIgnoreCase(a.tagName())) scroce += 5;
if (KeysEnum.tr.equalsIgnoreCase(a.tagName())) scroce += 10;
if (KeysEnum.td.equalsIgnoreCase(a.tagName())) scroce += 1;
if (KeysEnum.a.equalsIgnoreCase(a.tagName())) scroce += 1;
if (KeysEnum.p.equalsIgnoreCase(a.tagName())) scroce += 1;
try {
Date time = DateParser.parse(a.text());
if (null != time) scroce += 20;
} catch (Exception e) {
}
}
if (o.text().contains(KeysEnum.next_page)) scroce += 100;
if (o.text().contains(KeysEnum.start_page) || o.text().contains(KeysEnum.fisrt_page)) scroce += 100;
if (o.text().contains(KeysEnum.end_page) || o.text().contains(KeysEnum.last_page) || o.text().contains(KeysEnum.final_page))
scroce += 100;
o.attr(KeysEnum.attr_scroce, String.valueOf(scroce));
return scroce;
}
});
return nodes;
}
7.处理页面html，调用列表分析返回json结果
/**
* 提取页面列表元素的选择器以及页面分类标签
*
* @param document
* @param is_subitem
* @return
*/
public static Map<String, Object> dealListNode(Document document, boolean is_subitem) throws Exception {
Map<String, Object> result = new HashMap<String, Object>();
try {
ListAutoFire listAutoFire = new ListAutoFire();
Elements list_node = listAutoFire.autoFireListNodes(document);
List<Map<String, Object>> lists = new ArrayList();
if (null != list_node && list_node.size() > 0) {
for (Element list_sel_item : list_node) {
if (list_sel_item.hasAttr(KeysEnum.attr_list_tag_name) && StringUtils.isNotBlank(list_sel_item.attr(KeysEnum.attr_list_tag_name))) {
result.put(KeysEnum.tag_name, list_sel_item.attr(KeysEnum.attr_list_tag_name));
continue;
}
Map<String, Object> list_dom_frame = new HashMap<>();
list_dom_frame.put(KeysEnum.list_sel, list_sel_item.cssSelector());
if (is_subitem) {
Map<String, List<String>> listItem = new HashMap<String, List<String>>();
for (Element item : list_sel_item.children())
listItem.put(item.cssSelector(), getYeziNodeSel(item));
list_dom_frame.put(KeysEnum.list_dom, listItem);
}
list_dom_frame.put(KeysEnum.attr_scroce, list_sel_item.attr(KeysEnum.attr_scroce));
lists.add(list_dom_frame);
}
}
result.put(KeysEnum.list, lists);
} catch (Exception e) {
throw new Exception(KeysEnum.error_info, e.getCause());
}
return result;
} /**
* 处理网页结构
*
* @param home_url 入口地址
* @param list_index 列表元素获取数量
* @param is_subitem 是否处理列表元素子项抽取 true/false
* @param is_ifr 是否处理iframe true/false
* @return
*/
public static Map<String, Object> getWebSiteFrame(String home_url, int list_index, boolean is_subitem, boolean is_ifr) {
Map<String, Object> result = new HashMap<String, Object>();
if (StringUtils.isBlank(home_url)) return result;
try {
Document html = Jsoup.connect(home_url).ignoreContentType(true).validateTLSCertificates(false).timeout(5000).get();
if (null == html) throw new Exception(KeysEnum.open_fail);
Map<String, Object> mapNode = dealListNode(html, is_subitem);
List listNode = (List) mapNode.get(KeysEnum.list);
result.put(KeysEnum.home_url, home_url);
result.put(KeysEnum.tag_name, mapNode.get(KeysEnum.tag_name));
result.put(KeysEnum.list, listNode.subList(0, listNode.size() > list_index ? list_index : listNode.size()));
result.put(KeysEnum.ifrs, new ArrayList());
if (is_ifr) {
List<Map<String, Object>> ifrs = (List<Map<String, Object>>) result.get(KeysEnum.ifrs);
Elements iframe_nodes = html.getElementsByTag(KeysEnum.iframe);
if (null != iframe_nodes) {
for (Element iframe : iframe_nodes) {
String iframe_url = iframe.attr(KeysEnum.attr_src);
if (StringUtils.isBlank(iframe_url)) continue;
try {
Document iframe_html = Jsoup.connect(iframe_url).ignoreContentType(true).validateTLSCertificates(false).timeout(5000).get();
if (null == iframe_html) continue;
Map<String, Object> ifrMapNode = dealListNode(iframe_html, is_subitem);
List ifrListNode = (List) ifrMapNode.get(KeysEnum.list);
Map<String, Object> ifr_map = new HashMap();
ifr_map.put(KeysEnum.home_url, iframe_url);
ifr_map.put(KeysEnum.tag_name, ifrMapNode.get(KeysEnum.tag_name));
ifr_map.put(KeysEnum.list, ifrListNode.subList(0, ifrListNode.size() > list_index ? list_index : ifrListNode.size()));
ifrs.add(ifr_map);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
result.clear();
result.put(KeysEnum.home_url, home_url);
result.put(KeysEnum.error, KeysEnum.error_info);
result.put(KeysEnum.message, e.toString());
}
return result;
} /**
* 处理网页结构
*
* @param home_url 入口地址
* @param list_index 列表元素获取数量
* @param is_subitem 是否处理列表元素子项抽取 true/false
* @return
*/
public static Map<String, Object> getWebSiteFrame(String home_url, int list_index, boolean is_subitem) {
return getWebSiteFrame(home_url, list_index, is_subitem, false);
} /**
* 处理网页结构
*
* @param home_url 入口地址
* @param list_index 列表元素获取数量
* @return
*/
public static Map<String, Object> getWebSiteFrame(String home_url, int list_index) {
return getWebSiteFrame(home_url, list_index, false);
} /**
* 处理网页结构
*
* @param home_url 入口地址
* @return
*/
public static Map<String, Object> getWebSiteFrame(String home_url) {
return getWebSiteFrame(home_url, 10);
}
8.生成页面分析结果标记文件
public static void createMarkFile(Map siteFrame, String home_url, String path) {
try {
Document doc = Jsoup.connect(home_url).ignoreContentType(true).validateTLSCertificates(false).timeout(5000).get();
if (null == doc) return;
String style = ".mark_color {" +
"position:relative;" +
"pointer-events:none;" +
"left:0px;top:0px;" +
"display:inline-block;" +
"margin:-2px;width:100%;" +
"height:100%;" +
"border:dashed 2px #FF69B4;" +
"background-color: #43CD80;" +
"opacity:0.75;" +
"} " ;
List list = (List) siteFrame.get("list");
for (Object item : list) {
Map item_map = (Map) item;
String sel = (String) item_map.get("list_sel");
doc.select(sel).addClass("mark_color");
}
String content = doc.html();
content = content.contains("<base") ? content : content.replaceFirst("<head", "<base href='" + home_url + "'/><style>" + style + "</style><head");
FileUtils.writeStringToFile(new File(path), content, "UTF-8", false);
} catch (IOException e) {
e.printStackTrace();
}
}
9.上述第7步返回的结果实例：
拿cnblog首页做测试，返回结果：
字段解释：
home_url ：分析的页面地址
tag_name ：当前页面的类型，多数情况下不正确，我只是拿home_url和页面的url比对，取了对应的text
list：页面中疑似列表元素
list_sel：页面中疑似列表元素的选择器
list_dom：页面中疑似列表元素的 一级孩子节点元素，叶子元素选择器
ifrs：页面中包含iframe分析的结果，没有则为空
{
"home_url": "https://www.cnblogs.com/",
"tag_name": "1",
"list": [
{
"list_sel": "#post_list",
"list_dom": {
"#post_list > div.post_item:nth-child(7)": [
"#digg_count_9500831",
"#post_list > div.post_item:nth-child(7) > div.digg > div.clear",
"#digg_tip_9500831",
"#post_list > div.post_item:nth-child(7) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(7) > div.post_item_body > p.post_item_summary",
"#post_list > div.post_item:nth-child(7) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(7) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(7) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(7) > div.clear"
],
"#post_list > div.post_item:nth-child(19)": [
"#digg_count_9499348",
"#post_list > div.post_item:nth-child(19) > div.digg > div.clear",
"#digg_tip_9499348",
"#post_list > div.post_item:nth-child(19) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(19) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(19) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(19) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(19) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(19) > div.clear"
],
"#post_list > div.post_item:nth-child(6)": [
"#digg_count_9500833",
"#post_list > div.post_item:nth-child(6) > div.digg > div.clear",
"#digg_tip_9500833",
"#post_list > div.post_item:nth-child(6) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(6) > div.post_item_body > p.post_item_summary",
"#post_list > div.post_item:nth-child(6) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(6) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(6) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(6) > div.clear"
],
"#post_list > div.post_item:nth-child(9)": [
"#digg_count_9500757",
"#post_list > div.post_item:nth-child(9) > div.digg > div.clear",
"#digg_tip_9500757",
"#post_list > div.post_item:nth-child(9) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(9) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(9) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(9) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(9) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(9) > div.clear"
],
"#post_list > div.post_item:nth-child(17)": [
"#digg_count_9495616",
"#post_list > div.post_item:nth-child(17) > div.digg > div.clear",
"#digg_tip_9495616",
"#post_list > div.post_item:nth-child(17) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(17) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(17) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(17) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(17) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(17) > div.clear"
],
"#post_list > div.post_item:nth-child(8)": [
"#digg_count_9500822",
"#post_list > div.post_item:nth-child(8) > div.digg > div.clear",
"#digg_tip_9500822",
"#post_list > div.post_item:nth-child(8) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(8) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(8) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(8) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(8) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(8) > div.clear"
],
"#post_list > div.post_item:nth-child(18)": [
"#digg_count_9499454",
"#post_list > div.post_item:nth-child(18) > div.digg > div.clear",
"#digg_tip_9499454",
"#post_list > div.post_item:nth-child(18) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(18) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(18) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(18) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(18) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(18) > div.clear"
],
"#post_list > div.post_item:nth-child(3)": [
"#digg_count_9500944",
"#post_list > div.post_item:nth-child(3) > div.digg > div.clear",
"#digg_tip_9500944",
"#post_list > div.post_item:nth-child(3) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(3) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(3) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(3) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(3) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(3) > div.clear"
],
"#post_list > div.post_item:nth-child(2)": [
"#digg_count_9500357",
"#post_list > div.post_item:nth-child(2) > div.digg > div.clear",
"#digg_tip_9500357",
"#post_list > div.post_item:nth-child(2) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(2) > div.post_item_body > p.post_item_summary",
"#post_list > div.post_item:nth-child(2) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(2) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(2) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(2) > div.clear"
],
"#post_list > div.post_item:nth-child(5)": [
"#digg_count_9500890",
"#post_list > div.post_item:nth-child(5) > div.digg > div.clear",
"#digg_tip_9500890",
"#post_list > div.post_item:nth-child(5) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(5) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(5) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(5) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(5) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(5) > div.clear"
],
"#post_list > div.post_item:nth-child(4)": [
"#digg_count_9500935",
"#post_list > div.post_item:nth-child(4) > div.digg > div.clear",
"#digg_tip_9500935",
"#post_list > div.post_item:nth-child(4) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(4) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(4) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(4) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(4) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(4) > div.clear"
],
"#post_list > div.post_item:nth-child(1)": [
"#digg_count_9501071",
"#post_list > div.post_item:nth-child(1) > div.digg > div.clear",
"#digg_tip_9501071",
"#post_list > div.post_item:nth-child(1) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(1) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(1) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(1) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(1) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(1) > div.clear"
],
"#post_list > div.post_item:nth-child(15)": [
"#digg_count_9403762",
"#post_list > div.post_item:nth-child(15) > div.digg > div.clear",
"#digg_tip_9403762",
"#post_list > div.post_item:nth-child(15) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(15) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(15) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(15) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(15) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(15) > div.clear"
],
"#post_list > div.post_item:nth-child(16)": [
"#digg_count_9499534",
"#post_list > div.post_item:nth-child(16) > div.digg > div.clear",
"#digg_tip_9499534",
"#post_list > div.post_item:nth-child(16) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(16) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(16) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(16) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(16) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(16) > div.clear"
],
"#post_list > div.post_item:nth-child(13)": [
"#digg_count_9465698",
"#post_list > div.post_item:nth-child(13) > div.digg > div.clear",
"#digg_tip_9465698",
"#post_list > div.post_item:nth-child(13) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(13) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(13) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(13) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(13) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(13) > div.clear"
],
"#post_list > div.post_item:nth-child(14)": [
"#digg_count_9498410",
"#post_list > div.post_item:nth-child(14) > div.digg > div.clear",
"#digg_tip_9498410",
"#post_list > div.post_item:nth-child(14) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(14) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(14) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(14) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(14) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(14) > div.clear"
],
"#post_list > div.post_item:nth-child(11)": [
"#digg_count_9500633",
"#post_list > div.post_item:nth-child(11) > div.digg > div.clear",
"#digg_tip_9500633",
"#post_list > div.post_item:nth-child(11) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(11) > div.post_item_body > p.post_item_summary",
"#post_list > div.post_item:nth-child(11) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(11) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(11) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(11) > div.clear"
],
"#post_list > div.post_item:nth-child(12)": [
"#digg_count_9500352",
"#post_list > div.post_item:nth-child(12) > div.digg > div.clear",
"#digg_tip_9500352",
"#post_list > div.post_item:nth-child(12) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(12) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(12) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(12) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(12) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(12) > div.clear"
],
"#post_list > div.post_item:nth-child(20)": [
"#digg_count_9499225",
"#post_list > div.post_item:nth-child(20) > div.digg > div.clear",
"#digg_tip_9499225",
"#post_list > div.post_item:nth-child(20) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(20) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(20) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(20) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(20) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(20) > div.clear"
],
"#post_list > div.post_item:nth-child(10)": [
"#digg_count_9500632",
"#post_list > div.post_item:nth-child(10) > div.digg > div.clear",
"#digg_tip_9500632",
"#post_list > div.post_item:nth-child(10) > div.post_item_body > h3 > a.titlelnk",
"#post_list > div.post_item:nth-child(10) > div.post_item_body > p.post_item_summary > a > img.pfs",
"#post_list > div.post_item:nth-child(10) > div.post_item_body > div.post_item_foot > a.lightblue",
"#post_list > div.post_item:nth-child(10) > div.post_item_body > div.post_item_foot > span.article_comment > a.gray",
"#post_list > div.post_item:nth-child(10) > div.post_item_body > div.post_item_foot > span.article_view > a.gray",
"#post_list > div.post_item:nth-child(10) > div.clear"
]
},
"scroce": "9860"
},
{
"list_sel": "#cate_item",
"list_dom": {
"#cate_item_108705": [
"#cate_item_108705 > a"
],
"#cate_item_108704": [
"#cate_item_108704 > a"
],
"#cate_item_108703": [
"#cate_item_108703 > a"
],
"#cate_item_4": [
"#cate_item_4 > a"
],
"#cate_item_2": [
"#cate_item_2 > a"
],
"#cate_item_108709": [
"#cate_item_108709 > a"
],
"#cate_item_0": [
"#cate_item_0 > a"
],
"#cate_item_108698": [
"#cate_item_108698 > a"
],
"#cate_item_108724": [
"#cate_item_108724 > a"
],
"#cate_item_108701": [
"#cate_item_108701 > a"
],
"#cate_item_108712": [
"#cate_item_108712 > a"
],
"#cate_item_-1": [
"#cate_item_-1 > a"
]
},
"scroce": "1248"
},
{
"list_sel": "#friend_link",
"list_dom": {
"#friend_link > a:nth-child(15)": [
"#friend_link > a:nth-child(15)"
],
"#friend_link > a:nth-child(16)": [
"#friend_link > a:nth-child(16)"
],
"#friend_link > a:nth-child(17)": [
"#friend_link > a:nth-child(17)"
],
"#friend_link > a:nth-child(18)": [
"#friend_link > a:nth-child(18)"
],
"#friend_link > a:nth-child(1)": [
"#friend_link > a:nth-child(1)"
],
"#friend_link > a:nth-child(11)": [
"#friend_link > a:nth-child(11)"
],
"#friend_link > a:nth-child(12)": [
"#friend_link > a:nth-child(12)"
],
"#friend_link > a:nth-child(3)": [
"#friend_link > a:nth-child(3)"
],
"#friend_link > a:nth-child(13)": [
"#friend_link > a:nth-child(13)"
],
"#friend_link > a:nth-child(2)": [
"#friend_link > a:nth-child(2)"
],
"#friend_link > a:nth-child(14)": [
"#friend_link > a:nth-child(14)"
],
"#friend_link > a:nth-child(19)": [
"#friend_link > a:nth-child(19)"
],
"#friend_link > a:nth-child(5)": [
"#friend_link > a:nth-child(5)"
],
"#friend_link > a:nth-child(4)": [
"#friend_link > a:nth-child(4)"
],
"#friend_link > a:nth-child(7)": [
"#friend_link > a:nth-child(7)"
],
"#friend_link > a:nth-child(6)": [
"#friend_link > a:nth-child(6)"
],
"#friend_link > a:nth-child(10)": [
"#friend_link > a:nth-child(10)"
],
"#friend_link > a:nth-child(9)": [
"#friend_link > a:nth-child(9)"
],
"#friend_link > a:nth-child(8)": [
"#friend_link > a:nth-child(8)"
]
},
"scroce": "1197"
},
{
"list_sel": "#side_nav",
"list_dom": {
"#side_nav > div.w_l:nth-child(16)": [
"#side_nav > div.w_l:nth-child(16) > h4",
"#site_stats"
],
"#side_nav > p.r_l_1:nth-child(7)": [
"#side_nav > p.r_l_1:nth-child(7)"
],
"#side_nav > p.r_l_2:nth-child(8)": [
"#side_nav > p.r_l_2:nth-child(8)"
],
"#side_nav > p.r_l_3:nth-child(9)": [
"#side_nav > p.r_l_3:nth-child(9)"
],
"#side_nav > p.r_l_1:nth-child(5)": [
"#side_nav > p.r_l_1:nth-child(5)"
],
"#side_nav > p.r_l_3:nth-child(13)": [
"#side_nav > p.r_l_3:nth-child(13)"
],
"#side_nav > p.r_l_2:nth-child(4)": [
"#side_nav > p.r_l_2:nth-child(4)"
],
"#side_nav > p.r_l_3:nth-child(19)": [
"#side_nav > p.r_l_3:nth-child(19)"
],
"#side_nav > p.r_l_3:nth-child(3)": [
"#side_nav > p.r_l_3:nth-child(3)"
],
"#side_nav > div.w_l:nth-child(6)": [
"#side_nav > div.w_l:nth-child(6) > h4",
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(1) > a",
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(2) > a",
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(3) > a",
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(4) > a",
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(5) > a",
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(6) > a"
],
"#side_nav > p.r_l_2:nth-child(18)": [
"#side_nav > p.r_l_2:nth-child(18)"
],
"#side_nav > div.l_s:nth-child(12)": [
"#side_nav > div.l_s:nth-child(12)"
],
"#cate_title_block": [
"#cate_title_title > div.cate_title",
"#cate_item_108698 > a",
"#cate_item_2 > a",
"#cate_item_108701 > a",
"#cate_item_108703 > a",
"#cate_item_108704 > a",
"#cate_item_108705 > a",
"#cate_item_108709 > a",
"#cate_item_108712 > a",
"#cate_item_108724 > a",
"#cate_item_4 > a",
"#cate_item_0 > a",
"#cate_item_-1 > a",
"#cate_title_block > div.cate_bottom",
"#cate_sub_block",
"#cate_title_block > script"
],
"#side_nav > div.l_s:nth-child(2)": [
"#side_nav > div.l_s:nth-child(2)"
],
"#side_nav > p.r_l_1:nth-child(17)": [
"#side_nav > p.r_l_1:nth-child(17)"
],
"#side_nav > p.r_l_2:nth-child(14)": [
"#side_nav > p.r_l_2:nth-child(14)"
],
"#side_nav > p.r_l_1:nth-child(15)": [
"#side_nav > p.r_l_1:nth-child(15)"
],
"#user_stats": [
"#user_stats"
],
"#side_nav > div.l_s:nth-child(10)": [
"#side_nav > div.l_s:nth-child(10)"
]
},
"scroce": "975"
},
{
"list_sel": "#paging_block > div.pager",
"list_dom": {
"#paging_block > div.pager > a.p_9.middle": [
"#paging_block > div.pager > a.p_9.middle"
],
"#paging_block > div.pager > a.p_7.middle": [
"#paging_block > div.pager > a.p_7.middle"
],
"#paging_block > div.pager > a.p_8.middle": [
"#paging_block > div.pager > a.p_8.middle"
],
"#paging_block > div.pager > a:nth-child(14)": [
"#paging_block > div.pager > a:nth-child(14)"
],
"#paging_block > div.pager > a.p_11.middle": [
"#paging_block > div.pager > a.p_11.middle"
],
"#paging_block > div.pager > a.p_3.middle": [
"#paging_block > div.pager > a.p_3.middle"
],
"#paging_block > div.pager > a.p_4.middle": [
"#paging_block > div.pager > a.p_4.middle"
],
"#paging_block > div.pager > a.p_10.middle": [
"#paging_block > div.pager > a.p_10.middle"
],
"#paging_block > div.pager > a.p_2.middle": [
"#paging_block > div.pager > a.p_2.middle"
],
"#paging_block > div.pager > a.p_5.middle": [
"#paging_block > div.pager > a.p_5.middle"
],
"#paging_block > div.pager > a.p_6.middle": [
"#paging_block > div.pager > a.p_6.middle"
],
"#paging_block > div.pager > a.p_1.current": [
"#paging_block > div.pager > a.p_1.current"
],
"#paging_block > div.pager > span.ellipsis": [
"#paging_block > div.pager > span.ellipsis"
],
"#paging_block > div.pager > a.p_200.last": [
"#paging_block > div.pager > a.p_200.last"
]
},
"scroce": "865"
},
{
"list_sel": "#main > div.post_nav_block_wrapper > ul.post_nav_block",
"list_dom": {
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(1)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(1) > a.current_nav"
],
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(3)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(3) > a"
],
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(2)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(2) > a"
],
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(5)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(5) > a"
],
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(4)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(4) > a"
],
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(7)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(7) > a"
],
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(6)": [
"#main > div.post_nav_block_wrapper > ul.post_nav_block > li:nth-child(6) > a"
]
},
"scroce": "590"
},
{
"list_sel": "#nav_menu",
"list_dom": {
"#nav_menu > a:nth-child(3)": [
"#nav_menu > a:nth-child(3)"
],
"#nav_menu > a:nth-child(2)": [
"#nav_menu > a:nth-child(2)"
],
"#nav_menu > a:nth-child(5)": [
"#nav_menu > a:nth-child(5)"
],
"#nav_menu > a:nth-child(4)": [
"#nav_menu > a:nth-child(4)"
],
"#nav_menu > a:nth-child(1)": [
"#nav_menu > a:nth-child(1)"
],
"#nav_menu > a:nth-child(7)": [
"#nav_menu > a:nth-child(7)"
],
"#nav_menu > a:nth-child(6)": [
"#nav_menu > a:nth-child(6)"
],
"#nav_menu > a:nth-child(9)": [
"#nav_menu > a:nth-child(9)"
],
"#nav_menu > a:nth-child(8)": [
"#nav_menu > a:nth-child(8)"
]
},
"scroce": "486"
},
{
"list_sel": "#side_nav > div.w_l:nth-child(6) > ul",
"list_dom": {
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(3)": [
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(3) > a"
],
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(2)": [
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(2) > a"
],
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(1)": [
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(1) > a"
],
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(6)": [
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(6) > a"
],
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(5)": [
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(5) > a"
],
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(4)": [
"#side_nav > div.w_l:nth-child(6) > ul > li:nth-child(4) > a"
]
},
"scroce": "486"
},
{
"list_sel": "#headline_block > ul",
"list_dom": {
"#headline_block > ul > li:nth-child(4)": [
"#headline_block > ul > li:nth-child(4) > a:nth-child(1)",
"#headline_block > ul > li:nth-child(4) > a.right_more"
],
"#headline_block > ul > li.editor_pick": [
"#editor_pick_count",
"#headline_block > ul > li.editor_pick > a.right_more"
],
"#headline_block > ul > li:nth-child(3)": [
"#headline_block > ul > li:nth-child(3) > a:nth-child(1)",
"#headline_block > ul > li:nth-child(3) > a.right_more"
],
"#headline_block > ul > li:nth-child(2)": [
"#headline_block > ul > li:nth-child(2) > a:nth-child(1)",
"#headline_block > ul > li:nth-child(2) > a.right_more"
]
},
"scroce": "407"
},
{
"list_sel": "#header",
"list_dom": {
"#header > p.h_r_3:nth-child(1)": [
"#header > p.h_r_3:nth-child(1)"
],
"#header > p.h_r_2:nth-child(6)": [
"#header > p.h_r_2:nth-child(6)"
],
"#header > p.h_r_1:nth-child(3)": [
"#header > p.h_r_1:nth-child(3)"
],
"#header > p.h_r_2:nth-child(2)": [
"#header > p.h_r_2:nth-child(2)"
],
"#header > p.h_r_1:nth-child(5)": [
"#header > p.h_r_1:nth-child(5)"
],
"#header > p.h_r_3:nth-child(7)": [
"#header > p.h_r_3:nth-child(7)"
],
"#header_block": [
"#logo > h1 > a > img",
"#header_block > div.clear"
]
},
"scroce": "335"
}
],
"ifrs": []
}
10.上述第8步标记文件效果：
红色虚线框起来的是返回的json结果中list中的list_sel选择器选中的元素
分析结果统计：
处理了将近1万的网站发现，大致的网页列表结构可以发现，平时时间大致在2-3s左右，因为用的是jsoup访问的网页，包含了网页响应的时间，时间复杂度待优化，
分析结果对于一些比较复杂乱的网页支持有待加强，代码写的比较乱，有待优化，应该会有更好的处理方式，还请指教，相互学习交流。
转载请注明出处：https://www.cnblogs.com/jstarseven/p/9501210.html
源码地址：https://github.com/jstarseven/list-autofire
-END-
java实现网页结构分析列表发现的更多相关文章
- JAVA 异常类型结构分析
JAVA 异常类型结构分析 Throwable 是所有异常类型的基类,Throwable 下一层分为两个分支,Error 和 Exception. Error 和 Exception Error Er ...
- java中把list列表转为arrayList以及arraylist数组截取的简单方法
java中把list列表转为arrayList以及arraylist数组截取的简单方法 package xiaobai; import java.util.ArrayList; import java ...
- 用java编网页的学习流程,我的一些小心得(初学java到高深运用)
(1)java基础:首先得会写int,String,for循环,数组,**等等(熟练各种基础的关键字,各种java自带的排序,随即等等算法)什么是封装,继承,多态,然后private,public,p ...
- 如何以Java实现网页截图技术
转自 http://blog.csdn.net/cping1982/article/details/5353049 今天看到某网友关于“如何以Java实现网页截图技术”的咨询帖,由于出现该咨询的地 ...
- java读取网页图片路径并下载到本地
java读取网页图片路径并下载到本地 最近公司需要爬取一些网页上的数据,自己就简单的写了一个demo,其中有一些数据是图片,需要下载下来到本地并且 将图片的路径保存到数据库,示例代码如下: packa ...
- Java获取系统安装软件列表
/** * @author <a href="mailto:foohsinglong@gmail.com">kevin.long</a> * @descri ...
- [Python] - 使用chardet检查网页编码格式时发现的问题
最近在使用chardet检查网页编码格式时发现如下问题: 用urllib打开网页再检查编码格式和用urllib2打开网页检查编码格式结果不一样,所以urllib2打开可能导致问题,需要关注. 查看了相 ...
- java中如何使用列表数组
java中如何使用列表数组 觉得有用的话,欢迎一起讨论相互学习~Follow Me 转载链接 https://blog.csdn.net/hgtjcxy/article/details/8183519 ...
- JAVA字符串处理函数列表一览
JAVA字符串处理函数列表一览 Java中的字符串也是一连串的字符.但是与许多其他的计算机语言将字符串作为字符数组处理不同,Java将字符串作为String类型对象来处理.将字符串作为内置的对象处 ...
随机推荐
- [leetcode]58. Length of Last Word最后一个词的长度
Given a string s consists of upper/lower-case alphabets and empty space characters ' ', return the l ...
- [leetcode]23. Merge k Sorted Lists归并k个有序链表
Merge k sorted linked lists and return it as one sorted list. Analyze and describe its complexity. I ...
- ES6 Iterator
不同数据集合怎么用统一的方式读取 可以用for...of循环了
- SQL Server 2008 R2 链接 Oracle
参考网站: SP_addlinkedserver 小结 (oracle,sql server,access,excel) 64位SqlServer通过链接服务器与32位oracle通讯 SQL Ser ...
- c#关于Mysql MySqlBulkLoader 批量上传
有个list表有几万数据 用insert插入,速度跟蜗牛爬行, 几十个表,传起来可就需要时间了. 搜搜,发现有 MySqlBulkLoader 这个人家mysql 的dll 里边已经提供了这个方法 ...
- ES之五：ElasticSearch聚合
前言 说完了ES的索引与检索,接着再介绍一个ES高级功能API – 聚合(Aggregations),聚合功能为ES注入了统计分析的血统,使用户在面对大数据提取统计指标时变得游刃有余.同样的工作,你在 ...
- json字符转对象之new Function('return ' + str)
var jsonStr = '{"id":1,"name":"linda","hobbies":[{"id&q ...
- linux_文件夹实现挂载（必须在同一网段）
将外部想要挂载传输的目录开启共享文件夹 首先进行安装 yum install nfs-utils rpcbind yum install nfs* 建立想要挂载的目录 查看可以执行挂载的目录有哪些 s ...
- 判断jquery对象是否具有某指定属性或者方法的几种方法
1.typeof 运算符:返回一个用来表示表达式的数据类型的字符串.("number", "string", "boolean", &quo ...
- 【repost】jQuery笔记总结
第一节 jQuery初步认知 jQuery概述 JQuery概念 javascript概念 基于Js语言的API和语法组织逻辑,通过内置window和document对象,来操作内存中的DOM元素 J ...