tampermonkey，采用js解析自定义脚本，实现网页列表数据采集分析

最近一直在做数据采集的事情，目的是使用java开发一套分析指定采集规则，模拟用户动作做数据提取。
因此定义了一套动作脚本，open，click，get，list,opentab，closetab。。。
java解析脚本，调用phantomjs做数据提取，生成数据json文件，对外提供数据接口。
采集引擎终于写的差不多了，虽然还有很多问题需要修改，但是终于不用加班了，嘿嘿嘿。-------jstarseven

码字挺累的，转载请注明出处：http://www.cnblogs.com/jstarseven/p/6278197.html
言归正传，由于一直搞这些东西，突然想着拿js去写个采集玩一玩，就用tampermonkey，毕竟好久没玩了。

简介：针对一些网站的数据列表，定义采集脚本，模拟用户操作，做列表数据提取，生成json数据格式化展示。

json采集脚本定义：

 {
     "type": "list",
     "selector": "",//列表选择器
     "max_page": 1,//采集页数
     "page_selector": "",//翻页选择器
     "iframe_selector": "",//iframe 选择器
     "datas": [//采集字段定义
         {
             "selector": " ",//字段选择器<此处为针对列表的子选择器>
             "column": "title",//字段名称
             "from": "text",//采集类型
             "iframe_selector": "",//iframe选择器 防止一些网站怪异 一般不需要
             "open_tab": [//当前字段开新标签做采集
                 {
                     "selector": " ",//新标签字段选择器
                     "column": " ",
                     "from": "text",
                     "iframe_selector": ""
                 },
                 {
                     "selector": " ",
                     "column": " ",
                     "from": "text",
                     "iframe_selector": ""
                 },
                 {
                     "selector": " ",
                     "column": " ",
                     "from": "text",
                     "iframe_selector": ""
                 }
             ]
         },
         {
             "selector": " ",//字段选择器
             "column": " ",
             "from": "text",
             "iframe_selector": ""
         },
         {
             "selector": " ",//字段选择器
             "column": " ",
             "from": "text",
             "iframe_selector": ""
         }
     ]
 }

脚本定义好了，剩下的就是写js代码解析脚本，做数据采集，数据合并了。
那么怎么去解析实现呢，针对新开标签页的数据采集，怎么样要和之前的列表项数据做合并，保证数据的完整性呢？
.因为数据需要做存储，首先想到这么多数据该怎么存储呢，首先想到sessionStorage，但是sessionStorage在我新开标签页的时候数据不能共享，
那么就用localStorage,localStorage一般上限5m左右，足以存储一般列表的十几页数据。
2.详情页面的数据和列表项数据合并，既然上面说到localStorage，那么就在localStorage里面放入一个指定的map，存放列表数据
针对列表的每一项做一个key，然后再新开标签的时候传递key，提取详情的数据，将详情页面数据，放入map中指定key的数据中。

js实现map方便数据存储：

 /*
  * MAP对象，实现MAP功能
  *
  * 接口：
  * size()     获取MAP元素个数
  * isEmpty()    判断MAP是否为空
  * clear()     删除MAP所有元素
  * put(key, value)   向MAP中增加元素（key, value)
  * remove(key)    删除指定KEY的元素，成功返回True，失败返回False
  * get(key)    获取指定KEY的元素值VALUE，失败返回NULL
  * element(index)   获取指定索引的元素（使用element.key，element.value获取KEY和VALUE），失败返回NULL
  * containsKey(key)  判断MAP中是否含有指定KEY的元素
  * containsValue(value) 判断MAP中是否含有指定VALUE的元素
  * values()    获取MAP中所有VALUE的数组（ARRAY）
  * keys()     获取MAP中所有KEY的数组（ARRAY）
  */
 function Map() {
     this.elements = [];

     //获取MAP元素个数
     this.size = function () {
         return this.elements.length;
     };

     //判断MAP是否为空
     this.isEmpty = function () {
         return (this.elements.length < 1);
     };

     //删除MAP所有元素
     this.clear = function () {
         this.elements = [];
     };

     //向MAP中增加元素（key, value)
     this.put = function (_key, _value) {
         for (var i = 0; i < this.elements.length; i++) {
             if (this.elements[i].key == _key) {
                 this.elements[i].value = _value;
                 return;
             }
         }
         this.elements.push({
             key: _key,
             value: _value
         });
     };

     //删除指定KEY的元素，成功返回True，失败返回False
     this.remove = function (_key) {
         var bln = false;
         try {
             for (var i = 0; i < this.elements.length; i++) {
                 if (this.elements[i].key == _key) {
                     this.elements.splice(i, 1);
                     return true;
                 }
             }
         } catch (e) {
             bln = false;
         }
         return bln;
     };

     //获取指定KEY的元素值VALUE，失败返回NULL
     this.get = function (_key) {
         try {
             for (var i = 0; i < this.elements.length; i++) {
                 if (this.elements[i].key == _key) {
                     return this.elements[i].value;
                 }
             }
         } catch (e) {
             return null;
         }
     };

     //获取指定索引的元素（使用element.key，element.value获取KEY和VALUE），失败返回NULL
     this.element = function (_index) {
         if (_index < 0 || _index >= this.elements.length) {
             return null;
         }
         return this.elements[_index];
     };

     //判断MAP中是否含有指定KEY的元素
     this.containsKey = function (_key) {
         var bln = false;
         try {
             for (var i = 0; i < this.elements.length; i++) {
                 if (this.elements[i].key == _key) {
                     bln = true;
                 }
             }
         } catch (e) {
             bln = false;
         }
         return bln;
     };

     //判断MAP中是否含有指定VALUE的元素
     this.containsValue = function (_value) {
         var bln = false;
         try {
             for (var i = 0; i < this.elements.length; i++) {
                 if (this.elements[i].value == _value) {
                     bln = true;
                 }
             }
         } catch (e) {
             bln = false;
         }
         return bln;
     };

     //获取MAP中所有VALUE的数组（ARRAY）
     this.values = function () {
         var arr = [];
         for (var i = 0; i < this.elements.length; i++) {
             arr.push(this.elements[i].value);
         }
         return arr;
     };

     //获取MAP中所有KEY的数组（ARRAY）
     this.keys = function () {
         var arr = [];
         for (var i = 0; i < this.elements.length; i++) {
             arr.push(this.elements[i].key);
         }
         return arr;
     };
 }

js实现操作localStorage：

 /**
  *获取当前任务配置信息
  */
 function getTaskDataMap() {
     var data_maps = localStorage.getItem("data_maps");
     var datas = new Map();
     if (isNullParam(data_maps)) {
         data_maps = datas;
     } else {
         datas.elements = JSON.parse(data_maps).elements;
         return datas;
     }
     return data_maps;
 }

 /**
  *清空当前任务配置信息
  */
 function clearTaskDataMap() {
     localStorage.setItem("data_maps", "");
 }

 /**
  * 当前任务添加配置信息
  * @param step_id  脚本步骤id
  * @param config   [doms,json]
  */
 function addTaskDataMap(key, values) {
     if (isNullParam(key) || isNullParam(values))
         return;
     var data_maps = getTaskDataMap();
     data_maps.put(key, values);
     localStorage.setItem("data_maps", JSON.stringify(data_maps));
 }

采用jquery.simulate.js实现点击

 /*!
  * jQuery Simulate v@VERSION - simulate browser mouse and keyboard events
  * https://github.com/jquery/jquery-simulate
  *
  * Copyright jQuery Foundation and other contributors
  * Released under the MIT license.
  * http://jquery.org/license
  *
  * Date: @DATE
  */

 ;(function ($, undefined) {

     var rkeyEvent = /^key/,
         rmouseEvent = /^(?:mouse|contextmenu)|click/;

     $.fn.simulate = function (type, options) {
         return this.each(function () {
             new $.simulate(this, type, options);
         });
     };

     $.simulate = function (elem, type, options) {
         var method = $.camelCase("simulate-" + type);

         this.target = elem;
         this.options = options;

         if (this[method]) {
             this[method]();
         } else {
             this.simulateEvent(elem, type, options);
         }
     };

     $.extend($.simulate, {

         keyCode: {
             BACKSPACE: 8,
             COMMA: 188,
             DELETE: 46,
             DOWN: 40,
             END: 35,
             ENTER: 13,
             ESCAPE: 27,
             HOME: 36,
             LEFT: 37,
             NUMPAD_ADD: 107,
             NUMPAD_DECIMAL: 110,
             NUMPAD_DIVIDE: 111,
             NUMPAD_ENTER: 108,
             NUMPAD_MULTIPLY: 106,
             NUMPAD_SUBTRACT: 109,
             PAGE_DOWN: 34,
             PAGE_UP: 33,
             PERIOD: 190,
             RIGHT: 39,
             SPACE: 32,
             TAB: 9,
             UP: 38
         },

         buttonCode: {
             LEFT: 0,
             MIDDLE: 1,
             RIGHT: 2
         }
     });

     $.extend($.simulate.prototype, {

         simulateEvent: function (elem, type, options) {
             var event = this.createEvent(type, options);
             this.dispatchEvent(elem, type, event, options);
         },

         createEvent: function (type, options) {
             if (rkeyEvent.test(type)) {
                 return this.keyEvent(type, options);
             }

             if (rmouseEvent.test(type)) {
                 return this.mouseEvent(type, options);
             }
         },

         mouseEvent: function (type, options) {
             var event, eventDoc, doc, body;
             options = $.extend({
                 bubbles: true,
                 cancelable: (type !== "mousemove"),
                 view: window,
                 detail: 0,
                 screenX: 0,
                 screenY: 0,
                 clientX: 1,
                 clientY: 1,
                 ctrlKey: false,
                 altKey: false,
                 shiftKey: false,
                 metaKey: false,
                 button: 0,
                 relatedTarget: undefined
             }, options);

             if (document.createEvent) {
                 event = document.createEvent("MouseEvents");
                 event.initMouseEvent(type, options.bubbles, options.cancelable,
                     options.view, options.detail,
                     options.screenX, options.screenY, options.clientX, options.clientY,
                     options.ctrlKey, options.altKey, options.shiftKey, options.metaKey,
                     options.button, options.relatedTarget || document.body.parentNode);

                 // IE 9+ creates events with pageX and pageY set to 0.
                 // Trying to modify the properties throws an error,
                 // so we define getters to return the correct values.
                 if (event.pageX === 0 && event.pageY === 0 && Object.defineProperty) {
                     eventDoc = event.relatedTarget.ownerDocument || document;
                     doc = eventDoc.documentElement;
                     body = eventDoc.body;

                     Object.defineProperty(event, "pageX", {
                         get: function () {
                             return options.clientX +
                                 ( doc && doc.scrollLeft || body && body.scrollLeft || 0 ) -
                                 ( doc && doc.clientLeft || body && body.clientLeft || 0 );
                         }
                     });
                     Object.defineProperty(event, "pageY", {
                         get: function () {
                             return options.clientY +
                                 ( doc && doc.scrollTop || body && body.scrollTop || 0 ) -
                                 ( doc && doc.clientTop || body && body.clientTop || 0 );
                         }
                     });
                 }
             } else if (document.createEventObject) {
                 event = document.createEventObject();
                 $.extend(event, options);
                 // standards event.button uses constants defined here: http://msdn.microsoft.com/en-us/library/ie/ff974877(v=vs.85).aspx
                 // old IE event.button uses constants defined here: http://msdn.microsoft.com/en-us/library/ie/ms533544(v=vs.85).aspx
                 // so we actually need to map the standard back to oldIE
                 event.button = {
                         0: 1,
                         1: 4,
                         2: 2
                     }[event.button] || ( event.button === -1 ? 0 : event.button );
             }

             return event;
         },

         keyEvent: function (type, options) {
             var event;
             options = $.extend({
                 bubbles: true,
                 cancelable: true,
                 view: window,
                 ctrlKey: false,
                 altKey: false,
                 shiftKey: false,
                 metaKey: false,
                 keyCode: 0,
                 charCode: undefined
             }, options);

             if (document.createEvent) {
                 try {
                     event = document.createEvent("KeyEvents");
                     event.initKeyEvent(type, options.bubbles, options.cancelable, options.view,
                         options.ctrlKey, options.altKey, options.shiftKey, options.metaKey,
                         options.keyCode, options.charCode);
                     // initKeyEvent throws an exception in WebKit
                     // see: http://stackoverflow.com/questions/6406784/initkeyevent-keypress-only-works-in-firefox-need-a-cross-browser-solution
                     // and also https://bugs.webkit.org/show_bug.cgi?id=13368
                     // fall back to a generic event until we decide to implement initKeyboardEvent
                 } catch (err) {
                     event = document.createEvent("Events");
                     event.initEvent(type, options.bubbles, options.cancelable);
                     $.extend(event, {
                         view: options.view,
                         ctrlKey: options.ctrlKey,
                         altKey: options.altKey,
                         shiftKey: options.shiftKey,
                         metaKey: options.metaKey,
                         keyCode: options.keyCode,
                         charCode: options.charCode
                     });
                 }
             } else if (document.createEventObject) {
                 event = document.createEventObject();
                 $.extend(event, options);
             }

             if (!!/msie [\w.]+/.exec(navigator.userAgent.toLowerCase()) || (({}).toString.call(window.opera) === "[object Opera]")) {
                 event.keyCode = (options.charCode > 0) ? options.charCode : options.keyCode;
                 event.charCode = undefined;
             }

             return event;
         },

         dispatchEvent: function (elem, type, event) {
             if (elem.dispatchEvent) {
                 elem.dispatchEvent(event);
             } else if (type === "click" && elem.click && elem.nodeName.toLowerCase() === "input") {
                 elem.click();
             } else if (elem.fireEvent) {
                 elem.fireEvent("on" + type, event);
             }
         },

         simulateFocus: function () {
             var focusinEvent,
                 triggered = false,
                 element = $(this.target);

             function trigger() {
                 triggered = true;
             }

             element.bind("focus", trigger);
             element[0].focus();

             if (!triggered) {
                 focusinEvent = $.Event("focusin");
                 focusinEvent.preventDefault();
                 element.trigger(focusinEvent);
                 element.triggerHandler("focus");
             }
             element.unbind("focus", trigger);
         },

         simulateBlur: function () {
             var focusoutEvent,
                 triggered = false,
                 element = $(this.target);

             function trigger() {
                 triggered = true;
             }

             element.bind("blur", trigger);
             element[0].blur();

             // blur events are async in IE
             setTimeout(function () {
                 // IE won't let the blur occur if the window is inactive
                 if (element[0].ownerDocument.activeElement === element[0]) {
                     element[0].ownerDocument.body.focus();
                 }

                 // Firefox won't trigger events if the window is inactive
                 // IE doesn't trigger events if we had to manually focus the body
                 if (!triggered) {
                     focusoutEvent = $.Event("focusout");
                     focusoutEvent.preventDefault();
                     element.trigger(focusoutEvent);
                     element.triggerHandler("blur");
                 }
                 element.unbind("blur", trigger);
             }, 1);
         }
     });

     /** complex events **/

     function findCenter(elem) {
         var offset,
             document = $(elem.ownerDocument);
         elem = $(elem);
         offset = elem.offset();

         return {
             x: offset.left + elem.outerWidth() / 2 - document.scrollLeft(),
             y: offset.top + elem.outerHeight() / 2 - document.scrollTop()
         };
     }

     function findCorner(elem) {
         var offset,
             document = $(elem.ownerDocument);
         elem = $(elem);
         offset = elem.offset();

         return {
             x: offset.left - document.scrollLeft(),
             y: offset.top - document.scrollTop()
         };
     }

     $.extend($.simulate.prototype, {
         simulateDrag: function () {
             var i = 0,
                 target = this.target,
                 eventDoc = target.ownerDocument,
                 options = this.options,
                 center = options.handle === "corner" ? findCorner(target) : findCenter(target),
                 x = Math.floor(center.x),
                 y = Math.floor(center.y),
                 coord = {clientX: x, clientY: y},
                 dx = options.dx || ( options.x !== undefined ? options.x - x : 0 ),
                 dy = options.dy || ( options.y !== undefined ? options.y - y : 0 ),
                 moves = options.moves || 3;

             this.simulateEvent(target, "mousedown", coord);

             for (; i < moves; i++) {
                 x += dx / moves;
                 y += dy / moves;

                 coord = {
                     clientX: Math.round(x),
                     clientY: Math.round(y)
                 };

                 this.simulateEvent(eventDoc, "mousemove", coord);
             }

             if ($.contains(eventDoc, target)) {
                 this.simulateEvent(target, "mouseup", coord);
                 this.simulateEvent(target, "click", coord);
             } else {
                 this.simulateEvent(eventDoc, "mouseup", coord);
             }
         }
     });

 })(jQuery);

格式化json数据，高亮显示

 /**
  * 格式化json
  * @param json
  * @returns {string|XML}
  */
 function jsonSyntaxHighLight(json) {
     if (typeof json != 'string')
         json = JSON.stringify(json, undefined, 2);
     json = json.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
     return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g, function (match) {
         var cls = 'number';
         if (/^"/.test(match)) {
             if (/:$/.test(match)) {
                 cls = 'key';
             } else {
                 cls = 'string';
             }
         } else if (/true|false/.test(match)) {
             cls = 'boolean';
         } else if (/null/.test(match)) {
             cls = 'null';
         }
         return '<span class="' + cls + '">' + match + '</span>';
     });
 }

操作：
（以懒财网公告为例，测试）目前已经测试懒财，cnblog。。。
1.首先安装tampermonkey插件下载地址： http://tampermonkey.net/
2.新建脚本，复制web-extract-list.js 内容粘贴 ctrl+s
3.新建脚本，复制web-extract-detail.js 内容粘贴 ctrl+s
4.打开https://www.lancai.cn/about/notice.html 看执行效果

采集结束之后，json页面：

注意：根据采集的网站不同需要变更js文件里面的// @match 处匹配的url， 以及task_json的脚本配置信息

项目代码github地址：https://github.com/jstarseven/web-list-extract

码字挺累的，转载请注明出处：http://www.cnblogs.com/jstarseven/p/6278197.html

---

-END-