使用代码查看Nutch爬取的网站后生成的SequenceFile信息

必须针对data文件中的value类型来使用对应的类来查看(把这个data文件,放到了本地Windows的D盘根目录下).

代码:

 package cn.summerchill.nutch;
 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.SequenceFile;
 import org.apache.hadoop.io.Text;
 import org.apache.nutch.crawl.CrawlDatum;
 import org.apache.nutch.crawl.Inlinks;
 import org.apache.nutch.parse.ParseData;
 import org.apache.nutch.parse.ParseText;
 import org.apache.nutch.protocol.Content;
 /**
  * 读取nutch生成的sequencefile文件
  * @author Administrator
  *
  */
 public class SeFileReader {
     public static void main(String[] args) throws IOException {
         Configuration conf=new Configuration();
         Path dataPath=new Path("D:\\data");
         FileSystem fs=dataPath.getFileSystem(conf);
         SequenceFile.Reader reader=new SequenceFile.Reader(fs,dataPath,conf);
         Text key=new Text();
         CrawlDatum value=new CrawlDatum();
         //Content value = new Content();
         //Inlinks value = new Inlinks();
         //ParseText value = new ParseText();
         //ParseData value = new ParseData();
         while(reader.next(key,value)){
             System.out.println("key->\n"+key);
             System.err.println("value->\n"+value);
             try {
                 Thread.sleep(1000);
             } catch (InterruptedException e) {
                 e.printStackTrace();
             }
             System.out.println("=======================================");
         }
         reader.close();
     }
 }

运行结果:

key->
http://bbs.superwu.cn/
value->
Version: 7
Status: 2 (db_fetched)
Fetch time: Tue Nov 08 08:31:30 CST 2016
Modified time: Thu Jan 01 08:00:00 CST 1970
Retries since fetch: 0
Retry interval: 2592000 seconds (30 days)
Score: 1.6153846
Signature: 22defcd7cb4e7b1dc8a16a0a2f339ecb
Metadata:
     Content-Type=application/xhtml+xml
    _pst_=success(1), lastModified=0
    _rs_=610

=======================================
value->
Version: 7
Status: 1 (db_unfetched)
Fetch time: Sun Oct 09 08:31:35 CST 2016
Modified time: Thu Jan 01 08:00:00 CST 1970
Retries since fetch: 0
Retry interval: 2592000 seconds (30 days)
Score: 0.23076925
Signature: null
Metadata: 

key->
http://bbs.superwu.cn/archiver/
=======================================
key->
http://bbs.superwu.cn/forum.php
value->
Version: 7
Status: 1 (db_unfetched)
Fetch time: Sun Oct 09 08:31:35 CST 2016
Modified time: Thu Jan 01 08:00:00 CST 1970
Retries since fetch: 0
Retry interval: 2592000 seconds (30 days)
Score: 0.15384616
Signature: null
Metadata: 

=======================================