Lucene开发实例：Lucene中文分词(转载)

1、准备工作
下载lucene 3.6.1 ： http://lucene.apache.org/
下载中文分词IK Analyzer： http://code.google.com/p/ik-analyzer/downloads/list （注意下载的是IK Analyzer 2012_u5_source.zip，其他版本有bug）
下载solr 3.6.1：  http://lucene.apache.org/solr/（编译IK Analyzer时需引用包）
OK，将lucene 、solr 相关包（lucene-core-3.6.1.jar、lucene-highlighter-3.6.1.jar、lucene-analyzers-3.6.1.jar、apache-solr-core-3.6.1.jar、apache-solr-solrj-3.6.1.jar）拷贝到项目lib下，IK源码置于项目src下。

2、从Oracle数据库中取数据创建索引（使用IK分词）

  1 package lucene.util;
  2
  3 import org.apache.lucene.index.IndexWriter;
  4 import org.apache.lucene.index.IndexWriterConfig;
  5 import org.apache.lucene.index.CorruptIndexException;
  6 import org.apache.lucene.store.FSDirectory;
  7 import org.apache.lucene.store.Directory;
  8 import org.apache.lucene.analysis.Analyzer;
  9 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 10 import org.apache.lucene.util.Version;
 11 import org.apache.lucene.document.Document;
 12 import org.apache.lucene.document.Field;
 13 import org.wltea.analyzer.lucene.IKAnalyzer;
 14
 15 import java.sql.Connection;
 16 import java.io.File;
 17 import java.io.IOException;
 18 import java.util.ArrayList;
 19 import java.util.Date;
 20
 21 import modules.gk.Gk_info;
 22 import modules.gk.Gk_infoSub;
 23 import web.sys.Globals;
 24 import web.db.DBConnector;
 25 import web.db.ObjectCtl;
 26 import web.util.StringUtil;
 27 //Wizzer.cn
 28 public class LuceneIndex {
 29     IndexWriter writer = null;
 30     FSDirectory dir = null;
 31     boolean create = true;
 32
 33     public void init() {
 34         long a1 = System.currentTimeMillis();
 35         System.out.println("[Lucene 开始执行：" + new Date() + "]");
 36         Connection con = DBConnector.getconecttion(); //取得一个数据库连接
 37         try {
 38             final File docDir = new File(Globals.SYS_COM_CONFIG.get("sys.index.path").toString());//E:\lucene
 39             if (!docDir.exists()) {
 40                 docDir.mkdirs();
 41             }
 42             String cr = Globals.SYS_COM_CONFIG.get("sys.index.create").toString();//true or false
 43             if ("false".equals(cr.toLowerCase())) {
 44                 create = false;
 45             }
 46             Directory dir = FSDirectory.open(docDir);
 47 //            Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_36);
 48             Analyzer analyzer = new IKAnalyzer(true);
 49             IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_36, analyzer);
 50             if (create) {
 51                 // Create a new index in the directory, removing any
 52                 // previously indexed documents:
 53                 iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
 54             } else {
 55                 // Add new documents to an existing index:
 56                 iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
 57             }
 58             IndexWriter writer = new IndexWriter(dir, iwc);
 59             String sql = "SELECT indexno,title,describes,pdate,keywords FROM TABLEA WHERE STATE=1 AND SSTAG<>1 ";
 60             int rowCount = ObjectCtl.getRowCount(con, sql);
 61             int pageSize = StringUtil.StringToInt(Globals.SYS_COM_CONFIG.get("sys.index.size").toString());   //每页记录数
 62             int pages = (rowCount - 1) / pageSize + 1; //计算总页数
 63             ArrayList list = null;
 64             Gk_infoSub gk = null;
 65             for (int i = 1; i < pages+1; i++) {
 66                 long a = System.currentTimeMillis();
 67                 list = ObjectCtl.listPage(con, sql, i, pageSize, new Gk_infoSub());
 68                 for (int j = 0; j < list.size(); j++) {
 69                     gk = (Gk_infoSub) list.get(j);
 70                     Document doc = new Document();
 71                     doc.add(new Field("indexno", StringUtil.null2String(gk.getIndexno()), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));//主键不分词
 72                     doc.add(new Field("title", StringUtil.null2String(gk.getTitle()), Field.Store.YES, Field.Index.ANALYZED));
 73                     doc.add(new Field("describes", StringUtil.null2String(gk.getDescribes()), Field.Store.YES, Field.Index.ANALYZED));
 74                     doc.add(new Field("pdate", StringUtil.null2String(gk.getPdate()), Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS));//日期不分词
 75                     doc.add(new Field("keywords", StringUtil.null2String(gk.getKeywords()), Field.Store.YES, Field.Index.ANALYZED));
 76                     writer.addDocument(doc);
 77                     ObjectCtl.executeUpdateBySql(con,"UPDATE TABLEA SET SSTAG=1 WHERE indexno='"+gk.getIndexno()+"'");//更新已索引状态
 78                 }
 79
 80                 long b = System.currentTimeMillis();
 81                 long c = b - a;
 82                 System.out.println("[Lucene " + rowCount + "条，" + pages + "页，第" + i + "页花费时间：" + c + "毫秒]");
 83             }
 84             writer.commit();
 85
 86         } catch (Exception e) {
 87             e.printStackTrace();
 88         } finally {
 89             DBConnector.freecon(con); //释放数据库连接
 90             try {
 91                 if (writer != null) {
 92                     writer.close();
 93                 }
 94             } catch (CorruptIndexException e) {
 95                 e.printStackTrace();
 96             } catch (IOException e) {
 97                 e.printStackTrace();
 98             } finally {
 99                 try {
100                     if (dir != null && IndexWriter.isLocked(dir)) {
101                         IndexWriter.unlock(dir);//注意解锁
102                     }
103                 } catch (IOException e) {
104                     e.printStackTrace();
105                 }
106             }
107         }
108         long b1 = System.currentTimeMillis();
109         long c1 = b1 - a1;
110         System.out.println("[Lucene 执行完毕，花费时间：" + c1 + "毫秒，完成时间：" + new Date() + "]");
111     }
112 }

3、单字段查询以及多字段分页查询高亮显示

  1 package lucene.util;
  2
  3 import org.apache.lucene.store.FSDirectory;
  4 import org.apache.lucene.store.Directory;
  5 import org.apache.lucene.search.*;
  6 import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
  7 import org.apache.lucene.search.highlight.Highlighter;
  8 import org.apache.lucene.search.highlight.SimpleFragmenter;
  9 import org.apache.lucene.search.highlight.QueryScorer;
 10 import org.apache.lucene.queryParser.QueryParser;
 11 import org.apache.lucene.queryParser.MultiFieldQueryParser;
 12 import org.apache.lucene.analysis.TokenStream;
 13 import org.apache.lucene.analysis.Analyzer;
 14 import org.apache.lucene.analysis.KeywordAnalyzer;
 15 import org.apache.lucene.document.Document;
 16 import org.apache.lucene.index.IndexReader;
 17 import org.apache.lucene.index.Term;
 18 import org.apache.lucene.util.Version;
 19 import modules.gk.Gk_infoSub;
 20
 21 import java.util.ArrayList;
 22 import java.io.File;
 23 import java.io.StringReader;
 24 import java.lang.reflect.Constructor;
 25
 26 import web.util.StringUtil;
 27 import web.sys.Globals;
 28 import org.wltea.analyzer.lucene.IKAnalyzer;
 29 //Wizzer.cn
 30 public class LuceneQuery {
 31     private static String indexPath;// 索引生成的目录
 32     private int rowCount;// 记录数
 33     private int pages;// 总页数
 34     private int currentPage;// 当前页数
 35     private int pageSize;   //每页记录数
 36
 37     public LuceneQuery() {
 38         this.indexPath = Globals.SYS_COM_CONFIG.get("sys.index.path").toString();
 39     }
 40
 41     public int getRowCount() {
 42         return rowCount;
 43     }
 44
 45     public int getPages() {
 46         return pages;
 47     }
 48
 49     public int getPageSize() {
 50         return pageSize;
 51     }
 52
 53     public int getCurrentPage() {
 54         return currentPage;
 55     }
 56
 57     /**
 58      * 函数功能:根据字段查询索引
 59      */
 60     public ArrayList queryIndexTitle(String keyWord, int curpage, int pageSize) {
 61         ArrayList list = new ArrayList();
 62         try {
 63             if (curpage <= 0) {
 64                 curpage = 1;
 65             }
 66             if (pageSize <= 0) {
 67                 pageSize = 20;
 68             }
 69             this.pageSize = pageSize;   //每页记录数
 70             this.currentPage = curpage;   //当前页
 71             int start = (curpage - 1) * pageSize;
 72             Directory dir = FSDirectory.open(new File(indexPath));
 73             IndexReader reader = IndexReader.open(dir);
 74             IndexSearcher searcher = new IndexSearcher(reader);
 75             Analyzer analyzer = new IKAnalyzer(true);
 76             QueryParser queryParser = new QueryParser(Version.LUCENE_36, "title", analyzer);
 77             queryParser.setDefaultOperator(QueryParser.AND_OPERATOR);
 78             Query query = queryParser.parse(keyWord);
 79             int hm = start + pageSize;
 80             TopScoreDocCollector res = TopScoreDocCollector.create(hm, false);
 81             searcher.search(query, res);
 82
 83             SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<span style='color:red'>", "</span>");
 84             Highlighter highlighter = new Highlighter(simpleHTMLFormatter, new QueryScorer(query));
 85             this.rowCount = res.getTotalHits();
 86             this.pages = (rowCount - 1) / pageSize + 1; //计算总页数
 87             TopDocs tds = res.topDocs(start, pageSize);
 88             ScoreDoc[] sd = tds.scoreDocs;
 89             for (int i = 0; i < sd.length; i++) {
 90                 Document hitDoc = reader.document(sd[i].doc);
 91                 list.add(createObj(hitDoc, analyzer, highlighter));
 92             }
 93
 94         } catch (Exception e) {
 95             e.printStackTrace();
 96         }
 97
 98         return list;
 99
100     }
101     /**
102      * 函数功能:根据字段查询索引
103      */
104     public ArrayList queryIndexFields(String allkeyword, String onekeyword, String nokeyword, int curpage, int pageSize) {
105         ArrayList list = new ArrayList();
106         try {
107             if (curpage <= 0) {
108                 curpage = 1;
109             }
110             if (pageSize <= 0) {
111                 pageSize = 20;
112             }
113             this.pageSize = pageSize;   //每页记录数
114             this.currentPage = curpage;   //当前页
115             int start = (curpage - 1) * pageSize;
116             Directory dir = FSDirectory.open(new File(indexPath));
117             IndexReader reader = IndexReader.open(dir);
118             IndexSearcher searcher = new IndexSearcher(reader);
119             BooleanQuery bQuery = new BooleanQuery();  //组合查询
120             if (!"".equals(allkeyword)) {//包含全部关键词
121                 KeywordAnalyzer analyzer = new KeywordAnalyzer();
122                 BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD};//AND
123                 Query query = MultiFieldQueryParser.parse(Version.LUCENE_36, allkeyword, new String[]{"title", "describes", "keywords"}, flags, analyzer);
124                 bQuery.add(query, BooleanClause.Occur.MUST);  //AND
125             }
126             if (!"".equals(onekeyword)) { //包含任意关键词
127                 Analyzer analyzer = new IKAnalyzer(true);
128                 BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD};//OR
129                 Query query = MultiFieldQueryParser.parse(Version.LUCENE_36, onekeyword, new String[]{"title", "describes", "keywords"}, flags, analyzer);
130                 bQuery.add(query, BooleanClause.Occur.MUST);  //AND
131             }
132             if (!"".equals(nokeyword)) { //排除关键词
133                 Analyzer analyzer = new IKAnalyzer(true);
134                 BooleanClause.Occur[] flags = {BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD};//NOT
135                 Query query = MultiFieldQueryParser.parse(Version.LUCENE_36, nokeyword, new String[]{"title", "describes", "keywords"}, flags, analyzer);
136                 bQuery.add(query, BooleanClause.Occur.MUST_NOT);  //AND
137
138             }
139             int hm = start + pageSize;
140             TopScoreDocCollector res = TopScoreDocCollector.create(hm, false);
141             searcher.search(bQuery, res);
142             SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<span style='color:red'>", "</span>");
143             Highlighter highlighter = new Highlighter(simpleHTMLFormatter, new QueryScorer(bQuery));
144             this.rowCount = res.getTotalHits();
145             this.pages = (rowCount - 1) / pageSize + 1; //计算总页数
146             System.out.println("rowCount:" + rowCount);
147             TopDocs tds = res.topDocs(start, pageSize);
148             ScoreDoc[] sd = tds.scoreDocs;
149             Analyzer analyzer = new IKAnalyzer();
150             for (int i = 0; i < sd.length; i++) {
151                 Document hitDoc = reader.document(sd[i].doc);
152                 list.add(createObj(hitDoc, analyzer, highlighter));
153             }
154
155         } catch (Exception e) {
156             e.printStackTrace();
157         }
158
159         return list;
160
161     }
162
163     /**
164      * 创建返回对象（高亮）
165      */
166
167     private synchronized static Object createObj(Document doc, Analyzer analyzer, Highlighter highlighter) {
168
169         Gk_infoSub gk = new Gk_infoSub();
170         try {
171
172             if (doc != null) {
173                 gk.setIndexno(StringUtil.null2String(doc.get("indexno")));
174                 gk.setPdate(StringUtil.null2String(doc.get("pdate")));
175                 String title = StringUtil.null2String(doc.get("title"));
176                 gk.setTitle(title);
177                 if (!"".equals(title)) {
178                     highlighter.setTextFragmenter(new SimpleFragmenter(title.length()));
179                     TokenStream tk = analyzer.tokenStream("title", new StringReader(title));
180                     String htext = StringUtil.null2String(highlighter.getBestFragment(tk, title));
181                     if (!"".equals(htext)) {
182                         gk.setTitle(htext);
183                     }
184                 }
185                 String keywords = StringUtil.null2String(doc.get("keywords"));
186                 gk.setKeywords(keywords);
187                 if (!"".equals(keywords)) {
188                     highlighter.setTextFragmenter(new SimpleFragmenter(keywords.length()));
189                     TokenStream tk = analyzer.tokenStream("keywords", new StringReader(keywords));
190                     String htext = StringUtil.null2String(highlighter.getBestFragment(tk, keywords));
191                     if (!"".equals(htext)) {
192                         gk.setKeywords(htext);
193                     }
194                 }
195                 String describes = StringUtil.null2String(doc.get("describes"));
196                 gk.setDescribes(describes);
197                 if (!"".equals(describes)) {
198                     highlighter.setTextFragmenter(new SimpleFragmenter(describes.length()));
199                     TokenStream tk = analyzer.tokenStream("keywords", new StringReader(describes));
200                     String htext = StringUtil.null2String(highlighter.getBestFragment(tk, describes));
201                     if (!"".equals(htext)) {
202                         gk.setDescribes(htext);
203                     }
204                 }
205
206             }
207             return gk;
208         }
209         catch (Exception e) {
210
211             e.printStackTrace();
212             return null;
213         }
214         finally {
215             gk = null;
216         }
217
218     }
219
220     private synchronized static Object createObj(Document doc) {
221
222         Gk_infoSub gk = new Gk_infoSub();
223         try {
224
225             if (doc != null) {
226                 gk.setIndexno(StringUtil.null2String(doc.get("indexno")));
227                 gk.setPdate(StringUtil.null2String(doc.get("pdate")));
228                 gk.setTitle(StringUtil.null2String(doc.get("title")));
229                 gk.setKeywords(StringUtil.null2String(doc.get("keywords")));
230                 gk.setDescribes(StringUtil.null2String(doc.get("describes")));
231             }
232             return gk;
233         }
234         catch (Exception e) {
235
236             e.printStackTrace();
237             return null;
238         }
239         finally {
240             gk = null;
241         }
242
243     }
244 }

单字段查询：

 1 long a = System.currentTimeMillis();
 2         try {
 3             int curpage = StringUtil.StringToInt(StringUtil.null2String(form.get("curpage")));
 4             int pagesize = StringUtil.StringToInt(StringUtil.null2String(form.get("pagesize")));
 5             String title = StringUtil.replaceLuceneStr(StringUtil.null2String(form.get("title")));
 6             LuceneQuery lu = new LuceneQuery();
 7             form.addResult("list", lu.queryIndexTitle(title, curpage, pagesize));
 8             form.addResult("curPage", lu.getCurrentPage());
 9             form.addResult("pageSize", lu.getPageSize());
10             form.addResult("rowCount", lu.getRowCount());
11             form.addResult("pageCount", lu.getPages());
12         } catch (Exception e) {
13             e.printStackTrace();
14         }
15         long b = System.currentTimeMillis();
16         long c = b - a;
17         System.out.println("[搜索信息花费时间：" + c + "毫秒]");

多字段查询：

 1 long a = System.currentTimeMillis();
 2         try {
 3             int curpage = StringUtil.StringToInt(StringUtil.null2String(form.get("curpage")));
 4             int pagesize = StringUtil.StringToInt(StringUtil.null2String(form.get("pagesize")));
 5             String allkeyword = StringUtil.replaceLuceneStr(StringUtil.null2String(form.get("allkeyword")));
 6             String onekeyword = StringUtil.replaceLuceneStr(StringUtil.null2String(form.get("onekeyword")));
 7             String nokeyword = StringUtil.replaceLuceneStr(StringUtil.null2String(form.get("nokeyword")));
 8             LuceneQuery lu = new LuceneQuery();
 9             form.addResult("list", lu.queryIndexFields(allkeyword,onekeyword,nokeyword, curpage, pagesize));
10             form.addResult("curPage", lu.getCurrentPage());
11             form.addResult("pageSize", lu.getPageSize());
12             form.addResult("rowCount", lu.getRowCount());
13             form.addResult("pageCount", lu.getPages());
14         } catch (Exception e) {
15             e.printStackTrace();
16         }
17         long b = System.currentTimeMillis();
18         long c = b - a;
19         System.out.println("[高级检索花费时间：" + c + "毫秒]");

4、Lucene通配符查询

1 BooleanQuery bQuery = new BooleanQuery();  //组合查询
2 if (!"".equals(title)) {
3 WildcardQuery w1 = new WildcardQuery(new Term("title", title+ "*"));
4 bQuery.add(w1, BooleanClause.Occur.MUST);  //AND
5 }
6 int hm = start + pageSize;
7 TopScoreDocCollector res = TopScoreDocCollector.create(hm, false);
8 searcher.search(bQuery, res);

5、Lucene嵌套查询

实现SQL：(unitid like 'unitid%'  and idml like 'id2%') or (tounitid like 'unitid%' and tomlid like 'id2%' and tostate=1)

 1 BooleanQuery bQuery = new BooleanQuery();
 2         BooleanQuery b1 = new BooleanQuery();
 3         WildcardQuery w1 = new WildcardQuery(new Term("unitid", unitid + "*"));
 4         WildcardQuery w2 = new WildcardQuery(new Term("idml", id2 + "*"));
 5         b1.add(w1, BooleanClause.Occur.MUST);//AND
 6         b1.add(w2, BooleanClause.Occur.MUST);//AND
 7         bQuery.add(b1, BooleanClause.Occur.SHOULD);//OR
 8         BooleanQuery b2 = new BooleanQuery();
 9         WildcardQuery w3 = new WildcardQuery(new Term("tounitid", unitid + "*"));
10         WildcardQuery w4 = new WildcardQuery(new Term("tomlid", id2 + "*"));
11         WildcardQuery w5 = new WildcardQuery(new Term("tostate", "1"));
12         b2.add(w3, BooleanClause.Occur.MUST);//AND
13         b2.add(w4, BooleanClause.Occur.MUST);//AND
14         b2.add(w5, BooleanClause.Occur.MUST);//AND
15         bQuery.add(b2, BooleanClause.Occur.SHOULD);//OR

6、Lucene先根据时间排序后分页

 1 int hm = start + pageSize;
 2         Sort sort = new Sort(new SortField("pdate", SortField.STRING, true));
 3         TopScoreDocCollector res = TopScoreDocCollector.create(pageSize, false);
 4         searcher.search(bQuery, res);
 5         this.rowCount = res.getTotalHits();
 6         this.pages = (rowCount - 1) / pageSize + 1; //计算总页数
 7         TopDocs tds =searcher.search(bQuery,rowCount,sort);// res.topDocs(start, pageSize);
 8         ScoreDoc[] sd = tds.scoreDocs;
 9         System.out.println("rowCount:" + rowCount);
10         int i=0;
11         for (ScoreDoc scoreDoc : sd) {
12             i++;
13             if(i<start){
14                 continue;
15             }
16             if(i>hm){
17                 break;
18             }
19             Document doc = searcher.doc(scoreDoc.doc);
20             list.add(createObj(doc));
21         }

这个效率不高，正常的做法是创建索引的时候进行排序，之后使用分页方法，不要这样进行2次查询。