Lucene 3.6 make use of IKAnalyzer 3.2 stable (how to get token from tokenstream)

First things first, make sure all versions are compatible! Cos some interfaces are updated for lucene 3.6, meanwhile some methods are depreciated as well!


In this version: we use Lucene 3.6, IKAnalyzer 3.2. For built-in analyzers such as simple,  standard analyzers, this method also can be applied.

1. package com.segment;  
2.   
3. import java.io.StringReader;  
4. import org.apache.lucene.analysis.Analyzer;  
5. import org.apache.lucene.analysis.Token;  
6. import org.apache.lucene.analysis.TokenStream;  
7. import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;  
8. import org.apache.lucene.analysis.tokenattributes.TermAttribute;  
9. import org.apache.lucene.util.AttributeImpl;  
10. import org.wltea.analyzer.lucene.IKAnalyzer;  
11.   
12.   
13. public class Segment {  
14.     public static String show(Analyzer a, String s) throws Exception {  
15.   
16.         StringReader reader = new StringReader(s);  
17.         TokenStream ts = a.tokenStream(s, reader);  
18.         String s1 = "", s2 = "";  
19.         boolean hasnext= ts.incrementToken();  
20.         //Token t = ts.next();  
21.         while (hasnext) {  
22.             //AttributeImpl ta = new AttributeImpl();  
23.             CharTermAttribute ta = ts.getAttribute(CharTermAttribute.class);  
24.             //TermAttribute ta = ts.getAttribute(TermAttribute.class);  
25.               
26.             s2 = ta.toString() + " ";  
27.             s1 += s2;  
28.             hasnext = ts.incrementToken();  
29.         }  
30.         return s1;  
31.     }  
32.   
33.     public String segment(String s) throws Exception {  
34.         Analyzer a = new IKAnalyzer();  
35.         return show(a, s);  
36.     }  
37.     public static void main(String args[])  
38.     {  
39.         String name = "some test strings here!!!";  
40.         Segment s = new Segment();  
41.         String test = "";  
42.         try {  
43.             System.out.println(test+s.segment(name));  
44.         } catch (Exception e) {  
45.             // TODO Auto-generated catch block  
46.             e.printStackTrace();  
47.         }  
48.     }  
49.   
50. }  

reference: http://blog.csdn.net/hiphopmattshi/article/details/7226326