lucene中分词器的用法

  

   package
    com.ljq.analyzer;


   import
    java.io.StringReader;


   import
    jeasy.analysis.MMAnalyzer;


   import
    org.apache.lucene.analysis.Analyzer;

   import
    org.apache.lucene.analysis.SimpleAnalyzer;

   import
    org.apache.lucene.analysis.Token;

   import
    org.apache.lucene.analysis.TokenStream;

   import
    org.apache.lucene.analysis.cjk.CJKAnalyzer;

   import
    org.apache.lucene.analysis.standard.StandardAnalyzer;

   import
    org.junit.Test;


   public
    
   class
    AnalyzerTest {

 String ensaText 
   =
    
   "
   IndexWriter addDocument's a javadoc.txt
   "
   ;
 String ensa2Text 
   =
    
   "
   我们是中国人
   "
   ;
 String zhcjkaText 
   =
    
   "
   小笑话_总统的房间 Room .txt
   "
   ;
 String zhmnText 
   =
    
   "
   一位绅士到旅游胜地的一家饭店要开个房间
   "
   ;

 Analyzer ensa 
   =
    
   new
    StandardAnalyzer(); 
   //
    单字分词
   

    Analyzer ensa2 
   =
    
   new
    SimpleAnalyzer();
 Analyzer zhcjka 
   =
    
   new
    CJKAnalyzer(); 
   //
    二分法分词
   

    Analyzer zhmn 
   =
    
   new
    MMAnalyzer(); 
   //
    词库分词
   

   
 @Test
 
   public
    
   void
    test() 
   throws
    Exception {
 
   //
    单字分词
   

    
   /*
   (indexwriter,0,11,type=<ALPHANUM>)
 (adddocument,12,25,type=<APOSTROPHE>)
 (javadoc.txt,28,39,type=<HOST>)
   */
   
 
   //
   analyze(ensa, ensaText);
 
 
   //
   (我们是中国人,0,6)
 
   //
   analyze(ensa2, ensa2Text);
 
 
   //
    二分法分词
   

    
   /*
   (小笑,0,2,type=double)
 (笑话,1,3,type=double)
 (_,3,4,type=single)
 (总统,4,6,type=double)
 (统的,5,7,type=double)
 (的房,6,8,type=double)
 (房间,7,9,type=double)
 (room,10,14,type=single)
 (txt,16,19,type=single)
   */
   
 
   //
   analyze(zhcjka, zhcjkaText);
 
 
   //
    词库分词
   

    
   /*
   (一位,0,2)
 (绅士,2,4)
 (旅游胜地,5,9)
 (一家,10,12)
 (饭店,12,14)
 (要,14,15)
 (开个,15,17)
 (房间,17,19)
   */
   
 analyze(zhmn, zhmnText);
 }

 
   /**
   
 * 分词
 * 
 * 
   @param
    analyzer
 * 分词器
 * 
   @param
    text
 * 数据源
 * 
   @throws
    Exception
 
   */
   
 
   public
    
   void
    analyze(Analyzer analyzer, String text) 
   throws
    Exception {
 TokenStream tokenStream 
   =
    analyzer.tokenStream(
   "
   content
   "
   ,
 
   new
    StringReader(text));
 
   for
    (Token token 
   =
    
   new
    Token(); (token 
   =
    tokenStream.next(token)) 
   !=
    
   null
   ;) {
 System.out.println(token);
 }
 }
}