用开源Carrot2的后缀树算法做Web文本聚类

采用基于Java的开源搜索结果聚合引擎，Carrot2 2.0 中的后缀树算法
Carrot2 可以自动的把搜索结果归类到相应的语义类别中，这个功能是通过Carrot2一个现成的组件完成的，除此之外Carrot2 还包括了很多其他的搜索结果聚合聚类算法。

因为没有做中文分词，也没有中文的Stopword,所以我们用英文测试，实现代码

 1 SnippetTokenizer snippetTokenizer  =   new  SnippetTokenizer(); 
 2         List < DocReference >  documentReferences  =   new  ArrayList < DocReference > ();       
 3         List < TokenizedDocument >  documents  =   new  ArrayList < TokenizedDocument > ();        
 4         TokenizedDocument doc  =   null ;
 5         DocReference documentReference  =    null ;
 6         
 7          // 从搜索引擎google获取100篇数据
 8          {
 9            String url = "http://www.google.com/search?as_q=phone&num=100&hl=en&newwindow=1&btnG=Google+Search&as_epq=&as_oq=&as_eq=&lr=&as_ft=i&as_filetype=&as_qdr=all&as_nlo=&as_nhi=&as_occt=any&as_dt=i&as_sitesearch=&as_rights=&safe=images";
10            byte[] pageHtml = HttpUtil.getPage(url);
11            if(pageHtml == null ) return ;            
12            try {
13                String strHtml = new String(pageHtml, "utf-8");
14                String[][] result = StringUtil.splitByReg(strHtml,"<td class=j>(.*?)<br>");
15                 
16                if(result != null)
17                {      for(int i=0;i<result.length;i++)
18                        { 
19                         for(int j=0;j<result[i].length;j++)
20                         {
21                             doc = snippetTokenizer
22                                .tokenize(new RawDocumentSnippet(i+"sen"+j,result[i][j].replaceAll("<[^<>]+>",""), "en"));
23                                documentReference = new DocReference(doc);
24                                documentReferences.add(documentReference);
25                                documents.add(doc);                           
26                         }
27                        }
28                }
29            } catch (UnsupportedEncodingException e) { 
30                e.printStackTrace();
31            }
32        }
33
34         
35          // 构建后缀树
36          final  STCEngine stcEngine  =   new  STCEngine(documentReferences);
37         stcEngine.createSuffixTree();
38         HashMap < String,String >  defaults  =   new  HashMap < String,String > (); 
39         defaults.put( " lsi.threshold.clusterAssignment " ,  " 0.150 " );
40         defaults.put( " lsi.threshold.candidateCluster " ,  " 0.775 " ); 
41          final  StcParameters params  =  StcParameters.fromMap(defaults);
42         stcEngine.createBaseClusters(params);
43         stcEngine.createMergedClusters(params);
44
45          final  List clusters  =  stcEngine.getClusters();
46          int  max  =  params.getMaxClusters();
47
48          //  Convert STC's clusters to the format required by local interfaces.
49          final  List rawClusters  =   new  ArrayList();
50          for  (Iterator i  =  clusters.iterator(); i.hasNext()  &&  (max  >   0 ); max -- )
51          {
52            final MergedCluster b = (MergedCluster) i.next();
53            final RawClusterBase rawCluster = new RawClusterBase();
54
55            int maxPhr = 3; // TODO: This should be a configuration parameter moved to STCEngine perhaps.
56            final List phrases = b.getDescriptionPhrases();
57            for (Iterator j = phrases.iterator(); j.hasNext() && (maxPhr > 0); maxPhr--)
58            {
59                Phrase p = (Phrase) j.next();
60                rawCluster.addLabel(p.userFriendlyTerms().trim());
61            }
62
63            for (Iterator j = b.getDocuments().iterator(); j.hasNext();)
64            {
65                final int docIndex = ((Integer) j.next()).intValue();
66                final TokenizedDocument tokenizedDoc = (TokenizedDocument) documents.get(docIndex);
67                final RawDocument rawDoc = (RawDocument) tokenizedDoc.getProperty(TokenizedDocument.PROPERTY_RAW_DOCUMENT);
68                rawCluster.addDocument(rawDoc);
69            }
70
71            rawClusters.add(rawCluster);
72        }
73         
74          // 得到结果，输出
75          for  (Iterator iter  =  rawClusters.iterator(); iter.hasNext();)
76          {
77            RawCluster cluster = (RawCluster) iter.next();
78            final List phrases = cluster.getClusterDescription();
79            for(int i=0;i<phrases.size();i++)
80                System.out.print("#"+phrases.get(i)); 
81            System.out.println();
82            
83        }

下面是输出聚类phone的结果，还不错
#phone
#Phone Number
#yellow pages
#mobile phone
#cell phone
#Phone Book
#area code
#Business
#services
#Wireless
#people
#directory
#telephone
#address
#online