基于TextRank算法提取关键词——Java实现

依赖

 
     com.janeluo
     ikanalyzer
     2012_u6
 

代码

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.wltea.analyzer.lucene.IKAnalyzer;

import java.io.IOException;
import java.io.StringReader;
import java.util.*;

/**
 * @author yuyufeng
 * @date 2017/11/3
 */
public class Demo {
    public static void main(String[] args) {

        List keyWords = new ArrayList<>();
        int k = 2;  //窗口大小/2
        float d = 0.85f;
        /**
         * 标点符号、常用词、以及“名词、动词、形容词、副词之外的词”
         */
        Set stopWordSet = new HashSet();
        stopWordSet.add("是");
        stopWordSet.add("的");
        stopWordSet.add("地");
        stopWordSet.add("从");
        stopWordSet.add("将");
        stopWordSet.add("但");
        stopWordSet.add("都");
        stopWordSet.add("和");
        stopWordSet.add("为");
        stopWordSet.add("让");
        stopWordSet.add("在");
        stopWordSet.add("由");
        stopWordSet.add("上");
        String field = "PageRank近似于一个用户，是指在Internet上随机地单击链接将会到达特定网页的可能性。通常，能够从更多地方到达的网页更为重要，因此具有更高的PageRank。每个到其他网页的链接，都增加了该网页的PageRank。具有较高PageRank的网页一般都是通过更多其他网页的链接而提高的。";


        Analyzer analyzer = new IKAnalyzer(true);
        TokenStream ts = null;
        //分词
        try {
            ts = analyzer.tokenStream("myfield", new StringReader(field));
            OffsetAttribute offset = (OffsetAttribute) ts.addAttribute(OffsetAttribute.class);
            CharTermAttribute term = (CharTermAttribute) ts.addAttribute(CharTermAttribute.class);
            TypeAttribute type = (TypeAttribute) ts.addAttribute(TypeAttribute.class);
            ts.reset();

            while (ts.incrementToken()) {
                if (!stopWordSet.contains(term.toString())) {
                    keyWords.add(term.toString());
                }
            }
            ts.end();
        } catch (IOException var14) {
            var14.printStackTrace();
        } finally {
            if (ts != null) {
                try {
                    ts.close();
                } catch (IOException var13) {
                    var13.printStackTrace();
                }
            }

        }

        Map> relationWords = new HashMap<>();


        //获取每个关键词 前后k个的组合
        for (int i = 0; i < keyWords.size(); i++) {
            String keyword = keyWords.get(i);
            Set keySets = relationWords.get(keyword);
            if (keySets == null) {
                keySets = new HashSet<>();
                relationWords.put(keyword, keySets);
            }

            for (int j = i - k; j <= i + k; j++) {
                if (j < 0 || j >= keyWords.size() || j == i) {
                    continue;
                } else {
                    keySets.add(keyWords.get(j));
                }
            }
        }

       /* for (String s : relationWords.keySet()) {
            System.out.print(s+" ");
            for (String s1 : relationWords.get(s)) {
                System.out.print(s1+" ");
            }
            System.out.println();
        }*/


        Map score = new HashMap<>();
        float min_diff = 0.1f; //差值最小
        int max_iter = 100;//最大迭代次数

        //迭代
        for (int i = 0; i < max_iter; i++) {
            Map m = new HashMap<>();
            float max_diff = 0;
            for (String key : relationWords.keySet()) {
                Set value = relationWords.get(key);
                //先给每个关键词一个默认rank值
                m.put(key, 1 - d);
                //一个关键词的TextRank由其它成员投票出来
                for (String other : value) {
                    int size = relationWords.get(other).size();
                    if (key.equals(other) || size == 0) {
                        continue;
                    } else {
                        m.put(key, m.get(key) + d / size * (score.get(other) == null ? 0 : score.get(other)));
                    }
                }
//                System.out.println("m.get(key):"+m.get(key)+" score:"+(score.get(key) == null ? 0 : score.get(key)));
                max_diff = Math.max(max_diff, Math.abs(m.get(key) - (score.get(key) == null ? 0 : score.get(key))));
            }

            score = m;
            if (max_diff <= min_diff) {
                System.out.println("迭代次数：" + i);
                break;
            }
        }

        List scores = new ArrayList<>();
        for (String s : score.keySet()) {
            Score score1 = new Score();
            score1.key = s;
            score1.significance = score.get(s);
            scores.add(score1);
        }

        scores.sort(new Comparator() {
            @Override
            public int compare(Score o1, Score o2) {
                if (o2.significance - o1.significance > 0) {
                    return 1;
                } else {
                    return -1;
                }

            }
        });

        for (Score score1 : scores) {
            System.out.println(score1);
        }

    }
}

class Score {
    String key;
    float significance;

    @Override
    public String toString() {
        return "关键词=" + key +
                ", 重要程度=" + significance;
    }
}

运行结果
迭代次数：11

关键词=网页, 重要程度=2.8311346
关键词=链接, 重要程度=1.646728
关键词=pagerank, 重要程度=1.6038197
关键词=更多, 重要程度=1.2489531
关键词=到达, 重要程度=1.1083827
关键词=具有, 重要程度=0.98187566
关键词=其他, 重要程度=0.9651773
关键词=用户, 重要程度=0.81595975
关键词=指在, 重要程度=0.8086006
关键词=internet, 重要程度=0.80388165
关键词=一个, 重要程度=0.787644
关键词=随机, 重要程度=0.7764552
关键词=单击, 重要程度=0.76052386
关键词=将会, 重要程度=0.71690917
关键词=能够, 重要程度=0.7066941
关键词=可能性, 重要程度=0.70503104
关键词=更高, 重要程度=0.7045265
关键词=每个, 重要程度=0.7005399
关键词=特定, 重要程度=0.6963727
关键词=通过, 重要程度=0.69495517
关键词=因此, 重要程度=0.69311315
关键词=通常, 重要程度=0.69245243
关键词=该, 重要程度=0.6918771
关键词=一般, 重要程度=0.6895788
关键词=都是, 重要程度=0.686642
关键词=到, 重要程度=0.68152785
关键词=更为重要, 重要程度=0.68064004
关键词=地方, 重要程度=0.6771895
关键词=近似于, 重要程度=0.6137907
关键词=而, 重要程度=0.594995
关键词=增加了, 重要程度=0.5508093
关键词=较高, 重要程度=0.5392841
关键词=提高, 重要程度=0.44995427

在线体验

http://www.toptool.top/index.html?tool=keywords.html