未登录词识别

未登录词识别：不在词典中的词

---新词：杀马特

---命名实体：奥克兰


主要解决方案：基于规则合词，然后通过百度验证。

Start Char Char    1-2-Combine    #[图 n][里 f][市场 n][站 n]
Start Char Char Char    1-3-Combine    #
Start Char Char Char Char    1-4-Combine    #
Start Char Char Char Char Char    1-5-Combine    #
Start Char Char Char Char Char Char    1-6-Combine    #
Start Direction Char    1-2-Combine    #东澳站 南势站
Start Char Word    1-2-Combine    #[台 j][中港 nz][站 n]
Word Char Keyword    0-1-Combine    #[梨园 nz][寮 g][站 v][白沙 nz][屯 ng][站 n]
Char Char Keyword    0-1-Combine    #[商水县 ns][黄 a][寨 ng][站 n]
NumPrefix Num    0-1-Seq    #地五医院
Num NumSuffix    0-1-Seq    #93/号/酒家
Num Num    0-1-Combine #
Num Num Num    0-2-Combine #
Num Num Num Num    0-3-Combine #
Num Num Num Num Num    0-4-Combine #
Num Num Num Num Num Num    0-5-Combine #
Num Num Num Num Num Num Num    0-6-Combine #
Num Num Num Num Num Num Num Num    0-7-Combine #
Num Num Num Num Num Num Num Num Num    0-8-Combine #
Num Num Num Num Num Num Num Num Num Num    0-9-Combine #
Letter Letter Letter Letter Letter Letter Letter Letter Letter Letter Letter    0-10-Combine    #
Letter Letter Letter Letter Letter Letter Letter Letter Letter Letter    0-9-Combine    #
Letter Letter Letter Letter Letter Letter Letter Letter Letter    0-8-Combine    #
Letter Letter Letter Letter Letter Letter Letter Letter    0-7-Combine    #
Letter Letter Letter Letter Letter Letter Letter    0-6-Combine    #
Letter Letter Letter Letter Letter Letter    0-5-Combine    #
Letter Letter Letter Letter Letter    0-4-Combine    #
Letter Letter Letter Letter    0-3-Combine    #
Letter Letter Letter    0-2-Combine    #
Letter Letter    0-1-Combine    #
Num NumSuffix Keyword    0-1-Seq    #海口1号场BLACKSTONE球场
Num Char Char Keyword    0-2-Combine    #八里岔中学
Char Num Char Keyword    0-2-Combine    #八里岔中学
Char Char Num Keyword    0-2-Combine    #八里岔中学

ackage cn.tianditu.mt.common;



import java.io.BufferedReader;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.util.ArrayList;

import java.util.LinkedList;

import java.util.List;



import org.apache.commons.logging.Log;

import org.apache.commons.logging.LogFactory;



public class Grammar {

    

    protected static Log logger = LogFactory.getLog(Grammar.class);

    

    public final class TSTNode {

        public CombinRule data = null;

        protected TSTNode loNode;

        protected TSTNode eqNode;

        protected TSTNode hiNode;

        protected SegMarkType splitchar;



        public TSTNode(SegMarkType type) {

            this.splitchar = type;

        }

    }



    public TSTNode rootNode;

    

    public TSTNode add(List<SegMarkType> word) {

        if (null == word) {

            throw new NullPointerException("空指针异常");

        }



        int charIndex = 0;

        if (null == rootNode) {

            rootNode = new TSTNode(word.get(0));

        }

        TSTNode currentNode = rootNode;

        while (true) {

            int charComp = word.get(charIndex).compareTo(currentNode.splitchar);

            if (charComp == 0) {

                charIndex++;

                if (charIndex == word.size()) {

                    return currentNode;

                }

                if (null == currentNode.eqNode) {

                    currentNode.eqNode = new TSTNode(word.get(charIndex));

                }

                currentNode = currentNode.eqNode;

            } else if (charComp < 0) {

                if (null == currentNode.loNode) {

                    currentNode.loNode = new TSTNode(word.get(charIndex));

                }

                currentNode = currentNode.loNode;

            } else {

                if (null == currentNode.hiNode) {

                    currentNode.hiNode = new TSTNode(word.get(charIndex));

                }

                currentNode = currentNode.hiNode;

            }

        }

    }



    protected TSTNode getNode(List<SegMarkType> word) {

        if (null == word) {

            return null;

        }

        int len = word.size();

        if (len == 0)

            return null;

        TSTNode currentNode = rootNode; // 匹配过程中的当前节点的位置

        int charIndex = 0; // 表示当前要比较的字符在Key中的位置

        SegMarkType cmpChar = word.get(charIndex);

        int charComp;

        while (true) {

            if (currentNode == null) {// 没找到

                return null;

            }

            charComp = cmpChar.compareTo(currentNode.splitchar);

            if (charComp == 0) {// 相等往下走

                charIndex++;

                if (charIndex == len) {// 找到了

                    return currentNode;

                } else {

                    cmpChar = word.get(charIndex);// 词往下走

                }

                currentNode = currentNode.eqNode;

            } else if (charComp < 0) {// 小于往左走

                currentNode = currentNode.loNode;

            } else {// 大于往右走

                currentNode = currentNode.hiNode;

            }

        }

    }



    public MatchRet matchLong(List<WordInfo> tokens, int offset) {

        if (tokens == null || rootNode == null) {

            return null;

        }



        MatchRet ret = null;

        TSTNode currentNode = rootNode;

        int index = offset;

        while (currentNode != null) {

            int charComp = tokens.get(index).getType().compareTo(

                    currentNode.splitchar);

            if (charComp == 0) {

                index++;

                if (currentNode.data != null) {

                    ret = new MatchRet(currentNode, index);

                }

                if (index == tokens.size()) {

                    return ret;

                }

                currentNode = currentNode.eqNode;

            } else if (charComp < 0) {

                currentNode = currentNode.loNode;

            } else {

                currentNode = currentNode.hiNode;

            }

        }

        return ret;

    }



    /**

     * 根据语法规则进行合并

     * 支持多次合并

     * 且保留了源序列

     * @param tokens

     * @param rules

     * @return

     */

    private List<WordInfo> combineByRules(List<WordInfo> tokens,List<Combin> rules){

        if(rules==null){

            return tokens;

        }

        List<WordInfo> list=new ArrayList<WordInfo>();

        for (int i = 0; i < tokens.size();) {

            for (Combin com : rules) {

                if(i==com.getStart()){

                    int start=com.getStart();

                    int end=com.getEnd();

                    

                    List<WordInfo> sub=tokens.subList(start, end+1);//前闭后开

                    StringBuilder buff=new StringBuilder();

                    for (WordInfo wordInfo : sub) {

                        buff.append(wordInfo.getCn());

                    }                    

                    String cn=buff.toString();

                    SegMarkType type=com.getType();                    

                    WordInfo info=new WordInfo(cn,null,type,sub);                

                    list.add(info);                    

                    i=end+1;                    

                    continue;

                }        

            }

            list.add(tokens.get(i));

            i++;        

        }

        return list;

    }

    

    /**

     * 仅支持一次合并，不支持内部的多次合并，即无法达到有限状态机的效果

     * @param tokens

     * @param rules

     */

    @SuppressWarnings("unused")

    private void CombineOnce(LinkedList<WordInfo> tokens,

            List<Combin> rules) {



        for (Combin com : rules) {

            int start = com.getStart();

            int end = com.getEnd();

            SegMarkType type = com.getType();

            

            StringBuilder buff=new StringBuilder();

            for (int i = start; i <= end; i++) {

                WordInfo word=tokens.get(i);

                buff.append(word.getCn());

            }

            

            int dis=end-start+1;

            for (int i = 0; i < dis; i++) {

                tokens.remove(start);

            }

            

            String cn=buff.toString();

            WordInfo info=new WordInfo(cn,null,type);

            tokens.add(start, info);            

        }

    }



    public List<WordInfo> tag(List<WordInfo> tokens) {

        if (tokens == null || rootNode == null) {

            return null;

        }

        List<Combin> rules = new ArrayList<Combin>();

        for (int i = 0; i < tokens.size();) {

            MatchRet ret = matchLong(tokens, i);

            if (null != ret) {

                CombinRule rule = ret.getNode().data;//找到了树上的东西

                int indexCurrent = ret.getIndex()-1;

                List<Combin> list_com = rule.getPosition();

                for (Combin com : list_com) {

                    int start = indexCurrent - rule.getLen() + 1

                            + com.getStart();

                    int end = indexCurrent - rule.getLen() + 1 + com.getEnd();

                    Combin c = new Combin(start, end, com.getType());//拿到规则

                    rules.add(c);//放入规则列表

                }

                i = ret.getIndex();

            } else {

                i++;

            }

        }

        List<WordInfo> words= combineByRules(tokens,rules);//根据规则合并

        return words;

    }



    public Grammar(Config config){

        loadGrammar(config.getBasicGramFileName());

        loadGrammar(config.getGramFileName());

    }

    

    

    public void loadGrammar(String gramFileName){

        try {

            FileReader fileReader = new FileReader(gramFileName);

            BufferedReader reader = new BufferedReader(fileReader);

            String line;

            try {

                while ((line = reader.readLine()) != null) {

                    String[] arr=line.split("\t");

                    

                    List<SegMarkType> seq=FormSeq(arr[0]);

                    CombinRule rule=FormRule(arr[1],seq.size());                    

                    TSTNode node = this.add(seq);

                    node.data=rule;

                }

            } catch (NullPointerException e) {

                logger.info(e.getMessage());

                logger.info(e.getStackTrace());

            } catch (IllegalArgumentException e) {

                logger.info(e.getMessage());

                logger.info(e.getStackTrace());

            } catch (IOException e) {

                logger.info(e.getMessage());

                logger.info(e.getStackTrace());

            }

        } catch (FileNotFoundException e) {

            logger.info(e.getMessage());

            logger.info(e.getStackTrace());

        }

    }

    

    

    

    

    private CombinRule FormRule(String line,int size) {

        

        List<Combin> rec = new ArrayList<Combin>();

        String[] arr_1=line.split("#");

        for (String str : arr_1) {

            String[] arr_2=str.split("-");

            int start = Integer.parseInt(arr_2[0]);

            int end=Integer.parseInt(arr_2[1]);

            SegMarkType type=Enum.valueOf(SegMarkType.class, arr_2[2].trim());

            Combin pos = new Combin(start, end, type);

            rec.add(pos);

        }        

        CombinRule rule = new CombinRule(rec,size);

        return rule;

    }



    private List<SegMarkType> FormSeq(String string) {

        List<SegMarkType> list=new ArrayList<SegMarkType>();

        String[] arr=string.split(" ");

        for (String str : arr) {

            SegMarkType type=Enum.valueOf(SegMarkType.class, str);

            list.add(type);

        }

        return list;

    }

    

}