JAVA xml 流方式读取。数据挖掘大文件预处理。

import java.io.BufferedReader;

import java.io.BufferedWriter;

import java.io.File;

import java.io.FileReader;

import java.io.FileWriter;

import java.io.IOException;

import java.util.HashMap;

import java.util.Iterator;

import java.util.Map;



/**

 * @author gjf

 *db_pre.arff存储的是从xml文件中读取的xml信息

 */

public class ElmAuth {

    Map<String, Integer> map = new HashMap<String, Integer>();



    //第一步

    //从xml文件中提取  读取xml文件中的author信息，写到db_pre.arff，并且替换特殊字符

    public void settleXml(String src, String dst){//src=dblp.xml dst=db_pre.arff

        File file = new File(src);

        File fl = new File(dst);

        FileReader fr;

        try {

            fr = new FileReader(file);

            FileWriter fw = new FileWriter(fl);

            BufferedReader br = new BufferedReader(fr);

            BufferedWriter bw = new BufferedWriter(fw);

            String line = null;

            boolean flag = true;

            int loc_st;

            int loc_end;

            int len = 0, max = 0;

            while((line = br.readLine()) != null){

                if(line == null) 

                    break;    

                loc_st = line.indexOf("<author>");

                if(loc_st != -1){

                    loc_end = line.indexOf("</author>");

                    line = line.substring(loc_st + 8, loc_end);//在<author></author>之间的数据,一个作者的名字

                    line=line.replace('&', ' ');

                    line=line.replace('$', ' ');

                    line=line.replace("' "," ");

                    line=line.replace("'", " ");

                    /*flag以文章为界限,在同一篇文章内,flag=false,写入在同一行*/

                    if(flag){

                        bw.write("\n");

                        bw.write(line);

                        } else {

                            bw.write(",");

                            bw.write(line);

                            }

                    len++;//每写一个作者,计数加 +1 

                    flag = false;

                    } else {

                        flag = true;

                        if(max < len) max = len;//选择最大的len;

                        len = 0;

                        bw.flush();

                        }

                }

            System.out.println("第一步 论文中具有最大的作者数:" + max);

            } catch (IOException e) {

                e.printStackTrace();

                }    

    }



    //消除只有单个作item

    //第二步:将作者的信息db_pre.arff中只有一个作者的数据删除    

    public void elimate_one(String src, String dst){//src=db_pre.arff dst=db_elone.arff

        try {

            File file = new File(src);

            FileReader fr = new FileReader(file);

            BufferedReader br = new BufferedReader(fr);

            File filew = new File(dst);

            FileWriter fw = new FileWriter(filew);

            BufferedWriter bw = new BufferedWriter(fw);

            Map<String, Integer> map = new HashMap<String, Integer>();

            String line = null;

            int k = 1;

            int res = 0;

            while((line = br.readLine()) != null){

                String[] arrLine = line.split(",");

                //作者之间用","隔离,","的数量表示作者的个数,数量比一少,则不写入.

                if(arrLine.length > 1){

                    bw.write(line);

                    bw.write("\n");

                    res ++;

                }

            } 

            bw.flush();

            br.close();

            bw.close();

            fr.close();

            //System.out.println("The Row of the file is:" + res);

            System.out.println("这篇论文中去除单个作者后的行数:" + res);

        }catch (IOException e) {

            e.printStackTrace();

        }

    }



    //将剩余的作储再hashMap中，key值为人名，value为出现的次数,支持度数 

    public void createMap(String src){//srr=db_elone.arff

        try {

            File file = new File(src);

            FileReader fr = new FileReader(file);

            BufferedReader br = new BufferedReader(fr);



            String line = null;

            while((line = br.readLine()) != null){

                if(line == null)

                    break;

                String[] arrLine = line.split(",");

                for(int i = 0; i < arrLine.length; ++i){

                    if(map.get(arrLine[i]) == null){

                        map.put(arrLine[i], 1);

                    } else {

                        map.put(arrLine[i], map.get(arrLine[i]) + 1);

                    }

                }

            }

            fr.close();

            br.close();

        } catch (IOException e) {

            e.printStackTrace();

        } 

    }



    //从hashMap中删除小于支持度minsup的作者,本次的支持度数为100;

    public void settleMap(int minsup){

        Iterator it = map.keySet().iterator();

        while(it.hasNext()){

            String str = (String) it.next();

            if(map.get(str) < minsup){

                it.remove();

            }

        }

        System.out.println("Map的大小,支持度大于100的作者个数:" + map.size());

    }



    //将大于minsup的作者存储到文件 db_minsup.arff,存储的是符合筛选的作者

    public void updateMap(String src, String dst){//src=db_elone.arff dst=db_minsup.arff

        try {

            File filer = new File(src);

            FileReader fr = new FileReader(filer);

            BufferedReader br = new BufferedReader(fr);



            File filew = new File(dst);

            FileWriter fw = new FileWriter(filew);

            BufferedWriter bw = new BufferedWriter(fw);



            String line = null;

            int res = 0;

            boolean flag = true;

            while((line = br.readLine()) != null){

                if(line == null)break;

                String[] arrLine = line.split(",");

                if(flag == false)res++;

                flag = true;

                for(int i = 0; i < arrLine.length; ++i){

                    if(map.get(arrLine[i]) != null){

                        if(flag == true){

                            bw.write("\n" + arrLine[i]);

                            flag = false;

                        } else {

                            bw.write("," + arrLine[i]);

                        }

                    }

                }

            }

            bw.flush();

            System.out.println("符合筛选的作者合作写的论文篇数:" + res);

            fw.close();

            bw.close();

            fr.close();

            br.close();

        } catch (IOException e) {

            e.printStackTrace();

        } 

    }



    //生成weka识别的文  dst=db

    public void createWekaFile(String src, String dst){//src=db_minsup.arff dst=db

        try {

            File filer = new File(src);

            FileReader fr = new FileReader(filer);

            BufferedReader br = new BufferedReader(fr);



            File filew = new File(dst);

            FileWriter fw = new FileWriter(filew);

            BufferedWriter bw = new BufferedWriter(fw);

            bw.write("@relation db" + "\n");

            Iterator it = map.keySet().iterator();

            while(it.hasNext()){

                String str = (String) it.next();

                str.replace("'", "\'");

                bw.write("@attribute '" + str + "' { t}\n");

            }

            bw.write("@data" + "\n");

            

            String line = null;

            boolean flag = true;

            while((line = br.readLine()) != null){

                if(line == null)break;

                flag = true;

                char ch;

                it = map.keySet().iterator();

                while(it.hasNext()){

                    String str = (String)it.next();

                    if(line.indexOf(str) >= 0){

                        ch = 't';

                    } else {

                        ch = '?';

                    }

                    if(flag == true){

                        bw.write(ch);

                    } else {

                        bw.write("," + ch);

                    }

                    flag = false;

                }

                bw.write("\n");

            }

            bw.flush();

            fw.close();

            bw.close();

            fr.close();

            br.close();

        } catch (IOException e) {

            // TODO Auto-generated catch block

            e.printStackTrace();

        } 

    }

    

    public void clearMap(){

        map.clear();

    }

    

    public static void main(String args[]){

        ElmAuth elmauth = new ElmAuth();

        elmauth.settleXml("dblp.xml", "db_pre.arff");

        elmauth.elimate_one("db_pre.arff", "db_elone.arff");

        elmauth.createMap("db_elone.arff");

        elmauth.settleMap(100);//确定最小支持度数

        elmauth.updateMap("db_elone.arff", "db_minsup.arff");

        

        for(int i = 0; i < 20; ++i){

            System.out.println();

            elmauth.elimate_one("db_minsup.arff", "db_minsup_elone.arff");

            elmauth.clearMap();

            elmauth.createMap("db_minsup_elone.arff");

            elmauth.settleMap(100);

            elmauth.updateMap("db_minsup_elone.arff", "db_minsup.arff");

        }

                

        elmauth.createWekaFile("db_minsup.arff", "db.arff");

    }

}