按行读io优化

一、优化过程：

问题：所有文件读取慢。
优化1、之前：一个字节一个字节的读取。之后每次读取8*1024个字节。效率提高100倍。
问题：大文件读取慢。
优化2：之前内容用LinkedList，之后用ArrayList。LinkedList差寻慢，不是从头查就是从尾查
问题：读取大文件oom
原因：之前先一次性读取到StringBuilder（StringBuilder一开始被分配在young区，15次ygc后晋升到old区，晋升到old区的StringBuilder依然不断增大，直至oom）。已后期验证同样的数据StringBuilder将近比ArrayList大一倍。
优化3：之前先一次性读取到StringBuilder。之后每次读取8*1024个字节到StringBuilder（利用了young区的oom），同时加到ArrayList。

 

二、demo

package d;

import lombok.extern.slf4j.Slf4j;

import java.io.*;
import java.nio.charset.Charset;
import java.util.*;
import java.util.concurrent.CountDownLatch;

/**
 * @author sunxuejian
 * @date 2020/6/23
 */
@Slf4j
public class Test_IO {

    public final static String PARAMETERIZED_FILE = "parameterized.properties";
    protected static Charset charset = Charset.forName("utf-8");

    static {
        InputStream is = AbstractAsciiFileAccess.class.getClassLoader().getResourceAsStream(PARAMETERIZED_FILE);
        Properties properties = new Properties();
        try {
            properties.load(is);
            String encoding = properties.getProperty("file_encoding", "utf-8");
            System.out.println("===========>encoding" + encoding);
            if (!"".equals(encoding = encoding.trim())) {
                charset = Charset.forName(encoding);
            }
        } catch (IOException e) {
            e.printStackTrace();
        }

    }


    protected RandomAccessFile randomAccessFile;
    protected String separator;
    protected String readMode = "r";
    protected boolean enCycle;//是否回去读
    protected long lineCount;

    protected void setLineCount(File file) {
        LineNumberReader rf = null;
        try {
            rf = new LineNumberReader(new FileReader(file));
            if (rf != null) {
                rf.skip(file.length());

                this.lineCount = rf.getLineNumber() + 1;
            }
        } catch (IOException e) {
            log.error(e.getMessage(), e);
        } finally {
            if (rf != null) {
                try {
                    rf.close();
                } catch (IOException e) {
                    log.error(e.getMessage(), e);
                }
            }
        }
    }


    public String readLine() {
        try {
            return this.nextLine();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return null;
    }

    private volatile Boolean isReaded = false;
    private int readLineCount = 0;
    List lines = new ArrayList();

    private synchronized String nextLine() throws IOException {

        if(!isReaded) {
            StringBuilder builder = new StringBuilder();
            byte[] b = new byte[80 * 1024];
            int i;
            while(true) {
                i=randomAccessFile.read(b);
                if (i == -1) {
                    lines.add(builder.toString());
                    break;
                }
                builder.append(new String(b, 0, i, charset));
                List lineList = Arrays.asList(builder.toString().split(System.getProperty("line.separator")));
                int length = lineList.size();
                if (length == 1){
                    continue;
                }
                for (int j=0; j= lineCount) {
            if(enCycle){
                readLineCount=0;
            }else {
                return null;
            }
        }
        String s = lines.get(readLineCount++);
//        System.out.println("====>" + s);
        return s;
    }


    public Test_IO(String filepath) {
        this(filepath, null, false);
    }


    public Test_IO(String filepath, String separator, boolean enCycle) {
        try {
            File file = new File(filepath);
            if (file.exists() && file.isFile()) {
                randomAccessFile = new RandomAccessFile(file, readMode);
            } else {
                file = new File(AsciiFileAccessCircularly_type2.class.getClassLoader().getResource(filepath).getFile());
                randomAccessFile = new RandomAccessFile(file, readMode);
            }
            this.separator = separator;
            this.enCycle = enCycle;

            this.setLineCount(file);

        } catch (FileNotFoundException e) {
            e.printStackTrace();
        }

    }


    static Test_IO afc = new Test_IO("F:\\test-io\\random.txt");
    public static void main(String[] args) throws InterruptedException {

        CountDownLatch countDownLatch = new CountDownLatch(10);

        long x = System.currentTimeMillis();
        for(int i=0; i<10; i++){
            new Thread(new Runnable() {
                @Override
                public void run() {
                    while (true) {
                        String s = afc.readLine();
//                        System.out.println("==>" + s);
                        if(s == null) {
                            break;
                        }
                    }
                    countDownLatch.countDown();
                }
            },"thread" + i).start();
        }
        countDownLatch.await();
        long y = System.currentTimeMillis();
        System.out.println("耗时：" + (y-x));
    }

}