Hadoop javaAPI

1、读

package com.sdnware.start01.hadoop;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.List;
import java.util.function.Consumer;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class HDFSRead {

    public static void main(String[] args) {
        read1();
    }

    public static void read0(){
        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");
        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户
        Configuration conf = new Configuration();
        try {
            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);
            InputStream in = hdfs.open(new Path("/user/wordcount/My Father0.txt"));

            ByteArrayOutputStream out = new ByteArrayOutputStream();
            IOUtils.copyBytes(in, out, 4096,true);
            String str = out.toString();
            System.out.println(str);

        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    public static void read1(){
        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");
        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户
        Configuration conf = new Configuration();
        try {
            final FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);
            Path path = new Path("/user/wordcount");
            if(hdfs.exists(path)){
                List listStatus = Arrays.asList(hdfs.listStatus(path));
                listStatus.forEach(new Consumer() {
                    public void accept(FileStatus t) {
                        if(t.isFile()){
                            FSDataInputStream in = null;
                            try {
                                Path path2 = t.getPath();
                                in = hdfs.open(path2);
                                System.out.println("Read File："+ path2.getName());
                                ByteArrayOutputStream out = new ByteArrayOutputStream();
                                IOUtils.copyBytes(in, out, 4096,true);
                                String str = out.toString();
                                System.out.println(str);
                            } catch (IOException e) {
                                e.printStackTrace();
                            }finally{
                                IOUtils.closeStream(in);
                            }
                        }
                    }
                });
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }


}

2、写

package com.sdnware.start01.hadoop;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSWrite {

    public static void main(String[] args) {
        HDFSWrite writer = new HDFSWrite();
        writer.writer1();
    }

    public void writer0() {

        FSDataOutputStream out = null;
        FSDataInputStream in = null;

        try {
            System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");

            Configuration conf = new Configuration();
            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);
            FileSystem local = FileSystem.getLocal(conf);

            Path inputDir = new Path("D:/testfile/"); // 获取本地待被上传的文件
                                                        // 会把该文件夹下的内容都上传
            Path hdfsFile = new Path("/user/wordcount"); // 获取hdfs上需要获取文件的地址
            hdfs.mkdirs(hdfsFile); // 在hdfs上新建该文件夹
            FileStatus[] inputFiles = local.listStatus(inputDir); // 获取本地待上传文件夹的信息


            for (int i = 0; i < inputFiles.length; i++) { // i循环本地文件夹内的文件个数
                System.out.println(inputFiles[i].getPath().getName());
                in = local.open(inputFiles[i].getPath()); // 获取输入流
                out = hdfs.create(new Path("/user/wordcount/" + inputFiles[i].getPath().getName())); // 输出流位置获取
                byte buffer[] = new byte[256];
                int bytesRead = 0;
                while ((bytesRead = in.read(buffer)) > 0) { // 每次读取buffer大小的部分
                    out.write(buffer, 0, bytesRead); // 每次写入buffer中bytesRead大小的部分
                }
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }finally{
            IOUtils.closeQuietly(in);
            IOUtils.closeQuietly(out);
        }
    }


    public void writer1() {

        FSDataOutputStream out = null;
        FSDataInputStream in = null;

        try {
            System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");
            System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户
            Configuration conf = new Configuration();
            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);
            FileSystem local = FileSystem.getLocal(conf);

            Path inputDir = new Path("D:/testfile/"); // 获取本地待被上传的文件
                                                        // 会把该文件夹下的内容都上传
            Path hdfsFile = new Path("/user/wordcount"); // 获取hdfs上需要获取文件的地址
            hdfs.mkdirs(hdfsFile); // 在hdfs上新建该文件夹
            FileStatus[] inputFiles = local.listStatus(inputDir); // 获取本地待上传文件夹的信息


            for (int i = 0; i < inputFiles.length; i++) { // i循环本地文件夹内的文件个数
                System.out.println(inputFiles[i].getPath().getName());
                in = local.open(inputFiles[i].getPath()); // 获取输入流
                out = hdfs.create(new Path("/user/wordcount/" + inputFiles[i].getPath().getName())); // 输出流位置获取
                org.apache.hadoop.io.IOUtils.copyBytes(in, out, 4096,true);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }finally{
            org.apache.hadoop.io.IOUtils.closeStream(in);
            org.apache.hadoop.io.IOUtils.closeStream(out);
        }
    }

}

3、删除

package com.sdnware.start01.hadoop;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSDelete {

    public static void main(String[] args) {
        delete1();
    }

    public static void delete0(){//删除文件
        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");
        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户
        Configuration conf = new Configuration();
        try {
            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);
            Path hdfsFile = new Path("/user/wordcount/My Father.txt");
            hdfs.delete(hdfsFile, false);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

    public static void delete1(){//删除目录
        System.setProperty("hadoop.home.dir", "D:/soft/hadoop-2.6.0");
        System.setProperty("HADOOP_USER_NAME","root");//制定上传的用户
        Configuration conf = new Configuration();
        try {
            FileSystem hdfs = FileSystem.get(new URI("hdfs://f1:9000"), conf);
            Path hdfsFile = new Path("/usr");
            hdfs.delete(hdfsFile, true);
        } catch (IOException e) {
            e.printStackTrace();
        } catch (URISyntaxException e) {
            e.printStackTrace();
        }
    }

}