Hadoop HDFS 客户端测试

依赖

<dependencies>
    <dependency>
        <groupId>junitgroupId>
        <artifactId>junitartifactId>
        <version>RELEASEversion>
    dependency>
    <dependency>
        <groupId>org.apache.logging.log4jgroupId>
        <artifactId>log4j-coreartifactId>
        <version>2.8.2version>
    dependency>
    <dependency>
        <groupId>org.apache.hadoopgroupId>
        <artifactId>hadoop-commonartifactId>
        <version>2.7.2version>
    dependency>
    <dependency>
        <groupId>org.apache.hadoopgroupId>
        <artifactId>hadoop-clientartifactId>
        <version>2.7.2version>
    dependency>
    <dependency>
        <groupId>org.apache.hadoopgroupId>
        <artifactId>hadoop-hdfsartifactId>
        <version>2.7.2version>
    dependency>
dependencies>

测试demo1

代码

package com.yljphp.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;

public class Demo {

    public static void main(String[] args) throws IOException {

        Configuration configuration = new Configuration();

        configuration.set("fs.defaultFS", "hdfs://hadoop102:9000");

        //获取HDFS客户端对象
        FileSystem fs = FileSystem.get(configuration);

        //HDFS上创建文件夹
        fs.mkdirs(new Path("/demo/0331"));

        //关闭资源
        fs.close();

        System.out.println("ok~~~");
    }
}

配置VM options

客户端去操作 hdfs 时，是有一个用户身份的。默认情况下，hdfs 客户端 api 会从 jvm 中 获取一个参数来作为自己的用户身份:-DHADOOP_USER_NAME=root

如果不添加上面的配置，会报如下错误。

查看

测试demo2

代码

package com.yljphp.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

import java.io.IOException;
import java.net.URI;

public class Demo {

    public static void main(String[] args) throws Exception {

        Configuration configuration = new Configuration();

        configuration.set("fs.defaultFS", "hdfs://hadoop102:9000");

        //获取HDFS客户端对象
        //FileSystem fs = FileSystem.get(configuration);

        FileSystem fs = FileSystem.get(new URI("hdfs://hadoop102:9000"), configuration, "root");

        //HDFS上创建文件夹
        fs.mkdirs(new Path("/demo/0332"));

        //关闭资源
        fs.close();

        System.out.println("ok~~~");
    }
}

查看