8.4 HDFS常用API操作

为了测试方便, 我们使用但远程测试来完成下面的后面的代码.

8.4.1 HDFS文件上传

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.net.URI;


public class HDFSClient2 {

    private FileSystem fs;

    @Before
    public void initFs() throws Exception {
        Configuration conf = new Configuration();
        fs = FileSystem.get(new URI("hdfs://hadoop201:9000"), conf, "atguigu");
    }

    @Test
    public void uplaod() throws Exception {
        fs.copyFromLocalFile(new Path("c:/a.txt"), new Path("/java_test/aa.txt"));
    }

    @After
    public void closeFs() throws Exception {
        fs.close();
    }

}

给配置文件增加一行代码

package com.atguigu.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.net.URI;


public class HDFSClient2 {

    private FileSystem fs;

    @Before
    public void initFs() throws Exception {
        Configuration conf = new Configuration();
        // 在java代码中设置副本的数量
        conf.set("dfs.replication", "2");
        fs = FileSystem.get(new URI("hdfs://hadoop201:9000"), conf, "atguigu");
    }

    @Test
    public void uplaod() throws Exception {
        fs.copyFromLocalFile(new Path("c:/a.txt"), new Path("/java_test/aa.txt"));
    }

    @After
    public void closeFs() throws Exception {
        fs.close();
    }

}

在src/main/resources下增加配置文件: hdfs-site.xml

内容如下:

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
</configuration>

去掉conf.set("dfs.replication", "2");这行代码

总结:配置的优先级

客户端Java代码的设置值 > classpath下的用户定义的配置文件 > 集群上的配置文件 > 集群的默认配置