Hadoop3.0 Java API使用指南

0.开发环境概述

客户端环境

Windows 7 64位
Oracle JDK8 64位
Eclipse4.7

服务器伪分布式安装部署Hadoop3

1.Windows平台下Hadoop客户端运行环境搭建

-下载winutils

-解压缩到任意文件夹下

-新建环境变量HADOOP_HOME

-在环境变量PATH中添加%HADOOP_HOME%\bin

2.建立客户端工程

-新建Maven项目，POM文件如下:

<dependencies>
        <dependency>
            <groupId>jdk.tools</groupId>
            <artifactId>jdk.tools</artifactId>
            <version>1.8</version>
            <scope>system</scope>
            <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.0.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs-client</artifactId>
            <version>3.0.0</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.0.0</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.12</version>
            <scope>test</scope>
        </dependency>
    </dependencies>

-拷贝服务器端上的core-site.xml和hdfs-site.xml到maven工程的src文件夹下，开始码代码；

-建立与服务器端Hadoop的连接

    private static Configuration conf = new Configuration();
    private FileSystem hdfs;
    private final static String HDFSUri = "hdfs://192.168.1.111:9001";
    public FileSystem initHDFS() throws URISyntaxException {
        URI uri = new URI(HDFSUri);
        conf.addResource("hdfs-site.xml");
        conf.addResource("core-site.xml");
        try {
            hdfs = FileSystem.get(uri,conf);
        } catch (IOException e) {
            e.printStackTrace();
        }
        return hdfs;
    }

-检查文件是否存在

    /**
     * 检查文件是否存在
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void testCheckFile() throws IllegalArgumentException, IOException {
        Path path = new Path("hdfs://192.168.1.111:9001/user/root/test.txt");
        boolean result = hdfsDao.exists(path);
        System.out.println(result?"文件存在":"文件不存在");
    }

-在HDFS上新建文件夹

    /**
     * 新建文件夹
     * @throws IOException
     */
    @Test
    public void testMkdir() throws IOException {
        Path folderPath = new Path("hdfs://192.168.1.133:9000/user/root/tiffData");
        boolean result = hdfsDao.mkdirs(folderPath);
        System.out.println(result?"添加文件夹成功":"添加文件夹失败");
    }

-删除文件夹/文件

    /**
     * 删除文件夹/文件
     * @throws IOException
     */
    @Test
    public void testDeldir() throws IOException {
        Path folderPath = new Path("hdfs://192.168.1.133:9000/user/root/tiffData");
        boolean result = hdfsDao.delete(folderPath, false);
        System.out.println(result?"删除文件夹成功":"删除文件夹失败");
    }

-获取指定目录下所有文件(忽略目录)

/**
     * 获取指定目录下所有文件(忽略目录)
     * @throws FileNotFoundException
     * @throws IllegalArgumentException
     * @throws IOException
     */
    @Test
    public void testGetAllFile() throws FileNotFoundException, IllegalArgumentException, IOException {
        RemoteIterator<LocatedFileStatus> listFiles = hdfsDao.listFiles(new Path("/"), true);
        SimpleDateFormat sdf=new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
        while (listFiles.hasNext()) {
            LocatedFileStatus fileStatus = (LocatedFileStatus) listFiles.next();
            //权限
            FsPermission permission = fileStatus.getPermission();
            //拥有者
            String owner = fileStatus.getOwner();
            //组
            String group = fileStatus.getGroup();
            //文件大小byte
            long len = fileStatus.getLen();
            long modificationTime = fileStatus.getModificationTime();
            Path path = fileStatus.getPath();
            System.out.println("-------------------------------");
            System.out.println("permission:"+permission);
            System.out.println("owner:"+owner);
            System.out.println("group:"+group);
            System.out.println("len:"+len);
            System.out.println("modificationTime:"+sdf.format(new Date(modificationTime)));
            System.out.println("path:"+path);
        }
    }

-从HDFS上拷贝文件到本地

    /**
     * 从HDFS上拷贝文件到本地
     */
    @Test
    public void testDownLoadFile() {
        Path src = new Path("hdfs://192.168.1.133:9000/user/root/test.txt");
        Path des = new Path("D:\\workData\\test.txt");
        try {
            hdfsDao.copyToLocalFile(src, des);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

-本地文件上传

    @Test
    public void testFileUpload() {
        Path src = new Path("D:\\workData\\test1.txt");
        Path des = new Path("hdfs://192.168.1.133:9000/user/root/test1.txt");
        try {
            hdfsDao.copyFromLocalFile(src, des);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }