1.通过eclise或者idea的maven工具创建普通java项目
2.在pox.xml文件中添加依赖，下载hadoop client api需要依赖的jar包：
3.创建测试类，通过junit进行通过API操作hdfs系统文件的测试
package com.hadooptest;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
/**
* hdfs client api 通过hdfs 客户端api操作hdfs服务器
* @author Administrator
*/
public class HdfsAPITest {
FileSystem fileSystem = null;
Configuration configuration = null;
@Before
public void init() throws Exception {
configuration = new Configuration();
// 设置副本数 ： 取决于你的datanodes节点数
configuration.set("dfs.replication", "1");
fileSystem = FileSystem.get(new URI("hdfs://hadoop00:8020"), configuration, "root");
}
/**
* 通过hdsf client api创建文件目录
* @throws Exception
*/
@Test
public void mkdir() throws Exception {
boolean result = fileSystem.mkdirs(new Path("/hdfsapi/test"));
System.out.println(result);
}
/**
* 创建文件到hdfs服务器
* @throws Exception
*/
@Test
public void create() throws Exception {
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/test1.java"));
out.writeUTF("hello hadoop's hdfs!");
out.flush();
out.close();
}
/**
* 重命名
* @throws Exception
*/
@Test
public void rename() throws Exception {
boolean result = fileSystem.rename(new Path("/hdfsapi/test/test1.java"), new Path("/hdfsapi/test/test.py"));
System.out.println(result);
}
/**
* 本地文件上传到hdfs服务器
* @throws Exception
*/
@Test
public void copyfromlocalfile() throws Exception {
fileSystem.copyFromLocalFile(new Path("d://word.txt"), new Path("/hdfsapi/test"));
}
/**
* 带进度条的文件上传
* @throws Exception
*/
@Test
public void copyfromlocalfileprocess() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("D://software//navicat_premium12.zip")));
FSDataOutputStream out = fileSystem.create(new Path("/hdsfapi/test/navicat_premium12.zip"), new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out, 1024);
}
/**
* 复制文件到本地
* @throws Exception
*/
@Test
public void copytolocalfile() throws Exception {
fileSystem.copyToLocalFile(new Path("/hdfsapi/test/test.py"), new Path("D://hdfsapi//test.py"));
}
/**
* 查询文件列表
* @throws Exception
*/
@Test
public void fileList() throws Exception {
FileStatus[] listStatus = fileSystem.listStatus(new Path("/hdfsapi/test"));
for (FileStatus file : listStatus) {
if (file.isDirectory()) {
System.out.println("文件夹");
} else {
System.out.println("文件：" + file.getPath());
}
}
}
/**
* 递归查询子文件夹中的文件
* @throws Exception
*/
@Test
public void fileListRecursive() throws Exception {
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);
while (listFiles.hasNext()) {
LocatedFileStatus file = listFiles.next();
System.out.println(file.getPath().toString() + file.getLen() + file.getOwner() + file.getPermission()
+ file.getReplication());
}
}
/**
* 获取文件的块信息
* @throws Exception
*/
@Test
public void getblockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/test.java"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
for (BlockLocation block : blocks) {
for (String name : block.getNames()) {
System.out.println(name + block.getLength() + ":" + block.getOffset());
}
}
}
/**
* 递归删除文件目录及文件
*
* @throws Exception
*/
@Test
public void deletefile() throws Exception {
boolean result = fileSystem.delete(new Path("/hdfsapi/test/1"), true);
System.out.println(result);
}
@After
public void destory() throws Exception {
configuration = null;
fileSystem.close();
}
}
