Hadoop + Spring

Maven 依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>

   <groupId>org.baozi</groupId>
   <artifactId>hadoop-learning</artifactId>
   <version>1.0-SNAPSHOT</version>
   <packaging>jar</packaging>

   <name>hadoop-learning</name>
   <url>http://maven.apache.org</url>

   <properties>
      <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
   </properties>

   <repositories>
      <repository>
         <id>cloudera</id>
         <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
      </repository>
   </repositories>

   <dependencies>
      <!-- 添加hadoop依赖 -->
      <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-client</artifactId>
         <version>${hadoop.version}</version>
      </dependency>

      <!-- spring-data-hadoop -->
      <dependency>
         <groupId>org.springframework.data</groupId>
         <artifactId>spring-data-hadoop</artifactId>
         <version>2.5.0.RELEASE</version>
      </dependency>

      <!-- 添加单元测试的依赖 -->
      <dependency>
         <groupId>junit</groupId>
         <artifactId>junit</artifactId>
         <version>4.12</version>
         <scope>test</scope>
      </dependency>
   </dependencies>

</project>

属性文件
可有可无，没有直接写在配置文件中

fsUri=hdfs://host000:8020

Spring 配置文件

<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
       xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
       xmlns:hdp="http://www.springframework.org/schema/hadoop"
       xmlns:context="http://www.springframework.org/schema/context"
       xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
        http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
        http://www.springframework.org/schema/hadoop http://www.springframework.org/schema/hadoop/spring-hadoop.xsd">

    <context:property-placeholder location="hadoop.properties"/>

    <hdp:configuration id="hadoopConfiguration">
        fs.defaultFS=${fsUri}
    </hdp:configuration>

    <hdp:file-system id="fileSystem"
                     configuration-ref="hadoopConfiguration"
                     user="user000"
     />

</beans>

测试类

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;

import java.io.IOException;
import java.util.Arrays;


public class TestSpringHadoop {

    private ApplicationContext ctx;
    private FileSystem fileSystem;

    @Test
    public void test() throws IOException {
        System.out.println(
                Arrays.stream(fileSystem.listStatus(new Path("/")))
                .count()
        );
    }

    @Before
    public void setUp() {
        ctx = new ClassPathXmlApplicationContext("beans.xml");
        fileSystem = (FileSystem) ctx.getBean("fileSystem");
    }

    @After
    public void end() throws IOException {
        ctx = null;
        fileSystem.close();
    }

}

Hadoop + Spring Boot

添加依赖

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>

   <groupId>org.baozi</groupId>
   <artifactId>hadoop-boot-learning</artifactId>
   <version>0.0.1-SNAPSHOT</version>
   <packaging>jar</packaging>

   <name>hadoop-boot-learning</name>
   <description>Demo project for Spring Boot</description>

   <parent>
      <groupId>org.springframework.boot</groupId>
      <artifactId>spring-boot-starter-parent</artifactId>
      <version>1.5.10.RELEASE</version>
      <relativePath/> <!-- lookup parent from repository -->
   </parent>

   <repositories>
      <repository>
         <id>cloudera</id>
         <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
      </repository>
   </repositories>

   <properties>
      <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
   </properties>

   <dependencies>
      <dependency>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter</artifactId>
      </dependency>

      <!-- 添加hadoop依赖 -->
      <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-client</artifactId>
         <version>${hadoop.version}</version>
      </dependency>

      <!-- spring-data-hadoop-boot -->
      <dependency>
         <groupId>org.springframework.data</groupId>
         <artifactId>spring-data-hadoop-boot</artifactId>
         <version>2.5.0.RELEASE</version>
      </dependency>

      <dependency>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-test</artifactId>
         <scope>test</scope>
      </dependency>
   </dependencies>

   <build>
      <plugins>
         <plugin>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-maven-plugin</artifactId>
         </plugin>
      </plugins>
   </build>


</project>

属性文件

#spring.hadoop.fs-uri=hdfs://host000:8020
spring.hadoop.config.fs.defaultFS=hdfs://host000:8020
spring.hadoop.config.user=user000

测试类

package org.baozi;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.hadoop.fs.FsShell;
import org.springframework.test.context.junit4.SpringRunner;

@RunWith(SpringRunner.class)
@SpringBootTest
public class HadoopBootLearningApplicationTests {

   @Autowired
   private FsShell shell;

   @Test
   public void ls() {
      shell.ls("/").stream().forEach(e -> System.out.println(e.getPath()));
   }

}