Centos7安装Hadoop伪分布式集群

安装准备工作

#以此前提为例子
Centos7.x
JavaSE JDK包1.8：www.oracle.com:建议下载rpm包安装比较方便
Hadoop Stable:v2.9.0:下载地址：http://mirror.bit.edu.cn/apache/hadoop/common/

rpm -ivh jdk-8u151-linux-x64.rpm 
vim ~/.bashrc 

添加JAVA_HOME并生效：

#Add the Java Path
export JAVA_HOME=/usr/java/jdk1.8.0_151/
#保存退出
source ~/.bashrc

添加主机间互信
由于是单节点，故单机互信即可

#生成本机id_rsa.pub文件
ssh-keygen -t rsa 
cat ~/.ssh/id_rsa.pub >> authorized_keys
#尝试一下ssh root@localhost看看能不能进入另外一个shell

下载Hadoop Stable版本2.9.0

#下载2.9.0版本
wget http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.9.0/hadoop-2.9.0.tar.gz

将hadoop解压到指定目录：/usr/local (为例)

tar -zxvf hadoop-2.9.0.tar.gz -C /usr/local
mv /usr/local/hadoop-2.9.0 /usr/local/hadoop

提前添加好Hadoop环境变量

#Add the Hadoop Path
export HADOOP_HOME=/usr/local/hadoop
export HADOOP_PREFIX=$HADOOP_HOME
export HADOOP_INSTALL=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_YARN_HOME=$HADOOP_HOME
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_LIBEXEC_DIR=$HADOOP_HOME/libexec
export HADOOP_CONF_DIR=$HADOOP_PREFIX/etc/hadoop
export HADOOP_YARN_USER=root
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin

修改配置文件

cd $HADOOP_HOME/etc/hadoop
#用前先备份把
cp core-site.xml core-site.xml.backup
cp hdfs-site.xml hdfs-site.xml.backup
cp mapred-site.xml.template mapred-site.xml.
cp yarn-site.xml yarn-site.xml.backup

编辑core-site.xml文件

#vim core-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->
<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/hadoop/tmp</value>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>

编辑hdfs-site.xml文件

#vim hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/usr/local/hadoop/tmp/dfs/namenode_dir</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/usr/local/hadoop/tmp/dfs/datanode_dir</value>
    </property>
</configuration>

编辑mapred-site.xml文件

#vim mapred-site.xml
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>

编辑yarn-site.xml文件

#vim yarn-site.xml
<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>localhost</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>

递归生成配置文件中设置好的目录存放数据

mkdir -p /usr/local/hadoop/tmp/
mkdir -p /usr/local/hadoop/tmp/dfs/namenode_dir
mkdir -p /usr/local/hadoop/tmp/dfs/datanode_dir

namenode格式化

hadoop namenode -format

成功格式化

启动Hadoop集群

cd $HADOOP_HOME/sbin

诸多控制启停的脚本

首先看看当前运行在集群中的JAVA服务进程

启动HDFS：

./start-dfs.sh

Start HDFS

启动Yarn资源管理器：

./start-yarn.sh

Start Yarn

启动历史服务器：

mr-jobhistory-daemon.sh start historyserver

Stary historyserver

查看Hadoop集群启动后所运行的服务进程

• Namenode
• SecondaryNamenode
• Datanode
• ResourceManager
• NodeManager
• JobHistoryServer

可以通过访问webUI看看Hadoop集群是否完全启动

namenodeWebUI

历史服务器

部署完成～～如有不正，希望斧正～～