Hadoop 3.1.3 完全分布式集群搭建

1.解压Hadoop，zookeeper，配置环境变量

#JAVA_HOME

JAVA_HOME=/home/admin/modules/jdk1.8.0_144

export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:JAVA_HOME/lib/tools.jar

export PATH=$PATH:$JAVA_HOME/bin

#HADOOP_HOME

export HADOOP_HOME=/home/admin/modules/hadoop-3.1.3

export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

##ZOOKEEPER_HOME

export ZOOKEEPER_HOME=/home/admin/modules/zookeeper-3.4.10

export PATH=$PATH:$ZOOKEEPER_HOME/bin

2.修改配置文件

zoo.cfg

[root@node3 conf]# cat zoo.cfg

# The number of milliseconds of each tick

tickTime=2000

# The number of ticks that the initial

# synchronization phase can take

initLimit=10

# The number of ticks that can pass between

# sending a request and getting an acknowledgement

syncLimit=5

# the directory where the snapshot is stored.

# do not use /tmp for storage, /tmp here is just

# example sakes.

dataDir=/home/admin/modules/zookeeper-3.4.10/zkData

# the port at which the clients will connect

clientPort=2181

# the maximum number of client connections.

# increase this if you need to handle more clients

#maxClientCnxns=60

# Be sure to read the maintenance section of the

# administrator guide before turning on autopurge.

# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance

# The number of snapshots to retain in dataDir

#autopurge.snapRetainCount=3

# Purge task interval in hours

# Set to "0" to disable auto purge feature

#autopurge.purgeInterval=1

server.1=node1:2888:3888

server.2=node2:2888:3888

server.3=node3:2888:3888

#server.4=node4:2888:3888

3.在zookeeper解压文件夹创建zkData

touch myid

echo 1 > myid

4.配置Hadoop

修改.env文件

在hadoop-env.sh, mapred-env.sh,yarn-env.sh 添加JAVA-HOME

export JAVA_HOME=/home/admin/modules/jdk1.8.0_144

5.添加节点

修改workers，添加三个节点

node1

node2

node3

6.修改 core-site.xml

<name>fs.defaultFS</name>

<value>hdfs://mycluster</value>

</property>

<name>hadoop.tmp.dir</name>

<value>/home/admin/modules/hadoop-3.1.3/hadoop-data</value>

</property>

<name>ha.zookeeper.quorum</name>

</property>

修改hdfs-site.xml

<name>dfs.replication</name>

</property>

<name>dfs.nameservices</name>

<value>mycluster</value>

</property>

<name>dfs.ha.namenodes.mycluster</name>

</property>

<name>dfs.namenode.rpc-address.mycluster.nn1</name>

</property>

<name>dfs.namenode.rpc-address.mycluster.nn2</name>

</property>

<name>dfs.namenode.http-address.mycluster.nn1</name>

</property>

<name>dfs.namenode.http-address.mycluster.nn2</name>

</property>

<name>dfs.namenode.shared.edits.dir</name>

<value>qjournal://node1:8485;node2:8485;node3:8485/mycluster</value>

</property>

<name>dfs.client.failover.proxy.provider.mycluster</name>

<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>

</property>

<name>dfs.ha.fencing.methods</name>

<value>sshfence</value>

</property>

<name>dfs.ha.fencing.ssh.private-key-files</name>

</property>

<name>dfs.journalnode.edits.dir</name>

<value>/home/admin/modules/hadoop-3.1.3/journalnode</value>

</property>

<name>dfs.ha.automatic-failover.enabled</name>

</property>

<name>dfs.permissions.enable</name>

<value>false</value>

</property>

<name>dfs.webhdfs.enabled</name>

</property>

修改mapred-site.xml

<name>mapreduce.framework.name</name>

</property>

<name>mapreduce.jobhistory.address</name>

</property>

<name>mapreduce.jobhistory.webapp.address</name>

</property>

修改yarn-site.xml

<name>yarn.nodemanager.aux-services</name>

<value>mapreduce_shuffle</value>

</property>

<name>yarn.resourcemanager.ha.enabled</name>

</property>

<name>yarn.resourcemanager.cluster-id</name>

<value>cluster1</value>

</property>

<name>yarn.resourcemanager.ha.rm-ids</name>

</property>

<name>yarn.resourcemanager.hostname.rm1</name>

</property>

<name>yarn.resourcemanager.hostname.rm2</name>

</property>

<name>yarn.resourcemanager.webapp.address.rm1</name>

</property>

<name>yarn.resourcemanager.webapp.address.rm2</name>

</property>

<name>yarn.resourcemanager.zk-address</name>

</property>

<name>yarn.log-aggregation-enable</name>

</property>

<name>yarn.log-aggregation.retain-seconds</name>

</property>

<name>yarn.log.server.url</name>

<value>http://node1:19888/jobhistory/logs/</value>

</property>

7.修改sbin下面的

start（stop）-dfs.sh,在最开头添加如下

HDFS_DATANODE_USER=root

HDFS_DATANODE_SECURE_USER=hdfs

HDFS_NAMENODE_USER=root

HDFS_JOURNALNODE_USER=root

HDFS_ZKFC_USER=root

start(stop)-yarn.sh,在最开头添加如下

YARN_RESOURCEMANAGER_USER=root

HADOOP_SECURE_DN_USER=yarn

YARN_NODEMANAGER_USER=root

8.分发Hadoop，zookeeper到各个节点

9.每个节点启动zookeeper

zkServer.sh start

10.每个节点启动journalnode

hdfs --daemon start journalnode

11. 格式化主namenode

hdfs namenode -format

12.启动主namenode

启动主namenoe hdfs --daemon start namenode

13.从namenode同步主namenode

hdfs namenode -bootstrapStandby 同步主节点

14.启动从namenode

namenoe hdfs --daemon start namenode

15.初始化zookeeper

在主节点 hdfs zkfc -formatZK

16.主节点启动集群

sbin/start-dfs.sh

17.查看进程

[root@node1 hadoop-3.1.3]# jps

3824 DataNode

1413 QuorumPeerMain

4422 NodeManager

4729 Jps

3692 NameNode

4108 JournalNode

4287 DFSZKFailoverController

18.启动resource manager 高可用

sbin/start-yarn.sh

查看进程

[root@node2 hadoop-3.1.3]# jps

1713 NameNode

3971 Jps

1571 JournalNode

3350 NodeManager

1254 QuorumPeerMain

2438 DataNode

2606 DFSZKFailoverController

3231 ResourceManager

至此集群搭建完。

访问9870：

完美搭建！

谢谢大家~~~~~

Hadoop 3.1.3 完全分布式集群搭建

推荐阅读更多精彩内容