转载请注明原文出处
hadoop搭建
阿牛的资料下载
Hadoop-入门篇环境搭建（一）
Hadoop-入门篇环境搭建（二）
Hadoop-入门篇环境搭建（三）
Hadoop-入门篇环境搭建（四）
机器配置
+所有机器都有这个步骤
这里有4台机器
hadoop 在/opt/soft/hadoop 下面
node1
+ vi ~/.bash_profile
export HADOOP_PREFIX=/opt/soft/hadoop
+ source ~/.bash_profile
# cd /opt/soft/hadoop/etc/hadoop
# vi mapred-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_79
#vi hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.7.0_79
# vi slaves
node2
node3
node4
+ mkdir -p /opt/data/hadoop
+ mkdir -p /opt/data/journalnode
#vi hdfs-site.xml
#集群名字ID mycluster请替换为下面的value=hadoop
<property>
<name>dfs.nameservices</name>
<value>hadoop</value>
</property>
#集群有哪些机器
<property>
<name>dfs.ha.namenodes.hadoop</name>
<value>nn1,nn2</value>
#集群的namenode ip
</property>
<name>dfs.namenode.rpc-address.hadoop.nn1</name>
<value>node1:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.hadoop.nn2</name>
<value>node2:8020</value>
</property>
#连接页面
<property>
<name>dfs.namenode.http-address.hadoop.nn1</name>
<value>node1:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.hadoop.nn2</name>
<value>node2:50070</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://node2:8485;node3:8485;node4:8485/hadoop</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.hadoop</name>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_dsa</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.connect-timeout</name>
<value>60000</value>
</property>
#mkdir -p /opt/data/journalnode
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/opt/data/journalnode</value>
</property>
#自动选举
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
#zk节点的位置
<property>
<name>ha.zookeeper.quorum</name>
<value>node1:2181,node2:2181,node3:2181</value>
</property>
#in your core-site.xml file:
#<property>
# <name>fs.defaultFS</name>
# <value>hdfs://hadoop</value>
#</property>
#
vi core-site.xml
<property>
<name>ipc.client.connect.max.retries</name>
<value>20</value>
<description>
</description>
</property>
<property>
<name>ipc.client.connect.retry.interval</name>
<value>5000</value>
<description>
Indicates the number of milliseconds aclient will wait for before retrying to establish a server connection.
</description>
</property>
<property>
<name>ha.zookeeper.quorum</name>
<value>node1:2181,node2:2181,node3:2181</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/opt/data/hadoop</value>
</property>
<property>
<name>fs.defaultFS</name>
<value>hdfs://hadoop</value>
</property>
hdfs高可用搭建完成
yarn搭建
cd /opt/soft/hadoop/etc/hadoop
#vi yarn-site.xml
<property>
<name>yarn.resourcemanager.ha.enabled</name>
<value>true</value>
</property>
#this is yarn id not hdfs's id
<property>
<name>yarn.resourcemanager.cluster-id</name>
<value>hadoop-yarn</value>
</property>
#this is resourcemanager id how many resourcemanager ,if you want to set RSM please set this
<property>
<name>yarn.resourcemanager.ha.rm-ids</name>
<value>rm1,rm2</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm1</name>
<value>node3</value>
</property>
<property>
<name>yarn.resourcemanager.hostname.rm2</name>
<value>node4</value>
</property>
#there is ZK ,please synchronous your's ZK cluster
<property>
<name>yarn.resourcemanager.zk-address</name>
<value>node1:2181,node2:2181,node3:2181</value>
</property>
#this is a defects in yarn ,You have to remember
#please mapreduce_shuffle
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
16年的学习笔记，留言相互交流。
关注老铁的公众号，从小菜比到老菜比
你假笨462563010加QQ群一起学习
欢迎关注spring4all
