技能库

• docker 相关
• linux 相关
• shell 相关
• hadoop
• hive

准备文件

jdk 安装包：jdk-8u211-linux-x64.tar.gz
hadoop 安装包：hadoop-tar.gz
hive 安装包: hive-x-x.tar.gz

1 - dockerfile 【jdk】 文件

# CentOS with JDK 8
# Author        qiwei

# build a new image with basic  centos
FROM centos
# who is the author
MAINTAINER qiwei

# make a new directory to store the jdk files
RUN mkdir /usr/local/java

# copy the jdk  archive to the image,and it will automaticlly unzip the tar file
ADD jdk-8u211-linux-x64.tar.gz /usr/local/java/

# make a symbol link
RUN ln -s /usr/local/java/jdk1.8.0_211 /usr/local/java/jdk

# set environment variables
ENV JAVA_HOME /usr/local/java/jdk
ENV JRE_HOME ${JAVA_HOME}/jre
ENV CLASSPATH .:${JAVA_HOME}/lib:${JRE_HOME}/lib
ENV PATH ${JAVA_HOME}/bin:$PATH

3 - dockerfile 【container】 文件

# CentOS with JDK 8
# Author        qiwei

# build a new image with basic  centos
FROM centos-jdk
# who is the author
MAINTAINER qiwei


RUN mkdir -p /opt/data/hadoop/hdfs/name
RUN mkdir -p /opt/data/hadoop/hdfs/data
RUN mkdir -p /opt/data/hadoop/tmp
RUN mkdir -p /var/log/hadoop
RUN mkdir -p /var/log/yarn
RUN mkdir -p /var/log/hive

# install openssh
RUN yum -y  install openssh-server openssh-clients
RUN yum -y  install vim which net-tools

#generate key files
RUN ssh-keygen -q -t rsa -b 2048 -f /etc/ssh/ssh_host_rsa_key -N ''
RUN ssh-keygen -q -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key -N ''
RUN ssh-keygen -q -t dsa -f /etc/ssh/ssh_host_ed25519_key  -N ''

# login localhost without password
RUN ssh-keygen -f /root/.ssh/id_rsa -N ''
RUN touch /root/.ssh/authorized_keys
RUN cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys

# set password of root
RUN echo "root:123456" | chpasswd

# open the port 22
EXPOSE 22
# when start a container it will be executed
CMD ["/usr/sbin/sshd","-D"]

4 - docker compose 文件

version: '3'
services:
  federation-master1:
    image: hd-container:1.0
    hostname: hadoop-master1
    extra_hosts:
      - "hadoop-slave1:172.18.0.11"
      - "hadoop-slave2:172.18.0.12"
    networks:
      hd-net:
        ipv4_address: 172.18.0.10
    volumes:
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/hadoop-2.7.7:/opt/hadoop
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/etc/hadoop:/etc/hadoop
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/script:/opt/script
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/apache-hive-1.2.3:/opt/hive
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/etc/hive:/etc/hive
    environment:
       HADOOP_HOME: /opt/hadoop
       HADOOP_CONF_DIR: /etc/hadoop
       HADOOP_LOG_DIR: /var/log/hadoop
       YARN_LOG_DIR: /var/log/yarn
       HIVE_HOME: /opt/hive
       HIVE_CONF_DIR: /etc/hive
  federation-slave1:
    image: hd-container:1.0
    hostname: hadoop-slave1
    extra_hosts:
      - "hadoop-master1:172.18.0.10"
      - "hadoop-slave2:172.18.0.12"
    networks:
      hd-net:
        ipv4_address: 172.18.0.11
    volumes:
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/hadoop-2.7.7:/opt/hadoop
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/etc/hadoop:/etc/hadoop
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/script:/opt/script
    environment:
       HADOOP_HOME: /opt/hadoop
       HADOOP_CONF_DIR: /etc/hadoop
       HADOOP_LOG_DIR: /var/log/hadoop
       YARN_LOG_DIR: /var/log/yarn
  federation-slave2:
    image: hd-container:1.0
    hostname: hadoop-slave2
    extra_hosts:
      - "hadoop-master1:172.18.0.10"
      - "hadoop-slave1:172.18.0.11"
    networks:
      hd-net:
        ipv4_address: 172.18.0.12
    volumes:
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/hadoop-2.7.7:/opt/hadoop
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/etc/hadoop:/etc/hadoop
      - ~/workspace/docker/bigdata-docker/run-shell/hadoop-by-ssh/script:/opt/script
    environment:
       HADOOP_HOME: /opt/hadoop
       HADOOP_CONF_DIR: /etc/hadoop
       HADOOP_LOG_DIR: /var/log/hadoop
       YARN_LOG_DIR: /var/log/yarn
networks:
  hd-net:
    driver: bridge
    ipam:
      config:
        - subnet: 172.18.0.0/16

5 - 安装

5.1 生成docker 镜像命令，

docker build -t centos-jdk:1.0 #jdk 镜像
docker build -t hd-container:1.0 #hd 镜像

5.2 组织目录，修改对应配置

image.png

5.3 运行docker compose

#启动dockers
docker-compose -f docker-compose-hive.yml up -d

6 - 启动集群

6.1 启动hadoop 在hadoop bin目录下

hdfs namenode -format

hadoop-daemon.sh start namenode

hadoop-daemon.sh start datanode

yarn-daemon.sh start resourcemanager

yarn-daemon.sh start nodemanager

hdfs namenode -format

hadoop fs -mkdir -p /user/hive/warehouse

6.2 启动hiveserver

./hiveserver2