目录

Hadoop 网络

docker network create hadoop
docker network list

centos8

docker run -itd -h master --network hadoop --name hadoop_master -p 50070:50070 -p 8088:8088  -p 19888:19888 -p 9870:9870 -p 10000:10000  --privileged=true centos /sbin/init

docker exec -it hadoop_master bash

cd /etc/yum.repos.d/
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*

yum clean all && yum makecache

yum update -y

SSH

yum -y install openssh-clients openssh-server

生成3个key文件

ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key    
一路回车

ssh-keygen -t ecdsa -f /etc/ssh/ssh_host_ecdsa_key
一路回车
ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key
一路回车

启动sshd
/usr/sbin/sshd

修改root密码
yum install -y passwd
passwd root

设置ssh免密登录到本机
ssh-keygen
一路回车
ssh-copy-id localhost
输入root密码

免密登录测试
ssh localhost

安装JAVA

yum install java-1.8.0-openjdk* -y

搜索JAVA安装位置
find / -name 'java'

安装hadoop

## 下载
https://archive.apache.org/dist/hadoop/common/
https://archive.apache.org/dist/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz

cd /home/
tar -zxvf hadoop-3.3.6.tar.gz

## 复制
docker cp /home/hadoop-3.3.6/ hadoop_master:/usr/local/

docker exec -it hadoop_master bash
cd /usr/local

## 配置环境变量
vi /etc/profile  
export HADOOP_HOME=/usr/local/hadoop-3.3.6
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
## 配置文件生效
source /etc/profile

Hadoop配置文件修改

修改core-site.xml文件

vi /usr/local/hadoop-3.3.6/etc/hadoop/core-site.xml
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>file:/usr/local/hadoop-3.3.6/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>
    <property> 
        <name>hadoop.http.staticuser.user</name> 
        <value>root</value> 
    </property> 
    <!-- 整合hive -->
    <property> 
        <name>hadoop.proxyuser.root.hosts</name> 
        <value>*</value> 
    </property> 
    <property> 
        <name>hadoop.proxyuser.root.groups</name> 
        <value>*</value> 
    </property>
</configuration>

修改hdfs-site.xml文件

vi /usr/local/hadoop-3.3.6/etc/hadoop/hdfs-site.xml
<configuration>
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>master:50090</value>
  </property>
  <property>
    <name>dfs.namenode.name.dir</name>
    <value>file:/usr/local/hadoop-3.3.6/tmp/dfs/name</value>
  </property>
  <property>
    <name>dfs.datanode.data.dir</name>
    <value>file:/usr/local/hadoop-3.3.6/tmp/dfs/data</value>
  </property>
</configuration>

修改mapred-site.xml文件

vi /usr/local/hadoop-3.3.6/etc/hadoop/mapred-site.xml
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>master:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>master:19888</value>
    </property>
</configuration>

修改yarn-site.xml文件

vi /usr/local/hadoop-3.3.6/etc/hadoop/yarn-site.xml
<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>

修改hadoop-env.sh文件

vi /usr/local/hadoop-3.3.6/etc/hadoop/hadoop-env.sh
## 搜索JAVA安装位置
find / -name 'java'
## 修改JAVA_HOME
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk********.x86_64

修改workers文件（节点配置）

vi /usr/local/hadoop-3.3.6/etc/hadoop/workers
##删除原有内容，替换为下面内容
slave1
slave2

配置hadoop-env.sh

vi /usr/local/hadoop-3.3.6/etc/hadoop/hadoop-env.sh 
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

制作集群 Docker image

docker commit -a "somo" -m "Hadoop在centos上搭建的分布模式。" hadoop_master hadoop-centos:v1

启动集群主机

## hadoop_master 可以删除重建，也可以直接使用
docker run -itd --network hadoop -h master --privileged=true --name hadoop_master -p 50070:50070 -p 8088:8088  -p 19888:19888 -p 9870:9870 -p 10000:10000   hadoop-centos:v1 /sbin/init

## 节点
docker run -itd --network hadoop -h slave1 --privileged=true --name hadoop_slave1 hadoop-centos:v1 /sbin/init
docker run -itd --network hadoop -h slave2 --privileged=true --name hadoop_slave2 hadoop-centos:v1 /sbin/init

SSH

##-----------每个节点都要配置----------------
##启动sshd
/usr/sbin/sshd
##配置生效
source /etc/profile

## 配置HOSTS 
172.18.0.2      master
172.18.0.3      slave1
172.18.0.4      slave2


##-------------主节点运行------------------
ssh-copy-id slave1
ssh-copy-id slave2

## SSH 测试
ssh master
ssh slave1
ssh slave2

启动

## 格式化namenode
cd /usr/local/hadoop-3.3.6
hdfs namenode -format

## 启动
start-all.sh

http://master:9870
http://master:8088/

##停止
stop-all.sh