Hadoop集群安装-CDH5（3台服务器集群）

[root@Linux01 ~]# vim /etc/sysconfig/network
[root@Linux01 ~]# vim /etc/hosts
 
192.168.107.82 Hadoop-NN-01
192.168.107.83 Hadoop-DN-01 Zookeeper-01
192.168.107.84 Hadoop-DN-02 Zookeeper-01

[root@Linux01 ~]# useradd hadoopuser
[root@Linux01 ~]# passwd hadoopuser
[root@Linux01 ~]# su – hadoopuser        #切换用户

[hadoopuser@Linux05 hadoop-2.6.0-cdh5.6.0]$ ssh-keygen   #生成公私钥
[hadoopuser@Linux05 hadoop-2.6.0-cdh5.6.0]$ ssh-copy-id -i ~/.ssh/id_rsa.pub hadoopuser@Hadoop-NN-01

[hadoopuser@Linux05 hadoop-2.6.0-cdh5.6.0]$ ssh-copy-id Hadoop-NN-01（或写 IP：10.10.51.231）#将公钥扔到对方服务器
[hadoopuser@Linux05 hadoop-2.6.0-cdh5.6.0]$ ssh-copy-id”6000 Hadoop-NN-01”#如果带端口则这样写

[hadoopuser@Linux05 hadoop-2.6.0-cdh5.6.0]$ ssh Hadoop-NN-01  #验证（退出当前连接命令：exit、logout）
[hadoopuser@Linux05 hadoop-2.6.0-cdh5.6.0]$ ssh Hadoop-NN-01 –p 6000  #如果带端口这样写

[hadoopuser@Linux01 ~]$ vi ~/.bashrc
# hadoop cdh5
export HADOOP_HOME=/home/hadoopuser/hadoop-2.6.0-cdh5.6.0
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin
 
[hadoopuser@Linux01 ~]$ source ~/.bashrc  #生效

[hadoopuser@Linux01 ~]$ vi ~/.bashrc
# zookeeper cdh5
export ZOOKEEPER_HOME=/home/hadoopuser/zookeeper-3.4.5-cdh5.6.0
export PATH=$PATH:$ZOOKEEPER_HOME/bin
 
[hadoopuser@Linux01 ~]$ source ~/.bashrc  # 生效

[hadoopuser@Linux01 ~]$ vi $ZOOKEEPER_HOME/libexec/zkEnv.sh
56 行：找到如下位置修改语句：ZOO_LOG_DIR="$ZOOKEEPER_HOME/logs"

[hadoopuser@Linux01 ~]$ vi $ZOOKEEPER_HOME/conf/zoo.cfg
 
# zookeeper
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/home/hadoopuser/zookeeper-3.4.5-cdh5.6.0/data
clientPort=2181
 
# cluster
server.1=Zookeeper-01:2888:3888
server.2=Zookeeper-02:2888:3888

mkdir $ZOOKEEPER_HOME/data
echo 1 > $ZOOKEEPER_HOME/data/myid

mkdir $ZOOKEEPER_HOME/data
echo 2 > $ZOOKEEPER_HOME/data/myid

[hadoopuser@Linux01 ~]$ zkServer.sh start

[hadoopuser@Linux01 ~]$ jps
 
3051 Jps
2829 QuorumPeerMain

[hadoopuser@Linux01 ~]$ zkServer.sh status
 
JMX enabled by default
Using config: /home/zero/zookeeper/zookeeper-3.4.5-cdh5.0.1/bin/../conf/zoo.cfg
Mode: follower

Hadoop-NN-01

Hadoop-DN-01
Hadoop-DN-02

<configuration>
        <property>
               <name>fs.defaultFS</name>
               <value>hdfs://Hadoop-NN-01:9000</value>
               <description>定义 HadoopMaster 的 URI 和端口</description>
        </property>
        <property>
               <name>io.file.buffer.size</name>
               <value>131072</value>
               <description>用作序列化文件处理时读写 buffer 的大小</description>
        </property>
        <property>
               <name>hadoop.tmp.dir</name>
               <value>file:/home/hadoopuser/hadoop-2.6.0-cdh5.6.0/tmp</value>
               <description>临时数据存储目录设定</description>
        </property>
</configuration>

<configuration>
        <property>
               <name>dfs.namenode.name.dir</name>
               <value>file:/home/hadoopuser/hadoop-2.6.0-cdh5.6.0/dfs/name</value>
               <description> namenode 存放 name table(fsimage)本地目录（需要修改）</description>
        </property>
        <property>
               <name>dfs.namenode.data.dir</name>
               <value>file:/home/hadoopuser/hadoop-2.6.0-cdh5.6.0/dfs/data</value>
               <description>datanode 存放 block 本地目录（需要修改）</description>
        </property>
        <property>
               <name>dfs.replication</name>
               <value>1</value>
               <description>文件副本个数，默认为 3 </description>
        </property>
</configuration>

<configuration>
        <property>
               <name>yarn.resourcemanager.address</name>
               <value>Hadoop-NN-01:8032</value>
        </property>
        <property>
               <name>yarn.resourcemanager.scheduler.address</name>
               <value>Hadoop-NN-01:8030</value>
        </property>
        <property>
               <name>yarn.resourcemanager.resource-tracker.address</name>
               <value>Hadoop-NN-01:8031</value>
        </property>
        <property>
               <name>yarn.resourcemanager.admin.address</name>
               <value>Hadoop-NN-01:8033</value>
        </property>
        <property>
               <name>yarn.resourcemanager.webapp.address</name>
               <value>Hadoop-NN-01:8088</value>
        </property>
        <property>
               <name>yarn.nodemanager.aux-services</name>
               <value>mapreduce_shuffle</value>
        </property>
        <property>
               <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
               <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>
</configuration>

<configuration>
        <property>
               <name>mapreduce.framework.name</name>
               <value>yarn</value>
        </property>
        <property>
               <name>mapreduce.jobhistory.address</name>
               <value>Hadoop-NN-01:10020</value>
        </property>
        <property>
               <name>mapreduce.jobhistory.webapp.address</name>
               <value>Hadoop-NN-01:19888</value>
        </property>
</configuration>

#--------------------Java Env------------------------------
export JAVA_HOME="/usr/java/jdk1.8.0_73"
#--------------------Hadoop Env----------------------------
#export HADOOP_PID_DIR=${HADOOP_PID_DIR}
export HADOOP_PREFIX="/home/hadoopuser/hadoop-2.6.0-cdh5.6.0"
#--------------------Hadoop Daemon Options-----------------
# export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
# export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
#--------------------Hadoop Logs---------------------------
#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
#--------------------SSH PORT-------------------------------
export HADOOP_SSH_OPTS="-p 6000"        #如果你修改了 SSH 登录端口，一定要修改此配置。

#Yarn Daemon Options
#export YARN_RESOURCEMANAGER_OPTS
#export YARN_NODEMANAGER_OPTS
#export YARN_PROXYSERVER_OPTS
#export HADOOP_JOB_HISTORYSERVER_OPTS
#Yarn Logs
export YARN_LOG_DIR="/home/hadoopuser/hadoop-2.6.0-cdh5.6.0/logs"

scp -r /home/hadoopuser/hadoop-2.6.0-cdh5.6.0 hadoopuser@Hadoop-DN-01:/home/hadoopuser
scp -r /home/hadoopuser/hadoop-2.6.0-cdh5.6.0 hadoopuser@Hadoop-DN-02:/home/hadoopuser

[hadoopuser@Linux01 hadoop-2.6.0-cdh5.6.0]$ hadoop namenode -format

[hadoopuser@Linux01 hadoop-2.6.0-cdh5.6.0]$ hadoop-daemon.sh start journalnode starting journalnode, logging to /home/hadoopuser/hadoop-2.6.0-cdh5.6.0/logs/hadoop-puppet-journalnode-BigData-03.out

[hadoopuser@Linux01 hadoop-2.6.0-cdh5.6.0]$ jps
 
9076 Jps
9029 JournalNode

[hadoopuser@Linux01 hadoop-2.6.0-cdh5.6.0]$ start-dfs.sh

[hadoopuser@Linux01 hadoop-2.6.0-cdh5.6.0]$ start-yarn.sh