共计 15183 个字符,预计需要花费 38 分钟才能阅读完成。
下面关于 Hadoop 的文章您也可能喜欢,不妨看看:
Ubuntu14.04 下 Hadoop2.4.1 单机 / 伪分布式安装配置教程 http://www.linuxidc.com/Linux/2015-02/113487.htm
CentOS 安装和配置 Hadoop2.2.0 http://www.linuxidc.com/Linux/2014-01/94685.htm
Ubuntu 13.04 上搭建 Hadoop 环境 http://www.linuxidc.com/Linux/2013-06/86106.htm
Ubuntu 12.10 +Hadoop 1.2.1 版本集群配置 http://www.linuxidc.com/Linux/2013-09/90600.htm
Ubuntu 上搭建 Hadoop 环境(单机模式 + 伪分布模式)http://www.linuxidc.com/Linux/2013-01/77681.htm
Ubuntu 下 Hadoop 环境的配置 http://www.linuxidc.com/Linux/2012-11/74539.htm
单机版搭建 Hadoop 环境图文教程详解 http://www.linuxidc.com/Linux/2012-02/53927.htm
0、系统环境安装
1、同步机器时间
yum install -y ntp # 安装时间服务
ntpdate us.pool.ntp.org # 同步时间
vi /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
IPADDR=192.168.8.101
NETMASK=255.255.255.0
GATEWAY=192.168.8.2
HWADDR=00:0C:29:56:63:A1
TYPE=Ethernet
UUID=ecb7f947-8a93-488c-a118-ffb011421cac
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
service network restart
ifconfig eth0
eth0 Link encap:EthernetHWaddr00:0C:29:6C:20:2B
inet addr:192.168.8.101Bcast:192.168.8.255Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fe6c:202b/64Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500Metric:1
RX packets:777 errors:0 dropped:0 overruns:0 frame:0
TX packets:316 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:70611(68.9KiB) TX bytes:49955(48.7KiB)
rm -f /etc/udev/rules.d/70-persistent-net.rule
vim /etc/sysconfig/network-scripts/ifcfg-eth0
reboot
3、设置主机名
vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=hadoop01
NETWORKING_IPV6=no
vi /etc/hosts
127.0.0.1 localhost
192.168.8.101 hadoop01 192.168.8.102 hadoop02 192.168.8.103 hadoop03 192.168.8.104 hadoop04 192.168.8.105 hadoop05 192.168.8.106 hadoop06
关闭 ipv6
1、查看系统是否开启 ipv6
a)通过网卡属性查看
命令:ifconfig
注释:有“inet6 addr:。。。。。。。“的表示开启了 ipv6 功能
b)通过内核模块加载信息查看
命令:lsmod | grep ipv6
2、ipv6 关闭方法
在 /etc/modprobe.d/dist.conf 结尾添加
alias net-pf-10 off
alias ipv6 off
可用 vi 等编辑器,也可以通过命令:
cat <<EOF>>/etc/modprobe.d/dist.conf
alias net-pf-10 off
alias ipv6 off
EOF
关闭防火墙
chkconfig iptables stop
chkconfig iptables off
改好后重启服务器:
reboot
4、安装 JDK
vi /etc/proflie
export JAVA_HOME=/soft/jdk1.7.0_80/
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
source /etc/profile
更多详情见请继续阅读下一页的精彩内容:http://www.linuxidc.com/Linux/2016-01/127701p2.htm
5、SSH 免密码登录
cd /root/.ssh
ssh-keygen -t rsa #4 个回车
id_rsa id_rsa.pub
ssh-copy-id -i hadoop01
ssh-copy-id -i hadoop02
ssh-copy-id -i hadoop03
ssh-copy-id -i hadoop04
ssh-copy-id -i hadoop05
ssh-copy-id -i hadoop06
[root@hadoop01 .ssh]# ssh hadoop05
Last login:TueNov1017:43:412015 from 192.168.8.1
[root@hadoop05 ~]#
ssh-keygen -t rsa #4 个回车
ssh-copy-id -i hadoop01
6、安装 zookeeper
tar -zxvf zookeeper-3.4.6.tar.gz -C /root/soft
mv zoo.sample.cfg zoo.cfg
vi zoo.cfg
[root@hadoop04 conf]# vi zoo.cfg
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/soft/zookeeper-3.4.6/data
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
server.1=192.168.8.104:2888:3888
server.2=192.168.8.105:2888:3888
server.3=192.168.8.106:2888:3888
vi myid
bin/zkServer.sh start # 启动
bin/zkServer.sh status # 查看状态
[root@hadoop04 zookeeper-3.4.6]# bin/zkServer.sh status
JMX enabled by default
Using config:/soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: leader
[root@hadoop05 zookeeper-3.4.6]# bin/zkServer.sh status
JMX enabled by default
Using config:/soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: follower
[root@hadoop06 zookeeper-3.4.6]# bin/zkServer.sh status
JMX enabled by default
Using config:/soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: follower
bin/zkServer.sh stop
[root@hadoop04 zookeeper-3.4.6]# bin/zkServer.sh status
JMX enabled by default
Using config:/soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Error contacting service.It is probably not running.
[root@hadoop04 zookeeper-3.4.6]#
[root@hadoop05 zookeeper-3.4.6]# bin/zkServer.sh status
JMX enabled by default
Using config:/soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: follower
[root@hadoop06 zookeeper-3.4.6]# bin/zkServer.sh status
JMX enabled by default
Using config:/soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Mode: leader
7、hadoop 安装
[root@hadoop01 hadoop-2.7.1]# ls
bin etc include journal lib libexec LICENSE.txt logs NOTICE.txt README.txt sbin share
7.1、添加 hadoop 目录到环境变量
export JAVA_HOME=/soft/jdk1.7.0_80/
export HADOOP_HOME=/soft/hadoop-2.7.1
export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin
source /etc/profile
[root@hadoop01 hadoop-2.7.1]# which hadoop
/soft/hadoop-2.7.1/bin/hadoop
7.2、配置 hadoop-env.sh
vim hadoop-env.sh
export JAVA_HOME=/soft/jdk1.7.0_80/
7.3、配置 core-site.xml
<configuration>
<!-- 指定 hdfs 的 nameservice 为 ns1 -->
<property>
<name>fs.defaultFS</name>
<value>hdfs://ns1</value>
</property>
<!-- 指定 hadoop 临时目录 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/soft/hadoop-2.7.1/tmp</value>
</property>
<!-- 指定 zookeeper 地址 -->
<property>
<name>ha.zookeeper.quorum</name>
<value>hadoop04:2181,hadoop05:2181,hadoop06:2181</value>
</property>
</configuration>
7.4、配置 hdfs-site.xml
<configuration>
<!-- 指定 hdfs 的 nameservice 为 ns1,需要和 core-site.xml 中的保持一致 -->
<property>
<name>dfs.nameservices</name>
<value>ns1</value>
</property>
<!-- ns1 下面有两个 NameNode,分别是 nn1,nn2 -->
<property>
<name>dfs.ha.namenodes.ns1</name>
<value>nn1,nn2</value>
</property>
<!-- nn1 的 RPC 通信地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn1</name>
<value>hadoop01:9000</value>
</property>
<!-- nn1 的 http 通信地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn1</name>
<value>hadoop01:50070</value>
</property>
<!-- nn2 的 RPC 通信地址 -->
<property>
<name>dfs.namenode.rpc-address.ns1.nn2</name>
<value>hadoop02:9000</value>
</property>
<!-- nn2 的 http 通信地址 -->
<property>
<name>dfs.namenode.http-address.ns1.nn2</name>
<value>hadoop02:50070</value>
</property>
<!-- 指定 NameNode 的元数据在 JournalNode 上的存放位置 -->
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://hadoop04:8485;hadoop05:8485;hadoop06:8485/ns1</value>
</property>
<!-- 指定 JournalNode 在本地磁盘存放数据的位置 -->
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/soft/hadoop-2.7.1/journal</value>
</property>
<!-- 开启 NameNode 失败自动切换 -->
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
<!-- 配置失败自动切换实现方式 -->
<property>
<name>dfs.client.failover.proxy.provider.ns1</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!-- 配置隔离机制 -->
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<!-- 使用隔离机制时需要 ssh 免登陆 -->
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
</property>
</configuration>
7.4、配置 datanode 的配置文件 slaves
vi slaves
hadoop04
hadoop05
hadoop06
7.5、配置 mapreduce 文件 mapred-site.xml
mv mapred-site.xml.example mapred-site.xml
<configuration>
<!-- 指定 mr 框架为 yarn 方式 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
7.6、配置 yarn-site.xml
<configuration>
<!-- 指定 resourcemanager 地址 -->
<property>
<name>yarn.resourcemanager.hostname</name>
<value>hadoop03</value>
</property>
<!-- 指定 nodemanager 启动时加载 server 的方式为 shuffle server -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
</configuration>
scp -r hadoop2.7.1 hadoop02:/soft/
scp -r hadoop2.7.1 hadoop03:/soft/
scp -r hadoop2.7.1 hadoop04:/soft/
scp -r hadoop2.7.1 hadoop05:/soft/
scp -r hadoop2.7.1 hadoop06:/soft/
7.7、启动 Zookeeper 服务
[root@hadoop04 zookeeper-3.4.6]# bin/zkServer.sh start
JMX enabled by default
Using config: /soft/zookeeper-3.4.6/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
7.8、在 hadoop01 上启动 journalnode
[root@hadoop01 hadoop-2.7.1]# sbin/hadoop-daemons.sh start journalnode
[root@hadoop04 zookeeper-3.4.6]# jps
1532 JournalNode
1796 Jps
1470 QuorumPeerMain
7.9、在 hadoop01 上格式化 hadoop
hadoop namenode -format
scp -r tmp/ hadoop02:/soft/hadoop-2.7.1/
7.10、在 hadoop01 上格式化 ZK
hdfs zkfc -formatZK
7.11、在 hadoop01 上 启动 HDFS
sbin/start-dfs.sh
7.12 在 hadoop01 上启动 YARN
sbin/start-yarn.sh
8、使用 hadoop 集群测试
192.168.8.101 hadoop01
192.168.8.102 hadoop02
192.168.8.103 hadoop03
192.168.8.104 hadoop04
192.168.8.105 hadoop05
192.168.8.106 hadoop06
[root@hadoop01 hadoop-2.7.1]# jps
1614NameNode
2500Jps
1929DFSZKFailoverController
[root@hadoop01 hadoop-2.7.1]# kill -91614
本文永久更新链接地址:http://www.linuxidc.com/Linux/2016-01/127701.htm
下面关于 Hadoop 的文章您也可能喜欢,不妨看看:
Ubuntu14.04 下 Hadoop2.4.1 单机 / 伪分布式安装配置教程 http://www.linuxidc.com/Linux/2015-02/113487.htm
CentOS 安装和配置 Hadoop2.2.0 http://www.linuxidc.com/Linux/2014-01/94685.htm
Ubuntu 13.04 上搭建 Hadoop 环境 http://www.linuxidc.com/Linux/2013-06/86106.htm
Ubuntu 12.10 +Hadoop 1.2.1 版本集群配置 http://www.linuxidc.com/Linux/2013-09/90600.htm
Ubuntu 上搭建 Hadoop 环境(单机模式 + 伪分布模式)http://www.linuxidc.com/Linux/2013-01/77681.htm
Ubuntu 下 Hadoop 环境的配置 http://www.linuxidc.com/Linux/2012-11/74539.htm
单机版搭建 Hadoop 环境图文教程详解 http://www.linuxidc.com/Linux/2012-02/53927.htm
0、系统环境安装
1、同步机器时间
yum install -y ntp # 安装时间服务
ntpdate us.pool.ntp.org # 同步时间
vi /etc/sysconfig/network-scripts/ifcfg-eth0
DEVICE=eth0
IPADDR=192.168.8.101
NETMASK=255.255.255.0
GATEWAY=192.168.8.2
HWADDR=00:0C:29:56:63:A1
TYPE=Ethernet
UUID=ecb7f947-8a93-488c-a118-ffb011421cac
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=none
service network restart
ifconfig eth0
eth0 Link encap:EthernetHWaddr00:0C:29:6C:20:2B
inet addr:192.168.8.101Bcast:192.168.8.255Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fe6c:202b/64Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500Metric:1
RX packets:777 errors:0 dropped:0 overruns:0 frame:0
TX packets:316 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:70611(68.9KiB) TX bytes:49955(48.7KiB)
rm -f /etc/udev/rules.d/70-persistent-net.rule
vim /etc/sysconfig/network-scripts/ifcfg-eth0
reboot
3、设置主机名
vi /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=hadoop01
NETWORKING_IPV6=no
vi /etc/hosts
127.0.0.1 localhost
192.168.8.101 hadoop01 192.168.8.102 hadoop02 192.168.8.103 hadoop03 192.168.8.104 hadoop04 192.168.8.105 hadoop05 192.168.8.106 hadoop06
关闭 ipv6
1、查看系统是否开启 ipv6
a)通过网卡属性查看
命令:ifconfig
注释:有“inet6 addr:。。。。。。。“的表示开启了 ipv6 功能
b)通过内核模块加载信息查看
命令:lsmod | grep ipv6
2、ipv6 关闭方法
在 /etc/modprobe.d/dist.conf 结尾添加
alias net-pf-10 off
alias ipv6 off
可用 vi 等编辑器,也可以通过命令:
cat <<EOF>>/etc/modprobe.d/dist.conf
alias net-pf-10 off
alias ipv6 off
EOF
关闭防火墙
chkconfig iptables stop
chkconfig iptables off
改好后重启服务器:
reboot
4、安装 JDK
vi /etc/proflie
export JAVA_HOME=/soft/jdk1.7.0_80/
export PATH=$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$PATH
source /etc/profile
更多详情见请继续阅读下一页的精彩内容:http://www.linuxidc.com/Linux/2016-01/127701p2.htm