Hadoop2.6 全分布式安装,配置了联邦

Hadoop全分布式安装2.6   共4台机器
hadoop01  namenode             zookeeper                  zkfc(zookeeper failover controller)
hadoop02  namenode  datanode   zookeeper   journalnode    zkfc
hadoop03            datanode   zookeeper   journalnode
hadoop04            datanode               journalnode
修改主机名称 永久生效  重启
vi /etc/sysconfig/network
HOSTNAME=hadoop01
HOSTNAME=hadoop02
HOSTNAME=hadoop03
HOSTNAME=hadoop04
临时生效
hostname  hadoop01
设置主机名称
vi /etc/hosts
192.168.192.133  hadoop01
192.168.192.134  hadoop02
192.168.192.135  hadoop03
192.168.192.136  hadoop04
scp /etc/hosts root@hadoop02:/etc
scp /etc/hosts root@hadoop03:/etc
scp /etc/hosts root@hadoop04:/etc
ssh登陆其他机器
ssh 192.168.192.133  -l root 
输入密码
123456
配置ssh无密码登陆
cd ~
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa           生成公钥、私钥对
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys          导入公钥
ssh localhost    测试
登录其它机器 ssh hadoop01  -l root
cd ~
ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa           生成公钥、私钥对
然后用scp命令，把公钥文件发放给hadoop01
scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_02.pub
scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_03.pub
scp ~/.ssh/id_dsa.pub root@hadoop01:/root/.ssh/id_dsa_04.pub
登陆hadoop01   导入公钥
cat ~/.ssh/id_dsa_02.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa_03.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa_04.pub >> ~/.ssh/authorized_keys
将hadoop01上的“最全”公钥，复制到其它机器
scp ~/.ssh/authorized_keys root@hadoop02:/root/.ssh/authorized_keys
scp ~/.ssh/authorized_keys root@hadoop03:/root/.ssh/authorized_keys
scp ~/.ssh/authorized_keys root@hadoop04:/root/.ssh/authorized_keys
安装JDK 略过
上传并解压hadoop2.6
tar -zxvf hadoop-2.6.0-x64.tar.gz -C /opt/modules/
cd  /opt/modules/hadoop-2.6.0/etc/hadoop
vi hadoop-env.sh
export JAVA_HOME=/opt/modules/jdk1.7
vi yarn-env.sh
export JAVA_HOME=/opt/modules/jdk1.7
vi mapred-env.sh  
export JAVA_HOME=/opt/modules/jdk1.7
vi core-site.xml  tmp目录如不存在，则先mkdir手动创建
<configuration>
    <property>
    <name>hadoop.tmp.dir</name>
    <value>/opt/hadoop/tmp</value>
    </property>
    <property>
    <name>fs.defaultFS</name>
    <value>hdfs://mycluster</value>
    </property>
<property>
   <name>ha.zookeeper.quorum</name>
   <value>hadoop01:2181,hadoop02:2181,hadoop03:2181</value>
 </property>
</configuration>
rm -rf /opt/hadoop/
mkdir /opt/hadoop  
mkdir /opt/hadoop/tmp
mkdir /opt/hadoop/journal
vi hdfs-site.xml  dfs.namenode.name.dir和dfs.namenode.edits.dir所设置的目录都应该手工建立好
配置联邦两个namenode
<property>
  <name>dfs.nameservices</name>
  <value>mycluster</value>
</property>
<property>
  <name>dfs.ha.namenodes.mycluster</name>
  <value>nn1,nn2</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.mycluster.nn1</name>
  <value>hadoop01:8020</value>
</property>
<property>
  <name>dfs.namenode.rpc-address.mycluster.nn2</name>
  <value>hadoop02:8020</value>
</property>
<property>
  <name>dfs.namenode.http-address.mycluster.nn1</name>
  <value>hadoop01:50070</value>
</property>
<property>
  <name>dfs.namenode.http-address.mycluster.nn2</name>
  <value>hadoop02:50070</value>
</property>
<property>
  <name>dfs.namenode.shared.edits.dir</name>
  <value>qjournal://hadoop02:8485;hadoop03:8485;hadoop04:8485/mycluster</value>
</property>
<property>
  <name>dfs.client.failover.proxy.provider.mycluster</name>
  <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
  <name>dfs.ha.fencing.methods</name>
  <value>sshfence</value>
</property>
<property>
  <name>dfs.ha.fencing.ssh.private-key-files</name>
  <value>/root/.ssh/id_dsa</value>
</property>
<property>
  <name>dfs.journalnode.edits.dir</name>
  <value>/opt/hadoop/journal</value>
</property>
<property>
   <name>dfs.ha.automatic-failover.enabled</name>
   <value>true</value>
 </property> 
<property>
<name>dfs.replication</name>
<value>2</value>
</property>




cp /opt/modules/hadoop-2.6.0/etc/hadoop/mapred-site.xml.template mapred-site.xml
vi mapred-site.xml
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>


vi yarn-site.xml
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
vi slaves
hadoop02
hadoop03
hadoop04
安装zookeeper
tar -zxvf zookeeper-3.4.6.tar.gz -C /opt/modules/
cd /opt/modules/zookeeper-3.4.6/conf
cp zoo_sample.cfg zoo.cfg
vi zoo.cfg
mkdir /opt/zookeeper
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zookeeper
server.1=hadoop01:2888:3888
server.2=hadoop02:2888:3888
server.3=hadoop03:2888:3888
cd /opt/zookeeper       在hadoop01操作
vi myid
1
ssh hadoop02 -l root
scp -r /opt/zookeeper root@hadoop02:/opt/
vi myid
2
ssh hadoop03 -l root
scp -r /opt/zookeeper root@hadoop03:/opt/
vi myid
3
scp -r /opt/zookeeper root@hadoop03:/opt/
scp -r /opt/modules/zookeeper-3.4.6 root@hadoop02:/opt/modules
scp -r /opt/modules/zookeeper-3.4.6 root@hadoop03:/opt/modules
vi /etc/profile
export ZK_HOME=/opt/modules/zookeeper-3.4.6
export PATH=$PATH:$ZK_HOME/bin
source /etc/profile
scp /etc/profile root@hadoop02:/etc/
scp /etc/profile root@hadoop03:/etc/
启动zookeeper 01-02-03
zkServer.sh  start
停止ZooKeeper服务
zkServer.sh stop 
jps   查看是否启动
2516 QuorumPeerMain
拷贝hadoop
scp -r /opt/modules/hadoop-2.6.0 root@hadoop02:/opt/modules
scp -r /opt/modules/hadoop-2.6.0 root@hadoop03:/opt/modules
scp -r /opt/modules/hadoop-2.6.0 root@hadoop04:/opt/modules
02-03-04分别创建目录
mkdir /opt/hadoop
mkdir /opt/hadoop/tmp
mkdir /opt/hadoop/journal
mkdir /opt/hadoop/dfs
mkdir /opt/hadoop/dfs/data
mkdir /opt/hadoop/dfs/name
启动Hadoop
cd /opt/modules/hadoop-2.6.0/sbin/
02-03-04启动journalnode
./hadoop-daemon.sh start journalnode
格式化 zkfc(zookeeper failover controller)
01
cd /opt/modules/hadoop-2.6.0/bin/
./hdfs zkfc -formatZK
格式化namenode
01
cd /opt/modules/hadoop-2.6.0/bin/
./hdfs namenode -format
启动一个namenode
cd /opt/modules/hadoop-2.6.0/sbin/
./hadoop-daemon.sh start namenode
在没有格式化的namenode上执行
cd /opt/modules/hadoop-2.6.0/bin/
./hdfs namenode -bootstrapStandby
停止服务(zookeeper除外)
./stop-dfs.sh
全部启动
./start-dfs.sh
测试访问
hadoop01:50070
hadoop02:50070
启动yarn
./start-yarn.sh
查看日志前50行
tail -n50 /opt/modules/hadoop-2.6.0/logs/hadoop-root-  .log
配置hadoop环境变量
vi /etc/profile
##HADOOP
export HADOOP_HOME=/opt/modules/hadoop-2.6.0
export PATH=$PATH:/opt/modules/hadoop-2.6.0/sbin:/opt/modules/hadoop-2.6.0/bin
source /etc/profile
scp /etc/profile root@hadoop02:/etc/
scp /etc/profile root@hadoop03:/etc/
scp /etc/profile root@hadoop04:/etc/
拷贝hadoop配置
scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop02:/opt/modules/hadoop-2.6.0
scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop03:/opt/modules/hadoop-2.6.0
scp -r /opt/modules/hadoop-2.6.0/etc root@hadoop04:/opt/modules/hadoop-2.6.0
每次启动01-02-03
zkServer.sh  start
全部启动
./start-dfs.sh
启动yarn
./start-yarn.sh
停止服务(zookeeper除外)
./stop-dfs.sh
./stop-yarn.sh


批量关机小脚本

#!/bin/bash
ip_array=("hadoop02" "hadoop03" "hadoop04") 
for ip in ${ip_array[*]}  
do  
    ssh  root@$ip "shutdown -h now"  
done 
echo "shutdown02 03 04" 
ssh  root@hadoop01 "shutdown -h now"  


chmod +x ./test.sh