物联网架构之 Hadoop

修改/etc/hosts文件
192.168.107.197 node1
192.168.107.196 node2
192.168.107.195 node3

创建用户并加入组
groupadd hadoop
useradd -g hadoop hduser
passwd hduser
vim /etc/sudoers
hduser ALL=(ALL) ALL

安装JDK
rpm -ivh jdk-8u171-linux-x64.rpm

vim /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_171-amd64
export CLASSPATH=$JAV{A}_{H}OME/lib:$CLASSPATH
export PATH=$JAV{A}_{H}OME/bin:$PATH

source /etc/profile

java -version

配置本机SSH免密码登录
ssh-keygen -t rsa
ssh-copy-id node1
ssh-copy-id node2
ssh-copy-id node3

hadoop完全分布式安装

cd /home/hduser
tar zxf hadoop-2.6.5.tar.gz
mv hadoop-2.6.5 hadoop

hadoop的环境变量
vim /etc/profile
#hadoop
export HADOOP_HOME=/home/hduser/hadoop
export PATH=$HADOO{P}_{H}OME/bin:$PATH
source /etc/profile

配置Hadoop：
vim /home/hduser/hadoop/etc/hadoop/hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_171-amd64

vim /home/hduser/hadoop/etc/hadoop/yarn-env.sh
export JAVA_HOME=/usr/java/jdk1.8.0_171-amd64

vim /home/hduser/hadoop/etc/hadoop/slaves
node2
node3

vim /home/hduser/hadoop/etc/hadoop/core-site.xml


fs.defaultFS
hdfs://node1:9000


hadoop.tmp.dir
file:/home/hduser/hadoop/tmp

vim /home/hduser/hadoop/etc/hadoop/hdfs-site.xml


dfs.namenode.secondary.http-address
node1:50090


dfs.namenode.name.dir
file:/home/hduser/hadoop/dfs/name


dfs.datanode.data.dir
file:/home/hduser/hadoop/dfs/data


dfs.replication
2


dfs.webhdfs.enabled
true

vim /home/hduser/hadoop/etc/hadoop/mapred-site.xml


mapreduce.framework.name
yarn


mapreduce.jobhistory.address
node1:10020


mapreduce.jobhistory.webapp.address
node1:19888

vim /home/hduser/hadoop/etc/hadoop/yarn-site.xml


yarn.nodemanager.aux-services
mapreduce_shuffle


yarn.nodemanager.aux-services.mapreduce.shuffle.class
org.apache.hadoop.mapred.ShuffleHandler


yarn.resourcemanager.address
node1:8032


yarn.resourcemanager.scheduler.address
node1:8030


yarn.resourcemanager.resource-tracker.address
node1:8035


yarn.resourcemanager.admin.address
node1:8033


yarn.resourcemanager.webapp.address
node1:8088

scp -r /home/hduser/hadoop node2:/home/hduser
scp -r /home/hduser/hadoop node3:/home/hduser

验证安装配置：
cd /home/hduser/hadoop
bin/hdfs namenode -format
sbin/start-dfs.sh

jps

sbin/start-yarn.sh

sbin/start-all.sh

bin/hdfs dfsadmin -report
http://192.168.107.197:50070
sbin/stop-all.sh

mkdir file
cd file
echo “Hello World hi HADOOP” > file1.txt
echo “Hello hadoop hi CHINA” > file2.txt
sbin/start-all
bin/hadoop fs -mkdir /input2
bin/hadoop fs -put file* /input2
bin/hadoop fs -ls /input2
bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.6.5.jar wordcount /input2/ /output2/wordcount1
bin/hadoop fs -cat /output2/wordcount1/*

HDFS的相关命令：
hdfs fsck / -files -blocks
sbin/start-balancer.sh
hadoop fs -mkdir /user
hadoop fs -mkdir /user/hadoop/dir1 /user/hadoop/dir2
hadoop fs -ls /input2/file1.txt
hadoop fs -ls /input2/
hadoop fs -cat /input2/file1.txt /input2/file2.txt
文件转移
hadoop fs -put /home/hduser/file/file1.txt /input2
hadoop fs -put /home/hduser/file/file1.txt /home/hduser/file/file2.txt /input2
hadoop fs -get /input2/file1.txt $HOME/file.txt
hadoop fs -mv /input2/file1.txt /input2/file2.txt /user/hadoop/dir1
hadoop fs -cp /input2/file1.txt /input2/file2.txt /user/hadoop/dir1
hadoop fs -cp file:///file1.txt file:///file2.txt file:///tmp
hadoop fs -rm /input2/file3.txt
hadoop fs -rmr /input2#现在推荐使用 hadoop fs -rm -r /input2 命令
hadoop fs -test -e /input2/file3.txt
hadoop fs -test -z /input2/file1.txt