物联网架构之CDH

发布时间：2023年12月17日

1、常规初始化操作

三个节点都需要执行

hostnamectl set-hostname cdhmaster
yum -y install vim lrzsz net-tools

cat >>/etc/hosts<<EOF

192.168.180.210 cdhmaster
192.168.180.200 cdhslave01
192.168.180.190 cdhslave02
EOF

systemctl stop firewalld.service && systemctl disable firewalld.service

sed -i ‘/^SELINUX=/s/enforcing/disabled/’ /etc/selinux/config && setenforce 0

2、配置时间同步

cdhmaster:

yum install chrony -y

vim /etc/chrony.conf

server ntp6.aliyun.com iburst
allow 192.168.0.0/16

systemctl enable chronyd.service && systemctl restart chronyd.service

chronyc sources && chronyc -a makestep

cdhslave01/cdhslave02:

yum install chrony -y

vim /etc/chrony.conf
server cdhmaster iburst
systemctl enable chronyd.service && systemctl restart chronyd.service && chronyc sources

3、配置本机SSH免密码登录

ssh-keygen -t rsa

ssh-copy-id cdhmaster

ssh-copy-id cdhslave01

ssh-copy-id cdhslave02

4、安装jdk
三个节点都要执行

tar zxf jdk-8u171-linux-x64.tar.gz -C /opt/

mv /opt/jdk1.8.0_171/ /opt/jdk

cat >>/etc/profile<<EOF
export JAVA_HOME=/opt/jdk
export PATH= $JAVA_HOME/bin:$ PATH
EOF
source /etc/profile && java -version

5、禁用交换分区和透明大页
三个节点都要操作

sysctl -w vm.swappiness=0

echo “vm.swappiness=0” >> /etc/sysctl.conf

echo never > /sys/kernel/mm/transparent_hugepage/defrag

echo never > /sys/kernel/mm/transparent_hugepage/enabled

echo “echo never > /sys/kernel/mm/transparent_hugepage/defrag” >> /etc/rc.d/rc.local

echo “echo never > /sys/kernel/mm/transparent_hugepage/enabled” >> /etc/rc.d/rc.local

chmod +x /etc/rc.d/rc.local

6、安装数据库
只在主节点cdhmastar上操作

rpm -qa | grep mariadb

rpm -e --nodeps mariadb-libs-5.5.68-1.el7.x86_64

yum install -y libaio
useradd mysql -s /sbin/nologin

tar zvxf mysql-5.7.22-linux-glibc2.12-x86_64.tar.gz

mv mysql-5.7.22-linux-glibc2.12-x86_64 /usr/local/mysql

cd /usr/local/mysql/bin

./mysqld --initialize --user=mysql --datadir=/usr/local/mysql/data

自动生成数据库密码 IrszF>8vd(gy

cp /usr/local/mysql/support-files/mysql.server /etc/init.d/mysqld

vim /etc/my.cnf

[client]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
[mysqld]
port = 3306
socket = /usr/local/mysql/data/mysql.sock
pid_file = /usr/local/mysql/data/mysql.pid
datadir = /usr/local/mysql/data
default_storage_engine = InnoDB
max_allowed_packet = 512M
max_connections = 2048
open_files_limit = 65535
skip-name-resolve
lower_case_table_names=1
character-set-server = utf8mb4
collation-server = utf8mb4_unicode_ci
init_connect=‘SET NAMES utf8mb4’
innodb_buffer_pool_size = 1024M
innodb_log_file_size = 2048M
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit = 0
key_buffer_size = 64M
log-error = /usr/local/mysql/data/mysql_error.log
log-bin = /usr/local/mysql/data/mysql-bin
slow_query_log = 1
slow_query_log_file = /usr/local/mysql/data/mysql_slow_query.log
long_query_time = 5
tmp_table_size = 32M
max_heap_table_size = 32M
query_cache_type = 0
query_cache_size = 0
server-id=1

/etc/init.d/mysqld start

ln -s /usr/local/mysql/bin/* /usr/sbin/

mysqladmin -uroot -p password ‘123456’

mysql -u root -p123456

alter user ‘root’@‘localhost’;

alter user ‘root’@‘localhost’ identified by ‘123456’;

create database hive default charset utf8 collate utf8_general_ci;

create database oozie default charset utf8 collate utf8_general_ci;

grant all privileges on . to ‘cdh’@‘localhost’ identified by ‘123456’ with grant option;

grant all privileges on . to ‘cdh’@‘%’ identified by ‘123456’ with grant option;

grant all privileges on . to ‘root’@‘127.0.0.1’ identified by ‘123456’ with grant option;

grant all privileges on . to ‘scm’@‘127.0.0.1’ identified by ‘123456’ with grant option;

flush privileges;

7、安装CDH
所有节点上安装依赖包

yum install -y psmisc libxslt libxslt-python perl

上传cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz、mysql-connector-java-5.1.46.tar.gz到服务器

tar zxf cloudera-manager-centos7-cm5.14.0_x86_64.tar.gz -C /opt/

mv /opt/cm-5.14.0/ /opt/cm

tar zxf mysql-connector-java-5.1.46.tar.gz

cd mysql-connector-java-5.1.46

chmod +x mysql-connector-java-5.1.46-bin.jar

cp mysql-connector-java-5.1.46-bin.jar /opt/cm/share/cmf/lib/mysql-connector-java.jar

mkdir /usr/share/java

cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

在三台机器上创建cloudera-scm用户

useradd --system --home=/opt/cm/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment “Cloudera SCM User” cloudera-scm