基于Ambari的大数据平台搭建(HDP)

2020年9月9日 1672点热度 2人点赞 0条评论

简介

Apache Ambari 支持HDFS、MapReduce、Hive、Pig、Hbase、Zookeper、Sqoop和Hcatalog等的集中管理，类似于CDH，是一个集中部署、管理、监控 Hadoop 分布式集群的工具，主要由Ambari Server 和 Ambari Agent构成，支持Kerberos安全管理。
官网:http://ambari.apache.org/

前置

jdk部署、关闭防火墙、关闭selinux、关闭swap分区、python环境、ssh免密、chronyd时间同步
修改最大文件打开数为65535

echo "* soft nofile 65536"  >> /etc/security/limits.conf
echo "* hard nofile 65536"  >> /etc/security/limits.conf

UMASK 配置:
为当前登录会话设置umask：
umask 0022
检查当前的umask：
umask
永久更改所有交互式用户的umask：

echo umask 0022 >> /etc/profile

MySql安装

https://www.mysql.com/downloads/
https://dev.mysql.com/downloads/repo/yum/


wget http://dev.mysql.com/get/mysql57-community-release-el7-10.noarch.rpm
yum install mysql57-community-release-el7-10.noarch.rpm
yum  install mysql-community-server

systemctl start mysqld.service
systemctl status mysqld.service
初次启动密码：grep "password" /var/log/mysqld.log
密码修改


mysql -uroot -p
ALTER USER 'root'@'localhost' IDENTIFIED BY 'password';
# 降低密码安全要求
SHOW VARIABLES LIKE 'validate_password%';
set global validate_password_policy=0;
set global validate_password_length=1;

yum -y remove mysql57-community-release-el7-10.noarch
安装JDBC驱动
yum install -y mysql-connector-java
[collapse title="创建数据库"]

CREATE DATABASE ambari CHARACTER SET utf8 COLLATE utf8_general_ci; 
use ambari; 
CREATE USER 'ambari'@'%' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%'; 
CREATE USER 'ambari'@'localhost' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost'; 
CREATE USER 'ambari'@'master' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'master'; 
FLUSH PRIVILEGES; 
source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql 
show tables; 
use mysql; 
select Host User Password from user where user='ambari';

CREATE DATABASE hive CHARACTER SET utf8 COLLATE utf8_general_ci; 
use hive; 
CREATE USER 'hive'@'%' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%'; 
CREATE USER 'hive'@'localhost' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost'; 
CREATE USER 'hive'@'master' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'master'; 
FLUSH PRIVILEGES; 

CREATE DATABASE oozie CHARACTER SET utf8 COLLATE utf8_general_ci; 
use oozie; 
CREATE USER 'oozie'@'%' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'%'; 
CREATE USER 'oozie'@'localhost' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'localhost'; 
CREATE USER 'oozie'@'master' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'oozie'@'master'; 
FLUSH PRIVILEGES;

CREATE DATABASE ranger CHARACTER SET utf8 COLLATE utf8_general_ci; 
use ranger; 
CREATE USER 'rangeradmin'@'%' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'rangeradmin'@'%'; 
CREATE USER 'rangeradmin'@'localhost' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'rangeradmin'@'localhost'; 
CREATE USER 'rangeradmin'@'master' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'rangeradmin'@'master'; 
FLUSH PRIVILEGES;

CREATE DATABASE rangerkms CHARACTER SET utf8 COLLATE utf8_general_ci; 
use rangerkms; 
CREATE USER 'rangerkms'@'%' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'rangerkms'@'%'; 
CREATE USER 'rangerkms'@'localhost' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'rangerkms'@'localhost'; 
CREATE USER 'rangerkms'@'master' IDENTIFIED BY '123456'; 
GRANT ALL PRIVILEGES ON *.* TO 'rangerkms'@'master'; 
FLUSH PRIVILEGES;

[/collapse]

Ambari 2.7.4安装过程

wget -O /etc/yum.repos.d/ambari.repo http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.4.0/ambari.repo

安装PDF文件:https://docs.cloudera.com/HDPDocuments/Ambari-2.7.5.0/bk_ambari-installation/bk_ambari-installation.pdf

[hide reply_to_this="true"]wget https://www-eu.apache.org/dist/ambari/ambari-2.7.5/apache-ambari-2.7.5-src.tar.gz
可以通过yum安装，repo文件:

http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.4.0/ambari.repo

[/hide]
[successbox title="Ambari 本地Repo配置"]
软件包下载


wget http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.4.0/ambari-2.7.4.0-centos7.tar.gz
wget http://public-repo-1.hortonworks.com/HDP/centos7/3.x/updates/3.1.4.0/HDP-3.1.4.0-centos7-rpm.tar.gz
wget http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos7/HDP-UTILS-1.1.0.22-centos7.tar.gz
wget http://public-repo-1.hortonworks.com/HDP-GPL/centos7/3.x/updates/3.1.4.0/HDP-GPL-3.1.4.0-centos7-gpl.tar.gz

yum install -y epel-release
yum install -y nginx

vim /etc/nginx/nginx.conf

# For more information on configuration, see:
#   * Official English Documentation: http://nginx.org/en/docs/
#   * Official Russian Documentation: http://nginx.org/ru/docs/

user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;

# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;

events {
    worker_connections 1024;
}

http {
    autoindex on;
    autoindex_exact_size on;
    autoindex_localtime on;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log  /var/log/nginx/access.log  main;

    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;

    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;

    # Load modular configuration files from the /etc/nginx/conf.d directory.
    # See http://nginx.org/en/docs/ngx_core_module.html#include
    # for more information.
    include /etc/nginx/conf.d/*.conf;

    server {
        listen       80 default_server;
        listen       [::]:80 default_server;
        server_name  _;
#        root         /usr/share/nginx/html;

        # Load configuration files for the default server block.
        include /etc/nginx/default.d/*.conf;

        location / {
            root ambari;
        }

        error_page 404 /404.html;
            location = /40x.html {
        }

        error_page 500 502 503 504 /50x.html;
            location = /50x.html {
        }
    }

# Settings for a TLS enabled server.
#
#    server {
#        listen       443 ssl http2 default_server;
#        listen       [::]:443 ssl http2 default_server;
#        server_name  _;
#        root         /usr/share/nginx/html;
#
#        ssl_certificate "/etc/pki/nginx/server.crt";
#        ssl_certificate_key "/etc/pki/nginx/private/server.key";
#        ssl_session_cache shared:SSL:1m;
#        ssl_session_timeout  10m;
#        ssl_ciphers HIGH:!aNULL:!MD5;
#        ssl_prefer_server_ciphers on;
#
#        # Load configuration files for the default server block.
#        include /etc/nginx/default.d/*.conf;
#
#        location / {
#        }
#
#        error_page 404 /404.html;
#            location = /40x.html {
#        }
#
#        error_page 500 502 503 504 /50x.html;
#            location = /50x.html {
#        }
#    }

}

将下载的包解压至/usr/share/nginx/ambari

tar -zxvf ambari-2.7.4.0-centos7.tar.gz -C /usr/share/nginx/ambari/
tar -zxvf HDP-UTILS-1.1.0.22-centos7.tar.gz -C /usr/share/nginx/ambari/
tar -zxvf HDP-GPL-3.1.4.0-centos7-gpl.tar.gz -C /usr/share/nginx/ambari/

[hide reply_to_this="true"]yum install --downloadonly ambari-server --downloaddir=/usr/share/nginx/ambari
ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar
[/hide]
[warningbox title="repo文件修改"]
vim /etc/yum.repos.d/ambari.repo

#VERSION_NUMBER=2.7.4.0-118
[ambari-2.7.4.0]
#json.url = http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json
name=ambari Version - ambari-2.7.4.0
baseurl=http://node01/ambari/centos7/2.7.4.0-118/
gpgcheck=1
gpgkey=http://node01/ambari/centos7/2.7.4.0-118//RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

vim /etc/yum.repos.d/hdp.repo

#SION_NUMBER=3.1.4.0-315
[HDP-3.1.4.0-315]
name=HDP Version - HDP-3.1.4.0-315
baseurl=http://node01/HDP/centos7/3.1.4.0-315/
gpgcheck=1
gpgkey=http://node01/HDP/centos7/3.1.4.0-315/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

[HDP-UTILS-1.1.0.22]
name=HDP-UTILS Version - HDP-UTILS-1.1.0.22
baseurl=http://node01/HDP-UTILS/centos7/1.1.0.22/
gpgcheck=1
gpgkey=http://node01/HDP-UTILS/centos7/1.1.0.22/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

scp分发repo文件
[hide reply_to_this="true"]生成本地源
# createrepo /usr/share/nginx/ambari/HDP/centos7/
# createrepo /usr/share/nginx/ambari/HDP-UTILS/
[/hide]
[/warningbox]
[/successbox]
Master节点安装 Ambari-Server


yum install ambari-server
#按需求进行配置
ambari-server setup
#登录mysql 选择ambari数据库导入数据
source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql;

ambari-server start
ambari-server restart
ambari-server stop

user:admin password:admin

引用

https://blog.csdn.net/qq_40791253/article/details/102500998

基于Ambari的大数据平台搭建(HDP)

文章评论