07 . ELK Stack7.2一键多机部署脚本
时间:2022-07-25
本文章向大家介绍07 . ELK Stack7.2一键多机部署脚本,主要内容包括其使用实例、应用技巧、基本知识点总结和需要注意事项,具有一定的参考价值,需要的朋友可以参考一下。
一键部署脚本
目录结构
tree Log_Analysis_Platform_Document
Log_Analysis_Platform_Document
├── InstallES.sh
├── InstallFilebeat.sh
├── InstallKibana.sh
└── README.md
ES.sh
#!/usr/bin/env bash
# *************************************************************************************************************
# Author: ZhouJian
# Mail: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 AutoInstall Elasticsearchn-7.2 Deploy Script
# ****************************Elasticsearch Deplay Script******************************************************
clear
ESIP=`ip addr | grep "inet" | grep -v "127.0.0.1" | grep -v "inet6" | awk -F/ '{print $1}' | awk '{print $2}' `
echo -e " 33[32m ############################################################################# 33[0m"
echo -e " 33[32m # Auto Install ELK. ## 33[0m"
echo -e " 33[32m # Press Ctrl + C to cancel ## 33[0m"
echo -e " 33[32m # Any key to continue ## 33[0m"
echo -e " 33[32m # Softwae:elasticsearch-7.2.0/logstash-7.2.0/filebeat-7.2.0/kibana-7.2.0 ## 33[0m"
echo -e " 33[32m ############################################################################# 33[0m"
Read_Input() {
echo -e " 33[32m Please Input You Kibana Pass Key IP: 33[0m"
read -p "Please Input You HOST Pass Key IP:[192.168.244.55]" KibanaIP
read -p "Please Input You HOST Pass Key IP: Password:" KibanaPass
echo -e " 33[32m Please Input You Filebeat Pass Key IP: 33[0m"
read -p "Please Input You HOST Pass Key IP:[192.168.244.56]" FilebeatIP
read -p "Please Input You HOST Pass Key IP: Password:" FilebeatPass
}
Init_Yumsource()
{
if ! ping -c2 www.baidu.com &>/dev/null
then
echo "您无法上外网,不能配置yum源"
exit
fi
echo "配置yum源"
if [ ! -d /etc/yum.repos.d/backup ];then
mkdir /etc/yum.repos.d/backup
mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
fi
}
# *************************************************************************************************************
Init_Hostname()
{
hostnamectl set-hostname elk-1
echo "$ESIP elk-1" >> /etc/hosts
}
# *************************************************************************************************************
Init_SElinux()
{
echo "关闭防火墙"
systemctl stop firewalld
systemctl disable firewalld
echo "关闭selinux"
setenforce 0
sed -ri '/^SELINUX=/ s/enforcing/disabled/' /etc/selinux/config
echo "解决sshd远程连接慢的问题"
sed -ri '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
sed -ri '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
systemctl enable sshd crond &> /dev/null
}
# **************************************************************************************************************
Create_UserLogFile()
{
groupadd elk
useradd elk -g elk
mkdir -pv /data/elk/{data,logs}
chown -R elk:elk /data/
}
# **************************************************************************************************************
Unpackaged_Authorization()
{
yum -y install ntpdate
rpm -ivh /root/InstallELKB-Shell/jdk-8u121-linux-x64.rpm
tar xvf /root/InstallELKB-Shell/elasticsearch-7.2.0-linux-x86_64.tar.gz -C /opt/
chown -R elk:elk /opt/elasticsearch-7.2.0/
ntpdate -b ntp1.aliyun.com
}
# **************************************************************************************************************
Set_System_Parameter()
{
cat >> /etc/security/limits.conf <<EOF
* soft nproc 2048
* hard nproc 4096
* soft nofile 65536
* hard nofile 131072
EOF
echo "vm.max_map_count = 262144" >> /etc/sysctl.conf && sysctl -p
cat >> /etc/profile <<EOF
export HISTTIMEFORMAT="%Y-%m-%d %H:%M:%S "
EOF
source /etc/profile
cat >> /opt/elasticsearch-7.2.0/config/elasticsearch.yml <<EOF
cluster.name: elk
node.name: node-1
bootstrap.memory_lock: false
path.data: /data/elk/data
path.logs: /data/elk/logs
network.host: 0.0.0.0
http.port: 9200
discovery.seed_hosts: ["elk-1"]
cluster.initial_master_nodes: ["node-1"]
EOF
runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch ' &> /opt/elasticsearch.log &
}
Test_Service()
{
esport=`ss -antp |grep :::9200 | awk -F::: '{print $2}'`
if [ $esport -eq 9200 ];then
echo -e " 33[32m Elasticsearch is OK... 33[0m "
fi
}
# **********************PublicKeyKibana******************************************************************************
PublicKeyKibana()
{
if [ ! -f /usr/bin/expect ];then
yum -y install expect
fi
sed -i 's/# *StrictHostKeyChecking *ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config
systemctl restart sshd
cd /root/.ssh/
ssh-keygen -t rsa -N '' -f id_rsa -q
if [ $? -eq 0 ];then
/usr/bin/expect <<-EOF
set timeout 10
spawn ssh-copy-id $KibanaIP
expect {
"yes/no" { send "yesr"; exp_continue }
"password:" { send "$KibanaPassr"}
}
expect eof
EOF
fi
}
# **********************Kibana Deploy Script********************************************************************
Install_Kibana()
{
echo $ESIP > /root/InstallELKB-Shell/ESIP.txt
scp /root/InstallELKB-Shell/kibana-7.2.0-linux-x86_64.tar.gz $KibanaIP:
scp /root/InstallELKB-Shell/ESIP.txt $KibanaIP:
scp /root/InstallELKB-Shell/InstallKibana.sh $KibanaIP:
ssh root@$KibanaIP '
bash /root/InstallKibana.sh '
}
# *******************************************Filebeat Deploy Script***************************************************
PublicFilebeat()
{
if [ ! -f /usr/bin/expect ];then
yum -y install expect
fi
sed -i 's/# *StrictHostKeyChecking *ask/StrictHostKeyChecking no/g' /etc/ssh/ssh_config
systemctl restart sshd
cd /root/.ssh/
rm -rf /root/.ssh/*
ssh-keygen -t rsa -N '' -f id_rsa -q
if [ $? -eq 0 ];then
/usr/bin/expect <<-EOF
set timeout 10
spawn ssh-copy-id $FilebeatIP
expect {
"yes/no" { send "yesr"; exp_continue }
"password:" { send "$FilebeatPassr"}
}
expect eof
EOF
fi
}
Install_Filebeat()
{
scp /root/InstallELKB-Shell/filebeat-7.2.0-x86_64.rpm $FilebeatIP:
scp /root/InstallELKB-Shell/InstallFilebeat.sh $FilebeatIP:
ssh root@$FilebeatIP 'bash /root/InstallFilebeat.sh'
scp /root/InstallELKB-Shell/filebeat.yml $FilebeatIP:/etc/filebeat/
ssh root@$FilebeatIP 'systemctl restart filebeat && systemctl disable filebeat && rm -rf /root/InstallFilebeat.sh'
}
# ********************************************Logstash******************************************************************
Install_logstash()
{
tar xvf /root/InstallELKB-Shell/logstash-7.2.0.tar.gz -C /opt/
cp /root/InstallELKB-Shell/nginx.yml /opt/logstash-7.2.0/
/opt/logstash-7.2.0/bin/logstash -f /opt/logstash-7.2.0/nginx.yml &>/opt/logstash.log &
}
ES-StartUp_SelfStart()
{
cat >> /etc/init.d/elasticsearch.sh <<EOF
nohup runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch' &
nohup /opt/logstash-7.2.0/bin/logstash -f /opt/nginx.yml &
EOF
echo "/etc/init.d/elasticsearch.sh" >> /etc/rc.d/rc.local
chmod +x /etc/init.d/elasticsearch.sh
chmod +x /etc/rc.d/rc.local
}
main() {
#######Elasticsearch#######
Read_Input
Init_Yumsource
Init_Hostname
Init_SElinux
Create_UserLogFile
Unpackaged_Authorization
Set_System_Parameter
Test_Service
#########Kibana###########
PublicKeyKibana
Install_Kibana
########Filebeat#########
PublicFilebeat
Install_Filebeat
ES-StartUp_SelfStart
Kibana-StartUp_SelfStart
#######Logstash#########
Install_logstash
}
main
Kibana.sh
#!/usr/bin/env bash
# ***************************************************************************************************
# Author: ZhouJian
# MaiBox: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 Deploy Kibana Script
elastip=$(cat /root/ESIP.txt )
if [ ! -d /opt/kibana-7.2.0-linux-x86_64 ];then
tar xvf /root/kibana-7.2.0-linux-x86_64.tar.gz -C /opt/
fi
# ***************************************************************************************************
init_yumsource()
{
if ! ping -c2 www.baidu.com &>/dev/null
then
echo "您无法上外网,不能配置yum源"
exit
fi
echo "配置yum源"
if [ ! -d /etc/yum.repos.d/backup ];then
mkdir /etc/yum.repos.d/backup
mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
yum -y install ntpdate
ntpdate -b ntp1.aliyun.com
fi
}
# ***************************************************************************************************
init_SElinux()
{
echo "关闭防火墙"
systemctl stop firewalld
systemctl disable firewalld
echo "关闭selinux"
setenforce 0
sed -ri '/^SELINUX=/ s/enforcing/disabled/' /etc/selinux/config
echo "解决sshd远程连接慢的问题"
sed -ri '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
sed -ri '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
systemctl enable sshd crond &> /dev/null
}
# ***************************************************************************************************
SetKibanaParameter()
{
cat >> /opt/kibana-7.2.0-linux-x86_64/config/kibana.yml <<EOF
server.host: "0.0.0.0"
server.port: 5601
elasticsearch.hosts: ["http://$elastip:9200"]
EOF
}
# ***************************************************************************************************
StartKibana()
{
/opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root &>/opt/kibana.log &
}
Test_Service()
{
KibanaPort=` ss -antp | grep 5601 | awk '{print $4}' | awk -F*: '{print $NF}'`
if [ $KibanaPort -eq 5601 ];then
echo -e " 33[32m Kibana is OK... 33[0m "
fi
}
DeleteUselessFiles()
{
rm -rf /root/kibana-7.2.0-linux-x86_64.tar.gz
rm -rf /root/InstallKibana.sh
rm -rf /root/ESIP.txt
}
Kibana-StartUp_SelfStart()
{
echo "nohup /opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root &" >> /etc/init.d/kibana.sh
echo "/bin/bash /etc/init.d/kibana.sh" >> /etc/rc.local
chmod +x /etc/init.d/kibana.sh
chmod +x /etc/rc.local
}
init_SElinux
SetKibanaParameter
StartKibana
Test_Service
DeleteUselessFiles
Kibana-StartUp_SelfStart
Filebeat.sh
#!/usr/bin/env bash # *************************************************************
# Author: ZhouJian
# Mail: 18621048481@163.com
# Data: 2019-9-7
# Describe: CentOS 7 Deploy Filebeat7.2 Script
# *************************************************************
Init_Yumsource()
{
if ! ping -c2 www.baidu.com &>/dev/null
then
echo "您无法上外网,不能配置yum源"
exit
fi
echo "配置yum源"
if [ ! -d /etc/yum.repos.d/backup ];then
mkdir /etc/yum.repos.d/backup
mv /etc/yum.repos.d/* /etc/yum.repos.d/backup 2>/dev/null
curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo &>/dev/null
yum -y install ntpdate
ntpdate -b ntp1.aliyun.com
fi
}
Init_SElinux()
{
echo "关闭防火墙"
systemctl stop firewalld
systemctl disable firewalld
echo "关闭selinux"
setenforce 0
sed -i '/^SELINUX=/ s/enforcing/disabled/' /etc/selinux/config
echo "解决sshd远程连接慢的问题"
sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
systemctl enable sshd crond &> /dev/null
}
Install_Filebeat()
{
yum -y install ntpdate
ntpdate -b ntp1.aliyun.com
rpm -ivh /root/filebeat-7.2.0-x86_64.rpm
rm -rf /root/filebeat-7.2.0-x86_64.rpm
}
Init_Yumsource
Init_SElinux
Install_Filebeat
README.md
环境要求:
# CentOS7
# Javaa 1.8
IP |
hostname |
软件 |
内存要求 |
---|---|---|---|
192.168.122.3 |
elk-1 |
Elasticsearch、Logstash |
2G及以上 |
192.168.122.4 |
Kibana |
Kibana |
1G及以上 |
192.168.122.5 |
Filebeat |
Filebeat |
1G及以上 |
注意事项
# 1.一定要对时,时间校正,不然日志出不来;
# 2.java包最好用openjdk;
# 3.启动Elasticsearch必须切换成所创建的ELK用户启动,不然ES出于安全目的,会启动报错;
# 4.日志从Filebeat到Logstash再到ES检索到Kibana的读取速度取决于机器配置,注意用
# cat 日志文件* | wc -l 统计日志数量,然后到Elasticsearch去看总数量,确保日志都过来了在进行分析;
Elasticsearch安装
初始化
curl -o /etc/yum.repos.d/163.repo http://mirrors.163.com/.help/CentOS7-Base-163.repo &>/dev/null
curl -o /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
yum -y install ntpdate
ntpdate -b ntp1.aliyun.com
设置Hostname解析
hostnamectl set-hostname elk-1
## 修改/etc/hosts 增加如下内容
192.168.122.3 elk-1
java安装
# 安装java 1.8
yum -y install java-1.8.0-openjdk.x86_64
关闭防火墙,SeLinux
setenforce 0
sed -i '/^SELINUX=/ s/enforcing/disabled/' /etc/selinux/config
systemctl stop firewalld
systemctl disable firewalld
sed -i '/^GSSAPIAu/ s/yes/no/' /etc/ssh/sshd_config
sed -i '/^#UseDNS/ {s/^#//;s/yes/no/}' /etc/ssh/sshd_config
创建用户和组
# create user elk
groupadd elk
useradd elk -g elk
创建数据及日志文件并授权
mkdir -pv /data/elk/{data,logs}
chown -R elk:elk /data/elk/
软件包解压、授权
# 上传软件包
# 通过scp 或者FTP方式上传到/opt下
# 解压软件包到/opt目录
tar xvf elasticsearch-7.2.0-linux-x86_64.tar.gz -C /opt/
# 授权
chown -R elk:elk # 软件包名
elk-1配置文件
# 集群名
cluster.name: elk
# 节点名
node.name: node-1
# 存储数据
path.data: /data/elk/data
# 存放日志
path.logs: /data/elk/logs
# 锁内存,尽量不使用交换内存
bootstrap.memory_locak: false
# 网络地址
network.host: 0.0.0.0
http.port: 9200
# 发现集群hosts
discovery.sead_hosts: ["elk-1"]
# 设置集群master节点
cluster.inital_master_nodes: ["node-1"]
修改/etc/security/limits.conf
# *号不是注释
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 4096
修改/etc/sysctl.conf
echo "vm.max_map_count=262144" >> /etc/sysctl.conf
sysctl -p
ES启动
nohup runuser -l elk -c '/bin/bash /opt/elasticsearch-7.2.0/bin/elasticsearch' &
检查集群健康状态
curl -XGET 'elk-1:9200/_cluster/health?pretty'
Kibana安装使用
解压Kibana安装包
tar xvf kibana-7.2.0-linux-x86_64.tar.gz -C /opt/
修改Kibana配置文件
vim /opt/kibana-7.2.0-linux-x86_64/config/kibana.yml
server.port: 5601 # Port
server.host: 0.0.0.0 # 访问限制
elasticsearch.hosts: ["http://ESHostIP:9200"]
启动命令
/opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root
nohup /opt/kibana-7.2.0-linux-x86_64/bin/kibana --allow-root & 放入后台使用
tailf nohup.out # 实时查看服务运行状态
Filebeat 安装使用
下载安装
修改配置文件(修改/etc/filebeat/filebeat.yml)
filebeat.inputs:
- type: log
enabled: true
paths:
- /var/log/*.log # 抓取文件日志路径
# output.elasticsearch:
# hosts: ["ESHostIP:9200"] # 输出到ES
Filebeat到Lostash
Filebeat配置
(vim /etc/filebeat/filebeat.yml) shift + : 输入set nu 显示行号
24: enabled: true # 更改为true以启用输入配置
28: - /var/log/*.log # 替换为要抓取的日志文件路径
73: reload.enabled: true # 启动Filebeat模块
148: output.elasticsearch: # 加上注释;
150: hosts: ["localhost:9200"] # 加上注释;
158: output.logstash: # 去掉注释;
160: hosts: ["localhost:5044"] # 去掉注释,并修改localhost为logstash机器IP及对应端口号;
测试配置文件并启动
filebeat test config -e
systemctl start filebeat
systemctl enable filebeat
Logstash 安装使用
解压安装
上传包
tar xvf logstash-7.2.0.tar.gz -C /opt/
启动
/opt/logstash-7.2.0/bin/logstash -f /opt/配置文件名.yml
## 后台运行
nohup /opt/logstash-7.2.0/bin/logstash -f /opt/配置文件名.yml &
Logstash到Elasticsearch
主要看配置文件,配置文件对了,直接按照上面命令启动就可以了;
# Sample Logstash configuration for creating a simple
# Beats -> Logstash -> Elasticsearch pipeline.
input {
beats {
port => 5044
}
}
filter {
grok {
match => {
"message" => " %{DATA:log_date} %{TIME:log_localtime} %{JAVAFILE:name_file} %{WORD:workd}[%{WORD:ls}]: %{DATA:log_date2} %{TIME:log_localtime2} %{WORD:year_tmp}: %{WORD:name_2}: %{WORD:} %{WORD:}, %{JAVAFILE:}: %{JAVAFILE:app_id}, %{WORD}: %{IP:ip}, %{WORD:}: %{INT}, %{WORD}: %{USERNAME:device_id}"
}
}
}
output {
elasticsearch {
hosts => ["http://ElasticsearchHostIP:9200"]
index => "nginx_log-%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
}
}
- 已经重写,源码和文章请跳转http://www.cnblogs.com/ymnets/p/5621706.html
- 有趣 不用js也能创建silverlight
- Hadoop和Spark的异同
- ASP.NET MVC5+EF6+EasyUI 后台管理系统(62)-EF链接串加密
- sl 2.0 重要更新
- 云计算技术原理
- WCF技术剖析之五:利用ASP.NET兼容模式创建支持会话(Session)的WCF服务
- 进入AI时代,你准备好了吗?
- TiDB 在 G7 的实践和未来
- 投资钛值的你,知道钛链是什么吗?
- Is this a MS EnterLib DAAB BUG or not?
- Silverlight 2 has a Timer (DispatcherTimer)
- 难道调用ThreadPool.QueueUserWorkItem()的时候,真是必须调用Thread.Sleep(N)吗?
- silverlight 《Hands-On-Labs》教程系列
- JavaScript 教程
- JavaScript 编辑工具
- JavaScript 与HTML
- JavaScript 与Java
- JavaScript 数据结构
- JavaScript 基本数据类型
- JavaScript 特殊数据类型
- JavaScript 运算符
- JavaScript typeof 运算符
- JavaScript 表达式
- JavaScript 类型转换
- JavaScript 基本语法
- JavaScript 注释
- Javascript 基本处理流程
- Javascript 选择结构
- Javascript if 语句
- Javascript if 语句的嵌套
- Javascript switch 语句
- Javascript 循环结构
- Javascript 循环结构实例
- Javascript 跳转语句
- Javascript 控制语句总结
- Javascript 函数介绍
- Javascript 函数的定义
- Javascript 函数调用
- Javascript 几种特殊的函数
- JavaScript 内置函数简介
- Javascript eval() 函数
- Javascript isFinite() 函数
- Javascript isNaN() 函数
- parseInt() 与 parseFloat()
- escape() 与 unescape()
- Javascript 字符串介绍
- Javascript length属性
- javascript 字符串函数
- Javascript 日期对象简介
- Javascript 日期对象用途
- Date 对象属性和方法
- Javascript 数组是什么
- Javascript 创建数组
- Javascript 数组赋值与取值
- Javascript 数组属性和方法
- 搞它!!!linux远程控制 openssh
- 搞它!!!Linux构建远程YUM仓库与NFS共享存储服务
- 排障集锦:九九八十一难之第四难! yum下载软件发现已存在的 RPM 数据库问题,无法下载,
- 搞它!!!深入了解DNS域名解析服务,教你搭建一个属于自己的DNS服务器(正向解析、反向解析、泛域名解析、邮件交换解析、别名解析、分离解析,主从结构解析)
- Nginx Ingress 高并发实践
- 搞它!!!2020年了,你还不会PXE+kickstart 一键式部署安装系统么
- shell脚本快速入门系列—————— shell脚本编程规范
- shell脚本快速入门系列之------条件语句(if、case)
- 搞它!!!2020年了你还不会Cobbler自动装机么(装机步骤,优化内容详解,导入系统镜像步骤,cobbler-web管理认证方式
- 搞它!!!深入了解FTP文件传输服务
- 搞它!!!CentOS 7.6 安装和配置samba文件共享服务
- shell脚本快速入门系列之------变量
- 弄它!!! 深入了解STP生成树协议
- kali linux下的常用bash命令
- shell脚本快速入门之-----linux设置 自定义脚本开机启动,一键式部署网卡配置文件