ELK+Kafka 企业日志收集平台(二)这是原版
[root@es1 ~]# wget https://download.elastic.co/kibana/kibana/kibana-4.1.2-linux-x64.tar.gz [root@es1 ~]# tar -xf kibana-4.2.0-linux-x64.tar.gz -C /usr/local/
[root@es1 ~]# cd /usr/local/ [root@es1 local]# ln -sv kibana-4.1.2-linux-x64 kibana `kibana' -> `kibana-4.2.0-linux-x64' [root@es1 local]# cd kibana[root@es1 kibana]# vim config/kibana.yml server.port: 5601 #默认端口可以修改的 server.host: "0.0.0.0" #kibana监听的ip elasticsearch.url: "http://localhost:9200" #由于es在本地主机上面，所以这个选项打开注释即可
[root@es1 config]# cat /etc/init.d/kibana
#!/bin/bash
#chkconfig: 2345 55 24
#description: kibana service managerKIBBIN='/usr/local/kibana/bin/kibana'
LOCK='/usr/local/kibana/locks'START() {if [ -f $LOCK ];thenecho -e "kibana is already \033[32mrunning\033[0m, do nothing."elseecho -e "Start kibana service.\033[32mdone\033[m"cd /usr/local/kibana/binnohup ./kibana & >/dev/nulltouch $LOCKfi
}STOP() {if [ ! -f $LOCK ];thenecho -e "kibana is already stop, do nothing."elseecho -e "Stop kibana serivce \033[32mdone\033[m"rm -rf $LOCKps -ef | grep kibana | grep -v "grep" | awk '{print $2}' | xargs kill -s 9 >/dev/nullfi
}STATUS() {Port=$(netstat -tunl | grep ":5602")if [ "$Port" != "" ] && [ -f $LOCK ];thenecho -e "kibana is: \033[32mrunning\033[0m..."elseecho -e "kibana is: \033[31mstopped\033[0m..."fi
}case "$1" instart)START;;stop)STOP;;status)STATUS;;restart)STOP sleep 2START;;*)echo "Usage: /etc/init.d/kibana (|start|stop|status|restart)";;
esac
[root@es1 config]# chkconfig --add kibana [root@es1 config]# service kibana start Start kibana service.done [root@es1 config]#
[root@es1 config]# ss -tunl | grep "5601" tcp LISTEN 0 511 *:5601 *:* [root@es1 config]#
yum install -y nignx
[root@saltstack-node1 conf.d]# pwd
/etc/nginx/conf.d
[root@saltstack-node1 conf.d]# cat es.conf
upstream es {server 192.168.2.18:5601 max_fails=3 fail_timeout=30s;server 192.168.2.19:5601 max_fails=3 fail_timeout=30s;
}server {listen 80;server_name localhost;location / {proxy_pass http://es/;index index.html index.htm;#authauth_basic "ELK Private";auth_basic_user_file /etc/nginx/.htpasswd;}}
[root@saltstack-node1 conf.d]# htpasswd -cm /etc/nginx/.htpasswd elk New password: Re-type new password: Adding password for user elk-user [root@saltstack-node1 conf.d]# /etc/init.d/nginx restart Stopping nginx: [ OK ] Starting nginx: [ OK ] [root@saltstack-node1 conf.d]#
[root@webserver1 nginx]# vim nginx.conf
log_format json '{"@timestamp":"$time_iso8601",''"@version":"1",''"client":"$remote_addr",''"url":"$uri",''"status":"$status",''"domain":"$host",''"host":"$server_addr",''"size":$body_bytes_sent,''"responsetime":$request_time,''"referer": "$http_referer",''"ua": "$http_user_agent"''}';access_log /var/log/access_json.log json;
input {file { #从nginx日志读入type => "nginx-access"path => "/var/log/nginx/access.log"start_position => "beginning"codec => "json" #这里指定 codec格式为json}file { #从MySQL慢日志读入type => "slow-mysql"path => "/var/log/mysql/slow-mysql.log"start_position => "beginning"codec => multiline { #这里用到了logstash的插件功能，将本来属于一行的多行日志条目整合在一起，让他属于一条 pattern => "^# User@Host" #用到了正则去匹配negate => truewhat => "previous"}}
}output {
# stdout { codec=> rubydebug }if [type] == "nginx-access" { #通过判断input中定义的type，来让它在kafka集群中生成的主题名称kafka { #输出到kafka集群bootstrap_servers => "192.168.2.22:9092,192.168.2.23:9092,192.168.2.24:9092" #生产者们topic_id => "nginx-access" #主题名称compression_type => "snappy" #压缩类型}}if [type] == "slow-mysql" {kafka {bootstrap_servers => "192.168.2.22:9092,192.168.2.23:9092,192.168.2.24:9092"topic_id => "slow-mysql"compression_type => "snappy"}}
}
input {kafka {zk_connect => "192.168.2.22:2181,192.168.2.23:2181,192.168.2.24:2181"type => "nginx-access"topic_id => "nginx-access"codec => plainreset_beginning => falseconsumer_threads => 5decorate_events => true}kafka {zk_connect => "192.168.2.22:2181,192.168.2.23:2181,192.168.2.24:2181"type => "slow-mysql"topic_id => "slow-mysql"codec => plainreset_beginning => falseconsumer_threads => 5decorate_events => true}
}output {
# stdout { codec=> rubydebug }if [type] == "nginx-access" {elasticsearch {hosts => ["192.168.2.18:9200","192.168.2.19:9200"]index => "nginx-access-%{+YYYY-MM}"}}if [type] == "slow-mysql" {elasticsearch {hosts => ["192.168.2.18:9200","192.168.2.19:9200"]index => "slow-mysql-%{+YYYY-MM}"}}
}
Kibana报表展示
来源:http://blog.sctux.com/?p=451
本文来自互联网用户投稿，文章观点仅代表作者本人，不代表本站立场，不承担相关法律责任。如若转载，请注明出处。 如若内容造成侵权/违法违规/事实不符，请点击【内容举报】进行投诉反馈！