[ELK] ElasticSearch + Kibana + Logstash 설치

- Oracle JDK

1. Oracle JDK 다운로드

# wget --no-cookies --no-check-certificate --header "Cookie: gpw_e24=http://www.oracle.com/; oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/8u162-b12/0da788060d494f5095bf8624735fa2f1/jdk-8u162-linux-x64.rpm"

2. Oracle JDK 설치

# yum -y localinstall jdk-8u162-linux-x64.rpm

- ElasticSearch + Kibana + Logstash

1. RPM GPG KEY 임포트

# rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch

2. YUM Repo 파일 생성

# echo '[elasticsearch-6.x]

name=Elasticsearch repository for 6.x packages

baseurl=https://artifacts.elastic.co/packages/6.x/yum

gpgcheck=1

gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch

enabled=1

autorefresh=1

type=rpm-md

' | tee /etc/yum.repos.d/elasticsearch.repo

3. ElasticSearch 설치

# yum -y install elasticsearch

4. ElasticSearch 설정

# vi /etc/elasticsearch/elasticsearch.yml

...

#network.host: 192.168.0.1

network.host: localhost

5. ElasticSearch 기동

# systemctl start elasticsearch

# systemctl status elasticsearch

# systemctl enable elasticsearch

6. Kibana 설치

# yum -y install kibana

7. Kibana 설정

# vi /etc/kibana/kibana.yml

...

#server.host: "localhost"

server.host: "0.0.0.0"

8. Kibana 기동

# systemctl start kibana

# systemctl status kibana

# systemctl enable elasticsearch

9. EPEL Release 설치

EPEL 버전은 https://fedoraproject.org/wiki/EPEL에서 확인할 수 있다.

# rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm

10. NGINX 설치 (Apache 가능)

# yum -y install nginx httpd-tools

# htpasswd -c /etc/nginx/htpasswd.users admin

11. NGINX 설정

# cat /etc/nginx/conf.d/kibana.conf

server {

    listen 8080;

    listen [::]:8080;

    server_name _;

    location / {

        auth_basic "Restricted Access";

        auth_basic_user_file /etc/nginx/htpasswd.users;

        proxy_set_header X-Forwarded-Host $host;

        proxy_set_header X-Forwarded-Server $host;

        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

        proxy_pass http://localhost:5601;

    }

}

# cat /etc/nginx/nginx.conf

...

include /etc/nginx/conf.d/*.conf;

# systemctl start nginx && systemctl enable nginx

------------------------------------------------------------------------------

※ x-pack 설치 (skip)

# cd /usr/share/elasticsearch            #  base elasticsearch install directory

# bin/elasticsearch-plugin install x-pack

# bin/x-pack/setup-passwords interactive

-> 패스워드 설정

# systemctl restart elasticsearch

# bin/kibana-plugin install x-pack       #  base kibana install directory

# vi /etc/kibana/kibana.yml              #  Add credentials to the kibana.yml file

elasticsearch.username: "kibana"

elasticsearch.password:  "<pwd>"

# systemctl restart kibana

------------------------------------------------------------------------------

12. Logstash 설치

# yum -y install logstash

13. (Logstash) SSL 설정 (Optional)

# cd /etc/pki/tls

# cat /etc/pki/tls/openssl.conf

...

subjectAltName = IP:172.30.1.11        # ELK_server_private_ip

...

# openssl req -config /etc/pki/tls/openssl.cnf -x509 -days 3650 -batch -nodes -newkey rsa:2048 \

-keyout private/logstash-forwarder.key -out certs/logstash-forwarder.crt

이후에 위 logstash-forwarder.crt 파일은 Logstash 서버에 로그를 보내는 모든 서버들에 카피가 되어야 한다.

14. Logstash 설정

# vi /etc/logstash/conf.d/02-beats-input.conf

input { beats { port => 5044 ssl => true ssl_certificate => "/etc/pki/tls/certs/logstash-forwarder.crt" ssl_key => "/etc/pki/tls/private/logstash-forwarder.key" } }

# vi /etc/logstash/conf.d/10-syslog-filter.conf

filter { if [type] == "syslog" { grok { match => { "message" => "%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}\:%{SPACE}%{GREEDYDATA:syslog_message}" } add_field => [ "received_at", "%{@timestamp}" ] add_field => [ "received_from", "%{host}" ] } syslog_pri { } date { match => [ "syslog_timestamp", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ] } } }

# /etc/logstash/conf.d/30-elasticsearch-output.conf

output {

elasticsearch { hosts => ["localhost:9200"] sniffing => true manage_template => false index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" document_type => "%{[@metadata][type]}" } }

# systemctl restart logstash

# systemctl enable logstash

15. Beats-Dashboard 설치

# cd /tmp

# curl -L -O http://download.elastic.co/beats/dashboards/beats-dashboards-1.3.1.zip

# unzip beats-dashboards-1.3.1.zip

# cd beats-dashboards-*

# ./load.sh -url "http://localhost:9200" -user "admin:secret"                # x-pack 설치 시 kibana:암호

16. Filebeat 설치

# yum -y install filebeat

# vi /etc/filebeat/filebeat.yml

filebeat.prospectors:

- type: log

paths:

- /var/log/messages - /var/log/secure #- /var/log/*.log

...

#output.logstash:

  # The Logstash hosts

  #hosts: ["localhost:5044"]

  hosts: ["172.30.1.12:5044"]                        # ELK_server_private_IP

  bulk_max_size: 1024                                # 로그 버퍼 최대 사이즈 옵션? 확인 필요

  # Optional SSL. By default is off.

  # List of root certificates for HTTPS server verifications

  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]

  ssl.certificate_authorities: ["/etc/pki/tls/certs/logstash-forwarder.crt"]

# systemctl start filebeat

# systemctl enable filebeat

curl -XGET 'http://localhost:9200/filebeat-*/_search?pretty'

curl --user "kibana:rain2bow" -XGET 'http://localhost:9200/filebeat-*/_search?pretty'