[aws] EFK 스택 구성하기(efk stack setup)

EFK 스택 구성하기(efk stack setup)

Elasticsearch: Elasticsearch는 분산 검색 및 분석 엔진으로 대규모 데이터 저장 및 분석을 지원합니다. 로그 데이터를 저장하고 인덱싱합니다. Elasticsearch는 데이터 저장 및 검색에 최적화된 NoSQL 데이터베이스입니다.

Kibana: Kibana는 Elasticsearch의 데이터를 시각화하기 위한 웹 인터페이스입니다. Kibana를 사용하여 데이터를 검색, 시각화하고 대시보드를 생성할 수 있습니다.

Fluentd: Fluentd는 로그 수집 도구로, 로그 데이터를 수집, 처리 및 전송합니다. Fluentd는 다양한 데이터 소스에서 로그 데이터를 수집하고 Elasticsearch에 전송합니다.

Fluent Bit: Fluent Bit는 경량화된 로그 수집 도구로, Fluentd와 유사한 기능을 제공하지만, 메모리 및 CPU 사용량이 적습니다. Fluent Bit는 작은 규모의 인프라에서 사용하기 적합합니다.

아키텍처

[Web Server(Fluent Bit Agent)] --- [Fluentd] --- [Elasticsearch] --- [Kibana]

efk stack 구성

docker-compose.yml edit

version: '3.7'
services:

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
    restart: unless-stopped
    container_name: elasticsearch
    hostname: elasticsearch
    environment:
      - node.name=elasticsearch
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - ELASTIC_PASSWORD=elastic
      - xpack.security.enabled=true
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - /usr/share/zoneinfo/Asia/Seoul:/etc/localtime:ro
      - ./esdata:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
      - 9300:9300
    networks:
      - elastic

  kibana:
    image: docker.elastic.co/kibana/kibana:7.16.2
    restart: unless-stopped
    container_name: kibana
    hostname: kibana
    environment:
      - SERVER_NAME=kibana.scbyun.com
      - ELASTICSEARCH_URL=http://elasticsearch:9200
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
      - ELASTICSEARCH_USERNAME=elastic
      - ELASTICSEARCH_PASSWORD=elastic
      - xpack.security.enabled=true
    volumes:
      - /usr/share/zoneinfo/Asia/Seoul:/etc/localtime:ro
    depends_on:
      - elasticsearch
    ports:
      - 5601:5601
    networks:
      - elastic

  fluentd:
    image: anti1346/fluentd:v1.12-debian
    restart: unless-stopped
    container_name: fluentd
    hostname: fluentd
    volumes:
      - /usr/share/zoneinfo/Asia/Seoul:/etc/localtime:ro
      - ./fluentd/config/fluent.conf:/fluentd/etc/fluent.conf
      - ./fluentd/log:/fluentd/log
    depends_on:
      - elasticsearch
    ports:
      - "8888:8888"
      - "24224:24224"
      - "24224:24224/udp"
      - "5140:5140/udp"
    networks:
      - elastic

volumes:
  esdata:
    driver: local

networks:
  elastic:
    driver: bridge

mkdir -p esdata fluentd/log fluentd/config

chown -R 1000.1000 esdata

chown -R 999.999 fluentd/log

fluent.conf edit

-  기본 설정

$ cat fluent.conf
#fluentd/conf/fluent.conf
<source>
  @type  forward
  @id    input1
  @label @mainstream
  port  24224
</source>

<filter **>
  @type stdout
</filter>

<label @mainstream>
  <match docker.**>
    @type file
    @id   output_docker1
    path         /fluentd/log/docker.*.log
    symlink_path /fluentd/log/docker.log
    append       true
    time_slice_format %Y%m%d
    time_slice_wait   1m
    time_format       %Y%m%dT%H%M%S%z
  </match>
  <match **>
    @type file
    @id   output1
    path         /fluentd/log/data.*.log
    symlink_path /fluentd/log/data.log
    append       true
    time_slice_format %Y%m%d
    time_slice_wait   10m
    time_format       %Y%m%dT%H%M%S%z
  </match>
</label>

vim fluentd/config/fluent.conf

<source>
  @type http
  @id http_input
  @label @mainstream
  bind "0.0.0.0"
  port 8888
  tag http
</source>

<source>
  @type forward
  @label @mainstream
  bind "0.0.0.0"
  port 24224
</source>

<source>
  @type syslog
  @label @mainstream
  port 5140
  bind "0.0.0.0"
  tag syslog
</source>

<label @mainstream>
  <match syslog.**>
    @type elasticsearch
    suppress_type_name true
    host elasticsearch
    port 9200
    user elastic
    password elastic
    logstash_format true
    logstash_prefix syslog
    logstash_dateformat "%Y%m%d"
    flush_interval 1s
    <buffer>
      @type memory
      flush_mode interval
      flush_interval 1s
      flush_thread_count 1
    </buffer>
  </match>

  <match **>
    @type elasticsearch
    suppress_type_name true
    host elasticsearch
    port 9200
    user elastic
    password elastic
    logstash_format true
    logstash_prefix "efk-${tag}"
    logstash_dateformat "%Y%m%d"
    include_tag_key true
    tag_key @log_name
    flush_interval 1s
    <buffer>
      @type "file"
      path "/fluentd/log/buffer/aggregator.buffer"
      flush_mode interval
      flush_interval 1s
      chunk_limit_size 1m
      flush_thread_interval 0.1
      flush_thread_burst_interval 0.01
      flush_thread_count 15
      total_limit_size 2GB
      overflow_action throw_exception
      flush_at_shutdown true
      retry_max_times 30
      retry_max_interval 1h
    </buffer>
  </match>
</label>

docker-compose up -d

 

---

fluent-bit 설치 방법

- https://scbyun.com/1538

fluent-bit 설정

vim /etc/fluent-bit/fluent-bit.conf

[SERVICE]
    flush        1
    daemon       Off
    #log_level    info
    log_level    error
    parsers_file parsers.conf
    plugins_file plugins.conf

    http_server  On
    http_listen  0.0.0.0
    http_port    2020

    storage.metrics on

#[INPUT]
#    name cpu
#    tag  cpu.local
#    interval_sec 1

#########################
#####web access logs#####
[INPUT]
    Name tail
    Path /var/log/nginx/*.dev.scbyun.com-access.log
    Tag nginx-dev-<tag_prefix>
    Tag_Regex (?<tag_prefix>www|ssl|cms|pub)\.dev\.scbyun\.com-access\.log
    Parser nginx

[FILTER]
    Name grep
    Match nginx-dev-*
    Exclude agent Zabbix
    Exclude _request_uri /health_check.html

[OUTPUT]
    Name forward
    Match nginx-dev-*
    Host fluentd.scbyun.com
    Port 24224

#########################
#########syslog##########
[INPUT]
    Name   tail
    Path   /var/log/dmesg
    Tag    syslog-dmesg

[INPUT]
    Name   tail
    Path   /var/log/messages,/var/log/syslog
    Tag    syslog-messages

[INPUT]
    Name   tail
    Path   /var/log/secure
    Tag    syslog-secure

[OUTPUT]
    Name forward
    Match syslog-*
    Host fluentd.scbyun.com
    Port 24224

#[OUTPUT]
#    Name  es
#    Host  fluentd.scbyun.com
#    Port  9200
#    Match *
#    Index fluentbit
#    http_user elastic
#    http_passwd elastic

#[OUTPUT]
#    name  stdout
#    match *

fluent-bit -c /etc/fluent-bit/fluent-bit.conf --dry-run

systemctl restart fluent-bit

 

참고URL

- https://www.elastic.co/kr/blog/redesigning-product-logos-and-icons-while-building-a-design-hierarchy-at-elastic