EFK Composeをマルチノード化する

Docker Logging Efk Composeで、docker-composeを使ったELKスタックの例があるが、これをElasticSearchのマルチノード構成にする。

docker-compsoe.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
version: '3.7'
services:
  web:
    image: httpd
    ports:
      - "8080:80"
    links:
      - fluentd
    logging:
      driver: "fluentd"
      options:
        fluentd-address: "localhost:24224"
        fluentd-async-connect: "false"
        tag: httpd.access
    networks:
      - elastic

  fluentd:
    build: ./fluentd
    volumes:
      - ./fluentd/conf:/fluentd/etc
    links:
      - es01
      - es02
      - es03
    ports:
      - "24224:24224"
      - "24224:24224/udp"
    networks:
      - elastic

  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1
    container_name: es01
    environment:
      - node.name=es01
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es02,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./data/data01:/usr/share/elasticsearch/data:rw
    ports:
      - 9200:9200
    networks:
      - elastic

  es02:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1
    container_name: es02
    environment:
      - node.name=es02
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es03
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./data/data02:/usr/share/elasticsearch/data:rw
    networks:
      - elastic

  es03:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1
    container_name: es03
    environment:
      - node.name=es03
      - cluster.name=es-docker-cluster
      - discovery.seed_hosts=es01,es02
      - cluster.initial_master_nodes=es01,es02,es03
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    volumes:
      - ./data/data03:/usr/share/elasticsearch/data:rw
    networks:
      - elastic

  kib01:
    image: docker.elastic.co/kibana/kibana:7.12.1
    container_name: kib01
    ports:
      - 5601:5601
    links:
      - es01
      - es02
      - es03
    environment:
      ELASTICSEARCH_URL: http://es01:9200
      ELASTICSEARCH_HOSTS: '["http://es01:9200","http://es02:9200","http://es03:9200"]'
    networks:
      - elastic

networks:
  elastic:
    driver: bridge

fluentd/conf/fluent.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# fluentd/conf/fluent.conf
<source>
  @type forward
  port 24224
  bind 0.0.0.0
</source>
<match *.**>
  @type copy
  <store>
    @type elasticsearch
    host es01
    port 9200
    logstash_format true
    logstash_prefix fluentd
    logstash_dateformat %Y%m%d
    include_tag_key true
    type_name access_log
    tag_key @log_name
    flush_interval 1s
    reconnect_on_error true
    reload_on_failure true
    reload_connections false
  </store>
  <store>
    @type stdout
  </store>
</match>

fluentd/Dockerfile

1
2
3
# fluentd/Dockerfile
FROM fluent/fluentd:latest
RUN gem install fluent-plugin-elasticsearch --no-rdoc --no-ri

Kibanaの状態

Kibanaにアクセス。

Kibana width=640

左上のメニューからStack Managemetを選択。

Kibana width=640

Index Managementを選択。fluentd-YYYYMMDDの形式でインデックスが登録されている。

Kibana width=640