Upgrade Docker Graylog From 5.2 to 6.0

升級概要

首先，簡單描述升級方式的重點:

Mongo DB: mongo:5.0 -> mongo:6.0
根據官方自己寫的 Version Compatibility，Graylog 6.0.x 版可支援的 Mongo DB 版本為 5.0.7 ~ 7.x，秉持著有新不用舊的心態，一開始我是升 7.0，結果大暴死，Mongo DB container 陷入一個無限重啟的循環，看 log 好幾遍連怎麼死的都不知道，最後改成 6.0，啥也沒幹就啟動了。
Graylog Datanode: graylog/graylog-datanode:5.0 -> graylog/graylog-datanode:6.0
最一開始直接拿了官方給的 docker-compose 版本直接跑起來，現在要升級時，突然發現 compose file 裡面是跑 datanode，而不 ES 或是 OS，Version Compatibility 對於 datanode 也隻字未提。
Graylog: graylog/graylog:5.0 -> graylog/graylog:6.0
datanode 的 ENV VAR GRAYLOG_DATANODE_NODE_ID_FILE 似乎沒用處，可以刪掉或註解

升級前的 `docker-compose.yml`

目前的 docker-compose.yml 內容如下：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84


version: "3.8"

# For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
# Please take a look at the README at the top of this repo or the regular docs for more info.

services:
  mongodb:
    image: "mongo:5.0"
    environment:
      TZ: "Asia/Taipei"
    volumes:
      - "mongodb_data:/data/db"
    restart: "on-failure"

  # For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
  # Please take a look at the README at the top of this repo or the regular docs for more info.
  datanode:
    image: "${DATANODE_IMAGE:-graylog/graylog-datanode:5.2}"
    hostname: "datanode"
    environment:
      TZ: "Asia/Taipei"
      GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id"
      GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
      GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
      GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog"
    ulimits:
      memlock:
        hard: -1
        soft: -1
      nofile:
        soft: 65536
        hard: 65536
    ports:
      - "${IP_ADDRESS}:8999:8999/tcp"   # DataNode API
      - "${IP_ADDRESS}:9200:9200/tcp"
      - "${IP_ADDRESS}:9300:9300/tcp"
    volumes:
      - "graylog-datanode:/var/lib/graylog-datanode"
    restart: "on-failure"

  graylog:
    hostname: "server"
    image: "${GRAYLOG_IMAGE:-graylog/graylog:5.2}"
    depends_on:
      mongodb:
        condition: "service_started"
    entrypoint: "/usr/bin/tini --  /docker-entrypoint.sh"
    environment:
      TZ: "Asia/Taipei"
      GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id"
      GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
      GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
      GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
      GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
    ports:
    - "${IP_ADDRESS}:5044:5044/tcp"   # Beats
    - "${IP_ADDRESS}:5140:5140/udp"   # Syslog UDP
    - "${IP_ADDRESS}:5140:5140/tcp"   # Syslog TCP
    - "${IP_ADDRESS}:5555:5555/tcp"   # RAW TCP
    - "${IP_ADDRESS}:5555:5555/udp"   # RAW UDP
    - "${IP_ADDRESS}:9000:9000/tcp"   # Server API
    - "${IP_ADDRESS}:12201:12201/tcp" # GELF TCP
    - "${IP_ADDRESS}:12201:12201/udp" # GELF UDP
    #- "${IP_ADDRESS}:10000:10000/tcp" # Custom TCP port
    #- "${IP_ADDRESS}:10000:10000/udp" # Custom UDP port
    - "${IP_ADDRESS}:13301:13301/tcp" # Forwarder data
    - "${IP_ADDRESS}:13302:13302/tcp" # Forwarder config

    ##### temp perserve #####
    - "172.16.1.72:12201:12201/udp" # GELF UDP
    ##### temp perserve #####
    volumes:
      - "graylog_data:/usr/share/graylog/data/data"
      - "graylog_journal:/usr/share/graylog/data/journal"
      - "graylog_plugin:/usr/share/graylog/plugin"
    restart: "on-failure"

volumes:
  mongodb_data:
  graylog-datanode:
  graylog_data:
  graylog_journal:
  graylog_plugin:

注意：如果有使用到第三方外掛套件

這邊要特別注意，如果有使用到第三方外掛套件，要特別注意是否相容於 Graylog 6.0。

如果需要升級外掛版本的話，請先準備好相應的外掛版本，在升級過程中替換掉，否則 Graylog 升級後會陷入無限重啟的死循環內。

以這邊的例子，我有使用 wizecore/graylog2-output-syslog 這個 Syslog Output 外掛。

在 Graylog 5.2 時使用的外掛版本是 v4.2.6，對應 Graylog 6.0 的外掛版本是 v6.0.0。

因為外掛程式載入錯誤而陷入無限重啟的死循環時，log 大概長得像這樣：

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78


2024-06-26 15:55:58,286 INFO : org.mongodb.driver.cluster - Waiting for server to become available for operation with ID 11. Remaining time: 30000 ms. Selector: ReadPreferenceServerSelector{readPreference=primary}, topology description: {type=UNKNOWN, servers=[{address=mongodb:27017, type=UNKNOWN, state=CONNECTING}].
2024-06-26 15:55:59,015 INFO : org.graylog2.bootstrap.preflight.SearchDbPreflightCheck - Connected to (Elastic/Open)Search version <OpenSearch:2.12.0>
2024-06-26 15:55:59,095 INFO : org.hibernate.validator.internal.util.Version - HV000001: Hibernate Validator null
2024-06-26 15:56:00,376 ERROR: org.graylog2.bootstrap.CmdLineTool - Guice error (more detail on log level debug): No injectable constructor for type com.wizecore.graylog2.plugin.SyslogOutput.
2024-06-26 15:56:00,376 ERROR: org.graylog2.bootstrap.CmdLineTool - Guice error (more detail on log level debug): No implementation for com.wizecore.graylog2.plugin.SyslogOutput$Factory was bound.
2024-06-26 15:56:00,376 ERROR: org.graylog2.bootstrap.CmdLineTool - Startup error:
com.google.inject.CreationException: Unable to create injector, see the following errors:
1) [Guice/MissingConstructor]: No injectable constructor for type SyslogOutput.
class SyslogOutput does not have a @Inject annotated constructor or a no-arg constructor.
Requested by:
1  : SyslogOutput.class(SyslogOutput.java:43)
     while locating SyslogOutput
Exception in thread "main" com.google.inject.CreationException: Unable to create injector, see the following errors:
     at SyslogOutput$Factory.create(SyslogOutput.java:271)
1) [Guice/MissingConstructor]: No injectable constructor for type SyslogOutput.
Learn more:
class SyslogOutput does not have a @Inject annotated constructor or a no-arg constructor.
  https://github.com/google/guice/wiki/MISSING_CONSTRUCTOR
Requested by:
2) [Guice/MissingImplementation]: No implementation for SyslogOutput$Factory was bound.
1  : SyslogOutput.class(SyslogOutput.java:43)
     while locating SyslogOutput
     at SyslogOutput$Factory.create(SyslogOutput.java:271)
Learn more:
  https://github.com/google/guice/wiki/MISSING_CONSTRUCTOR
Requested by:
2) [Guice/MissingImplementation]: No implementation for SyslogOutput$Factory was bound.
1  : Graylog2Module.installOutput(Graylog2Module.java:267)
Requested by:
      \_ installed by: PluginBindings -> SyslogOutputModule
1  : Graylog2Module.installOutput(Graylog2Module.java:267)
Learn more:
      \_ installed by: PluginBindings -> SyslogOutputModule
  https://github.com/google/guice/wiki/MISSING_IMPLEMENTATION
Learn more:
2 errors
======================
  https://github.com/google/guice/wiki/MISSING_IMPLEMENTATION
2 errors
======================
Full classname legend:
Full classname legend:
======================
======================
Graylog2Module:       "org.graylog2.plugin.inject.Graylog2Module"
Graylog2Module:       "org.graylog2.plugin.inject.Graylog2Module"
PluginBindings:       "org.graylog2.shared.bindings.PluginBindings"
PluginBindings:       "org.graylog2.shared.bindings.PluginBindings"
SyslogOutput:         "com.wizecore.graylog2.plugin.SyslogOutput"
SyslogOutput:         "com.wizecore.graylog2.plugin.SyslogOutput"
SyslogOutput$Factory: "com.wizecore.graylog2.plugin.SyslogOutput$Factory"
SyslogOutput$Factory: "com.wizecore.graylog2.plugin.SyslogOutput$Factory"
SyslogOutputModule:   "com.wizecore.graylog2.plugin.SyslogOutputModule"
========================
End of classname legend:
========================
	at com.google.inject.internal.Errors.throwCreationExceptionIfErrorsExist(Errors.java:589) ~[graylog.jar:?]
SyslogOutputModule:   "com.wizecore.graylog2.plugin.SyslogOutputModule"
========================
End of classname legend:
========================
	at com.google.inject.internal.Errors.throwCreationExceptionIfErrorsExist(Errors.java:589)
	at com.google.inject.internal.InternalInjectorCreator.initializeStatically(InternalInjectorCreator.java:163)
	at com.google.inject.internal.InternalInjectorCreator.build(InternalInjectorCreator.java:110)
	at com.google.inject.Guice.createInjector(Guice.java:87)
	at org.graylog2.shared.bindings.GuiceInjectorHolder.createInjector(GuiceInjectorHolder.java:34)
	at org.graylog2.bootstrap.CmdLineTool.setupInjector(CmdLineTool.java:530)
	at org.graylog2.bootstrap.CmdLineTool.doRun(CmdLineTool.java:317)
	at org.graylog2.bootstrap.CmdLineTool.run(CmdLineTool.java:267)
	at org.graylog2.bootstrap.Main.main(Main.java:55)
	at com.google.inject.internal.InternalInjectorCreator.initializeStatically(InternalInjectorCreator.java:163) ~[graylog.jar:?]
	at com.google.inject.internal.InternalInjectorCreator.build(InternalInjectorCreator.java:110) ~[graylog.jar:?]
	at com.google.inject.Guice.createInjector(Guice.java:87) ~[graylog.jar:?]
	at org.graylog2.shared.bindings.GuiceInjectorHolder.createInjector(GuiceInjectorHolder.java:34) ~[graylog.jar:?]
	at org.graylog2.bootstrap.CmdLineTool.setupInjector(CmdLineTool.java:530) ~[graylog.jar:?]
	at org.graylog2.bootstrap.CmdLineTool.doRun(CmdLineTool.java:317) ~[graylog.jar:?]
	at org.graylog2.bootstrap.CmdLineTool.run(CmdLineTool.java:267) [graylog.jar:?]
	at org.graylog2.bootstrap.Main.main(Main.java:55) [graylog.jar:?]

開始升級

Step.1 升級 MongoDB

第一步先單獨升級 MongoDB 到 6.0，升級時不會影響另外兩個 container 運作。

將 docker-compose.yml 內 MongoDB 的版本改成 6.0 後執行 docker-compose up -d，container 啟動後透過以下指令升級功能相容性版本：

1

docker-compose exec mongodb mongosh --eval 'db.adminCommand( { setFeatureCompatibilityVersion: "6.0" } )'

正常情況得到的輸出：

1

{ ok: 1 }

Step.2 升級第三方外掛套件版本 (沒有使用的話可直接略過此步驟)

需要升級第三方外掛套件的話，不需要停止 container，直接將新版本第三方外掛套件複製到 plugin volume，並移除舊版本即可。

1
2


docker cp ./graylog-output-syslog-6.0.0.jar graylog-graylog-1:/usr/share/graylog/plugin/
docker exec graylog-graylog-1 rm -f /usr/share/graylog/plugin/graylog-output-syslog-4.2.6.jar

此步驟不需要先停止 container 再進行，如果是在停止的狀態下，需要透過其他 running container 掛載 volume 後才能將舊版外掛套件刪除。

，下載後丟到 graylog_plugin volume 裡即可，舊版本先留著，升級完成後再刪除。

Step.3 升級 Graylog 和 Datanode Image

把 docker-compose.yml 內的 graylog/graylog:5.2 與 graylog/graylog-datanode:5.2 版號改成 6.0 即可。

如果要將 Community Edition 轉換為 Enterprise Edition 的話，也可以把 graylog/graylog:5.2 直接改成 graylog/graylog-enterprise:6.0，我自己測試同時升級 Major Version 與轉換到 Enterprise Edition 是沒問題，不過升級前一律建議完整備份再升級，中間會不會出什麼差錯誰也說不準。

感謝 Chris 的紀錄，照著官方的指引走到 step.7 就會卡關，不得不說 Graylog 東西好，但是文件真的有點糟糕 = =。

最後檢視環境大部分都正常，除了 Elasticsearch cluster 狀態有警告：

Elasticsearch cluster datanode-cluster is yellow. Shards: 42 active, 0 initializing, 0 relocating, 2 unassigned, What does this mean?

看了一下官方說明，可以手動介入也可以試著放置 play，在放置兩天後它自己修復了。

Elasticsearch cluster datanode-cluster is green. Shards: 47 active, 0 initializing, 0 relocating, 0 unassigned, What does this mean?

以下為升級完成後的完整 docker-compose.yml 檔案內容

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82


version: "3.8"

# For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
# Please take a look at the README at the top of this repo or the regular docs for more info.

services:
  mongodb:
    image: "mongo:6.0"
    environment:
      TZ: "Asia/Taipei"
    volumes:
      - "mongodb_data:/data/db"
    restart: "on-failure"

  # For DataNode setup, graylog starts with a preflight UI, this is a change from just using OpenSearch/Elasticsearch.
  # Please take a look at the README at the top of this repo or the regular docs for more info.
  datanode:
    image: "${DATANODE_IMAGE:-graylog/graylog-datanode:6.0}"
    hostname: "datanode"
    environment:
      TZ: "Asia/Taipei"
      # deprecated on 6.0
      # GRAYLOG_DATANODE_NODE_ID_FILE: "/var/lib/graylog-datanode/node-id"
      GRAYLOG_DATANODE_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
      GRAYLOG_DATANODE_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
      GRAYLOG_DATANODE_MONGODB_URI: "mongodb://mongodb:27017/graylog"
    ulimits:
      memlock:
        hard: -1
        soft: -1
      nofile:
        soft: 65536
        hard: 65536
    ports:
      - "${IP_ADDRESS}:8999:8999/tcp"  # Graylog Data Node REST API
      - "${IP_ADDRESS}:9200:9200/tcp"  # OpenSearch REST API
      - "${IP_ADDRESS}:9300:9300/tcp"  # OpenSearch Transport API
    volumes:
      - "graylog-datanode:/var/lib/graylog-datanode"
    restart: "on-failure"

  graylog:
    hostname: "server"
    image: "${GRAYLOG_IMAGE:-graylog/graylog:6.0}"
    depends_on:
      mongodb:
        condition: "service_started"
    entrypoint: "/usr/bin/tini --  /docker-entrypoint.sh"
    environment:
      TZ: "Asia/Taipei"
      GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/data/node-id"
      GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
      GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
      GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
      GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
      GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
    ports:
    - "${IP_ADDRESS}:5044:5044/tcp"   # Beats
    - "${IP_ADDRESS}:5140:5140/udp"   # Syslog UDP
    - "${IP_ADDRESS}:5140:5140/tcp"   # Syslog TCP
    - "${IP_ADDRESS}:5555:5555/tcp"   # RAW TCP
    - "${IP_ADDRESS}:5555:5555/udp"   # RAW UDP
    - "${IP_ADDRESS}:9000:9000/tcp"   # Server API
    - "${IP_ADDRESS}:12201:12201/tcp" # GELF TCP
    - "${IP_ADDRESS}:12201:12201/udp" # GELF UDP
    #- "${IP_ADDRESS}:10000:10000/tcp" # Custom TCP port
    #- "${IP_ADDRESS}:10000:10000/udp" # Custom UDP port
    - "${IP_ADDRESS}:13301:13301/tcp" # Forwarder data
    - "${IP_ADDRESS}:13302:13302/tcp" # Forwarder config

    volumes:
      - "graylog_data:/usr/share/graylog/data/data"
      - "graylog_journal:/usr/share/graylog/data/journal"
      - "graylog_plugin:/usr/share/graylog/plugin"
    restart: "on-failure"

volumes:
  mongodb_data:
  graylog-datanode:
  graylog_data:
  graylog_journal:
  graylog_plugin: