chore: configure Logger overload handling

config/deps/logger.exs

@@ -1,28 +1,46 @@
 import Config
 
 # Include referrer in Logster request log
-config :logster, :allowed_headers, ["referer"]
+config :logster,
+  headers: ["referer"],
+  formatter: :string
 
 if config_env() == :prod do
   # Do not print debug messages in production
   config :logger,
     level: :info,
-    handle_sasl_reports: true,
-    backends: [:console]
+    handle_sasl_reports: true
 
-  config :logger, :console,
-    level: :info,
+  config :logger, :default_formatter,
     format: "$dateT$time [$level]$levelpad node=$node $metadata$message\n",
     metadata: [:ip, :mbta_id, :request_id]
+
+  # Drop aggressively and early to prevent the message queue from growing large.
+  # Also enable terminating overloaded handlers based on memory or queue length.
+  # https://www.erlang.org/docs/28/apps/kernel/logger_chapter.html#overload_protection
+  config :logger, :default_handler,
+    config: [
+      # default
+      sync_mode_qlen: 10,
+      # lower than default of 200
+      drop_mode_qlen: 50,
+      # lower than default of 1000
+      flush_qlen: 200,
+      # changed from false
+      overload_kill_enable: true,
+      # lower than default of 20_000
+      overload_kill_qlen: 500,
+      # default
+      overload_kill_mem_size: 3_000_000
+    ]
 end
 
 if config_env() == :dev do
-  config :logger, truncate: :infinity
-
-  config :logger, :console,
+  config :logger, :default_formatter,
     format: "$date $time [$level] $metadata$message\n",
-    level: :notice,
     metadata: [:ip, :mbta_id, :request_id]
+
+  config :logger, level: :notice
 end
 
 if config_env() == :test do

config/deps/phoenix.exs

@@ -3,6 +3,9 @@ import Config
 config :phoenix, :gzippable_exts, ~w(.txt .html .js .css .svg)
 config :phoenix, :json_library, Poison
 
+# Using Logster instead
+config :phoenix, :logger, false
+
 if config_env() == :prod do
   config :dotcom,
     dev_server?: false

config/runtime.exs

@@ -202,7 +202,6 @@ config :dotcom,
 if System.get_env("LOGGER_LEVEL") in ~w(emergency alert critical error warning notice info debug all none) &&
      config_env() != :test do
   config :logger, level: String.to_atom(System.get_env("LOGGER_LEVEL"))
-  config :logger, :console, level: String.to_atom(System.get_env("LOGGER_LEVEL"))
 end
 
 default_key_base = "yK6hUINZWlq04EPu3SJjAHNDYgka8MZqgXZykF+AQ2PvWs4Ua4IELdFl198aMvw0"

lib/dotcom/application.ex

@@ -69,6 +69,8 @@ defmodule Dotcom.Application do
           []
         end
 
+    :ok = Logster.attach_phoenix_logger()
+
     opts = [strategy: :one_for_one, name: Dotcom.Supervisor]
 
     Supervisor.start_link(children, opts)

lib/dotcom_web/endpoint.ex

@@ -28,7 +28,7 @@ defmodule DotcomWeb.Endpoint do
 
   plug(DotcomWeb.Plugs.RemoteIp)
   plug(Plug.RequestId)
-  plug(Logster.Plugs.Logger, formatter: Dotcom.Logster.SafeStringFormatter)
+  plug(Logster.Plug)
 
   plug(
     Plug.Parsers,

mix.exs

@@ -140,7 +140,7 @@ defmodule DotCom.Mixfile do
       {:jason, "1.4.4", override: true},
       {:lazy_html, "0.1.10", only: [:test]},
       {:live_isolated_component, "0.10.0", only: [:test]},
-      {:logster, "1.1.1"},
+      {:logster, "~> 2.0.0-rc.5"},
       # reverted from 0.4
       {:mail, "0.3.1"},
       {:mbta_metro, "1.1.2", runtime: false},

mix.lock

@@ -75,7 +75,7 @@
   "laboratory": {:git, "https://github.com/mbta/laboratory.git", "2d9ed1cf4471c0001d2f97d483a219c58158e141", [ref: "master"]},
   "lazy_html": {:hex, :lazy_html, "0.1.10", "ffe42a0b4e70859cf21a33e12a251e0c76c1dff76391609bd56702a0ef5bc429", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "50f67e5faa09d45a99c1ddf3fac004f051997877dc8974c5797bb5ccd8e27058"},
   "live_isolated_component": {:hex, :live_isolated_component, "0.10.0", "e38b0b8dc5ff39f1d581fa2a05c7928d0475a854e246aee87fc2bc22726ff356", [:mix], [{:phoenix, "~> 1.7.0 or ~> 1.8.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.0.0 or ~> 1.1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "170bb70fa89db7e94f95407523352cd619d0f17886017c188f93cfe728550725"},
-  "logster": {:hex, :logster, "1.1.1", "d6fddac540dd46adde0c894024500867fe63b0043713f842c62da5815e21db10", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d18e852c430812ad1c9756998ebe46ec814c724e6eb551a512d7e3f8dee24cef"},
+  "logster": {:hex, :logster, "2.0.0-rc.5", "5ae9e58b15f03cbe1f71203808ed0db6b0a2b60c0dbd8f4ed76d1b769ec9d440", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "dddafb22aabbbd7dbef83356d3de66b564a894e19615b049ca1b704a59a032ba"},
   "mail": {:hex, :mail, "0.3.1", "cb0a14e4ed8904e4e5a08214e686ccf6f9099346885db17d8c309381f865cc5c", [:mix], [], "hexpm", "1db701e89865c1d5fa296b2b57b1cd587587cca8d8a1a22892b35ef5a8e352a6"},
   "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},
   "makeup_eex": {:hex, :makeup_eex, "2.0.2", "88983b72aadb2e8408b06f7c9413804ce7eae2ca2a5a35cb738c6a9cb393c155", [:mix], [{:makeup, "~> 1.2.1 or ~> 1.3", [hex: :makeup, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_html, "~> 0.2.0 or ~> 1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 1.2", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "30ac121dda580298ff3378324ffaec94aad5a5b67e0cc6af177c67d5f45629b9"},
@@ -117,7 +117,7 @@
   "phoenix_view": {:hex, :phoenix_view, "2.0.4", "b45c9d9cf15b3a1af5fb555c674b525391b6a1fe975f040fb4d913397b31abf4", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "4e992022ce14f31fe57335db27a28154afcc94e9983266835bb3040243eb620b"},
   "plug": {:hex, :plug, "1.20.1", "82cdee1d7535d4f4db5c5602a7fd49512d64690be54fd62374856ee70e62eb29", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "892d2a1a7a3f5368c5a3b9067bba1050c031495f48c430ec00b09691dbf211b7"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.8.0", "07789e9c03539ee51bb14a07839cc95aa96999fd8846ebfd28c97f0b50c7b612", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9cbfaaf17463334ca31aed38ea7e08a68ee37cabc077b1e9be6d2fb68e0171d0"},
-  "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"},
+  "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"},
   "poison": {:hex, :poison, "6.0.0", "9bbe86722355e36ffb62c51a552719534257ba53f3271dacd20fbbd6621a583a", [:mix], [{:decimal, "~> 2.1", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "bb9064632b94775a3964642d6a78281c07b7be1319e0016e1643790704e739a2"},
   "pollution": {:hex, :pollution, "0.9.2", "3f67542631071c99f807d2a8f9da799c07cd983c902f5357b9e1569c20a26e76", [:mix], [], "hexpm", "6399fd8ffd97dcc3d9d277f60542a234d644d7bcc0d48c8fda93d6be4801bac2"},
   "polyline": {:hex, :polyline, "1.6.0", "f34a460e2c38a8a0930d2c5d5300f2046e885da8c53dd041d7f5eccc398eb177", [:mix], [], "hexpm", "68d30940121d6191f21060cfe75c7dd2cee92a3c280d95aaebaa0b9c8aa2262d"},