From 8af8a9bce5319ebf99573433f9f89d94767e173c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 13 Apr 2022 20:50:08 +0100 Subject: [PATCH] Dockerfile-workers: give the master its own log config (#12466) When we run a worker-mode synapse under docker, everything gets logged to stdout. Currently, output from the workers is tacked with a worker name, for example: ``` 2022-04-13 15:27:56,810 - worker:frontend_proxy1 - synapse.util.caches.lrucache - 154 - INFO - LruCache._expire_old_entries-0 - Dropped 0 items from caches ``` - note `worker:frontend_proxy1`. No such tag is applied to log lines from the master, which makes somewhat confusing reading. To fix this, we generate a dedicated log config file for the master in the same way that we do for the workers, and use that. --- changelog.d/12466.misc | 1 + docker/configure_workers_and_start.py | 48 +++++++++++++++++---------- 2 files changed, 32 insertions(+), 17 deletions(-) create mode 100644 changelog.d/12466.misc diff --git a/changelog.d/12466.misc b/changelog.d/12466.misc new file mode 100644 index 0000000000..b0c2c950fe --- /dev/null +++ b/changelog.d/12466.misc @@ -0,0 +1 @@ +Dockerfile-workers: give the master its own log config. diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 3e91024e8c..8f1e61930e 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -29,7 +29,7 @@ import os import subprocess import sys -from typing import Any, Dict, Set +from typing import Any, Dict, Mapping, Set import jinja2 import yaml @@ -341,7 +341,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # base shared worker jinja2 template. # # This config file will be passed to all workers, included Synapse's main process. - shared_config = {"listeners": listeners} + shared_config: Dict[str, Any] = {"listeners": listeners} # The supervisord config. The contents of which will be inserted into the # base supervisord jinja2 template. @@ -446,21 +446,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Write out the worker's logging config file - # Check whether we should write worker logs to disk, in addition to the console - extra_log_template_args = {} - if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): - extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( - dir=data_dir, name=worker_name - ) - - # Render and write the file - log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) - convert( - "/conf/log.config", - log_config_filepath, - worker_name=worker_name, - **extra_log_template_args, - ) + log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir) # Then a worker config file convert( @@ -496,6 +482,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str): # Finally, we'll write out the config files. + # log config for the master process + master_log_config = generate_worker_log_config(environ, "master", data_dir) + shared_config["log_config"] = master_log_config + # Shared homeserver config convert( "/conf/shared.yaml.j2", @@ -532,6 +522,30 @@ def generate_worker_files(environ, config_path: str, data_dir: str): os.mkdir(log_dir) +def generate_worker_log_config( + environ: Mapping[str, str], worker_name: str, data_dir: str +) -> str: + """Generate a log.config file for the given worker. + + Returns: the path to the generated file + """ + # Check whether we should write worker logs to disk, in addition to the console + extra_log_template_args = {} + if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"): + extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format( + dir=data_dir, name=worker_name + ) + # Render and write the file + log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name) + convert( + "/conf/log.config", + log_config_filepath, + worker_name=worker_name, + **extra_log_template_args, + ) + return log_config_filepath + + def start_supervisord(): """Starts up supervisord which then starts and monitors all other necessary processes