Dockerfile-workers: give the master its own log config (#12466)

When we run a worker-mode synapse under docker, everything gets logged to stdout. Currently, output from the workers is tacked with a worker name, for example:

```
2022-04-13 15:27:56,810 - worker:frontend_proxy1 - synapse.util.caches.lrucache - 154 - INFO - LruCache._expire_old_entries-0 - Dropped 0 items from caches
```

- note `worker:frontend_proxy1`. No such tag is applied to log lines from the master, which makes somewhat confusing reading.

To fix this, we generate a dedicated log config file for the master in the same way that we do for the workers, and use that.
This commit is contained in:
Richard van der Hoff 2022-04-13 20:50:08 +01:00 committed by GitHub
parent 8e2759f2d8
commit 8af8a9bce5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 32 additions and 17 deletions

1
changelog.d/12466.misc Normal file
View file

@ -0,0 +1 @@
Dockerfile-workers: give the master its own log config.

View file

@ -29,7 +29,7 @@
import os import os
import subprocess import subprocess
import sys import sys
from typing import Any, Dict, Set from typing import Any, Dict, Mapping, Set
import jinja2 import jinja2
import yaml import yaml
@ -341,7 +341,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
# base shared worker jinja2 template. # base shared worker jinja2 template.
# #
# This config file will be passed to all workers, included Synapse's main process. # This config file will be passed to all workers, included Synapse's main process.
shared_config = {"listeners": listeners} shared_config: Dict[str, Any] = {"listeners": listeners}
# The supervisord config. The contents of which will be inserted into the # The supervisord config. The contents of which will be inserted into the
# base supervisord jinja2 template. # base supervisord jinja2 template.
@ -446,21 +446,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
# Write out the worker's logging config file # Write out the worker's logging config file
# Check whether we should write worker logs to disk, in addition to the console log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)
extra_log_template_args = {}
if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
dir=data_dir, name=worker_name
)
# Render and write the file
log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
convert(
"/conf/log.config",
log_config_filepath,
worker_name=worker_name,
**extra_log_template_args,
)
# Then a worker config file # Then a worker config file
convert( convert(
@ -496,6 +482,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
# Finally, we'll write out the config files. # Finally, we'll write out the config files.
# log config for the master process
master_log_config = generate_worker_log_config(environ, "master", data_dir)
shared_config["log_config"] = master_log_config
# Shared homeserver config # Shared homeserver config
convert( convert(
"/conf/shared.yaml.j2", "/conf/shared.yaml.j2",
@ -532,6 +522,30 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
os.mkdir(log_dir) os.mkdir(log_dir)
def generate_worker_log_config(
environ: Mapping[str, str], worker_name: str, data_dir: str
) -> str:
"""Generate a log.config file for the given worker.
Returns: the path to the generated file
"""
# Check whether we should write worker logs to disk, in addition to the console
extra_log_template_args = {}
if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
dir=data_dir, name=worker_name
)
# Render and write the file
log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
convert(
"/conf/log.config",
log_config_filepath,
worker_name=worker_name,
**extra_log_template_args,
)
return log_config_filepath
def start_supervisord(): def start_supervisord():
"""Starts up supervisord which then starts and monitors all other necessary processes """Starts up supervisord which then starts and monitors all other necessary processes