Create healthcheck script for synapse-workers container (#11429)

The intent is to iterate through all the worker ports and only
report healthy when all are healthy, starting with the main process.
This commit is contained in:
Michael Kaye 2021-11-26 14:05:20 +00:00 committed by GitHub
parent c675a18071
commit e2c300e7e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 23 additions and 0 deletions

1
changelog.d/11429.docker Normal file
View file

@ -0,0 +1 @@
Update `Dockerfile-workers` to healthcheck all workers in container.

View file

@ -21,3 +21,6 @@ VOLUME ["/data"]
# files to run the desired worker configuration. Will start supervisord. # files to run the desired worker configuration. Will start supervisord.
COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
ENTRYPOINT ["/configure_workers_and_start.py"] ENTRYPOINT ["/configure_workers_and_start.py"]
HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
CMD /bin/sh /healthcheck.sh

View file

@ -0,0 +1,6 @@
#!/bin/sh
# This healthcheck script is designed to return OK when every
# host involved returns OK
{%- for healthcheck_url in healthcheck_urls %}
curl -fSs {{ healthcheck_url }} || exit 1
{%- endfor %}

View file

@ -474,10 +474,16 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
# Determine the load-balancing upstreams to configure # Determine the load-balancing upstreams to configure
nginx_upstream_config = "" nginx_upstream_config = ""
# At the same time, prepare a list of internal endpoints to healthcheck
# starting with the main process which exists even if no workers do.
healthcheck_urls = ["http://localhost:8080/health"]
for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items(): for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
body = "" body = ""
for port in upstream_worker_ports: for port in upstream_worker_ports:
body += " server localhost:%d;\n" % (port,) body += " server localhost:%d;\n" % (port,)
healthcheck_urls.append("http://localhost:%d/health" % (port,))
# Add to the list of configured upstreams # Add to the list of configured upstreams
nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format( nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@ -510,6 +516,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
worker_config=supervisord_config, worker_config=supervisord_config,
) )
# healthcheck config
convert(
"/conf/healthcheck.sh.j2",
"/healthcheck.sh",
healthcheck_urls=healthcheck_urls,
)
# Ensure the logging directory exists # Ensure the logging directory exists
log_dir = data_dir + "/logs" log_dir = data_dir + "/logs"
if not os.path.exists(log_dir): if not os.path.exists(log_dir):