From 95e02b856b2ac1409dc0d5575ee0b7be55105e9a Mon Sep 17 00:00:00 2001 From: Robert Swain Date: Thu, 28 Sep 2017 12:12:47 +0200 Subject: [PATCH 01/47] docker: Initial Dockerfile and docker-compose.yaml --- Dockerfile | 58 ++++++++++++++++++ docker/README.md | 70 ++++++++++++++++++++++ docker/docker-compose.yaml | 39 ++++++++++++ docker/rootfs/etc/service/synapse/finish | 17 ++++++ docker/rootfs/etc/service/synapse/run | 75 ++++++++++++++++++++++++ 5 files changed, 259 insertions(+) create mode 100644 Dockerfile create mode 100644 docker/README.md create mode 100644 docker/docker-compose.yaml create mode 100755 docker/rootfs/etc/service/synapse/finish create mode 100755 docker/rootfs/etc/service/synapse/run diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..9b11a143f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,58 @@ +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM phusion/baseimage:0.9.22 + +COPY ./ /synapse/source/ + +RUN apt-get update -y \ + && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxslt1-dev \ + python-pip \ + python-setuptools \ + python-virtualenv \ + python2.7-dev \ + sqlite3 \ + && virtualenv -p python2.7 /synapse \ + && . /synapse/bin/activate \ + && pip install --upgrade pip \ + && pip install --upgrade setuptools \ + && pip install --upgrade psycopg2 \ + && cd /synapse/source \ + && pip install --upgrade ./ \ + && cd / \ + && rm -rf /synapse/source \ + && apt-get autoremove -y \ + build-essential \ + libffi-dev \ + libjpeg-dev \ + libpq-dev \ + libssl-dev \ + libxslt1-dev \ + python2.7-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY docker/rootfs/ / + +VOLUME /synapse/config/ +VOLUME /synapse/data/ + +CMD ["/sbin/my_init"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..c9e6fd216d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,70 @@ +# Synapse Docker + +## Build + +Build the docker image with the `docker build` command from the root of the synapse repository. + +``` +docker build -t matrix-org/synapse:v0.22.1 . +``` + +The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:` where `` is the same as the release tag in the synapse git repository. + +## Configure + +Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment. + +The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command. + +Docker container environment variables: +* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten. +* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking. +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`] +* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`] +* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`] + +``` +CONFIG_PATH=/my/magical/config/path/ +mkdir -p ${CONFIG_PATH} +docker run \ + --rm \ + -e GENERATE_CONFIG=yes \ + -e POSTGRES_PASSWORD=MyVerySecretPassword \ + -e REPORT_STATS=yes \ + -e SERVER_NAME=example.com \ + -v ${CONFIG_PATH}:/synapse/config/ \ + matrix-org/synapse:develop +``` + +This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. + +## Run + +**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead. + +### Docker Compose + +A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition. + +From the `docker/` subdirectory of the synapse repository: +``` +CONFIG_PATH=/my/magical/config/path/ +POSTGRES_PASSWORD=MyVerySecretPassword \ +docker-compose \ + -p synapse \ + up -d +``` + +### Docker + +Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. + +``` +docker run \ + -d \ + --name synapse \ + -v ${CONFIG_PATH}:/synapse/config/ \ + matrix-org/synapse:v0.22.1 +``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000000..ff36081a9b --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,39 @@ +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3' + +services: + postgres: + image: postgres:9.6.5-alpine + environment: + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: synapse + expose: + - 5432 + restart: unless-stopped + volumes: + - postgres-data:/var/lib/postgresql/data/ + + synapse: + image: matrix-org/synapse:develop + ports: + - 8008:8008 + - 8448:8448 + restart: unless-stopped + volumes: + - ${CONFIG_PATH}:/synapse/config/ + +volumes: + postgres-data: diff --git a/docker/rootfs/etc/service/synapse/finish b/docker/rootfs/etc/service/synapse/finish new file mode 100755 index 0000000000..2aace581a1 --- /dev/null +++ b/docker/rootfs/etc/service/synapse/finish @@ -0,0 +1,17 @@ +#!/bin/bash +# +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kill -TERM 1 diff --git a/docker/rootfs/etc/service/synapse/run b/docker/rootfs/etc/service/synapse/run new file mode 100755 index 0000000000..dd797d3ef9 --- /dev/null +++ b/docker/rootfs/etc/service/synapse/run @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Copyright 2017 Vector Creations Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +: ${CONFIG_PATH:="/synapse/config"} +: ${POSTGRES_DATABASE:="synapse"} +: ${POSTGRES_HOST:="postgres"} +: ${POSTGRES_USER:="postgres"} +: ${REPORT_STATS:="yes"} +: ${SERVER_NAME:="localhost"} + +DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml" +HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml" +SYNAPSE_COMMAND="python -m synapse.app.homeserver" + +. /synapse/bin/activate +cd /synapse + +if [[ -n "${GENERATE_CONFIG}" ]]; then + ${SYNAPSE_COMMAND} \ + --server-name ${SERVER_NAME} \ + --config-path ${HOMESERVER_CONFIG_PATH} \ + --generate-config \ + --report-stats=${REPORT_STATS} + + if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then + echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated." + else + echo "Generating ${DATABASE_CONFIG_PATH}..." + if [[ -n "${POSTGRES_PASSWORD}" ]]; then + (cat > ${DATABASE_CONFIG_PATH}) < ${DATABASE_CONFIG_PATH}) < Date: Fri, 29 Sep 2017 11:40:15 +0200 Subject: [PATCH 02/47] docker: s/matrix-org/matrixdotorg/g --- docker/README.md | 8 ++++---- docker/docker-compose.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/README.md b/docker/README.md index c9e6fd216d..c15517d0e0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,10 +5,10 @@ Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrix-org/synapse:v0.22.1 . +docker build -t matrixdotorg/synapse:v0.22.1 . ``` -The `-t` option sets the image tag. Official images are tagged `matrix-org/synapse:` where `` is the same as the release tag in the synapse git repository. +The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. ## Configure @@ -35,7 +35,7 @@ docker run \ -e REPORT_STATS=yes \ -e SERVER_NAME=example.com \ -v ${CONFIG_PATH}:/synapse/config/ \ - matrix-org/synapse:develop + matrixdotorg/synapse:v0.22.1 ``` This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. @@ -66,5 +66,5 @@ docker run \ -d \ --name synapse \ -v ${CONFIG_PATH}:/synapse/config/ \ - matrix-org/synapse:v0.22.1 + matrixdotorg/synapse:v0.22.1 ``` diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ff36081a9b..73cc29f8fd 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -27,7 +27,7 @@ services: - postgres-data:/var/lib/postgresql/data/ synapse: - image: matrix-org/synapse:develop + image: matrixdotorg/synapse:v0.22.1 ports: - 8008:8008 - 8448:8448 From 431476fbc4ef0c740e33e19ccc73996c2412e4f9 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 3 Feb 2018 20:18:36 +0100 Subject: [PATCH 03/47] Initial commit including a Dockerfile for synapse --- Dockerfile | 16 ++++++++++++++++ contrib/docker/start.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 Dockerfile create mode 100755 contrib/docker/start.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..5f0433004f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:2-alpine + +RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers + +COPY synapse /usr/local/src/synapse +COPY setup.py setup.cfg README.rst synctl /usr/local/src/ + +RUN cd /usr/local/src \ + && pip install --upgrade --process-dependency-links . \ + && rm -rf setup.py setup.cfg synapse + +COPY contrib/docker / + +VOLUME ["/data"] + +ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/start.py b/contrib/docker/start.py new file mode 100755 index 0000000000..4f63ea1ad5 --- /dev/null +++ b/contrib/docker/start.py @@ -0,0 +1,29 @@ +#!/usr/local/bin/python + +import jinja2 +import os +import sys +import socket + +convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +mode = sys.argv[1] if len(sys.argv) > 1 else None + +if "SYNAPSE_SERVER_NAME" not in os.environ: + print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") + sys.exit(2) + +params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] + +if mode == "generate": + params.append("--generate-config") + +# Parse the configuration file +if not os.path.exists("/compiled"): + os.mkdir("/compiled") +convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") +convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) + +# TODO, replace with a call to synapse.app.homeserver.run() +os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params) From d434ae33875b4c7d5ee04c45e454ce237ce578f8 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 3 Feb 2018 20:30:08 +0100 Subject: [PATCH 04/47] Add template config files for the Docker image --- contrib/docker/conf/homeserver.yaml | 181 ++++++++++++++++++++++++++++ contrib/docker/conf/log.config | 36 ++++++ 2 files changed, 217 insertions(+) create mode 100644 contrib/docker/conf/homeserver.yaml create mode 100644 contrib/docker/conf/log.config diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml new file mode 100644 index 0000000000..851c389c19 --- /dev/null +++ b/contrib/docker/conf/homeserver.yaml @@ -0,0 +1,181 @@ +# vim:ft=yaml + +## TLS ## + +tls_certificate_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.crt" +tls_private_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.key" +tls_dh_params_path: "/data/{{ SYNAPSE_SERVER_NAME }}.tls.dh" +no_tls: {{ "True" if SYNAPSE_NO_TLS else "False" }} +tls_fingerprints: [] + +## Server ## + +server_name: "{{ SYNAPSE_SERVER_NAME }}" +pid_file: /homeserver.pid +web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }} +soft_file_limit: 0 + +## Ports ## + +listeners: + {% if not SYNAPSE_NO_TLS %} + - + port: 8448 + bind_addresses: ['0.0.0.0'] + type: http + tls: true + x_forwarded: false + resources: + - names: [client, webclient] + compress: true + - names: [federation] # Federation APIs + compress: false + {% endif %} + + - port: 8008 + tls: false + bind_addresses: ['0.0.0.0'] + type: http + x_forwarded: false + + resources: + - names: [client, webclient] + compress: true + - names: [federation] + compress: false + +## Database ## + +{% if SYNAPSE_DB_HOST %} + +{% else %} +database: + name: "sqlite3" + args: + database: "/data/homeserver.db" +{% endif %} + +## Performance ## + +event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" +verbose: 0 +log_file: "/data/homeserver.log" +log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config" + +## Ratelimiting ## + +rc_messages_per_second: 0.2 +rc_message_burst_count: 10.0 +federation_rc_window_size: 1000 +federation_rc_sleep_limit: 10 +federation_rc_sleep_delay: 500 +federation_rc_reject_limit: 50 +federation_rc_concurrent: 3 + +## Files ## + +media_store_path: "/data/media" +uploads_path: "/data/uploads" +max_upload_size: "10M" +max_image_pixels: "32M" +dynamic_thumbnails: false + +# List of thumbnail to precalculate when an image is uploaded. +thumbnail_sizes: +- width: 32 + height: 32 + method: crop +- width: 96 + height: 96 + method: crop +- width: 320 + height: 240 + method: scale +- width: 640 + height: 480 + method: scale +- width: 800 + height: 600 + method: scale + +url_preview_enabled: False +max_spider_size: "10M" + +## Captcha ## + +recaptcha_public_key: "YOUR_PUBLIC_KEY" +recaptcha_private_key: "YOUR_PRIVATE_KEY" +enable_registration_captcha: False +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" + +## Turn ## + +turn_uris: [] +turn_shared_secret: "YOUR_SHARED_SECRET" +turn_user_lifetime: "1h" +turn_allow_guests: True + +## Registration ## + +enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} +registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" +bcrypt_rounds: 12 +allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} + +# The list of identity servers trusted to verify third party +# identifiers by this server. +trusted_third_party_id_servers: + - matrix.org + - vector.im + - riot.im + +## Metrics ### + +enable_metrics: False +report_stats: False + +## API Configuration ## + +room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.name" + +app_service_config_files: [] +macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" +expire_access_token: False + +## Signing Keys ## + +signing_key_path: "/data/{{ SYNAPSE_SERVER_NAME }}.signing.key" +old_signing_keys: {} +key_refresh_interval: "1d" # 1 Day. + +# The trusted servers to download signing keys from. +perspectives: + servers: + "matrix.org": + verify_keys: + "ed25519:auto": + key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw" + +password_config: + enabled: true + +#email: +# enable_notifs: false +# smtp_host: "localhost" +# smtp_port: 25 +# smtp_user: "exampleusername" +# smtp_pass: "examplepassword" +# require_transport_security: False +# notif_from: "Your Friendly %(app)s Home Server " +# app_name: Matrix +# template_dir: res/templates +# notif_template_html: notif_mail.html +# notif_template_text: notif_mail.txt +# notif_for_new_users: True +# riot_base_url: "http://localhost/riot" + +enable_group_creation: true diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config new file mode 100644 index 0000000000..45e7eef953 --- /dev/null +++ b/contrib/docker/conf/log.config @@ -0,0 +1,36 @@ +version: 1 + +formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + +filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + +handlers: + file: + class: logging.handlers.RotatingFileHandler + formatter: precise + filename: /data/homeserver.log + maxBytes: 104857600 + backupCount: 10 + filters: [context] + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + +loggers: + synapse: + level: INFO + + synapse.storage.SQL: + # beware: increasing this to DEBUG will make synapse log sensitive + # information such as access tokens. + level: INFO + +root: + level: INFO + handlers: [file, console] From 48bc22f89dadb8278cf2b8c940604534999d246f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 10:58:07 +0100 Subject: [PATCH 05/47] Allow for a wheel cache and include missing files in the build --- .dockerignore | 5 +++++ Dockerfile | 13 +++++++------ contrib/docker/start.py | 1 + 3 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..f36f86fbb7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +Dockerfile +.travis.yml +.gitignore +demo/etc +tox.ini diff --git a/Dockerfile b/Dockerfile index 5f0433004f..277246b697 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,15 +2,16 @@ FROM python:2-alpine RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers -COPY synapse /usr/local/src/synapse -COPY setup.py setup.cfg README.rst synctl /usr/local/src/ +COPY . /synapse -RUN cd /usr/local/src \ - && pip install --upgrade --process-dependency-links . \ +# A wheel cache may be provided in ./cache for faster build +RUN cd /synapse \ + && pip install --upgrade pip setuptools \ + && mkdir -p /synapse/cache \ + && pip install -f /synapse/cache --upgrade --process-dependency-links . \ + && mv /synapse/contrib/docker/* / \ && rm -rf setup.py setup.cfg synapse -COPY contrib/docker / - VOLUME ["/data"] ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 4f63ea1ad5..2c427ba1b7 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -22,6 +22,7 @@ if mode == "generate": # Parse the configuration file if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) From 6d1e28a8426da9e954a3edec25a8717376c583f3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 11:14:06 +0100 Subject: [PATCH 06/47] Generate any missing keys before starting synapse --- contrib/docker/start.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 2c427ba1b7..e50d23be5f 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -3,7 +3,7 @@ import jinja2 import os import sys -import socket +import subprocess convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -12,19 +12,20 @@ if "SYNAPSE_SERVER_NAME" not in os.environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) -params = ["--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] - -if mode == "generate": - params.append("--generate-config") +args = ["python", "-m", "synapse.app.homeserver", + "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] # Parse the configuration file -if not os.path.exists("/compiled"): - os.mkdir("/compiled") - +if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) -# TODO, replace with a call to synapse.app.homeserver.run() -os.execv("/usr/local/bin/python", ["python", "-m", "synapse.app.homeserver"] + params) +# In generate mode, generate a configuration, missing keys, then exit +if mode == "generate": + os.execv("/usr/local/bin/python", args + ["--generate-config"]) +# In normal mode, generate missing keys if any, then run synapse +else: + subprocess.check_output(args + ["--generate-keys"]) + os.execv("/usr/local/bin/python", args) From f2bf0cda02fef358172033b28dab5f1805c31cad Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 11:40:20 +0100 Subject: [PATCH 07/47] Generate shared secrets if not defined in the environment --- contrib/docker/start.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index e50d23be5f..7057f85f61 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -5,10 +5,11 @@ import os import sys import subprocess -convert = lambda src, dst: open(dst, "w").write(jinja2.Template(open(src).read()).render(**os.environ)) +convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None +environ = os.environ.copy() -if "SYNAPSE_SERVER_NAME" not in os.environ: +if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) @@ -17,10 +18,16 @@ args = ["python", "-m", "synapse.app.homeserver", "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] +# Generate any missing shared secret +for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): + if secret not in environ: + print("Generating a random secret for {}".format(secret)) + environ[secret] = os.urandom(32).encode("hex") + # Parse the configuration file if not os.path.exists("/compiled"): os.mkdir("/compiled") -convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml") -convert("/conf/log.config", "/compiled/%s.log.config" % os.environ.get("SYNAPSE_SERVER_NAME")) +convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) +convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": From 886c2d50197bb3558168ce45b6975777780c7aad Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:20:29 +0100 Subject: [PATCH 08/47] Support an external postgresql config in the Docker image --- contrib/docker/conf/homeserver.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 851c389c19..1a685320c0 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -46,8 +46,17 @@ listeners: ## Database ## -{% if SYNAPSE_DB_HOST %} - +{% if SYNAPSE_DB_PASSWORD %} +database: + name: "psycopg2" + args: + user: "{{ SYNAPSE_DB_USER or "matrix" }}" + password: "{{ SYNAPSE_DB_PASSWORD }}" + database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}" + host: "{{ SYNAPSE_DB_HOST or "db" }}" + port: "{{ SYNAPSE_DB_PORT or "5432" }}" + cp_min: 5 + cp_max: 10 {% else %} database: name: "sqlite3" From 042757feb25ba9f0cccfdaa6c8775957880c2d2e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:28:42 +0100 Subject: [PATCH 09/47] Install the postgres dependencies --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 277246b697..881c25c243 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,12 @@ FROM python:2-alpine -RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers +RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse # A wheel cache may be provided in ./cache for faster build RUN cd /synapse \ - && pip install --upgrade pip setuptools \ + && pip install --upgrade pip setuptools psycopg2 \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ && mv /synapse/contrib/docker/* / \ From 1ba2fe114c79dd0f2f2fb8868f8cb0a9f5893652 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 12:55:20 +0100 Subject: [PATCH 10/47] Provide an example docker compose file --- Dockerfile | 2 ++ contrib/docker/docker-compose.yml | 32 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 contrib/docker/docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 881c25c243..25f3746303 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,4 +14,6 @@ RUN cd /synapse \ VOLUME ["/data"] +EXPOSE 8448 + ENTRYPOINT ["/start.py"] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml new file mode 100644 index 0000000000..659bee2979 --- /dev/null +++ b/contrib/docker/docker-compose.yml @@ -0,0 +1,32 @@ +# This compose file is compatible with Copose itself, it might need some +# adjustments to run properly with stack. + +version: '3' + +services: + + synapse: + image: matrix/synapse + # See the readme for a full documentation of the environment settings + environment: + - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_DB_PASSWORD=changeme + volumes: + - ./files:/data + # One may either expose ports directly + ports: + - 8448:8448/tcp + # ... or use a reverse proxy, here is an example for traefik + labels: + - traefik.enable=true + - traefik.frontend.rule=Host:my.matrix.Host + - traefik.port=8448 + + db: + image: postgres:latest + # Change that password, of course! + environment: + - POSTGRES_USER=matrix + - POSTGRES_PASSWORD=changeme + volumes: + - ./schemas:/var/lib/postgres From a207cccb059451682564cddf0e39c7c45b06cb72 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:04:26 +0100 Subject: [PATCH 11/47] Reuse environment variables of the postgres container --- contrib/docker/conf/homeserver.yaml | 12 ++++++------ contrib/docker/docker-compose.yml | 11 ++++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 1a685320c0..7450cc1228 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -46,15 +46,15 @@ listeners: ## Database ## -{% if SYNAPSE_DB_PASSWORD %} +{% if POSTGRES_PASSWORD %} database: name: "psycopg2" args: - user: "{{ SYNAPSE_DB_USER or "matrix" }}" - password: "{{ SYNAPSE_DB_PASSWORD }}" - database: "{{ SYNAPSE_DB_DATABASE or "matrix" }}" - host: "{{ SYNAPSE_DB_HOST or "db" }}" - port: "{{ SYNAPSE_DB_PORT or "5432" }}" + user: "{{ POSTGRES_USER or "matrix" }}" + password: "{{ POSTGRES_PASSWORD }}" + database: "{{ POSTGRES_DB or "matrix" }}" + host: "{{ POSTGRES_HOST or "db" }}" + port: "{{ POSTGRES_PORT or "5432" }}" cp_min: 5 cp_max: 10 {% else %} diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 659bee2979..b8f9741f05 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,13 +6,18 @@ version: '3' services: synapse: - image: matrix/synapse + image: synapse + # Since snyapse does not retry to connect to the database, restart upon + # failure + restart: unless-stopped # See the readme for a full documentation of the environment settings environment: - SYNAPSE_SERVER_NAME=my.matrix.host - - SYNAPSE_DB_PASSWORD=changeme + - SYNAPSE_ENABLE_REGISTRATION=yes volumes: - ./files:/data + depends_on: + - db # One may either expose ports directly ports: - 8448:8448/tcp @@ -29,4 +34,4 @@ services: - POSTGRES_USER=matrix - POSTGRES_PASSWORD=changeme volumes: - - ./schemas:/var/lib/postgres + - ./schemas:/var/lib/postgresql/data From 84a9209ba7294243e5bbefede46a3dee7994da9b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:08:43 +0100 Subject: [PATCH 12/47] Remove etc/service files from rob's branch --- contrib/docker/rob/docker-compose.yaml | 39 ---------- .../rob/rootfs/etc/service/synapse/finish | 17 ----- .../docker/rob/rootfs/etc/service/synapse/run | 75 ------------------- 3 files changed, 131 deletions(-) delete mode 100644 contrib/docker/rob/docker-compose.yaml delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/finish delete mode 100755 contrib/docker/rob/rootfs/etc/service/synapse/run diff --git a/contrib/docker/rob/docker-compose.yaml b/contrib/docker/rob/docker-compose.yaml deleted file mode 100644 index 73cc29f8fd..0000000000 --- a/contrib/docker/rob/docker-compose.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '3' - -services: - postgres: - image: postgres:9.6.5-alpine - environment: - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_DB: synapse - expose: - - 5432 - restart: unless-stopped - volumes: - - postgres-data:/var/lib/postgresql/data/ - - synapse: - image: matrixdotorg/synapse:v0.22.1 - ports: - - 8008:8008 - - 8448:8448 - restart: unless-stopped - volumes: - - ${CONFIG_PATH}:/synapse/config/ - -volumes: - postgres-data: diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/finish b/contrib/docker/rob/rootfs/etc/service/synapse/finish deleted file mode 100755 index 2aace581a1..0000000000 --- a/contrib/docker/rob/rootfs/etc/service/synapse/finish +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kill -TERM 1 diff --git a/contrib/docker/rob/rootfs/etc/service/synapse/run b/contrib/docker/rob/rootfs/etc/service/synapse/run deleted file mode 100755 index dd797d3ef9..0000000000 --- a/contrib/docker/rob/rootfs/etc/service/synapse/run +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# -# Copyright 2017 Vector Creations Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -: ${CONFIG_PATH:="/synapse/config"} -: ${POSTGRES_DATABASE:="synapse"} -: ${POSTGRES_HOST:="postgres"} -: ${POSTGRES_USER:="postgres"} -: ${REPORT_STATS:="yes"} -: ${SERVER_NAME:="localhost"} - -DATABASE_CONFIG_PATH="${CONFIG_PATH}/database.yaml" -HOMESERVER_CONFIG_PATH="${CONFIG_PATH}/homeserver.yaml" -SYNAPSE_COMMAND="python -m synapse.app.homeserver" - -. /synapse/bin/activate -cd /synapse - -if [[ -n "${GENERATE_CONFIG}" ]]; then - ${SYNAPSE_COMMAND} \ - --server-name ${SERVER_NAME} \ - --config-path ${HOMESERVER_CONFIG_PATH} \ - --generate-config \ - --report-stats=${REPORT_STATS} - - if [[ -f "${DATABASE_CONFIG_PATH}" ]]; then - echo "Config file '${DATABASE_CONFIG_PATH}' already exists. Remove it if you want it to be generated." - else - echo "Generating ${DATABASE_CONFIG_PATH}..." - if [[ -n "${POSTGRES_PASSWORD}" ]]; then - (cat > ${DATABASE_CONFIG_PATH}) < ${DATABASE_CONFIG_PATH}) < Date: Sun, 4 Feb 2018 15:27:32 +0100 Subject: [PATCH 13/47] Update sumperdump Docker readme to match this image properties --- contrib/docker/README.md | 108 ++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 41 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index c15517d0e0..8ba5f79692 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -1,5 +1,12 @@ # Synapse Docker +This Docker image will run Synapse as a single process. It does not provide any +database server or TURN server that you should run separately. + +If you run a Postgres server, you should simply have it in the same Compose +project or set the proper environment variables and the image will automatically +use that server. + ## Build Build the docker image with the `docker build` command from the root of the synapse repository. @@ -10,54 +17,29 @@ docker build -t matrixdotorg/synapse:v0.22.1 . The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. -## Configure - -Synapse provides a command for generating homeserver configuration files. These are a good starting point for setting up your own deployment. - -The documentation below will refer to a `CONFIG_PATH` shell variable. This is a path to a directory where synapse configuration will be stored. It needs to be mapped into the container as a volume at `/synapse/config/` as can be seen in the example `docker run` command. - -Docker container environment variables: -* `GENERATE_CONFIG` - Set this to any non-empty string, such as `yes`, to trigger generation of configuration files. Existing files in the `CONFIG_PATH` will **not** be overwritten. -* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `synapse`] -* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `postgres` which is useful when using a container on the same docker network in a compose file where the postgres service is called `postgres`] **NOTE**: `localhost` and `127.0.0.1` refer to the container itself unless running the container with `host` networking. -* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. -* `POSTGRES_USER` - The user for the synapse postgres database. [default: `postgres`] -* `REPORT_STATS` - Whether to send anonymous usage statistics back to the Matrix project which helps us to get funding! Must be `yes` or `no`. [default: `yes`] -* `SERVER_NAME` - The domain used for the Matrix homeserver. If you intend to run this synapse instance on a public domain, use that domain. [default: `localhost`] - -``` -CONFIG_PATH=/my/magical/config/path/ -mkdir -p ${CONFIG_PATH} -docker run \ - --rm \ - -e GENERATE_CONFIG=yes \ - -e POSTGRES_PASSWORD=MyVerySecretPassword \ - -e REPORT_STATS=yes \ - -e SERVER_NAME=example.com \ - -v ${CONFIG_PATH}:/synapse/config/ \ - matrixdotorg/synapse:v0.22.1 -``` - -This will create a temporary container from the image and use the synapse code for generating configuration files and TLS keys and certificates for the specified `SERVER_NAME` domain. The files are written to `CONFIG_PATH`. +You may have a local Python wheel cache available, in which case copy the relevant packages in the ``cache/`` directory at the root of the project. ## Run -**NOTE**: If you are not using postgresql and are using sqlite3 as your database, you will need to make a directory to store the sqlite3 database file in and then mount this volume into the container at `/synapse/data/`. As it is so easy to use postgresql, when using Docker containers, this is not documented to somewhat discourage it. Choose a `POSTGRES_PASSWORD` instead. +It is recommended that you use Docker Compose to run your containers, including +this image and a Postgres server. A sample ``docker-compose.yml`` is provided, +with example labels for a reverse proxy and other artifacts. -### Docker Compose +Then, to run the server: -A `docker-compose.yaml` file is included to ease deployment of the basic synapse and postgres setup. Remember to set a `POSTGRES_PASSWORD` when generating your configuration above. You will need it for running the containers in the composition. - -From the `docker/` subdirectory of the synapse repository: ``` -CONFIG_PATH=/my/magical/config/path/ -POSTGRES_PASSWORD=MyVerySecretPassword \ -docker-compose \ - -p synapse \ - up -d +docker-compose up -d ``` -### Docker +In the case you specified a custom path for you configuration file and wish to +generate a fresh ``homeserver.yaml``, simply run: + +``` +docker-compose run synapse generate +``` + +If you do not wish to use Compose, you may still run this image using plain +Docker commands: Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. @@ -65,6 +47,50 @@ Note that the following is just a guideline and you may need to add parameters t docker run \ -d \ --name synapse \ - -v ${CONFIG_PATH}:/synapse/config/ \ + -v ${DATA_PATH}:/data \ + -e SYNAPSE_SERVER_NAME=my.matrix.host \ matrixdotorg/synapse:v0.22.1 ``` + + +## Volumes + +The image expects a single volue, located at ``/data``, that will hold: + +* temporary files during uploads; +* uploaded media and thumbnais; +* the SQLite database if you do not configure postgres. + +## Environment + +If you do not specify a custom path for the configuration file, a very generic +file will be generated, based on the following environment settings. +These are a good starting point for setting up your own deployment. + +Synapse specific settings: + +* ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all + other options then). +* ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if + you run your own TLS-capable reverse proxy). +* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. +* ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on + the Synapse instance. +* ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. +* ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. + +Shared secrets, these will be initialized to random values if not set: + +* ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if + registration is disable. +* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon. + +Database specific values (will use SQLite if not set): + +* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`] +* `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] +* `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. +* `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] From b8ab78b82c60ab9c897891cf9f53c9cc44873a25 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 15:41:54 +0100 Subject: [PATCH 14/47] Add the build cache/ folder to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 491047c352..d13017f400 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ demo/media_store.* demo/etc uploads +cache .idea/ media_store/ From f72c9c1fb650c7f3bac4d77d4e24fa0469d09ebb Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 16:18:40 +0100 Subject: [PATCH 15/47] Fix multiple typos --- MANIFEST.in | 1 + contrib/docker/README.md | 19 ++++++++++++------- contrib/docker/docker-compose.yml | 7 ++++--- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index afb60e12ee..8c9a57a9ca 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,6 +25,7 @@ recursive-include synapse/static *.js exclude jenkins.sh exclude jenkins*.sh exclude jenkins* +exclude Dockerfile recursive-exclude jenkins *.sh prune .github diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 8ba5f79692..73e53e4306 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -23,7 +23,7 @@ You may have a local Python wheel cache available, in which case copy the releva It is recommended that you use Docker Compose to run your containers, including this image and a Postgres server. A sample ``docker-compose.yml`` is provided, -with example labels for a reverse proxy and other artifacts. +including example labels for reverse proxying and other artifacts. Then, to run the server: @@ -35,13 +35,13 @@ In the case you specified a custom path for you configuration file and wish to generate a fresh ``homeserver.yaml``, simply run: ``` -docker-compose run synapse generate +docker-compose run --rm synapse generate ``` If you do not wish to use Compose, you may still run this image using plain -Docker commands: - -Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation with your postgres database. +Docker commands. Note that the following is just a guideline and you may need +to add parameters to the docker run command to account for the network situation +with your postgres database. ``` docker run \ @@ -55,7 +55,7 @@ docker run \ ## Volumes -The image expects a single volue, located at ``/data``, that will hold: +The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; * uploaded media and thumbnais; @@ -63,10 +63,15 @@ The image expects a single volue, located at ``/data``, that will hold: ## Environment -If you do not specify a custom path for the configuration file, a very generic +Unless you specify a custom path for the configuration file, a very generic file will be generated, based on the following environment settings. These are a good starting point for setting up your own deployment. +Global settings: + +* ``UID``, the user id Synapse will run as [default 991] +* ``GID``, the group id Synapse will run as [default 991] + Synapse specific settings: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b8f9741f05..727743effa 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -1,4 +1,4 @@ -# This compose file is compatible with Copose itself, it might need some +# This compose file is compatible with Compose itself, it might need some # adjustments to run properly with stack. version: '3' @@ -18,10 +18,11 @@ services: - ./files:/data depends_on: - db - # One may either expose ports directly + # In order to expose Synapse, remove one of the following, you might for + # instance expose the TLS port directly: ports: - 8448:8448/tcp - # ... or use a reverse proxy, here is an example for traefik + # ... or use a reverse proxy, here is an example for traefik: labels: - traefik.enable=true - traefik.frontend.rule=Host:my.matrix.Host From e9021e16c49c4224782040449b43fb0015c9f05c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sun, 4 Feb 2018 23:19:08 +0100 Subject: [PATCH 16/47] Run the server as an unprivileged user --- Dockerfile | 2 +- contrib/docker/start.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 25f3746303..f687a4f2bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:2-alpine -RUN apk add --no-cache --virtual .nacl_deps build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 7057f85f61..8bc72bf428 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -9,14 +9,16 @@ convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(sr mode = sys.argv[1] if len(sys.argv) > 1 else None environ = os.environ.copy() +# Check mandatory parameters and build the base start arguments if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) +permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver", - "--server-name", os.environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", os.environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", os.environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] + "--server-name", environ.get("SYNAPSE_SERVER_NAME"), + "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), + "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] # Generate any missing shared secret for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): @@ -35,4 +37,5 @@ if mode == "generate": # In normal mode, generate missing keys if any, then run synapse else: subprocess.check_output(args + ["--generate-keys"]) - os.execv("/usr/local/bin/python", args) + subprocess.check_output(["chown", "-R", permissions, "/data"]) + os.execv("/sbin/su-exec", ["su-exec", permissions] + args) From 8db84e9b2198ad4a80f6e9e66e2fc8c2d44fbeca Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 20:08:35 +0100 Subject: [PATCH 17/47] Remove docker related files from the python manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 8c9a57a9ca..e2a6623a63 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -26,6 +26,7 @@ exclude jenkins.sh exclude jenkins*.sh exclude jenkins* exclude Dockerfile +exclude .dockerignore recursive-exclude jenkins *.sh prune .github From 81010a126e9bbcce018b104921ff5221a67f99cc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 21:28:15 +0100 Subject: [PATCH 18/47] Add dynamic recaptcha configuration in the Docker image --- contrib/docker/README.md | 4 ++++ contrib/docker/conf/homeserver.yaml | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 73e53e4306..f4bc78908d 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -86,6 +86,10 @@ Synapse specific settings: * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public + key in order to enable recaptcha upon registration +* ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private + key in order to enable recaptcha upon registration Shared secrets, these will be initialized to random values if not set: diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 7450cc1228..6f1de24aad 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -112,10 +112,17 @@ max_spider_size: "10M" ## Captcha ## +{% if SYNAPSE_RECAPTCHA_PUBLIC_KEY %} +recaptcha_public_key: "{{ SYNAPSE_RECAPTCHA_PUBLIC_KEY }}" +recaptcha_private_key: "{{ SYNAPSE_RECAPTCHA_PRIVATE_KEY }}" +enable_registration_captcha: True +recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% else %} recaptcha_public_key: "YOUR_PUBLIC_KEY" recaptcha_private_key: "YOUR_PRIVATE_KEY" enable_registration_captcha: False recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" +{% endif %} ## Turn ## From cd51931b62aef63dacf4d79cdfa5de56da4eeda6 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 21:53:53 +0100 Subject: [PATCH 19/47] Add dynamic TURN configuration in the Docker image --- contrib/docker/README.md | 7 +++++-- contrib/docker/conf/homeserver.yaml | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index f4bc78908d..0da7b56628 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -87,9 +87,12 @@ Synapse specific settings: * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public - key in order to enable recaptcha upon registration + key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private - key in order to enable recaptcha upon registration + key in order to enable recaptcha upon registration. +* ``SYNAPSE_TURN_URIS``, set this variable to the coma-separated list of TURN + uris to enable TURN for this homeserver. +* ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. Shared secrets, these will be initialized to random values if not set: diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 6f1de24aad..6f8fb24e5f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -126,10 +126,19 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" ## Turn ## +{% if SYNAPSE_TURN_URIS %} +turn_uris: +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - {{ uri }} +{% endfor %} +turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" +turn_user_lifetime: "1h" +turn_allow_guests: True +{% else %} turn_uris: [] turn_shared_secret: "YOUR_SHARED_SECRET" turn_user_lifetime: "1h" turn_allow_guests: True +{% endif %} ## Registration ## From cf4ef60e287a6a61844ab260606721db2b7ee0cd Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:10:03 +0100 Subject: [PATCH 20/47] Document the cache factor environment variable for Docker --- contrib/docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 0da7b56628..8b1e0afee6 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -84,6 +84,7 @@ Synapse specific settings: the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. +* ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. * ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public From d8c7da5dca907bf65293b6b967200141cad69410 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:12:50 +0100 Subject: [PATCH 21/47] Fix a typo in the Docker README --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 8b1e0afee6..624dca747a 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -58,7 +58,7 @@ docker run \ The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; -* uploaded media and thumbnais; +* uploaded media and thumbnails; * the SQLite database if you do not configure postgres. ## Environment From f5364b47ec4b67a552976b9dbb9594d448744e30 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:14:40 +0100 Subject: [PATCH 22/47] Point to the 'latest' tag in the Docker documentation --- contrib/docker/README.md | 2 +- contrib/docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 624dca747a..87354b9bc3 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -49,7 +49,7 @@ docker run \ --name synapse \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ - matrixdotorg/synapse:v0.22.1 + matrixdotorg/synapse:latest ``` diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 727743effa..3fb156db47 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: synapse + image: matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped From 630573a9325f826e81f04650fd83e64d1c831035 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:57:22 +0100 Subject: [PATCH 23/47] Do not copy documentation files to the Docker root folder --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f687a4f2bb..2ae503ce79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN cd /synapse \ && pip install --upgrade pip setuptools psycopg2 \ && mkdir -p /synapse/cache \ && pip install -f /synapse/cache --upgrade --process-dependency-links . \ - && mv /synapse/contrib/docker/* / \ + && mv /synapse/contrib/docker/start.py /synapse/contrib/docker/conf / \ && rm -rf setup.py setup.cfg synapse VOLUME ["/data"] From ee3b160a2ad375223b4304184304605a35f1b406 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 22:57:35 +0100 Subject: [PATCH 24/47] Only generate configuration files when necessary --- contrib/docker/start.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 8bc72bf428..d3364e4226 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -26,16 +26,18 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE print("Generating a random secret for {}".format(secret)) environ[secret] = os.urandom(32).encode("hex") -# Parse the configuration file -if not os.path.exists("/compiled"): os.mkdir("/compiled") -convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) -convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) - # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": os.execv("/usr/local/bin/python", args + ["--generate-config"]) + # In normal mode, generate missing keys if any, then run synapse else: + # Parse the configuration file + if "SYNAPSE_CONFIG_PATH" not in environ: + if not os.path.exists("/compiled"): os.mkdir("/compiled") + convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) + convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) + # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) subprocess.check_output(["chown", "-R", permissions, "/data"]) os.execv("/sbin/su-exec", ["su-exec", permissions] + args) From 107a5c94418de55d52a0232115d5c3c7efc6d285 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 23:02:33 +0100 Subject: [PATCH 25/47] Add the non-tls port to the expose list --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2ae503ce79..bec884ce9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,6 @@ RUN cd /synapse \ VOLUME ["/data"] -EXPOSE 8448 +EXPOSE 8008/tcp 8448/tcp ENTRYPOINT ["/start.py"] From 1ffd9cb93617fe9bb2367d575786c0ff222cd415 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Mon, 5 Feb 2018 23:13:27 +0100 Subject: [PATCH 26/47] Support loading application service files from /data/appservices/ --- contrib/docker/README.md | 7 ++++++- contrib/docker/conf/homeserver.yaml | 9 ++++++++- contrib/docker/start.py | 4 ++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 87354b9bc3..c1724fe269 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -59,7 +59,12 @@ The image expects a single volume, located at ``/data``, that will hold: * temporary files during uploads; * uploaded media and thumbnails; -* the SQLite database if you do not configure postgres. +* the SQLite database if you do not configure postgres; +* the appservices configuration. + +In order to setup an application service, simply create an ``appservices`` +directory in the data volume and write the application service Yaml +configuration file there. Multiple application services are supported. ## Environment diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 6f8fb24e5f..e5d3f965e4 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -128,7 +128,7 @@ recaptcha_siteverify_api: "https://www.google.com/recaptcha/api/siteverify" {% if SYNAPSE_TURN_URIS %} turn_uris: -{% for uri in SYNAPSE_TURN_URIS.split(',') %} - {{ uri }} +{% for uri in SYNAPSE_TURN_URIS.split(',') %} - "{{ uri }}" {% endfor %} turn_shared_secret: "{{ SYNAPSE_TURN_SECRET }}" turn_user_lifetime: "1h" @@ -167,7 +167,14 @@ room_invite_state_types: - "m.room.avatar" - "m.room.name" +{% if SYNAPSE_APPSERVICES %} +app_service_config_files: +{% for appservice in SYNAPSE_APPSERVICES %} - "{{ appservice }}" +{% endfor %} +{% else %} app_service_config_files: [] +{% endif %} + macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}" expire_access_token: False diff --git a/contrib/docker/start.py b/contrib/docker/start.py index d3364e4226..8ade0f227d 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -4,6 +4,7 @@ import jinja2 import os import sys import subprocess +import glob convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -26,6 +27,9 @@ for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KE print("Generating a random secret for {}".format(secret)) environ[secret] = os.urandom(32).encode("hex") +# Load appservices configurations +environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") + # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": os.execv("/usr/local/bin/python", args + ["--generate-config"]) From 63fd148724399d52f3435b1c172435d7cabcde4c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:46:11 +0100 Subject: [PATCH 27/47] Make it clear that two modes are avaiable in the documentation, improve the compose file --- contrib/docker/README.md | 17 ++++++++++------- contrib/docker/docker-compose.yml | 12 ++++++++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index c1724fe269..197bad103e 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -12,7 +12,7 @@ use that server. Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrixdotorg/synapse:v0.22.1 . +docker build -t matrixdotorg/synapse . ``` The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. @@ -76,12 +76,17 @@ Global settings: * ``UID``, the user id Synapse will run as [default 991] * ``GID``, the group id Synapse will run as [default 991] +* ``SYNAPSE_CONFIG_PATH``, path to a custom config file -Synapse specific settings: +If ``SYNAPSE_CONFIG_PATH`` is set, you should generate a configuration file +then customize it manually. No other environment variable is required. + +Otherwise, a dynamic configuration file will be used. The following environment +variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. -* ``SYNAPSE_CONFIG_PATH``, path to a custom config file (will ignore all - other options then). +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous + statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. @@ -90,8 +95,6 @@ Synapse specific settings: * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. * ``SYNAPSE_EVENT_CACHE_SIZE``, the event cache size [default `10K`]. * ``SYNAPSE_CACHE_FACTOR``, the cache factor [default `0.5`]. -* ``SYNAPSE_REPORT_STATS``, set this variable to `yes` to enable anonymous - statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_RECAPTCHA_PUBLIC_KEY``, set this variable to the recaptcha public key in order to enable recaptcha upon registration. * ``SYNAPSE_RECAPTCHA_PRIVATE_KEY``, set this variable to the recaptcha private @@ -100,7 +103,7 @@ Synapse specific settings: uris to enable TURN for this homeserver. * ``SYNAPSE_TURN_SECRET``, set this to the TURN shared secret if required. -Shared secrets, these will be initialized to random values if not set: +Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3fb156db47..b07984ea34 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: matrixdotorg/synapse:latest + image: docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped @@ -15,7 +15,12 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_ENABLE_REGISTRATION=yes volumes: + # You may either store all the files in a local folder - ./files:/data + # .. or you may split this between different storage points + # - ./files:/data + # - /path/to/ssd:/data/uploads + # - /path/to/large_hdd:/data/media depends_on: - db # In order to expose Synapse, remove one of the following, you might for @@ -29,10 +34,13 @@ services: - traefik.port=8448 db: - image: postgres:latest + image: postgres:10-alpine # Change that password, of course! environment: - POSTGRES_USER=matrix - POSTGRES_PASSWORD=changeme volumes: + # You may store the database tables in a local folder.. - ./schemas:/var/lib/postgresql/data + # .. or store them on some high performance storage for better results + # - /path/to/ssd/storage:/var/lib/postfesql/data From 58df3a8c5dcf5de12bcf9190551dc8241d32b8a1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:48:53 +0100 Subject: [PATCH 28/47] Add some documentation about high performance storage --- contrib/docker/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 197bad103e..3710afb0cf 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -31,11 +31,11 @@ Then, to run the server: docker-compose up -d ``` -In the case you specified a custom path for you configuration file and wish to +In the case you specify a custom path for you configuration file and wish to generate a fresh ``homeserver.yaml``, simply run: ``` -docker-compose run --rm synapse generate +docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate ``` If you do not wish to use Compose, you may still run this image using plain @@ -62,6 +62,11 @@ The image expects a single volume, located at ``/data``, that will hold: * the SQLite database if you do not configure postgres; * the appservices configuration. +You are free to use separate volumes depending on storage endpoints at your +disposal. For instance, ``/data/media`` coud be stored on a large but low +performance hdd storage while other files could be stored on high performance +endpoints. + In order to setup an application service, simply create an ``appservices`` directory in the data volume and write the application service Yaml configuration file there. Multiple application services are supported. From 084afbb6a06f2661bed503bf49b0291ce999c6c1 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 19:50:04 +0100 Subject: [PATCH 29/47] Rename the permissions variable to avoid confusion --- contrib/docker/start.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 8ade0f227d..13a10a11bb 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -15,7 +15,7 @@ if "SYNAPSE_SERVER_NAME" not in environ: print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") sys.exit(2) -permissions = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) +ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) args = ["python", "-m", "synapse.app.homeserver", "--server-name", environ.get("SYNAPSE_SERVER_NAME"), "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), @@ -43,5 +43,5 @@ else: convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) - subprocess.check_output(["chown", "-R", permissions, "/data"]) - os.execv("/sbin/su-exec", ["su-exec", permissions] + args) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From b8a4dceb3cee6b69d1b1b882cef1f96a3ff6249f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:41:41 +0100 Subject: [PATCH 30/47] Refactor the start script to better handle mandatory parameters --- contrib/docker/README.md | 2 +- contrib/docker/docker-compose.yml | 1 + contrib/docker/start.py | 56 ++++++++++++++++++------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 3710afb0cf..0493d2ee6e 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -90,7 +90,7 @@ Otherwise, a dynamic configuration file will be used. The following environment variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. -* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``not``), enable anonymous +* ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index b07984ea34..3d0b3c0ea4 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -13,6 +13,7 @@ services: # See the readme for a full documentation of the environment settings environment: - SYNAPSE_SERVER_NAME=my.matrix.host + - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes volumes: # You may either store all the files in a local folder diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 13a10a11bb..32142bbe00 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -6,42 +6,52 @@ import sys import subprocess import glob +# Utility functions convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ)) + +def check_arguments(environ, args): + for argument in args: + if argument not in environ: + print("Environment variable %s is mandatory, exiting." % argument) + sys.exit(2) + +def generate_secrets(environ, secrets): + for secret in secrets: + if secret not in environ: + print("Generating a random secret for {}".format(secret)) + environ[secret] = os.urandom(32).encode("hex") + +# Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None environ = os.environ.copy() - -# Check mandatory parameters and build the base start arguments -if "SYNAPSE_SERVER_NAME" not in environ: - print("Environment variable SYNAPSE_SERVER_NAME is mandatory, exiting.") - sys.exit(2) - ownership = "{}:{}".format(environ.get("UID", 991), environ.get("GID", 991)) -args = ["python", "-m", "synapse.app.homeserver", - "--server-name", environ.get("SYNAPSE_SERVER_NAME"), - "--report-stats", environ.get("SYNAPSE_REPORT_STATS", "no"), - "--config-path", environ.get("SYNAPSE_CONFIG_PATH", "/compiled/homeserver.yaml")] - -# Generate any missing shared secret -for secret in ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY"): - if secret not in environ: - print("Generating a random secret for {}".format(secret)) - environ[secret] = os.urandom(32).encode("hex") - -# Load appservices configurations -environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") +args = ["python", "-m", "synapse.app.homeserver"] # In generate mode, generate a configuration, missing keys, then exit if mode == "generate": - os.execv("/usr/local/bin/python", args + ["--generate-config"]) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_CONFIG_PATH")) + args += [ + "--server-name", environ["SYNAPSE_SERVER_NAME"], + "--report-stats", environ["SYNAPSE_REPORT_STATS"], + "--config-path", environ["SYNAPSE_CONFIG_PATH"], + "--generate-config" + ] + os.execv("/usr/local/bin/python", args) # In normal mode, generate missing keys if any, then run synapse else: # Parse the configuration file - if "SYNAPSE_CONFIG_PATH" not in environ: + if "SYNAPSE_CONFIG_PATH" in environ: + args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] + else: + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY")) + environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) - convert("/conf/log.config", "/compiled/%s.log.config" % environ.get("SYNAPSE_SERVER_NAME"), environ) + convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ) + subprocess.check_output(["chown", "-R", ownership, "/data"]) + args += ["--config-path", "/compiled/homeserver.yaml"] # Generate missing keys and start synapse subprocess.check_output(args + ["--generate-keys"]) - subprocess.check_output(["chown", "-R", ownership, "/data"]) os.execv("/sbin/su-exec", ["su-exec", ownership] + args) From e174c46a295ca6e06b217b5dcbcf995b890e6d07 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:42:57 +0100 Subject: [PATCH 31/47] Use 'synapse' as a default postgres user in Docker examples --- contrib/docker/conf/homeserver.yaml | 4 ++-- contrib/docker/docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index e5d3f965e4..1ca1fe991f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -50,9 +50,9 @@ listeners: database: name: "psycopg2" args: - user: "{{ POSTGRES_USER or "matrix" }}" + user: "{{ POSTGRES_USER or "synapse" }}" password: "{{ POSTGRES_PASSWORD }}" - database: "{{ POSTGRES_DB or "matrix" }}" + database: "{{ POSTGRES_DB or "synapse" }}" host: "{{ POSTGRES_HOST or "db" }}" port: "{{ POSTGRES_PORT or "5432" }}" cp_min: 5 diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3d0b3c0ea4..e447bf1212 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -38,7 +38,7 @@ services: image: postgres:10-alpine # Change that password, of course! environment: - - POSTGRES_USER=matrix + - POSTGRES_USER=synapse - POSTGRES_PASSWORD=changeme volumes: # You may store the database tables in a local folder.. From 914a59cb8c12b25d77ed9a81e4543c23d7e10b5e Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:43:45 +0100 Subject: [PATCH 32/47] Disable the Web client in the Docker image --- contrib/docker/README.md | 1 - contrib/docker/conf/homeserver.yaml | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 0493d2ee6e..9f40dc0d58 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -94,7 +94,6 @@ variables are available for configuration: statistics reporting back to the Matrix project which helps us to get funding. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). -* ``SYNAPSE_WEB_CLIENT``, set this variable to enable the embedded Web client. * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on the Synapse instance. * ``SYNAPSE_ALLOW_GUEST``, set this variable to allow guest joining this server. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 1ca1fe991f..19a2cbad29 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -12,7 +12,7 @@ tls_fingerprints: [] server_name: "{{ SYNAPSE_SERVER_NAME }}" pid_file: /homeserver.pid -web_client: {{ "True" if SYNAPSE_WEB_CLIENT else "False" }} +web_client: False soft_file_limit: 0 ## Ports ## @@ -26,7 +26,7 @@ listeners: tls: true x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] # Federation APIs compress: false @@ -39,7 +39,7 @@ listeners: x_forwarded: false resources: - - names: [client, webclient] + - names: [client] compress: true - names: [federation] compress: false From a0af0054ec91e92a6843c121cd27e92ea63c1034 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:46:11 +0100 Subject: [PATCH 33/47] Honor the SYNAPSE_REPORT_STATS parameter in the Docker image --- contrib/docker/conf/homeserver.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 19a2cbad29..3b57f7174d 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -156,8 +156,13 @@ trusted_third_party_id_servers: ## Metrics ### +{% if SYNAPSE_REPORT_STATS.lower() == "yes" %} +enable_metrics: True +report_stats: True +{% else %} enable_metrics: False report_stats: False +{% endif %} ## API Configuration ## From ef1f8d4be6e970043b8283f5caa1ca764652ad56 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 20:53:12 +0100 Subject: [PATCH 34/47] Enable email server configuration from environment variables --- contrib/docker/README.md | 7 ++++++ contrib/docker/conf/homeserver.yaml | 33 +++++++++++++++-------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 9f40dc0d58..b74c72698c 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -119,3 +119,10 @@ Database specific values (will use SQLite if not set): * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] + +Mail server specific values (will not send emails if not set): + +* ``SYNAPSE_SMTP_HOST``, hostname to the mail server. +* ``SYNAPSE_SMTP_PORT``, TCP port for accessing the mail server [default ``25``]. +* ``SYNAPSE_SMTP_USER``, username for authenticating against the mail server if any. +* ``SYNAPSE_SMTP_PASSWORD``, password for authenticating against the mail server if any. diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 3b57f7174d..198b8ddee7 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -146,6 +146,7 @@ enable_registration: {{ "True" if SYNAPSE_ENABLE_REGISTRATION else "False" }} registration_shared_secret: "{{ SYNAPSE_REGISTRATION_SHARED_SECRET }}" bcrypt_rounds: 12 allow_guest_access: {{ "True" if SYNAPSE_ALLOW_GUEST else "False" }} +enable_group_creation: true # The list of identity servers trusted to verify third party # identifiers by this server. @@ -200,19 +201,19 @@ perspectives: password_config: enabled: true -#email: -# enable_notifs: false -# smtp_host: "localhost" -# smtp_port: 25 -# smtp_user: "exampleusername" -# smtp_pass: "examplepassword" -# require_transport_security: False -# notif_from: "Your Friendly %(app)s Home Server " -# app_name: Matrix -# template_dir: res/templates -# notif_template_html: notif_mail.html -# notif_template_text: notif_mail.txt -# notif_for_new_users: True -# riot_base_url: "http://localhost/riot" - -enable_group_creation: true +{% if SYNAPSE_SMTP_HOST %} +email: + enable_notifs: false + smtp_host: "{{ SYNAPSE_SMTP_HOST }}" + smtp_port: {{ SYNAPSE_SMTP_PORT or "25" }} + smtp_user: "{{ SYNAPSE_SMTP_USER }}" + smtp_pass: "{{ SYNAPSE_SMTP_PASSWORD }}" + require_transport_security: False + notif_from: "{{ SYNAPSE_SMTP_FROM or "hostmaster@" + SYNAPSE_SERVER_NAME }}" + app_name: Matrix + template_dir: res/templates + notif_template_html: notif_mail.html + notif_template_text: notif_mail.txt + notif_for_new_users: True + riot_base_url: "https://{{ SYNAPSE_SERVER_NAME }}" +{% endif %} From b9b668e4bb6ec3c0ae6c696a4b7abf6f537e0504 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:39:36 +0100 Subject: [PATCH 35/47] Update to Alpine 3.7 and switch to libressl --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bec884ce9b..8085f3d354 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:2-alpine +FROM docker.io/python:2-alpine3.7 -RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev openssl-dev libjpeg-turbo-dev linux-headers postgresql-dev +RUN apk add --no-cache --virtual .nacl_deps su-exec build-base libffi-dev zlib-dev libressl-dev libjpeg-turbo-dev linux-headers postgresql-dev COPY . /synapse From d8680c969bb2e8248436d7352ebf0f5cd1daea7b Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:55:35 +0100 Subject: [PATCH 36/47] Make it clear that the image has two modes of operation --- contrib/docker/README.md | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index b74c72698c..27e25afcef 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -21,23 +21,42 @@ You may have a local Python wheel cache available, in which case copy the releva ## Run +This image is designed to run either with an automatically generated configuration +file or with a custom configuration that requires manual edition. + +### Automated configuration + It is recommended that you use Docker Compose to run your containers, including this image and a Postgres server. A sample ``docker-compose.yml`` is provided, including example labels for reverse proxying and other artifacts. -Then, to run the server: +Read the section about environment variables and set at least mandatory variables, +then run the server: ``` docker-compose up -d ``` -In the case you specify a custom path for you configuration file and wish to -generate a fresh ``homeserver.yaml``, simply run: +### Manual configuration + +A sample ``docker-compose.yml`` is provided, including example labels for +reverse proxying and other artifacts. + +Specify a ``SYNAPSE_CONFIG_PATH``, preferably to a persistent path, +to use manual configuration. To generate a fresh ``homeserver.yaml``, simply run: ``` docker-compose run --rm -e SYNAPSE_SERVER_NAME=my.matrix.host synapse generate ``` +Then, customize your configuration and run the server: + +``` +docker-compose up -d +``` + +### Without Compose + If you do not wish to use Compose, you may still run this image using plain Docker commands. Note that the following is just a guideline and you may need to add parameters to the docker run command to account for the network situation @@ -49,10 +68,10 @@ docker run \ --name synapse \ -v ${DATA_PATH}:/data \ -e SYNAPSE_SERVER_NAME=my.matrix.host \ - matrixdotorg/synapse:latest + -e SYNAPSE_REPORT_STATS=yes \ + docker.io/matrixdotorg/synapse:latest ``` - ## Volumes The image expects a single volume, located at ``/data``, that will hold: From 48e2c641b80ac57d16701e4a27f348ea4f2c66cc Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 21:58:12 +0100 Subject: [PATCH 37/47] Specify the Docker registry in the build tag --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 27e25afcef..221d9c53b5 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -12,7 +12,7 @@ use that server. Build the docker image with the `docker build` command from the root of the synapse repository. ``` -docker build -t matrixdotorg/synapse . +docker build -t docker.io/matrixdotorg/synapse . ``` The `-t` option sets the image tag. Official images are tagged `matrixdotorg/synapse:` where `` is the same as the release tag in the synapse git repository. From a03c382966a3219acd26851db9cc6558e5c53310 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Thu, 8 Feb 2018 22:00:43 +0100 Subject: [PATCH 38/47] Specify the Docker registry for the postgres image --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index e447bf1212..1d2aebbcd3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -35,7 +35,7 @@ services: - traefik.port=8448 db: - image: postgres:10-alpine + image: docker.io/postgres:10-alpine # Change that password, of course! environment: - POSTGRES_USER=synapse From e511979fe6c4a03da3e9c1d16672e263f54ee2d3 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 9 Feb 2018 00:13:26 +0100 Subject: [PATCH 39/47] Make SYNAPSE_MACAROON_SECRET_KEY a mandatory option --- contrib/docker/README.md | 3 ++- contrib/docker/start.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 221d9c53b5..25c358c847 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -111,6 +111,8 @@ variables are available for configuration: * ``SYNAPSE_SERVER_NAME`` (mandatory), the current server public hostname. * ``SYNAPSE_REPORT_STATS``, (mandatory, ``yes`` or ``no``), enable anonymous statistics reporting back to the Matrix project which helps us to get funding. +* ``SYNAPSE_MACAROON_SECRET_KEY`` (mandatory) secret for signing access tokens + to the server, set this to a proper random key. * ``SYNAPSE_NO_TLS``, set this variable to disable TLS in Synapse (use this if you run your own TLS-capable reverse proxy). * ``SYNAPSE_ENABLE_REGISTRATION``, set this variable to enable registration on @@ -130,7 +132,6 @@ Shared secrets, that will be initialized to random values if not set: * ``SYNAPSE_REGISTRATION_SHARED_SECRET``, secret for registrering users if registration is disable. -* ``SYNAPSE_MACAROON_SECRET_KEY``, secret for Macaroon. Database specific values (will use SQLite if not set): diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 32142bbe00..d4c1140b1d 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -44,8 +44,8 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) - generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET", "SYNAPSE_MACAROON_SECRET_KEY")) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY")) + generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",)) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) From ca70148c0569295a2b9ecdd1cd9cd85a203f20e7 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 9 Feb 2018 00:23:19 +0100 Subject: [PATCH 40/47] Fix the path to the log config file --- contrib/docker/conf/homeserver.yaml | 2 +- contrib/docker/start.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/docker/conf/homeserver.yaml b/contrib/docker/conf/homeserver.yaml index 198b8ddee7..6bc25bb45f 100644 --- a/contrib/docker/conf/homeserver.yaml +++ b/contrib/docker/conf/homeserver.yaml @@ -69,7 +69,7 @@ database: event_cache_size: "{{ SYNAPSE_EVENT_CACHE_SIZE or "10K" }}" verbose: 0 log_file: "/data/homeserver.log" -log_config: "/data/{{ SYNAPSE_SERVER_NAME }}.log.config" +log_config: "/compiled/log.config" ## Ratelimiting ## diff --git a/contrib/docker/start.py b/contrib/docker/start.py index d4c1140b1d..75c30b8ac0 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -49,7 +49,7 @@ else: environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) - convert("/conf/log.config", "/compiled/%s.log.config" % environ["SYNAPSE_SERVER_NAME"], environ) + convert("/conf/log.config", "/compiled/log.config", environ) subprocess.check_output(["chown", "-R", ownership, "/data"]) args += ["--config-path", "/compiled/homeserver.yaml"] # Generate missing keys and start synapse From 6f0b1f85f9f34401219eab4b4977a63c698ce987 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 00:05:03 +0100 Subject: [PATCH 41/47] Generate macaroon and registration secrets, then store the results to the data dir --- contrib/docker/docker-compose.yml | 2 +- contrib/docker/start.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1d2aebbcd3..9e32dd87de 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: docker.io/matrixdotorg/synapse:latest + image: synapse #docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped diff --git a/contrib/docker/start.py b/contrib/docker/start.py index 75c30b8ac0..90e8b9c51a 100755 --- a/contrib/docker/start.py +++ b/contrib/docker/start.py @@ -16,10 +16,16 @@ def check_arguments(environ, args): sys.exit(2) def generate_secrets(environ, secrets): - for secret in secrets: + for name, secret in secrets.items(): if secret not in environ: - print("Generating a random secret for {}".format(secret)) - environ[secret] = os.urandom(32).encode("hex") + filename = "/data/%s.%s.key" % (environ["SYNAPSE_SERVER_NAME"], name) + if os.path.exists(filename): + with open(filename) as handle: value = handle.read() + else: + print("Generating a random secret for {}".format(name)) + value = os.urandom(32).encode("hex") + with open(filename, "w") as handle: handle.write(value) + environ[secret] = value # Prepare the configuration mode = sys.argv[1] if len(sys.argv) > 1 else None @@ -44,8 +50,11 @@ else: if "SYNAPSE_CONFIG_PATH" in environ: args += ["--config-path", environ["SYNAPSE_CONFIG_PATH"]] else: - check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS", "SYNAPSE_MACAROON_SECRET_KEY")) - generate_secrets(environ, ("SYNAPSE_REGISTRATION_SHARED_SECRET",)) + check_arguments(environ, ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS")) + generate_secrets(environ, { + "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET", + "macaroon": "SYNAPSE_MACAROON_SECRET_KEY" + }) environ["SYNAPSE_APPSERVICES"] = glob.glob("/data/appservices/*.yaml") if not os.path.exists("/compiled"): os.mkdir("/compiled") convert("/conf/homeserver.yaml", "/compiled/homeserver.yaml", environ) From b815aa0e2db8f50116b1443b559ca13fe6ad1750 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 21:59:58 +0100 Subject: [PATCH 42/47] Remove an accidentally committed test configuration --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 9e32dd87de..1d2aebbcd3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,7 +6,7 @@ version: '3' services: synapse: - image: synapse #docker.io/matrixdotorg/synapse:latest + image: docker.io/matrixdotorg/synapse:latest # Since snyapse does not retry to connect to the database, restart upon # failure restart: unless-stopped From 07f1b7181997dca91b67dca7561ce4c532caf253 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 23:57:36 +0100 Subject: [PATCH 43/47] Explicitely provide the postgres password to synapse in the Compose example --- contrib/docker/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 1d2aebbcd3..46e72601d3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,6 +15,7 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes + - POSTGRES_PASSWORD=changeme volumes: # You may either store all the files in a local folder - ./files:/data From f44b7c022f6bc8b30cb8c446e0922b26b8b8eb5a Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 10 Feb 2018 23:57:51 +0100 Subject: [PATCH 44/47] Disable logging to file and rely on the console when using Docker --- contrib/docker/conf/log.config | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config index 45e7eef953..b5c907c4f9 100644 --- a/contrib/docker/conf/log.config +++ b/contrib/docker/conf/log.config @@ -10,13 +10,6 @@ filters: request: "" handlers: - file: - class: logging.handlers.RotatingFileHandler - formatter: precise - filename: /data/homeserver.log - maxBytes: 104857600 - backupCount: 10 - filters: [context] console: class: logging.StreamHandler formatter: precise @@ -33,4 +26,4 @@ loggers: root: level: INFO - handlers: [file, console] + handlers: [console] From d4c14e143825c679a7909b009aec2e51f829857f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 1 May 2018 20:47:58 +0200 Subject: [PATCH 45/47] Fix the documentation about 'POSTGRES_DB' --- contrib/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/README.md b/contrib/docker/README.md index 25c358c847..aed56646c2 100644 --- a/contrib/docker/README.md +++ b/contrib/docker/README.md @@ -135,7 +135,7 @@ Shared secrets, that will be initialized to random values if not set: Database specific values (will use SQLite if not set): -* `POSTGRES_DATABASE` - The database name for the synapse postgres database. [default: `matrix`] +* `POSTGRES_DB` - The database name for the synapse postgres database. [default: `synapse`] * `POSTGRES_HOST` - The host of the postgres database if you wish to use postgresql instead of sqlite3. [default: `db` which is useful when using a container on the same docker network in a compose file where the postgres service is called `db`] * `POSTGRES_PASSWORD` - The password for the synapse postgres database. **If this is set then postgres will be used instead of sqlite3.** [default: none] **NOTE**: You are highly encouraged to use postgresql! Please use the compose file to make it easier to deploy. * `POSTGRES_USER` - The user for the synapse postgres database. [default: `matrix`] From 4f2e898c29de5b51e8c5893afa64e5b43d25048f Mon Sep 17 00:00:00 2001 From: kaiyou Date: Tue, 1 May 2018 20:49:52 +0200 Subject: [PATCH 46/47] Make the logging level configurable --- contrib/docker/conf/log.config | 6 +++--- contrib/docker/docker-compose.yml | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/docker/conf/log.config b/contrib/docker/conf/log.config index b5c907c4f9..1851995802 100644 --- a/contrib/docker/conf/log.config +++ b/contrib/docker/conf/log.config @@ -17,13 +17,13 @@ handlers: loggers: synapse: - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} synapse.storage.SQL: # beware: increasing this to DEBUG will make synapse log sensitive # information such as access tokens. - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} root: - level: INFO + level: {{ SYNAPSE_LOG_LEVEL or "WARNING" }} handlers: [console] diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 46e72601d3..0b531949e0 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -15,6 +15,7 @@ services: - SYNAPSE_SERVER_NAME=my.matrix.host - SYNAPSE_REPORT_STATS=no - SYNAPSE_ENABLE_REGISTRATION=yes + - SYNAPSE_LOG_LEVEL=INFO - POSTGRES_PASSWORD=changeme volumes: # You may either store all the files in a local folder From 5addeaa02c432241af282900b19172ba7158118c Mon Sep 17 00:00:00 2001 From: kaiyou Date: Fri, 4 May 2018 21:23:01 +0200 Subject: [PATCH 47/47] Add Docker packaging in the author list --- AUTHORS.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index 3dcb1c2a89..e13ac5ad34 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -60,3 +60,6 @@ Niklas Riekenbrauck Christoph Witzany * Add LDAP support for authentication + +Pierre Jaury +* Docker packaging \ No newline at end of file