Compare ported to unported PG schemas in portdb test job (#13808)

This commit is contained in:
David Robertson 2022-09-16 16:25:54 +01:00 committed by GitHub
parent 5e84461653
commit 642c4b253d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 50 additions and 47 deletions

View file

@ -1,31 +0,0 @@
#!/usr/bin/env python
# Copyright 2019 The Matrix.org Foundation C.I.C.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import sys
import psycopg2
# a very simple replacment for `psql`, to make up for the lack of the postgres client
# libraries in the synapse docker image.
# We use "postgres" as a database because it's bound to exist and the "synapse" one
# doesn't exist yet.
db_conn = psycopg2.connect(
user="postgres", host="localhost", password="postgres", dbname="postgres"
)
db_conn.autocommit = True
cur = db_conn.cursor()
for c in sys.argv[1:]:
cur.execute(c)

View file

@ -32,7 +32,7 @@ else
fi fi
# Create the PostgreSQL database. # Create the PostgreSQL database.
poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse" psql -c "CREATE DATABASE synapse"
# Port the SQLite databse to postgres so we can check command works against postgres # Port the SQLite databse to postgres so we can check command works against postgres
echo "+++ Port SQLite3 databse to postgres" echo "+++ Port SQLite3 databse to postgres"

View file

@ -2,27 +2,27 @@
# #
# Test script for 'synapse_port_db'. # Test script for 'synapse_port_db'.
# - configures synapse and a postgres server. # - configures synapse and a postgres server.
# - runs the port script on a prepopulated test sqlite db # - runs the port script on a prepopulated test sqlite db. Checks that the
# - also runs it against an new sqlite db # return code is zero.
# - reruns the port script on the same sqlite db, targetting the same postgres db.
# Checks that the return code is zero.
# - runs the port script against a new sqlite db. Checks the return code is zero.
# #
# Expects Synapse to have been already installed with `poetry install --extras postgres`. # Expects Synapse to have been already installed with `poetry install --extras postgres`.
# Expects `poetry` to be available on the `PATH`. # Expects `poetry` to be available on the `PATH`.
set -xe set -xe -o pipefail
cd "$(dirname "$0")/../.." cd "$(dirname "$0")/../.."
echo "--- Generate the signing key" echo "--- Generate the signing key"
# Generate the server's signing key.
poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
echo "--- Prepare test database" echo "--- Prepare test database"
# Make sure the SQLite3 database is using the latest schema and has no pending background updates.
# Make sure the SQLite3 database is using the latest schema and has no pending background update.
poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# Create the PostgreSQL database. # Create the PostgreSQL database.
poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse" psql -c "CREATE DATABASE synapse"
echo "+++ Run synapse_port_db against test database" echo "+++ Run synapse_port_db against test database"
# TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`, # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
@ -45,9 +45,23 @@ rm .ci/test_db.db
poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
# re-create the PostgreSQL database. # re-create the PostgreSQL database.
poetry run .ci/scripts/postgres_exec.py \ psql \
"DROP DATABASE synapse" \ -c "DROP DATABASE synapse" \
"CREATE DATABASE synapse" -c "CREATE DATABASE synapse"
echo "+++ Run synapse_port_db against empty database" echo "+++ Run synapse_port_db against empty database"
poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
echo "--- Create a brand new postgres database from schema"
cp .ci/postgres-config.yaml .ci/postgres-config-unported.yaml
sed -i -e 's/database: synapse/database: synapse_unported/' .ci/postgres-config-unported.yaml
psql -c "CREATE DATABASE synapse_unported"
poetry run update_synapse_database --database-config .ci/postgres-config-unported.yaml --run-background-updates
echo "+++ Comparing ported schema with unported schema"
# Ignore the tables that portdb creates. (Should it tidy them up when the porting is completed?)
psql synapse -c "DROP TABLE port_from_sqlite3;"
pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse_unported > unported.sql
pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse > ported.sql
# By default, `diff` returns zero if there are no changes and nonzero otherwise
diff -u unported.sql ported.sql | tee schema_diff

View file

@ -362,18 +362,22 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- run: sudo apt-get -qq install xmlsec1 - run: sudo apt-get -qq install xmlsec1 postgresql-client
- uses: matrix-org/setup-python-poetry@v1 - uses: matrix-org/setup-python-poetry@v1
with: with:
extras: "postgres" extras: "postgres"
- run: .ci/scripts/test_export_data_command.sh - run: .ci/scripts/test_export_data_command.sh
env:
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
portdb: portdb:
if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
needs: linting-done needs: linting-done
runs-on: ubuntu-latest runs-on: ubuntu-latest
env:
TOP: ${{ github.workspace }}
strategy: strategy:
matrix: matrix:
include: include:
@ -399,12 +403,27 @@ jobs:
steps: steps:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- run: sudo apt-get -qq install xmlsec1 - run: sudo apt-get -qq install xmlsec1 postgresql-client
- uses: matrix-org/setup-python-poetry@v1 - uses: matrix-org/setup-python-poetry@v1
with: with:
python-version: ${{ matrix.python-version }} python-version: ${{ matrix.python-version }}
extras: "postgres" extras: "postgres"
- run: .ci/scripts/test_synapse_port_db.sh - run: .ci/scripts/test_synapse_port_db.sh
id: run_tester_script
env:
PGHOST: localhost
PGUSER: postgres
PGPASSWORD: postgres
PGDATABASE: postgres
- name: "Upload schema differences"
uses: actions/upload-artifact@v3
if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
with:
name: Schema dumps
path: |
unported.sql
ported.sql
schema_diff
complement: complement:
if: "${{ !failure() && !cancelled() }}" if: "${{ !failure() && !cancelled() }}"

1
changelog.d/13808.misc Normal file
View file

@ -0,0 +1 @@
Check that portdb generates the same postgres schema as that in the source tree.