Updates to the schema dump script (#13770)

This commit is contained in:
David Robertson 2022-09-13 10:53:11 +01:00 committed by GitHub
parent 540afb0bfc
commit b60d47ab2c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 21 additions and 32 deletions

1
changelog.d/13770.misc Normal file
View file

@ -0,0 +1 @@
Update the script which makes full schema dumps.

View file

@ -9,8 +9,10 @@
export PGHOST="localhost" export PGHOST="localhost"
POSTGRES_DB_NAME="synapse_full_schema.$$" POSTGRES_DB_NAME="synapse_full_schema.$$"
SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" SQLITE_SCHEMA_FILE="schema.sql.sqlite"
POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" SQLITE_ROWS_FILE="rows.sql.sqlite"
POSTGRES_SCHEMA_FILE="full.sql.postgres"
POSTGRES_ROWS_FILE="rows.sql.postgres"
REQUIRED_DEPS=("matrix-synapse" "psycopg2") REQUIRED_DEPS=("matrix-synapse" "psycopg2")
@ -22,7 +24,7 @@ usage() {
echo " Username to connect to local postgres instance. The password will be requested" echo " Username to connect to local postgres instance. The password will be requested"
echo " during script execution." echo " during script execution."
echo "-c" echo "-c"
echo " CI mode. Enables coverage tracking and prints every command that the script runs." echo " CI mode. Prints every command that the script runs."
echo "-o <path>" echo "-o <path>"
echo " Directory to output full schema files to." echo " Directory to output full schema files to."
echo "-h" echo "-h"
@ -37,11 +39,6 @@ while getopts "p:co:h" opt; do
c) c)
# Print all commands that are being executed # Print all commands that are being executed
set -x set -x
# Modify required dependencies for coverage
REQUIRED_DEPS+=("coverage" "coverage-enable-subprocess")
COVERAGE=1
;; ;;
o) o)
command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1) command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1)
@ -102,6 +99,7 @@ SQLITE_DB=$TMPDIR/homeserver.db
POSTGRES_CONFIG=$TMPDIR/postgres.conf POSTGRES_CONFIG=$TMPDIR/postgres.conf
# Ensure these files are delete on script exit # Ensure these files are delete on script exit
# TODO: the trap should also drop the temp postgres DB
trap 'rm -rf $TMPDIR' EXIT trap 'rm -rf $TMPDIR' EXIT
cat > "$SQLITE_CONFIG" <<EOF cat > "$SQLITE_CONFIG" <<EOF
@ -147,48 +145,34 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
# Make sure the SQLite3 database is using the latest schema and has no pending background update. # Make sure the SQLite3 database is using the latest schema and has no pending background update.
echo "Running db background jobs..." echo "Running db background jobs..."
synapse/_scripts/update_synapse_database.py --database-config --run-background-updates "$SQLITE_CONFIG" synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
# Create the PostgreSQL database. # Create the PostgreSQL database.
echo "Creating postgres database..." echo "Creating postgres database..."
createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME" createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
echo "Copying data from SQLite3 to Postgres with synapse_port_db..." echo "Running db background jobs..."
if [ -z "$COVERAGE" ]; then synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
# No coverage needed
synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
else
# Coverage desired
coverage run synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
fi
# Delete schema_version, applied_schema_deltas and applied_module_schemas tables # Delete schema_version, applied_schema_deltas and applied_module_schemas tables
# Also delete any shadow tables from fts4 # Also delete any shadow tables from fts4
# This needs to be done after synapse_port_db is run
echo "Dropping unwanted db tables..." echo "Dropping unwanted db tables..."
SQL=" SQL="
DROP TABLE schema_version; DROP TABLE schema_version;
DROP TABLE applied_schema_deltas; DROP TABLE applied_schema_deltas;
DROP TABLE applied_module_schemas; DROP TABLE applied_module_schemas;
DROP TABLE event_search_content;
DROP TABLE event_search_segments;
DROP TABLE event_search_segdir;
DROP TABLE event_search_docsize;
DROP TABLE event_search_stat;
DROP TABLE user_directory_search_content;
DROP TABLE user_directory_search_segments;
DROP TABLE user_directory_search_segdir;
DROP TABLE user_directory_search_docsize;
DROP TABLE user_directory_search_stat;
" "
sqlite3 "$SQLITE_DB" <<< "$SQL" sqlite3 "$SQLITE_DB" <<< "$SQL"
psql "$POSTGRES_DB_NAME" -w <<< "$SQL" psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..." echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_SCHEMA_FILE' and '$OUTPUT_DIR/$SQLITE_ROWS_FILE'..."
sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" sqlite3 "$SQLITE_DB" ".schema --indent" > "$OUTPUT_DIR/$SQLITE_SCHEMA_FILE"
sqlite3 "$SQLITE_DB" ".dump --data-only --nosys" > "$OUTPUT_DIR/$SQLITE_ROWS_FILE"
echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..." echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE' and '$OUTPUT_DIR/$POSTGRES_ROWS_FILE'..."
pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE"
pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_ROWS_FILE"
echo "Cleaning up temporary Postgres database..." echo "Cleaning up temporary Postgres database..."
dropdb $POSTGRES_DB_NAME dropdb $POSTGRES_DB_NAME

View file

@ -26,6 +26,10 @@
* (event, state) pair, we can use that stream_ordering to identify when * (event, state) pair, we can use that stream_ordering to identify when
* the new state was assigned for the event. * the new state was assigned for the event.
*/ */
/* NB: This table belongs to the `main` logical database; it should not be present
* in `state`.
*/
CREATE TABLE IF NOT EXISTS ex_outlier_stream( CREATE TABLE IF NOT EXISTS ex_outlier_stream(
event_stream_ordering BIGINT PRIMARY KEY NOT NULL, event_stream_ordering BIGINT PRIMARY KEY NOT NULL,
event_id TEXT NOT NULL, event_id TEXT NOT NULL,