Add a sytest blacklist file (#5611)

* Add a sytest blacklist file

* Add changelog

* Add blacklist to manifest
This commit is contained in:
Andrew Morgan 2019-07-04 16:24:13 +01:00 committed by Amber Brown
parent 822a0f0435
commit ee91ac179c
5 changed files with 43 additions and 148 deletions

View file

@ -173,11 +173,12 @@ steps:
queue: "medium" queue: "medium"
command: command:
- "bash .buildkite/merge_base_branch.sh" - "bash .buildkite/merge_base_branch.sh"
- "bash .buildkite/synapse_sytest.sh" - "bash /synapse_sytest.sh"
plugins: plugins:
- docker#v3.0.1: - docker#v3.0.1:
image: "matrixdotorg/sytest-synapse:py35" image: "matrixdotorg/sytest-synapse:py35"
propagate-environment: true propagate-environment: true
always-pull: true
retry: retry:
automatic: automatic:
- exit_status: -1 - exit_status: -1
@ -192,11 +193,12 @@ steps:
POSTGRES: "1" POSTGRES: "1"
command: command:
- "bash .buildkite/merge_base_branch.sh" - "bash .buildkite/merge_base_branch.sh"
- "bash .buildkite/synapse_sytest.sh" - "bash /synapse_sytest.sh"
plugins: plugins:
- docker#v3.0.1: - docker#v3.0.1:
image: "matrixdotorg/sytest-synapse:py35" image: "matrixdotorg/sytest-synapse:py35"
propagate-environment: true propagate-environment: true
always-pull: true
retry: retry:
automatic: automatic:
- exit_status: -1 - exit_status: -1
@ -212,11 +214,12 @@ steps:
WORKERS: "1" WORKERS: "1"
command: command:
- "bash .buildkite/merge_base_branch.sh" - "bash .buildkite/merge_base_branch.sh"
- "bash .buildkite/synapse_sytest.sh" - "bash /synapse_sytest.sh"
plugins: plugins:
- docker#v3.0.1: - docker#v3.0.1:
image: "matrixdotorg/sytest-synapse:py35" image: "matrixdotorg/sytest-synapse:py35"
propagate-environment: true propagate-environment: true
always-pull: true
soft_fail: true soft_fail: true
retry: retry:
automatic: automatic:

View file

@ -1,145 +0,0 @@
#!/bin/bash
#
# Fetch sytest, and then run the tests for synapse. The entrypoint for the
# sytest-synapse docker images.
set -ex
if [ -n "$BUILDKITE" ]
then
SYNAPSE_DIR=`pwd`
else
SYNAPSE_DIR="/src"
fi
# Attempt to find a sytest to use.
# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
if [ -d "/sytest" ]; then
# If the user has mounted in a SyTest checkout, use that.
echo "Using local sytests..."
# create ourselves a working directory and dos2unix some scripts therein
mkdir -p /work/jenkins
for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
dos2unix -n "/sytest/$i" "/work/$i"
done
ln -sf /sytest/tests /work
ln -sf /sytest/keys /work
SYTEST_LIB="/sytest/lib"
else
if [ -n "BUILDKITE_BRANCH" ]
then
branch_name=$BUILDKITE_BRANCH
else
# Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
fi
# Try and fetch the branch
echo "Trying to get same-named sytest branch..."
wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
# Probably a 404, fall back to develop
echo "Using develop instead..."
wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
}
mkdir -p /work
tar -C /work --strip-components=1 -xf sytest.tar.gz
SYTEST_LIB="/work/lib"
fi
cd /work
# PostgreSQL setup
if [ -n "$POSTGRES" ]
then
export PGUSER=postgres
export POSTGRES_DB_1=pg1
export POSTGRES_DB_2=pg2
# Start the database
su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
# Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
jenkins/prep_sytest_for_postgres.sh
# Make the test databases for the two Synapse servers that will be spun up
su -c 'psql -c "CREATE DATABASE pg1;"' postgres
su -c 'psql -c "CREATE DATABASE pg2;"' postgres
fi
if [ -n "$OFFLINE" ]; then
# if we're in offline mode, just put synapse into the virtualenv, and
# hope that the deps are up-to-date.
#
# (`pip install -e` likes to reinstall setuptools even if it's already installed,
# so we just run setup.py explicitly.)
#
(cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
else
# We've already created the virtualenv, but lets double check we have all
# deps.
/venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
/venv/bin/pip install -q --upgrade --no-cache-dir \
lxml psycopg2 coverage codecov tap.py
# Make sure all Perl deps are installed -- this is done in the docker build
# so will only install packages added since the last Docker build
./install-deps.pl
fi
# Run the tests
>&2 echo "+++ Running tests"
RUN_TESTS=(
perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
)
TEST_STATUS=0
if [ -n "$WORKERS" ]; then
RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
else
RUN_TESTS+=(-I Synapse)
fi
"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
if [ $TEST_STATUS -ne 0 ]; then
>&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
else
>&2 echo -e "run-tests \e[32mPASSED\e[0m"
fi
>&2 echo "--- Copying assets"
# Copy out the logs
mkdir -p /logs
cp results.tap /logs/results.tap
rsync --ignore-missing-args --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
# Upload coverage to codecov and upload files, if running on Buildkite
if [ -n "$BUILDKITE" ]
then
/venv/bin/coverage combine || true
/venv/bin/coverage xml || true
/venv/bin/codecov -X gcov -f coverage.xml
wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
tar xvf buildkite.tar.gz
chmod +x ./buildkite-agent
# Upload the files
./buildkite-agent artifact upload "/logs/**/*.log*"
./buildkite-agent artifact upload "/logs/results.tap"
if [ $TEST_STATUS -ne 0 ]; then
# Annotate, if failure
/venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
fi
fi
exit $TEST_STATUS

View file

@ -7,6 +7,7 @@ include demo/README
include demo/demo.tls.dh include demo/demo.tls.dh
include demo/*.py include demo/*.py
include demo/*.sh include demo/*.sh
include sytest-blacklist
recursive-include synapse/storage/schema *.sql recursive-include synapse/storage/schema *.sql
recursive-include synapse/storage/schema *.sql.postgres recursive-include synapse/storage/schema *.sql.postgres

1
changelog.d/5611.misc Normal file
View file

@ -0,0 +1 @@
Add a blacklist file to the repo to blacklist certain sytests from failing CI.

35
sytest-blacklist Normal file
View file

@ -0,0 +1,35 @@
# This file serves as a blacklist for SyTest tests that we expect will fail in
# Synapse.
#
# Each line of this file is scanned by sytest during a run and if the line
# exactly matches the name of a test, it will be marked as "expected fail",
# meaning the test will still run, but failure will not mark the entire test
# suite as failing.
#
# Test names are encouraged to have a bug accompanied with them, serving as an
# explanation for why the test has been excluded.
# Blacklisted due to https://github.com/matrix-org/synapse/issues/1679
Remote room members also see posted message events
# Blacklisted due to https://github.com/matrix-org/synapse/issues/2065
Guest users can accept invites to private rooms over federation
# Blacklisted due to https://github.com/vector-im/riot-web/issues/7211
The only membership state included in a gapped incremental sync is for senders in the timeline
# Blacklisted due to https://github.com/matrix-org/synapse/issues/1658
Newly created users see their own presence in /initialSync (SYT-34)
# Blacklisted due to https://github.com/matrix-org/synapse/issues/1396
Should reject keys claiming to belong to a different user
# Blacklisted due to https://github.com/matrix-org/synapse/issues/2306
Users appear/disappear from directory when join_rules are changed
Users appear/disappear from directory when history_visibility are changed
# Blacklisted due to https://github.com/matrix-org/synapse/issues/1531
Enabling an unknown default rule fails with 404
# Blacklisted due to https://github.com/matrix-org/synapse/issues/1663
New federated private chats get full presence information (SYN-115)