Poetry: select olddeps using poetry (#12407)

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
This commit is contained in:
David Robertson 2022-04-12 11:50:11 +01:00 committed by GitHub
parent 3ad74b63e5
commit 4e13743738
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 70 additions and 29 deletions

View file

@ -1,6 +1,9 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# this script is run by GitHub Actions in a plain `focal` container; it installs the # this script is run by GitHub Actions in a plain `focal` container; it
# minimal requirements for tox and hands over to the py3-old tox environment. # - installs the minimal system requirements, and poetry;
# - patches the project definition file to refer to old versions only;
# - creates a venv with these old versions using poetry; and finally
# - invokes `trial` to run the tests with old deps.
# Prevent tzdata from asking for user input # Prevent tzdata from asking for user input
export DEBIAN_FRONTEND=noninteractive export DEBIAN_FRONTEND=noninteractive
@ -9,12 +12,70 @@ set -ex
apt-get update apt-get update
apt-get install -y \ apt-get install -y \
python3 python3-dev python3-pip python3-venv \ python3 python3-dev python3-pip python3-venv pipx \
libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
export LANG="C.UTF-8" export LANG="C.UTF-8"
# Prevent virtualenv from auto-updating pip to an incompatible version # Prevent virtualenv from auto-updating pip to an incompatible version
export VIRTUALENV_NO_DOWNLOAD=1 export VIRTUALENV_NO_DOWNLOAD=1
exec tox -e py3-old # TODO: in the future, we could use an implementation of
# https://github.com/python-poetry/poetry/issues/3527
# https://github.com/pypa/pip/issues/8085
# to select the lowest possible versions, rather than resorting to this sed script.
# Patch the project definitions in-place:
# - Replace all lower and tilde bounds with exact bounds
# - Make the pyopenssl 17.0, which is the oldest version that works with
# a `cryptography` compiled against OpenSSL 1.1.
# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
# - Omit systemd: we're not logging to journal here.
# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
# runs on 3.8 (#12343).
sed -i \
-e "s/[~>]=/==/g" \
-e "/psycopg2/d" \
-e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
-e '/systemd/d' \
pyproject.toml
# Use poetry to do the installation. This ensures that the versions are all mutually
# compatible (as far the package metadata declares, anyway); pip's package resolver
# is more lax.
#
# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
# toml file. This means we don't have to ensure compatibility between old deps and
# dev tools.
pip install --user toml
REMOVE_DEV_DEPENDENCIES="
import toml
with open('pyproject.toml', 'r') as f:
data = toml.loads(f.read())
del data['tool']['poetry']['dev-dependencies']
with open('pyproject.toml', 'w') as f:
toml.dump(data, f)
"
python3 -c "$REMOVE_DEV_DEPENDENCIES"
pipx install poetry==1.1.12
~/.local/bin/poetry lock
echo "::group::Patched pyproject.toml"
cat pyproject.toml
echo "::endgroup::"
echo "::group::Lockfile after patch"
cat poetry.lock
echo "::endgroup::"
~/.local/bin/poetry install -E "all test"
~/.local/bin/poetry run trial --jobs=2 tests

View file

@ -128,6 +128,7 @@ jobs:
|| true || true
trial-olddeps: trial-olddeps:
# Note: sqlite only; no postgres
if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
needs: linting-done needs: linting-done
runs-on: ubuntu-latest runs-on: ubuntu-latest
@ -135,11 +136,11 @@ jobs:
- uses: actions/checkout@v2 - uses: actions/checkout@v2
- name: Test with old deps - name: Test with old deps
uses: docker://ubuntu:focal # For old python and sqlite uses: docker://ubuntu:focal # For old python and sqlite
# Note: focal seems to be using 3.8, but the oldest is 3.7?
# See https://github.com/matrix-org/synapse/issues/12343
with: with:
workdir: /github/workspace workdir: /github/workspace
entrypoint: .ci/scripts/test_old_deps.sh entrypoint: .ci/scripts/test_old_deps.sh
env:
TRIAL_FLAGS: "--jobs=2"
- name: Dump logs - name: Dump logs
# Logs are most useful when the command fails, always include them. # Logs are most useful when the command fails, always include them.
if: ${{ always() }} if: ${{ always() }}

1
changelog.d/12407.misc Normal file
View file

@ -0,0 +1 @@
Run the olddeps CI job using Poetry.

22
tox.ini
View file

@ -105,29 +105,7 @@ commands =
# ) # )
usedevelop=true usedevelop=true
# A test suite for the oldest supported versions of Python libraries, to catch
# any uses of APIs not available in them.
[testenv:py3-old]
skip_install = true
usedevelop = false
deps =
Automat == 0.8.0
lxml
# markupsafe 2.1 introduced a change that breaks Jinja 2.x. Since we depend on
# Jinja >= 2.9, it means this test suite will fail if markupsafe >= 2.1 is installed.
markupsafe < 2.1
{[base]deps}
commands =
# Make all greater-thans equals so we test the oldest version of our direct
# dependencies, but make the pyopenssl 17.0, which can work against an
# OpenSSL 1.1 compiled cryptography (as older ones don't compile on Travis).
/bin/sh -c 'python -m synapse.python_dependencies | sed -e "s/>=/==/g" -e "/psycopg2/d" -e "s/pyopenssl==16.0.0/pyopenssl==17.0.0/" | xargs -d"\n" pip install'
# Install Synapse itself. This won't update any libraries.
pip install -e ".[test]"
{[testenv]commands}
[testenv:benchmark] [testenv:benchmark]
deps = deps =