From 01d8997a45d96bb111e312278080cb7648394bde Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 3 Jun 2024 14:16:35 +0100 Subject: [PATCH] WIP --- pyproject.toml | 101 +++++++++++++++++++------------------ synapse/logging/context.py | 83 ++++++++++++++++++++++++++++-- 2 files changed, 131 insertions(+), 53 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1485016a5a..4c3959584e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,38 +1,38 @@ [tool.towncrier] - package = "synapse" - filename = "CHANGES.md" - directory = "changelog.d" - issue_format = "[\\#{issue}](https://github.com/element-hq/synapse/issues/{issue})" +package = "synapse" +filename = "CHANGES.md" +directory = "changelog.d" +issue_format = "[\\#{issue}](https://github.com/element-hq/synapse/issues/{issue})" - [[tool.towncrier.type]] - directory = "feature" - name = "Features" - showcontent = true +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true - [[tool.towncrier.type]] - directory = "bugfix" - name = "Bugfixes" - showcontent = true +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bugfixes" +showcontent = true - [[tool.towncrier.type]] - directory = "docker" - name = "Updates to the Docker image" - showcontent = true +[[tool.towncrier.type]] +directory = "docker" +name = "Updates to the Docker image" +showcontent = true - [[tool.towncrier.type]] - directory = "doc" - name = "Improved Documentation" - showcontent = true +[[tool.towncrier.type]] +directory = "doc" +name = "Improved Documentation" +showcontent = true - [[tool.towncrier.type]] - directory = "removal" - name = "Deprecations and Removals" - showcontent = true +[[tool.towncrier.type]] +directory = "removal" +name = "Deprecations and Removals" +showcontent = true - [[tool.towncrier.type]] - directory = "misc" - name = "Internal Changes" - showcontent = true +[[tool.towncrier.type]] +directory = "misc" +name = "Internal Changes" +showcontent = true [tool.black] target-version = ['py38', 'py39', 'py310', 'py311'] @@ -51,11 +51,7 @@ line-length = 88 # flake8-bugbear compatible checks. Its error codes are described at # https://beta.ruff.rs/docs/rules/#flake8-bugbear-b # B023: Functions defined inside a loop must not use variables redefined in the loop -ignore = [ - "B023", - "E501", - "E731", -] +ignore = ["B023", "E501", "E731"] select = [ # pycodestyle "E", @@ -80,7 +76,15 @@ select = [ [tool.isort] line_length = 88 -sections = ["FUTURE", "STDLIB", "THIRDPARTY", "TWISTED", "FIRSTPARTY", "TESTS", "LOCALFOLDER"] +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "TWISTED", + "FIRSTPARTY", + "TESTS", + "LOCALFOLDER", +] default_section = "THIRDPARTY" known_first_party = ["synapse"] known_tests = ["tests"] @@ -102,9 +106,7 @@ authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" readme = "README.rst" repository = "https://github.com/element-hq/synapse" -packages = [ - { include = "synapse" }, -] +packages = [{ include = "synapse" }] classifiers = [ "Development Status :: 5 - Production/Stable", "Topic :: Communications :: Chat", @@ -120,7 +122,7 @@ include = [ { path = "INSTALL.md", format = "sdist" }, { path = "mypy.ini", format = "sdist" }, { path = "scripts-dev", format = "sdist" }, - { path = "synmark", format="sdist" }, + { path = "synmark", format = "sdist" }, { path = "sytest-blacklist", format = "sdist" }, { path = "tests", format = "sdist" }, { path = "UPGRADE.rst", format = "sdist" }, @@ -130,9 +132,7 @@ include = [ { path = "rust/build.rs", format = "sdist" }, { path = "rust/src/**", format = "sdist" }, ] -exclude = [ - { path = "synapse/*.so", format = "sdist"} -] +exclude = [{ path = "synapse/*.so", format = "sdist" }] [tool.poetry.build] script = "build_rust.py" @@ -171,9 +171,8 @@ canonicaljson = "^2.0.0" signedjson = "^1.1.0" # validating SSL certs for IP addresses requires service_identity 18.1. service-identity = ">=18.1.0" -# Twisted 18.9 introduces some logger improvements that the structured -# logger utilises -Twisted = {extras = ["tls"], version = ">=18.9.0"} +# Twisted 21.2 introduces contextvar support +Twisted = { extras = ["tls", "contextvars"], version = ">=21.2.0" } treq = ">=15.1" # Twisted has required pyopenssl 16.0 since about Twisted 16.6. pyOpenSSL = ">=16.0.0" @@ -290,7 +289,9 @@ all = [ # matrix-synapse-ldap3 "matrix-synapse-ldap3", # postgres - "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", + "psycopg2", + "psycopg2cffi", + "psycopg2cffi-compat", # saml2 "pysaml2", # oidc and jwt @@ -300,9 +301,11 @@ all = [ # sentry "sentry-sdk", # opentracing - "jaeger-client", "opentracing", + "jaeger-client", + "opentracing", # redis - "txredisapi", "hiredis", + "txredisapi", + "hiredis", # cache-memory "pympler", # improved user search @@ -386,8 +389,8 @@ build-backend = "poetry.core.masonry.api" skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64" # We need a rust compiler -before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal" -environment= { PATH = "$PATH:$HOME/.cargo/bin" } +before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal" +environment = { PATH = "$PATH:$HOME/.cargo/bin" } # For some reason if we don't manually clean the build directory we # can end up polluting the next build with a .so that is for the wrong diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 4650b60962..5e7cc4c3b7 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -33,9 +33,12 @@ import logging import threading import typing import warnings +from collections.abc import Coroutine, Generator +from contextvars import ContextVar from types import TracebackType from typing import ( TYPE_CHECKING, + Any, Awaitable, Callable, Optional, @@ -657,13 +660,14 @@ class PreserveLoggingContext: ) -_thread_local = threading.local() -_thread_local.current_context = SENTINEL_CONTEXT +_CURRENT_CONTEXT_VAR: ContextVar[LoggingContextOrSentinel] = ContextVar( + "current_context", default=SENTINEL_CONTEXT +) def current_context() -> LoggingContextOrSentinel: """Get the current logging context from thread local storage""" - return getattr(_thread_local, "current_context", SENTINEL_CONTEXT) + return _CURRENT_CONTEXT_VAR.get() def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSentinel: @@ -684,7 +688,7 @@ def set_current_context(context: LoggingContextOrSentinel) -> LoggingContextOrSe if current is not context: rusage = get_thread_resource_usage() current.stop(rusage) - _thread_local.current_context = context + _CURRENT_CONTEXT_VAR.set(context) context.start(rusage) return current @@ -971,3 +975,74 @@ def defer_to_threadpool( return f(*args, **kwargs) return make_deferred_yieldable(threads.deferToThreadPool(reactor, threadpool, g)) + + +_T = TypeVar("_T") + + +@attr.s(frozen=True, slots=True, auto_attribs=True) +class _ResourceTracker(Generator[defer.Deferred[Any], Any, _T]): + gen: Generator[defer.Deferred[Any], Any, _T] + + def send(self, val: Any) -> defer.Deferred[_T]: + try: + return self.gen.send(val) + finally: + pass + + @overload + def throw( + self, + a: Type[BaseException], + b: object = ..., + c: Optional[TracebackType] = ..., + /, + ) -> defer.Deferred[Any]: ... + + @overload + def throw( + self, a: BaseException, v: None = ..., c: Optional[TracebackType] = ..., / + ) -> defer.Deferred[Any]: ... + + def throw(self, a: Any, b: Any = None, c: Any = None) -> defer.Deferred[Any]: + try: + return self.throw(a, b, c) + finally: + pass + + +@attr.s(frozen=True, slots=True, auto_attribs=True) +class _ResourceTracker2(Coroutine[defer.Deferred[Any], Any, _T]): + gen: Coroutine[defer.Deferred[Any], Any, _T] + + def send(self, val: Any) -> defer.Deferred[_T]: + try: + return self.gen.send(val) + finally: + pass + + @overload + def throw( + self, + a: Type[BaseException], + b: object = ..., + c: Optional[TracebackType] = ..., + /, + ) -> defer.Deferred[Any]: ... + + @overload + def throw( + self, a: BaseException, v: None = ..., c: Optional[TracebackType] = ..., / + ) -> defer.Deferred[Any]: ... + + def throw(self, a: Any, b: Any = None, c: Any = None) -> defer.Deferred[Any]: + try: + return self.throw(a, b, c) + finally: + pass + + def __await__(self) -> Generator[defer.Deferred[Any], Any, _T]: + return _ResourceTracker(self.gen.__await__()) + + def close(self) -> None: + return self.gen.close()