From 4db07f9aefbd2b93df8f8b5c5153183ea1539b32 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 3 Mar 2021 18:49:08 +0000 Subject: [PATCH 1/3] Set X-Forwarded-Proto header when frontend-proxy proxies a request (#9539) Should fix some remaining warnings --- changelog.d/9539.feature | 1 + synapse/app/generic_worker.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 changelog.d/9539.feature diff --git a/changelog.d/9539.feature b/changelog.d/9539.feature new file mode 100644 index 0000000000..06cfd5d199 --- /dev/null +++ b/changelog.d/9539.feature @@ -0,0 +1 @@ +Add support for `X-Forwarded-Proto` header when using a reverse proxy. diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index dc0d3eb725..274d582d07 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -23,6 +23,7 @@ from typing_extensions import ContextManager from twisted.internet import address from twisted.web.resource import IResource +from twisted.web.server import Request import synapse import synapse.events @@ -190,7 +191,7 @@ class KeyUploadServlet(RestServlet): self.http_client = hs.get_simple_http_client() self.main_uri = hs.config.worker_main_http_uri - async def on_POST(self, request, device_id): + async def on_POST(self, request: Request, device_id: Optional[str]): requester = await self.auth.get_user_by_req(request, allow_guest=True) user_id = requester.user.to_string() body = parse_json_object_from_request(request) @@ -223,10 +224,12 @@ class KeyUploadServlet(RestServlet): header: request.requestHeaders.getRawHeaders(header, []) for header in (b"Authorization", b"User-Agent") } - # Add the previous hop the the X-Forwarded-For header. + # Add the previous hop to the X-Forwarded-For header. x_forwarded_for = request.requestHeaders.getRawHeaders( b"X-Forwarded-For", [] ) + # we use request.client here, since we want the previous hop, not the + # original client (as returned by request.getClientAddress()). if isinstance(request.client, (address.IPv4Address, address.IPv6Address)): previous_host = request.client.host.encode("ascii") # If the header exists, add to the comma-separated list of the first @@ -239,6 +242,14 @@ class KeyUploadServlet(RestServlet): x_forwarded_for = [previous_host] headers[b"X-Forwarded-For"] = x_forwarded_for + # Replicate the original X-Forwarded-Proto header. Note that + # XForwardedForRequest overrides isSecure() to give us the original protocol + # used by the client, as opposed to the protocol used by our upstream proxy + # - which is what we want here. + headers[b"X-Forwarded-Proto"] = [ + b"https" if request.isSecure() else b"http" + ] + try: result = await self.http_client.post_json_get_json( self.main_uri + request.uri.decode("ascii"), body, headers=headers From 33a02f0f52a5cd793c0d123463d8a7a1e3f4f198 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 3 Mar 2021 15:47:38 -0500 Subject: [PATCH 2/3] Fix additional type hints from Twisted upgrade. (#9518) --- changelog.d/9518.misc | 1 + .../federation/matrix_federation_agent.py | 18 +++++----- synapse/http/matrixfederationclient.py | 6 ++-- synapse/http/server.py | 29 +++++++++------ synapse/http/site.py | 35 ++++++++++++------- synapse/logging/_remote.py | 6 ++-- synapse/metrics/__init__.py | 11 +++--- synapse/module_api/__init__.py | 4 +-- synapse/push/httppusher.py | 5 +-- synapse/replication/tcp/client.py | 4 +-- synapse/server.py | 3 +- tests/rest/client/v1/test_login.py | 35 +++++++++++++------ 12 files changed, 96 insertions(+), 61 deletions(-) create mode 100644 changelog.d/9518.misc diff --git a/changelog.d/9518.misc b/changelog.d/9518.misc new file mode 100644 index 0000000000..14c7b78dd9 --- /dev/null +++ b/changelog.d/9518.misc @@ -0,0 +1 @@ +Fix incorrect type hints. diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 2e83fa6773..b07aa59c08 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -14,7 +14,7 @@ # limitations under the License. import logging import urllib.parse -from typing import List, Optional +from typing import Any, Generator, List, Optional from netaddr import AddrFormatError, IPAddress, IPSet from zope.interface import implementer @@ -116,7 +116,7 @@ class MatrixFederationAgent: uri: bytes, headers: Optional[Headers] = None, bodyProducer: Optional[IBodyProducer] = None, - ) -> defer.Deferred: + ) -> Generator[defer.Deferred, Any, defer.Deferred]: """ Args: method: HTTP method: GET/POST/etc @@ -177,17 +177,17 @@ class MatrixFederationAgent: # We need to make sure the host header is set to the netloc of the # server and that a user-agent is provided. if headers is None: - headers = Headers() + request_headers = Headers() else: - headers = headers.copy() + request_headers = headers.copy() - if not headers.hasHeader(b"host"): - headers.addRawHeader(b"host", parsed_uri.netloc) - if not headers.hasHeader(b"user-agent"): - headers.addRawHeader(b"user-agent", self.user_agent) + if not request_headers.hasHeader(b"host"): + request_headers.addRawHeader(b"host", parsed_uri.netloc) + if not request_headers.hasHeader(b"user-agent"): + request_headers.addRawHeader(b"user-agent", self.user_agent) res = yield make_deferred_yieldable( - self._agent.request(method, uri, headers, bodyProducer) + self._agent.request(method, uri, request_headers, bodyProducer) ) return res diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index cde42e9f5e..0f107714ea 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -1049,14 +1049,14 @@ def check_content_type_is_json(headers: Headers) -> None: RequestSendFailed: if the Content-Type header is missing or isn't JSON """ - c_type = headers.getRawHeaders(b"Content-Type") - if c_type is None: + content_type_headers = headers.getRawHeaders(b"Content-Type") + if content_type_headers is None: raise RequestSendFailed( RuntimeError("No Content-Type header received from remote server"), can_retry=False, ) - c_type = c_type[0].decode("ascii") # only the first header + c_type = content_type_headers[0].decode("ascii") # only the first header val, options = cgi.parse_header(c_type) if val != "application/json": raise RequestSendFailed( diff --git a/synapse/http/server.py b/synapse/http/server.py index 845db9b78d..fa89260850 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -21,6 +21,7 @@ import logging import types import urllib from http import HTTPStatus +from inspect import isawaitable from io import BytesIO from typing import ( Any, @@ -30,6 +31,7 @@ from typing import ( Iterable, Iterator, List, + Optional, Pattern, Tuple, Union, @@ -79,10 +81,12 @@ def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: """Sends a JSON error response to clients.""" if f.check(SynapseError): - error_code = f.value.code - error_dict = f.value.error_dict() + # mypy doesn't understand that f.check asserts the type. + exc = f.value # type: SynapseError # type: ignore + error_code = exc.code + error_dict = exc.error_dict() - logger.info("%s SynapseError: %s - %s", request, error_code, f.value.msg) + logger.info("%s SynapseError: %s - %s", request, error_code, exc.msg) else: error_code = 500 error_dict = {"error": "Internal server error", "errcode": Codes.UNKNOWN} @@ -91,7 +95,7 @@ def return_json_error(f: failure.Failure, request: SynapseRequest) -> None: "Failed handle request via %r: %r", request.request_metrics.name, request, - exc_info=(f.type, f.value, f.getTracebackObject()), + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore ) # Only respond with an error response if we haven't already started writing, @@ -128,7 +132,8 @@ def return_html_error( `{msg}` placeholders), or a jinja2 template """ if f.check(CodeMessageException): - cme = f.value + # mypy doesn't understand that f.check asserts the type. + cme = f.value # type: CodeMessageException # type: ignore code = cme.code msg = cme.msg @@ -142,7 +147,7 @@ def return_html_error( logger.error( "Failed handle request %r", request, - exc_info=(f.type, f.value, f.getTracebackObject()), + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore ) else: code = HTTPStatus.INTERNAL_SERVER_ERROR @@ -151,7 +156,7 @@ def return_html_error( logger.error( "Failed handle request %r", request, - exc_info=(f.type, f.value, f.getTracebackObject()), + exc_info=(f.type, f.value, f.getTracebackObject()), # type: ignore ) if isinstance(error_template, str): @@ -278,7 +283,7 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): raw_callback_return = method_handler(request) # Is it synchronous? We'll allow this for now. - if isinstance(raw_callback_return, (defer.Deferred, types.CoroutineType)): + if isawaitable(raw_callback_return): callback_return = await raw_callback_return else: callback_return = raw_callback_return # type: ignore @@ -399,8 +404,10 @@ class JsonResource(DirectServeJsonResource): A tuple of the callback to use, the name of the servlet, and the key word arguments to pass to the callback """ + # At this point the path must be bytes. + request_path_bytes = request.path # type: bytes # type: ignore + request_path = request_path_bytes.decode("ascii") # Treat HEAD requests as GET requests. - request_path = request.path.decode("ascii") request_method = request.method if request_method == b"HEAD": request_method = b"GET" @@ -551,7 +558,7 @@ class _ByteProducer: request: Request, iterator: Iterator[bytes], ): - self._request = request + self._request = request # type: Optional[Request] self._iterator = iterator self._paused = False @@ -563,7 +570,7 @@ class _ByteProducer: """ Send a list of bytes as a chunk of a response. """ - if not data: + if not data or not self._request: return self._request.write(b"".join(data)) diff --git a/synapse/http/site.py b/synapse/http/site.py index 30153237e3..47754aff43 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -14,7 +14,7 @@ import contextlib import logging import time -from typing import Optional, Union +from typing import Optional, Type, Union import attr from zope.interface import implementer @@ -57,7 +57,7 @@ class SynapseRequest(Request): def __init__(self, channel, *args, **kw): Request.__init__(self, channel, *args, **kw) - self.site = channel.site + self.site = channel.site # type: SynapseSite self._channel = channel # this is used by the tests self.start_time = 0.0 @@ -96,25 +96,34 @@ class SynapseRequest(Request): def get_request_id(self): return "%s-%i" % (self.get_method(), self.request_seq) - def get_redacted_uri(self): - uri = self.uri + def get_redacted_uri(self) -> str: + """Gets the redacted URI associated with the request (or placeholder if the URI + has not yet been received). + + Note: This is necessary as the placeholder value in twisted is str + rather than bytes, so we need to sanitise `self.uri`. + + Returns: + The redacted URI as a string. + """ + uri = self.uri # type: Union[bytes, str] if isinstance(uri, bytes): - uri = self.uri.decode("ascii", errors="replace") + uri = uri.decode("ascii", errors="replace") return redact_uri(uri) - def get_method(self): - """Gets the method associated with the request (or placeholder if not - method has yet been received). + def get_method(self) -> str: + """Gets the method associated with the request (or placeholder if method + has not yet been received). Note: This is necessary as the placeholder value in twisted is str rather than bytes, so we need to sanitise `self.method`. Returns: - str + The request method as a string. """ - method = self.method + method = self.method # type: Union[bytes, str] if isinstance(method, bytes): - method = self.method.decode("ascii") + return self.method.decode("ascii") return method def render(self, resrc): @@ -432,7 +441,9 @@ class SynapseSite(Site): assert config.http_options is not None proxied = config.http_options.x_forwarded - self.requestFactory = XForwardedForRequest if proxied else SynapseRequest + self.requestFactory = ( + XForwardedForRequest if proxied else SynapseRequest + ) # type: Type[Request] self.access_logger = logging.getLogger(logger_name) self.server_version_string = server_version_string.encode("ascii") diff --git a/synapse/logging/_remote.py b/synapse/logging/_remote.py index f8e9112b56..174ca7be5a 100644 --- a/synapse/logging/_remote.py +++ b/synapse/logging/_remote.py @@ -32,7 +32,7 @@ from twisted.internet.endpoints import ( TCP4ClientEndpoint, TCP6ClientEndpoint, ) -from twisted.internet.interfaces import IPushProducer, ITransport +from twisted.internet.interfaces import IPushProducer, IStreamClientEndpoint, ITransport from twisted.internet.protocol import Factory, Protocol from twisted.python.failure import Failure @@ -121,7 +121,9 @@ class RemoteHandler(logging.Handler): try: ip = ip_address(self.host) if isinstance(ip, IPv4Address): - endpoint = TCP4ClientEndpoint(_reactor, self.host, self.port) + endpoint = TCP4ClientEndpoint( + _reactor, self.host, self.port + ) # type: IStreamClientEndpoint elif isinstance(ip, IPv6Address): endpoint = TCP6ClientEndpoint(_reactor, self.host, self.port) else: diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index a8cb49d5b4..3b499efc07 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -527,7 +527,7 @@ class ReactorLastSeenMetric: REGISTRY.register(ReactorLastSeenMetric()) -def runUntilCurrentTimer(func): +def runUntilCurrentTimer(reactor, func): @functools.wraps(func) def f(*args, **kwargs): now = reactor.seconds() @@ -590,13 +590,14 @@ def runUntilCurrentTimer(func): try: # Ensure the reactor has all the attributes we expect - reactor.runUntilCurrent - reactor._newTimedCalls - reactor.threadCallQueue + reactor.seconds # type: ignore + reactor.runUntilCurrent # type: ignore + reactor._newTimedCalls # type: ignore + reactor.threadCallQueue # type: ignore # runUntilCurrent is called when we have pending calls. It is called once # per iteratation after fd polling. - reactor.runUntilCurrent = runUntilCurrentTimer(reactor.runUntilCurrent) + reactor.runUntilCurrent = runUntilCurrentTimer(reactor, reactor.runUntilCurrent) # type: ignore # We manually run the GC each reactor tick so that we can get some metrics # about time spent doing GC, diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index 2e3b311c4a..db2d400b7e 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Any, Generator, Iterable, Optional, Tuple from twisted.internet import defer @@ -307,7 +307,7 @@ class ModuleApi: @defer.inlineCallbacks def get_state_events_in_room( self, room_id: str, types: Iterable[Tuple[str, Optional[str]]] - ) -> defer.Deferred: + ) -> Generator[defer.Deferred, Any, defer.Deferred]: """Gets current state events for the given room. (This is exposed for compatibility with the old SpamCheckerApi. We should diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index f4d7e199e9..eb6de8ba72 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -15,11 +15,12 @@ # limitations under the License. import logging import urllib.parse -from typing import TYPE_CHECKING, Any, Dict, Iterable, Union +from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Union from prometheus_client import Counter from twisted.internet.error import AlreadyCalled, AlreadyCancelled +from twisted.internet.interfaces import IDelayedCall from synapse.api.constants import EventTypes from synapse.events import EventBase @@ -71,7 +72,7 @@ class HttpPusher(Pusher): self.data = pusher_config.data self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC self.failing_since = pusher_config.failing_since - self.timed_call = None + self.timed_call = None # type: Optional[IDelayedCall] self._is_processing = False self._group_unread_count_by_room = hs.config.push_group_unread_count_by_room self._pusherpool = hs.get_pusherpool() diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 2618eb1e53..3455839d67 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -108,9 +108,7 @@ class ReplicationDataHandler: # Map from stream to list of deferreds waiting for the stream to # arrive at a particular position. The lists are sorted by stream position. - self._streams_to_waiters = ( - {} - ) # type: Dict[str, List[Tuple[int, Deferred[None]]]] + self._streams_to_waiters = {} # type: Dict[str, List[Tuple[int, Deferred]]] async def on_rdata( self, stream_name: str, instance_name: str, token: int, rows: list diff --git a/synapse/server.py b/synapse/server.py index 1d4370e0ba..afd7cd72e7 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -38,6 +38,7 @@ from typing import ( import twisted.internet.base import twisted.internet.tcp +from twisted.internet import defer from twisted.mail.smtp import sendmail from twisted.web.iweb import IPolicyForHTTPS @@ -403,7 +404,7 @@ class HomeServer(metaclass=abc.ABCMeta): return RoomShutdownHandler(self) @cache_in_self - def get_sendmail(self) -> sendmail: + def get_sendmail(self) -> Callable[..., defer.Deferred]: return sendmail @cache_in_self diff --git a/tests/rest/client/v1/test_login.py b/tests/rest/client/v1/test_login.py index 744d8d0941..20af3285bd 100644 --- a/tests/rest/client/v1/test_login.py +++ b/tests/rest/client/v1/test_login.py @@ -522,7 +522,9 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): shorthand=False, ) self.assertEqual(channel.code, 302, channel.result) - cas_uri = channel.headers.getRawHeaders("Location")[0] + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + cas_uri = location_headers[0] cas_uri_path, cas_uri_query = cas_uri.split("?", 1) # it should redirect us to the login page of the cas server @@ -545,7 +547,9 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): + "&idp=saml", ) self.assertEqual(channel.code, 302, channel.result) - saml_uri = channel.headers.getRawHeaders("Location")[0] + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + saml_uri = location_headers[0] saml_uri_path, saml_uri_query = saml_uri.split("?", 1) # it should redirect us to the login page of the SAML server @@ -567,17 +571,21 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): + "&idp=oidc", ) self.assertEqual(channel.code, 302, channel.result) - oidc_uri = channel.headers.getRawHeaders("Location")[0] + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + oidc_uri = location_headers[0] oidc_uri_path, oidc_uri_query = oidc_uri.split("?", 1) # it should redirect us to the auth page of the OIDC server self.assertEqual(oidc_uri_path, TEST_OIDC_AUTH_ENDPOINT) # ... and should have set a cookie including the redirect url - cookies = dict( - h.split(";")[0].split("=", maxsplit=1) - for h in channel.headers.getRawHeaders("Set-Cookie") - ) + cookie_headers = channel.headers.getRawHeaders("Set-Cookie") + assert cookie_headers + cookies = {} # type: Dict[str, str] + for h in cookie_headers: + key, value = h.split(";")[0].split("=", maxsplit=1) + cookies[key] = value oidc_session_cookie = cookies["oidc_session"] macaroon = pymacaroons.Macaroon.deserialize(oidc_session_cookie) @@ -590,9 +598,9 @@ class MultiSSOTestCase(unittest.HomeserverTestCase): # that should serve a confirmation page self.assertEqual(channel.code, 200, channel.result) - self.assertTrue( - channel.headers.getRawHeaders("Content-Type")[-1].startswith("text/html") - ) + content_type_headers = channel.headers.getRawHeaders("Content-Type") + assert content_type_headers + self.assertTrue(content_type_headers[-1].startswith("text/html")) p = TestHtmlParser() p.feed(channel.text_body) p.close() @@ -806,6 +814,7 @@ class CASTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, 302) location_headers = channel.headers.getRawHeaders("Location") + assert location_headers self.assertEqual(location_headers[0][: len(redirect_url)], redirect_url) @override_config({"sso": {"client_whitelist": ["https://legit-site.com/"]}}) @@ -1248,7 +1257,9 @@ class UsernamePickerTestCase(HomeserverTestCase): # that should redirect to the username picker self.assertEqual(channel.code, 302, channel.result) - picker_url = channel.headers.getRawHeaders("Location")[0] + location_headers = channel.headers.getRawHeaders("Location") + assert location_headers + picker_url = location_headers[0] self.assertEqual(picker_url, "/_synapse/client/pick_username/account_details") # ... with a username_mapping_session cookie @@ -1291,6 +1302,7 @@ class UsernamePickerTestCase(HomeserverTestCase): ) self.assertEqual(chan.code, 302, chan.result) location_headers = chan.headers.getRawHeaders("Location") + assert location_headers # send a request to the completion page, which should 302 to the client redirectUrl chan = self.make_request( @@ -1300,6 +1312,7 @@ class UsernamePickerTestCase(HomeserverTestCase): ) self.assertEqual(chan.code, 302, chan.result) location_headers = chan.headers.getRawHeaders("Location") + assert location_headers # ensure that the returned location matches the requested redirect URL path, query = location_headers[0].split("?", 1) From 3f49d80dcf3043f6c5a06ca99aff3acb1fe1da22 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Mar 2021 10:12:53 +0000 Subject: [PATCH 3/3] 1.29.0rc1 --- CHANGES.md | 54 ++++++++++++++++++++++++++++++++++++++++ changelog.d/8957.feature | 1 - changelog.d/8978.feature | 1 - changelog.d/9203.feature | 1 - changelog.d/9285.bugfix | 1 - changelog.d/9372.feature | 1 - changelog.d/9383.feature | 1 - changelog.d/9385.feature | 1 - changelog.d/9402.bugfix | 1 - changelog.d/9416.bugfix | 1 - changelog.d/9432.misc | 1 - changelog.d/9436.bugfix | 1 - changelog.d/9438.feature | 1 - changelog.d/9440.bugfix | 1 - changelog.d/9449.bugfix | 1 - changelog.d/9462.misc | 1 - changelog.d/9463.doc | 1 - changelog.d/9464.misc | 1 - changelog.d/9465.bugfix | 1 - changelog.d/9466.bugfix | 1 - changelog.d/9470.bugfix | 1 - changelog.d/9472.feature | 1 - changelog.d/9479.bugfix | 1 - changelog.d/9496.misc | 1 - changelog.d/9497.bugfix | 1 - changelog.d/9498.bugfix | 1 - changelog.d/9501.feature | 1 - changelog.d/9502.misc | 1 - changelog.d/9503.bugfix | 1 - changelog.d/9506.bugfix | 1 - changelog.d/9512.feature | 1 - changelog.d/9515.misc | 1 - changelog.d/9516.bugfix | 1 - changelog.d/9518.misc | 1 - changelog.d/9519.misc | 1 - changelog.d/9521.misc | 1 - changelog.d/9529.misc | 1 - changelog.d/9530.bugfix | 1 - changelog.d/9536.bugfix | 1 - changelog.d/9537.bugfix | 1 - changelog.d/9539.feature | 1 - synapse/__init__.py | 2 +- 42 files changed, 55 insertions(+), 41 deletions(-) delete mode 100644 changelog.d/8957.feature delete mode 100644 changelog.d/8978.feature delete mode 100644 changelog.d/9203.feature delete mode 100644 changelog.d/9285.bugfix delete mode 100644 changelog.d/9372.feature delete mode 100644 changelog.d/9383.feature delete mode 100644 changelog.d/9385.feature delete mode 100644 changelog.d/9402.bugfix delete mode 100644 changelog.d/9416.bugfix delete mode 100644 changelog.d/9432.misc delete mode 100644 changelog.d/9436.bugfix delete mode 100644 changelog.d/9438.feature delete mode 100644 changelog.d/9440.bugfix delete mode 100644 changelog.d/9449.bugfix delete mode 100644 changelog.d/9462.misc delete mode 100644 changelog.d/9463.doc delete mode 100644 changelog.d/9464.misc delete mode 100644 changelog.d/9465.bugfix delete mode 100644 changelog.d/9466.bugfix delete mode 100644 changelog.d/9470.bugfix delete mode 100644 changelog.d/9472.feature delete mode 100644 changelog.d/9479.bugfix delete mode 100644 changelog.d/9496.misc delete mode 100644 changelog.d/9497.bugfix delete mode 100644 changelog.d/9498.bugfix delete mode 100644 changelog.d/9501.feature delete mode 100644 changelog.d/9502.misc delete mode 100644 changelog.d/9503.bugfix delete mode 100644 changelog.d/9506.bugfix delete mode 100644 changelog.d/9512.feature delete mode 100644 changelog.d/9515.misc delete mode 100644 changelog.d/9516.bugfix delete mode 100644 changelog.d/9518.misc delete mode 100644 changelog.d/9519.misc delete mode 100644 changelog.d/9521.misc delete mode 100644 changelog.d/9529.misc delete mode 100644 changelog.d/9530.bugfix delete mode 100644 changelog.d/9536.bugfix delete mode 100644 changelog.d/9537.bugfix delete mode 100644 changelog.d/9539.feature diff --git a/CHANGES.md b/CHANGES.md index d9ecbac440..cd830e5178 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,57 @@ +Synapse 1.29.0rc1 (2021-03-04) +============================== + +Features +-------- + +- Add rate limiters to cross-user key sharing requests. ([\#8957](https://github.com/matrix-org/synapse/issues/8957)) +- Add `order_by` to the admin API `GET /_synapse/admin/v1/users//media`. Contributed by @dklimpel. ([\#8978](https://github.com/matrix-org/synapse/issues/8978)) +- Add some configuration settings to make users' profile data more private. ([\#9203](https://github.com/matrix-org/synapse/issues/9203)) +- The `no_proxy` and `NO_PROXY` environment variables are now respected in proxied HTTP clients with the lowercase form taking precedence if both are present. Additionally, the lowercase `https_proxy` environment variable is now respected in proxied HTTP clients on top of existing support for the uppercase `HTTPS_PROXY` form and takes precedence if both are present. Contributed by Timothy Leung. ([\#9372](https://github.com/matrix-org/synapse/issues/9372)) +- Add a configuration option, `user_directory.prefer_local_users`, which when enabled will make it more likely for users on the same server as you to appear above other users. ([\#9383](https://github.com/matrix-org/synapse/issues/9383), [\#9385](https://github.com/matrix-org/synapse/issues/9385)) +- Add support for regenerating thumbnails if they have been deleted but the original image is still stored. ([\#9438](https://github.com/matrix-org/synapse/issues/9438)) +- Add support for `X-Forwarded-Proto` header when using a reverse proxy. ([\#9472](https://github.com/matrix-org/synapse/issues/9472), [\#9501](https://github.com/matrix-org/synapse/issues/9501), [\#9512](https://github.com/matrix-org/synapse/issues/9512), [\#9539](https://github.com/matrix-org/synapse/issues/9539)) + + +Bugfixes +-------- + +- Fix a bug where users' pushers were not all deleted when they deactivated their account. ([\#9285](https://github.com/matrix-org/synapse/issues/9285), [\#9516](https://github.com/matrix-org/synapse/issues/9516)) +- Fix a bug where a lot of unnecessary presence updates were sent when joining a room. ([\#9402](https://github.com/matrix-org/synapse/issues/9402)) +- Fix a bug that caused multiple calls to the experimental `shared_rooms` endpoint to return stale results. ([\#9416](https://github.com/matrix-org/synapse/issues/9416)) +- Fix a bug in single sign-on which could cause a "No session cookie found" error. ([\#9436](https://github.com/matrix-org/synapse/issues/9436)) +- Fix bug introduced in v1.27.0 where allowing a user to choose their own username when logging in via single sign-on did not work unless an `idp_icon` was defined. ([\#9440](https://github.com/matrix-org/synapse/issues/9440)) +- Fix a bug introduced in v1.26.0 where some sequences were not properly configured when running `synapse_port_db`. ([\#9449](https://github.com/matrix-org/synapse/issues/9449)) +- Fix deleting pushers when using sharded pushers. ([\#9465](https://github.com/matrix-org/synapse/issues/9465), [\#9466](https://github.com/matrix-org/synapse/issues/9466), [\#9479](https://github.com/matrix-org/synapse/issues/9479), [\#9536](https://github.com/matrix-org/synapse/issues/9536)) +- Fix missing startup checks for the consistency of certain PostgreSQL sequences. ([\#9470](https://github.com/matrix-org/synapse/issues/9470)) +- Fix a long-standing bug where the media repository could leak file descriptors while previewing media. ([\#9497](https://github.com/matrix-org/synapse/issues/9497)) +- Properly purge the event chain cover index when purging history. ([\#9498](https://github.com/matrix-org/synapse/issues/9498)) +- Fix missing chain cover index due to a schema delta not being applied correctly. Only affected servers that ran development versions. ([\#9503](https://github.com/matrix-org/synapse/issues/9503)) +- Fix a bug introduced in v1.25.0 where `/_synapse/admin/join/` would fail when given a room alias. ([\#9506](https://github.com/matrix-org/synapse/issues/9506)) +- Prevent presence background jobs from running when presence is disabled. ([\#9530](https://github.com/matrix-org/synapse/issues/9530)) +- Fix rare edge case that caused a background update to fail if the server had rejected an event that had duplicate auth events. ([\#9537](https://github.com/matrix-org/synapse/issues/9537)) + + +Improved Documentation +---------------------- + +- Update the example systemd config to propagate reloads to individual units. ([\#9463](https://github.com/matrix-org/synapse/issues/9463)) + + +Internal Changes +---------------- + +- Add documentation and type hints to `parse_duration`. ([\#9432](https://github.com/matrix-org/synapse/issues/9432)) +- Remove vestiges of `uploads_path` configuration setting. ([\#9462](https://github.com/matrix-org/synapse/issues/9462)) +- Add a comment about systemd-python. ([\#9464](https://github.com/matrix-org/synapse/issues/9464)) +- Test that we require validated email for email pushers. ([\#9496](https://github.com/matrix-org/synapse/issues/9496)) +- Allow python to generate bytecode for synapse. ([\#9502](https://github.com/matrix-org/synapse/issues/9502)) +- Fix incorrect type hints. ([\#9515](https://github.com/matrix-org/synapse/issues/9515), [\#9518](https://github.com/matrix-org/synapse/issues/9518)) +- Add type hints to device and event report admin API. ([\#9519](https://github.com/matrix-org/synapse/issues/9519)) +- Add type hints to user admin API. ([\#9521](https://github.com/matrix-org/synapse/issues/9521)) +- Bump the versions of mypy and mypy-zope used for static type checking. ([\#9529](https://github.com/matrix-org/synapse/issues/9529)) + + Synapse 1.xx.0 ============== diff --git a/changelog.d/8957.feature b/changelog.d/8957.feature deleted file mode 100644 index fa8961f840..0000000000 --- a/changelog.d/8957.feature +++ /dev/null @@ -1 +0,0 @@ -Add rate limiters to cross-user key sharing requests. diff --git a/changelog.d/8978.feature b/changelog.d/8978.feature deleted file mode 100644 index 042e257bf0..0000000000 --- a/changelog.d/8978.feature +++ /dev/null @@ -1 +0,0 @@ -Add `order_by` to the admin API `GET /_synapse/admin/v1/users//media`. Contributed by @dklimpel. \ No newline at end of file diff --git a/changelog.d/9203.feature b/changelog.d/9203.feature deleted file mode 100644 index 36b66a47a8..0000000000 --- a/changelog.d/9203.feature +++ /dev/null @@ -1 +0,0 @@ -Add some configuration settings to make users' profile data more private. diff --git a/changelog.d/9285.bugfix b/changelog.d/9285.bugfix deleted file mode 100644 index 81188c5473..0000000000 --- a/changelog.d/9285.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where users' pushers were not all deleted when they deactivated their account. diff --git a/changelog.d/9372.feature b/changelog.d/9372.feature deleted file mode 100644 index 3cb01004c9..0000000000 --- a/changelog.d/9372.feature +++ /dev/null @@ -1 +0,0 @@ -The `no_proxy` and `NO_PROXY` environment variables are now respected in proxied HTTP clients with the lowercase form taking precedence if both are present. Additionally, the lowercase `https_proxy` environment variable is now respected in proxied HTTP clients on top of existing support for the uppercase `HTTPS_PROXY` form and takes precedence if both are present. Contributed by Timothy Leung. diff --git a/changelog.d/9383.feature b/changelog.d/9383.feature deleted file mode 100644 index 8957c9cc5e..0000000000 --- a/changelog.d/9383.feature +++ /dev/null @@ -1 +0,0 @@ -Add a configuration option, `user_directory.prefer_local_users`, which when enabled will make it more likely for users on the same server as you to appear above other users. \ No newline at end of file diff --git a/changelog.d/9385.feature b/changelog.d/9385.feature deleted file mode 100644 index cbe3922de8..0000000000 --- a/changelog.d/9385.feature +++ /dev/null @@ -1 +0,0 @@ - Add a configuration option, `user_directory.prefer_local_users`, which when enabled will make it more likely for users on the same server as you to appear above other users. \ No newline at end of file diff --git a/changelog.d/9402.bugfix b/changelog.d/9402.bugfix deleted file mode 100644 index 7729225ba2..0000000000 --- a/changelog.d/9402.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where a lot of unnecessary presence updates were sent when joining a room. diff --git a/changelog.d/9416.bugfix b/changelog.d/9416.bugfix deleted file mode 100644 index 4d79cb2228..0000000000 --- a/changelog.d/9416.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug that caused multiple calls to the experimental `shared_rooms` endpoint to return stale results. \ No newline at end of file diff --git a/changelog.d/9432.misc b/changelog.d/9432.misc deleted file mode 100644 index 1e07da2033..0000000000 --- a/changelog.d/9432.misc +++ /dev/null @@ -1 +0,0 @@ -Add documentation and type hints to `parse_duration`. diff --git a/changelog.d/9436.bugfix b/changelog.d/9436.bugfix deleted file mode 100644 index a530516eed..0000000000 --- a/changelog.d/9436.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug in single sign-on which could cause a "No session cookie found" error. diff --git a/changelog.d/9438.feature b/changelog.d/9438.feature deleted file mode 100644 index a1f6be5563..0000000000 --- a/changelog.d/9438.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for regenerating thumbnails if they have been deleted but the original image is still stored. diff --git a/changelog.d/9440.bugfix b/changelog.d/9440.bugfix deleted file mode 100644 index 47b9842b37..0000000000 --- a/changelog.d/9440.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug introduced in v1.27.0 where allowing a user to choose their own username when logging in via single sign-on did not work unless an `idp_icon` was defined. diff --git a/changelog.d/9449.bugfix b/changelog.d/9449.bugfix deleted file mode 100644 index 54214a7e4a..0000000000 --- a/changelog.d/9449.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in v1.26.0 where some sequences were not properly configured when running `synapse_port_db`. diff --git a/changelog.d/9462.misc b/changelog.d/9462.misc deleted file mode 100644 index 1b245bf85d..0000000000 --- a/changelog.d/9462.misc +++ /dev/null @@ -1 +0,0 @@ -Remove vestiges of `uploads_path` configuration setting. diff --git a/changelog.d/9463.doc b/changelog.d/9463.doc deleted file mode 100644 index c9cedd147d..0000000000 --- a/changelog.d/9463.doc +++ /dev/null @@ -1 +0,0 @@ -Update the example systemd config to propagate reloads to individual units. diff --git a/changelog.d/9464.misc b/changelog.d/9464.misc deleted file mode 100644 index 39fcf85d40..0000000000 --- a/changelog.d/9464.misc +++ /dev/null @@ -1 +0,0 @@ -Add a comment about systemd-python. diff --git a/changelog.d/9465.bugfix b/changelog.d/9465.bugfix deleted file mode 100644 index 2ab4f315c1..0000000000 --- a/changelog.d/9465.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix deleting pushers when using sharded pushers. diff --git a/changelog.d/9466.bugfix b/changelog.d/9466.bugfix deleted file mode 100644 index 2ab4f315c1..0000000000 --- a/changelog.d/9466.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix deleting pushers when using sharded pushers. diff --git a/changelog.d/9470.bugfix b/changelog.d/9470.bugfix deleted file mode 100644 index c1b7dbb17d..0000000000 --- a/changelog.d/9470.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix missing startup checks for the consistency of certain PostgreSQL sequences. diff --git a/changelog.d/9472.feature b/changelog.d/9472.feature deleted file mode 100644 index 06cfd5d199..0000000000 --- a/changelog.d/9472.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for `X-Forwarded-Proto` header when using a reverse proxy. diff --git a/changelog.d/9479.bugfix b/changelog.d/9479.bugfix deleted file mode 100644 index 2ab4f315c1..0000000000 --- a/changelog.d/9479.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix deleting pushers when using sharded pushers. diff --git a/changelog.d/9496.misc b/changelog.d/9496.misc deleted file mode 100644 index d5866c56f7..0000000000 --- a/changelog.d/9496.misc +++ /dev/null @@ -1 +0,0 @@ -Test that we require validated email for email pushers. diff --git a/changelog.d/9497.bugfix b/changelog.d/9497.bugfix deleted file mode 100644 index 598bcaab67..0000000000 --- a/changelog.d/9497.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where the media repository could leak file descriptors while previewing media. diff --git a/changelog.d/9498.bugfix b/changelog.d/9498.bugfix deleted file mode 100644 index dce0ad0920..0000000000 --- a/changelog.d/9498.bugfix +++ /dev/null @@ -1 +0,0 @@ -Properly purge the event chain cover index when purging history. diff --git a/changelog.d/9501.feature b/changelog.d/9501.feature deleted file mode 100644 index 06cfd5d199..0000000000 --- a/changelog.d/9501.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for `X-Forwarded-Proto` header when using a reverse proxy. diff --git a/changelog.d/9502.misc b/changelog.d/9502.misc deleted file mode 100644 index c5c29672b6..0000000000 --- a/changelog.d/9502.misc +++ /dev/null @@ -1 +0,0 @@ -Allow python to generate bytecode for synapse. \ No newline at end of file diff --git a/changelog.d/9503.bugfix b/changelog.d/9503.bugfix deleted file mode 100644 index 0868691389..0000000000 --- a/changelog.d/9503.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix missing chain cover index due to a schema delta not being applied correctly. Only affected servers that ran development versions. diff --git a/changelog.d/9506.bugfix b/changelog.d/9506.bugfix deleted file mode 100644 index cc0d410e0f..0000000000 --- a/changelog.d/9506.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in v1.25.0 where `/_synapse/admin/join/` would fail when given a room alias. diff --git a/changelog.d/9512.feature b/changelog.d/9512.feature deleted file mode 100644 index 06cfd5d199..0000000000 --- a/changelog.d/9512.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for `X-Forwarded-Proto` header when using a reverse proxy. diff --git a/changelog.d/9515.misc b/changelog.d/9515.misc deleted file mode 100644 index 14c7b78dd9..0000000000 --- a/changelog.d/9515.misc +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect type hints. diff --git a/changelog.d/9516.bugfix b/changelog.d/9516.bugfix deleted file mode 100644 index 81188c5473..0000000000 --- a/changelog.d/9516.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug where users' pushers were not all deleted when they deactivated their account. diff --git a/changelog.d/9518.misc b/changelog.d/9518.misc deleted file mode 100644 index 14c7b78dd9..0000000000 --- a/changelog.d/9518.misc +++ /dev/null @@ -1 +0,0 @@ -Fix incorrect type hints. diff --git a/changelog.d/9519.misc b/changelog.d/9519.misc deleted file mode 100644 index caccc88a19..0000000000 --- a/changelog.d/9519.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to device and event report admin API. \ No newline at end of file diff --git a/changelog.d/9521.misc b/changelog.d/9521.misc deleted file mode 100644 index 1424d9c188..0000000000 --- a/changelog.d/9521.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to user admin API. \ No newline at end of file diff --git a/changelog.d/9529.misc b/changelog.d/9529.misc deleted file mode 100644 index b9021a26b4..0000000000 --- a/changelog.d/9529.misc +++ /dev/null @@ -1 +0,0 @@ -Bump the versions of mypy and mypy-zope used for static type checking. diff --git a/changelog.d/9530.bugfix b/changelog.d/9530.bugfix deleted file mode 100644 index bb4db675d9..0000000000 --- a/changelog.d/9530.bugfix +++ /dev/null @@ -1 +0,0 @@ -Prevent presence background jobs from running when presence is disabled. \ No newline at end of file diff --git a/changelog.d/9536.bugfix b/changelog.d/9536.bugfix deleted file mode 100644 index 2ab4f315c1..0000000000 --- a/changelog.d/9536.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix deleting pushers when using sharded pushers. diff --git a/changelog.d/9537.bugfix b/changelog.d/9537.bugfix deleted file mode 100644 index 033ab1c939..0000000000 --- a/changelog.d/9537.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix rare edge case that caused a background update to fail if the server had rejected an event that had duplicate auth events. diff --git a/changelog.d/9539.feature b/changelog.d/9539.feature deleted file mode 100644 index 06cfd5d199..0000000000 --- a/changelog.d/9539.feature +++ /dev/null @@ -1 +0,0 @@ -Add support for `X-Forwarded-Proto` header when using a reverse proxy. diff --git a/synapse/__init__.py b/synapse/__init__.py index 869e860fb0..2c24d4ae03 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -48,7 +48,7 @@ try: except ImportError: pass -__version__ = "1.28.0" +__version__ = "1.29.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when