matrix-synapse 1.139.2__cp39-abi3-manylinux_2_28_aarch64.whl → 1.140.0rc1__cp39-abi3-manylinux_2_28_aarch64.whl

synapse/_scripts/generate_workers_map.py

@@ -157,7 +157,12 @@ def get_registered_paths_for_default(
     # TODO We only do this to avoid an error, but don't need the database etc
     hs.setup()
     registered_paths = get_registered_paths_for_hs(hs)
-    hs.cleanup()
+    # NOTE: a more robust implementation would properly shutdown/cleanup each server
+    # to avoid resource buildup.
+    # However, the call to `shutdown` is `async` so it would require additional complexity here.
+    # We are intentionally skipping this cleanup because this is a short-lived, one-off
+    # utility script where the simpler approach is sufficient and we shouldn't run into
+    # any resource buildup issues.
 
     return registered_paths
 

synapse/_scripts/synapse_port_db.py

@@ -98,7 +98,6 @@ from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStor
 from synapse.storage.engines import create_engine
 from synapse.storage.prepare_database import prepare_database
 from synapse.types import ISynapseReactor
-from synapse.util import SYNAPSE_VERSION
 
 # Cast safety: Twisted does some naughty magic which replaces the
 # twisted.internet.reactor module with a Reactor instance at runtime.
@@ -325,7 +324,6 @@ class MockHomeserver(HomeServer):
             hostname=config.server.server_name,
             config=config,
             reactor=reactor,
-            version_string=f"Synapse/{SYNAPSE_VERSION}",
         )
 
 

synapse/_scripts/update_synapse_database.py

@@ -28,11 +28,9 @@ import yaml
 from twisted.internet import defer, reactor as reactor_
 
 from synapse.config.homeserver import HomeServerConfig
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.server import HomeServer
 from synapse.storage import DataStore
 from synapse.types import ISynapseReactor
-from synapse.util import SYNAPSE_VERSION
 
 # Cast safety: Twisted does some naughty magic which replaces the
 # twisted.internet.reactor module with a Reactor instance at runtime.
@@ -48,12 +46,10 @@ class MockHomeserver(HomeServer):
             hostname=config.server.server_name,
             config=config,
             reactor=reactor,
-            version_string=f"Synapse/{SYNAPSE_VERSION}",
         )
 
 
 def run_background_updates(hs: HomeServer) -> None:
-    server_name = hs.hostname
     main = hs.get_datastores().main
     state = hs.get_datastores().state
 
@@ -67,9 +63,8 @@ def run_background_updates(hs: HomeServer) -> None:
     def run() -> None:
         # Apply all background updates on the database.
         defer.ensureDeferred(
-            run_as_background_process(
+            hs.run_as_background_process(
                 "background_updates",
-                server_name,
                 run_background_updates,
             )
         )

synapse/api/auth/base.py

@@ -354,12 +354,10 @@ class BaseAuth:
                 effective_user_id, effective_device_id
             )
             if device_opt is None:
-                # For now, use 400 M_EXCLUSIVE if the device doesn't exist.
-                # This is an open thread of discussion on MSC3202 as of 2021-12-09.
                 raise AuthError(
                     400,
                     f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
-                    Codes.EXCLUSIVE,
+                    Codes.UNKNOWN_DEVICE,
                 )
 
         return create_requester(

synapse/api/auth/mas.py

@@ -33,7 +33,6 @@ from synapse.api.errors import (
     UnrecognizedRequestError,
 )
 from synapse.http.site import SynapseRequest
-from synapse.logging.context import PreserveLoggingContext
 from synapse.logging.opentracing import (
     active_span,
     force_tracing,
@@ -229,13 +228,12 @@ class MasDelegatedAuth(BaseAuth):
         try:
             with start_active_span("mas-introspect-token"):
                 inject_request_headers(raw_headers)
-                with PreserveLoggingContext():
-                    resp_body = await self._rust_http_client.post(
-                        url=self._introspection_endpoint,
-                        response_limit=1 * 1024 * 1024,
-                        headers=raw_headers,
-                        request_body=body,
-                    )
+                resp_body = await self._rust_http_client.post(
+                    url=self._introspection_endpoint,
+                    response_limit=1 * 1024 * 1024,
+                    headers=raw_headers,
+                    request_body=body,
+                )
         except HttpResponseException as e:
             end_time = self._clock.time()
             introspection_response_timer.labels(

synapse/api/auth/msc3861_delegated.py

@@ -38,7 +38,6 @@ from synapse.api.errors import (
     UnrecognizedRequestError,
 )
 from synapse.http.site import SynapseRequest
-from synapse.logging.context import PreserveLoggingContext
 from synapse.logging.opentracing import (
     active_span,
     force_tracing,
@@ -327,13 +326,12 @@ class MSC3861DelegatedAuth(BaseAuth):
         try:
             with start_active_span("mas-introspect-token"):
                 inject_request_headers(raw_headers)
-                with PreserveLoggingContext():
-                    resp_body = await self._rust_http_client.post(
-                        url=uri,
-                        response_limit=1 * 1024 * 1024,
-                        headers=raw_headers,
-                        request_body=body,
-                    )
+                resp_body = await self._rust_http_client.post(
+                    url=uri,
+                    response_limit=1 * 1024 * 1024,
+                    headers=raw_headers,
+                    request_body=body,
+                )
         except HttpResponseException as e:
             end_time = self._clock.time()
             introspection_response_timer.labels(

synapse/api/errors.py

@@ -149,6 +149,9 @@ class Codes(str, Enum):
     )
     MSC4306_NOT_IN_THREAD = "IO.ELEMENT.MSC4306.M_NOT_IN_THREAD"
 
+    # Part of MSC4326
+    UNKNOWN_DEVICE = "ORG.MATRIX.MSC4326.M_UNKNOWN_DEVICE"
+
 
 class CodeMessageException(RuntimeError):
     """An exception with integer code, a message string attributes and optional headers.

synapse/app/_base.py

@@ -28,6 +28,7 @@ import sys
 import traceback
 import warnings
 from textwrap import indent
+from threading import Thread
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -40,6 +41,7 @@ from typing import (
     Tuple,
     cast,
 )
+from wsgiref.simple_server import WSGIServer
 
 from cryptography.utils import CryptographyDeprecationWarning
 from typing_extensions import ParamSpec
@@ -73,7 +75,6 @@ from synapse.events.presence_router import load_legacy_presence_router
 from synapse.handlers.auth import load_legacy_password_auth_providers
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, PreserveLoggingContext
-from synapse.logging.opentracing import init_tracer
 from synapse.metrics import install_gc_manager, register_threadpool
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.metrics.jemalloc import setup_jemalloc_stats
@@ -98,22 +99,47 @@ reactor = cast(ISynapseReactor, _reactor)
 
 logger = logging.getLogger(__name__)
 
-# list of tuples of function, args list, kwargs dict
-_sighup_callbacks: List[
-    Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]
-] = []
+_instance_id_to_sighup_callbacks_map: Dict[
+    str, List[Tuple[Callable[..., None], Tuple[object, ...], Dict[str, object]]]
+] = {}
+"""
+Map from homeserver instance_id to a list of callbacks.
+
+We use `instance_id` instead of `server_name` because it's possible to have multiple
+workers running in the same process with the same `server_name`.
+"""
 P = ParamSpec("P")
 
 
-def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs) -> None:
+def register_sighup(
+    homeserver_instance_id: str,
+    func: Callable[P, None],
+    *args: P.args,
+    **kwargs: P.kwargs,
+) -> None:
     """
     Register a function to be called when a SIGHUP occurs.
 
     Args:
+        homeserver_instance_id: The unique ID for this Synapse process instance
+            (`hs.get_instance_id()`) that this hook is associated with.
         func: Function to be called when sent a SIGHUP signal.
         *args, **kwargs: args and kwargs to be passed to the target function.
     """
-    _sighup_callbacks.append((func, args, kwargs))
+
+    _instance_id_to_sighup_callbacks_map.setdefault(homeserver_instance_id, []).append(
+        (func, args, kwargs)
+    )
+
+
+def unregister_sighups(instance_id: str) -> None:
+    """
+    Unregister all sighup functions associated with this Synapse instance.
+
+    Args:
+        instance_id: Unique ID for this Synapse process instance.
+    """
+    _instance_id_to_sighup_callbacks_map.pop(instance_id, [])
 
 
 def start_worker_reactor(
@@ -282,7 +308,9 @@ def register_start(
     clock.call_when_running(lambda: defer.ensureDeferred(wrapper()))
 
 
-def listen_metrics(bind_addresses: StrCollection, port: int) -> None:
+def listen_metrics(
+    bind_addresses: StrCollection, port: int
+) -> List[Tuple[WSGIServer, Thread]]:
     """
     Start Prometheus metrics server.
 
@@ -295,14 +323,22 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None:
     bytecode at a time), this still works because the metrics thread can preempt the
     Twisted reactor thread between bytecode boundaries and the metrics thread gets
     scheduled with roughly equal priority to the Twisted reactor thread.
+
+    Returns:
+        List of WSGIServer with the thread they are running on.
     """
     from prometheus_client import start_http_server as start_http_server_prometheus
 
     from synapse.metrics import RegistryProxy
 
+    servers: List[Tuple[WSGIServer, Thread]] = []
     for host in bind_addresses:
         logger.info("Starting metrics listener on %s:%d", host, port)
-        start_http_server_prometheus(port, addr=host, registry=RegistryProxy)
+        server, thread = start_http_server_prometheus(
+            port, addr=host, registry=RegistryProxy
+        )
+        servers.append((server, thread))
+    return servers
 
 
 def listen_manhole(
@@ -310,7 +346,7 @@ def listen_manhole(
     port: int,
     manhole_settings: ManholeConfig,
     manhole_globals: dict,
-) -> None:
+) -> List[Port]:
     # twisted.conch.manhole 21.1.0 uses "int_from_bytes", which produces a confusing
     # warning. It's fixed by https://github.com/twisted/twisted/pull/1522), so
     # suppress the warning for now.
@@ -322,7 +358,7 @@ def listen_manhole(
 
     from synapse.util.manhole import manhole
 
-    listen_tcp(
+    return listen_tcp(
         bind_addresses,
         port,
         manhole(settings=manhole_settings, globals=manhole_globals),
@@ -385,17 +421,26 @@ def listen_http(
     context_factory: Optional[IOpenSSLContextFactory],
     reactor: ISynapseReactor = reactor,
 ) -> List[Port]:
+    """
+    Args:
+        listener_config: TODO
+        root_resource: TODO
+        version_string: A string to present for the Server header
+        max_request_body_size: TODO
+        context_factory: TODO
+        reactor: TODO
+    """
     assert listener_config.http_options is not None
 
     site_tag = listener_config.get_site_tag()
 
     site = SynapseSite(
-        "synapse.access.%s.%s"
+        logger_name="synapse.access.%s.%s"
         % ("https" if listener_config.is_tls() else "http", site_tag),
-        site_tag,
-        listener_config,
-        root_resource,
-        version_string,
+        site_tag=site_tag,
+        config=listener_config,
+        resource=root_resource,
+        server_version_string=version_string,
         max_request_body_size=max_request_body_size,
         reactor=reactor,
         hs=hs,
@@ -499,7 +544,7 @@ def refresh_certificate(hs: "HomeServer") -> None:
         logger.info("Context factories updated.")
 
 
-async def start(hs: "HomeServer") -> None:
+async def start(hs: "HomeServer", freeze: bool = True) -> None:
     """
     Start a Synapse server or worker.
 
@@ -510,6 +555,11 @@ async def start(hs: "HomeServer") -> None:
 
     Args:
         hs: homeserver instance
+        freeze: whether to freeze the homeserver base objects in the garbage collector.
+            May improve garbage collection performance by marking objects with an effectively
+            static lifetime as frozen so they don't need to be considered for cleanup.
+            If you ever want to `shutdown` the homeserver, this needs to be
+            False otherwise the homeserver cannot be garbage collected after `shutdown`.
     """
     server_name = hs.hostname
     reactor = hs.get_reactor()
@@ -542,12 +592,17 @@ async def start(hs: "HomeServer") -> None:
                 # we're not using systemd.
                 sdnotify(b"RELOADING=1")
 
-                for i, args, kwargs in _sighup_callbacks:
-                    i(*args, **kwargs)
+            for sighup_callbacks in _instance_id_to_sighup_callbacks_map.values():
+                for func, args, kwargs in sighup_callbacks:
+                    func(*args, **kwargs)
 
                 sdnotify(b"READY=1")
 
-            return run_as_background_process(
+            # It's okay to ignore the linter error here and call
+            # `run_as_background_process` directly because `_handle_sighup` operates
+            # outside of the scope of a specific `HomeServer` instance and holds no
+            # references to it which would prevent a clean shutdown.
+            return run_as_background_process(  # type: ignore[untracked-background-process]
                 "sighup",
                 server_name,
                 _handle_sighup,
@@ -565,8 +620,8 @@ async def start(hs: "HomeServer") -> None:
 
         signal.signal(signal.SIGHUP, run_sighup)
 
-        register_sighup(refresh_certificate, hs)
-        register_sighup(reload_cache_config, hs.config)
+        register_sighup(hs.get_instance_id(), refresh_certificate, hs)
+        register_sighup(hs.get_instance_id(), reload_cache_config, hs.config)
 
     # Apply the cache config.
     hs.config.caches.resize_all_caches()
@@ -574,9 +629,6 @@ async def start(hs: "HomeServer") -> None:
     # Load the certificate from disk.
     refresh_certificate(hs)
 
-    # Start the tracer
-    init_tracer(hs)  # noqa
-
     # Instantiate the modules so they can register their web resources to the module API
     # before we start the listeners.
     module_api = hs.get_module_api()
@@ -603,11 +655,15 @@ async def start(hs: "HomeServer") -> None:
     hs.get_pusherpool().start()
 
     def log_shutdown() -> None:
-        with LoggingContext("log_shutdown"):
+        with LoggingContext(name="log_shutdown", server_name=server_name):
             logger.info("Shutting down...")
 
     # Log when we start the shut down process.
-    hs.get_clock().add_system_event_trigger("before", "shutdown", log_shutdown)
+    hs.register_sync_shutdown_handler(
+        phase="before",
+        eventType="shutdown",
+        shutdown_func=log_shutdown,
+    )
 
     setup_sentry(hs)
     setup_sdnotify(hs)
@@ -636,18 +692,24 @@ async def start(hs: "HomeServer") -> None:
         # `REQUIRED_ON_BACKGROUND_TASK_STARTUP`
         start_phone_stats_home(hs)
 
-    # We now freeze all allocated objects in the hopes that (almost)
-    # everything currently allocated are things that will be used for the
-    # rest of time. Doing so means less work each GC (hopefully).
-    #
-    # PyPy does not (yet?) implement gc.freeze()
-    if hasattr(gc, "freeze"):
-        gc.collect()
-        gc.freeze()
-
-        # Speed up shutdowns by freezing all allocated objects. This moves everything
-        # into the permanent generation and excludes them from the final GC.
-        atexit.register(gc.freeze)
+    if freeze:
+        # We now freeze all allocated objects in the hopes that (almost)
+        # everything currently allocated are things that will be used for the
+        # rest of time. Doing so means less work each GC (hopefully).
+        #
+        # Note that freezing the homeserver object means that it won't be able to be
+        # garbage collected in the case of attempting an in-memory `shutdown`. This only
+        # needs to be considered if such a case is desirable. Exiting the entire Python
+        # process will function expectedly either way.
+        #
+        # PyPy does not (yet?) implement gc.freeze()
+        if hasattr(gc, "freeze"):
+            gc.collect()
+            gc.freeze()
+
+            # Speed up process exit by freezing all allocated objects. This moves everything
+            # into the permanent generation and excludes them from the final GC.
+            atexit.register(gc.freeze)
 
 
 def reload_cache_config(config: HomeServerConfig) -> None:

synapse/app/admin_cmd.py

@@ -65,7 +65,6 @@ from synapse.storage.databases.main.stream import StreamWorkerStore
 from synapse.storage.databases.main.tags import TagsWorkerStore
 from synapse.storage.databases.main.user_erasure_store import UserErasureWorkerStore
 from synapse.types import JsonMapping, StateMap
-from synapse.util import SYNAPSE_VERSION
 from synapse.util.logcontext import LoggingContext
 
 logger = logging.getLogger("synapse.app.admin_cmd")
@@ -316,7 +315,6 @@ def start(config: HomeServerConfig, args: argparse.Namespace) -> None:
     ss = AdminCmdServer(
         config.server.server_name,
         config=config,
-        version_string=f"Synapse/{SYNAPSE_VERSION}",
     )
 
     setup_logging(ss, config, use_worker_options=True)
@@ -329,7 +327,7 @@ def start(config: HomeServerConfig, args: argparse.Namespace) -> None:
     # command.
 
     async def run() -> None:
-        with LoggingContext(name="command"):
+        with LoggingContext(name="command", server_name=config.server.server_name):
             await _base.start(ss)
             await args.func(ss, args)
 
@@ -342,5 +340,5 @@ def start(config: HomeServerConfig, args: argparse.Namespace) -> None:
 
 if __name__ == "__main__":
     homeserver_config, args = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config, args)

synapse/app/appservice.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/client_reader.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/event_creator.py

@@ -26,7 +26,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/federation_reader.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/federation_sender.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/frontend_proxy.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/generic_worker.py

@@ -49,6 +49,7 @@ from synapse.config.server import ListenerConfig, TCPListenerConfig
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.server import JsonResource, OptionsResource
 from synapse.logging.context import LoggingContext
+from synapse.logging.opentracing import init_tracer
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.rest import ClientRestResource, admin
@@ -111,7 +112,6 @@ from synapse.storage.databases.main.transactions import TransactionWorkerStore
 from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore
 from synapse.storage.databases.main.user_directory import UserDirectoryStore
 from synapse.storage.databases.main.user_erasure_store import UserErasureWorkerStore
-from synapse.util import SYNAPSE_VERSION
 from synapse.util.httpresourcetree import create_resource_tree
 
 logger = logging.getLogger("synapse.app.generic_worker")
@@ -277,11 +277,13 @@ class GenericWorkerServer(HomeServer):
                 self._listen_http(listener)
             elif listener.type == "manhole":
                 if isinstance(listener, TCPListenerConfig):
-                    _base.listen_manhole(
-                        listener.bind_addresses,
-                        listener.port,
-                        manhole_settings=self.config.server.manhole_settings,
-                        manhole_globals={"hs": self},
+                    self._listening_services.extend(
+                        _base.listen_manhole(
+                            listener.bind_addresses,
+                            listener.port,
+                            manhole_settings=self.config.server.manhole_settings,
+                            manhole_globals={"hs": self},
+                        )
                     )
                 else:
                     raise ConfigError(
@@ -295,9 +297,11 @@ class GenericWorkerServer(HomeServer):
                     )
                 else:
                     if isinstance(listener, TCPListenerConfig):
-                        _base.listen_metrics(
-                            listener.bind_addresses,
-                            listener.port,
+                        self._metrics_listeners.extend(
+                            _base.listen_metrics(
+                                listener.bind_addresses,
+                                listener.port,
+                            )
                         )
                     else:
                         raise ConfigError(
@@ -354,11 +358,13 @@ def start(config: HomeServerConfig) -> None:
     hs = GenericWorkerServer(
         config.server.server_name,
         config=config,
-        version_string=f"Synapse/{SYNAPSE_VERSION}",
     )
 
     setup_logging(hs, config, use_worker_options=True)
 
+    # Start the tracer
+    init_tracer(hs)  # noqa
+
     try:
         hs.setup()
 
@@ -382,7 +388,7 @@ def start(config: HomeServerConfig) -> None:
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)