matrix-synapse 1.139.2__cp39-abi3-manylinux_2_28_aarch64.whl → 1.140.0rc1__cp39-abi3-manylinux_2_28_aarch64.whl

synapse/app/homeserver.py

@@ -22,14 +22,13 @@
 import logging
 import os
 import sys
-from typing import Dict, Iterable, List
+from typing import Dict, Iterable, List, Optional
 
 from twisted.internet.tcp import Port
 from twisted.web.resource import EncodingResourceWrapper, Resource
 from twisted.web.server import GzipEncoderFactory
 
 import synapse
-import synapse.config.logger
 from synapse import events
 from synapse.api.urls import (
     CLIENT_API_PREFIX,
@@ -50,6 +49,7 @@ from synapse.app._base import (
 )
 from synapse.config._base import ConfigError, format_config_error
 from synapse.config.homeserver import HomeServerConfig
+from synapse.config.logger import setup_logging
 from synapse.config.server import ListenerConfig, TCPListenerConfig
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.additional_resource import AdditionalResource
@@ -60,6 +60,7 @@ from synapse.http.server import (
     StaticResource,
 )
 from synapse.logging.context import LoggingContext
+from synapse.logging.opentracing import init_tracer
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
 from synapse.rest import ClientRestResource, admin
@@ -69,7 +70,8 @@ from synapse.rest.synapse.client import build_synapse_client_resource_tree
 from synapse.rest.well_known import well_known_resource
 from synapse.server import HomeServer
 from synapse.storage import DataStore
-from synapse.util.check_dependencies import VERSION, check_requirements
+from synapse.types import ISynapseReactor
+from synapse.util.check_dependencies import check_requirements
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.module_loader import load_module
 
@@ -81,6 +83,10 @@ def gz_wrap(r: Resource) -> Resource:
 
 
 class SynapseHomeServer(HomeServer):
+    """
+    Homeserver class for the main Synapse process.
+    """
+
     DATASTORE_CLASS = DataStore
 
     def _listener_http(
@@ -276,11 +282,13 @@ class SynapseHomeServer(HomeServer):
                 )
             elif listener.type == "manhole":
                 if isinstance(listener, TCPListenerConfig):
-                    _base.listen_manhole(
-                        listener.bind_addresses,
-                        listener.port,
-                        manhole_settings=self.config.server.manhole_settings,
-                        manhole_globals={"hs": self},
+                    self._listening_services.extend(
+                        _base.listen_manhole(
+                            listener.bind_addresses,
+                            listener.port,
+                            manhole_settings=self.config.server.manhole_settings,
+                            manhole_globals={"hs": self},
+                        )
                     )
                 else:
                     raise ConfigError(
@@ -293,9 +301,11 @@ class SynapseHomeServer(HomeServer):
                     )
                 else:
                     if isinstance(listener, TCPListenerConfig):
-                        _base.listen_metrics(
-                            listener.bind_addresses,
-                            listener.port,
+                        self._metrics_listeners.extend(
+                            _base.listen_metrics(
+                                listener.bind_addresses,
+                                listener.port,
+                            )
                         )
                     else:
                         raise ConfigError(
@@ -339,12 +349,17 @@ def load_or_generate_config(argv_options: List[str]) -> HomeServerConfig:
     return config
 
 
-def setup(config: HomeServerConfig) -> SynapseHomeServer:
+def create_homeserver(
+    config: HomeServerConfig,
+    reactor: Optional[ISynapseReactor] = None,
+) -> SynapseHomeServer:
     """
-    Create and setup a Synapse homeserver instance given a configuration.
+    Create a homeserver instance for the Synapse main process.
 
     Args:
         config: The configuration for the homeserver.
+        reactor: Optionally provide a reactor to use. Can be useful in different
+            scenarios that you want control over the reactor, such as tests.
 
     Returns:
         A homeserver instance.
@@ -352,10 +367,9 @@ def setup(config: HomeServerConfig) -> SynapseHomeServer:
 
     if config.worker.worker_app:
         raise ConfigError(
-            "You have specified `worker_app` in the config but are attempting to start a non-worker "
+            "You have specified `worker_app` in the config but are attempting to setup a non-worker "
             "instance. Please use `python -m synapse.app.generic_worker` instead (or remove the option if this is the main process)."
         )
-        sys.exit(1)
 
     events.USE_FROZEN_DICTS = config.server.use_frozen_dicts
     synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage
@@ -363,61 +377,84 @@ def setup(config: HomeServerConfig) -> SynapseHomeServer:
     if config.server.gc_seconds:
         synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds
 
-    if (
-        config.registration.enable_registration
-        and not config.registration.enable_registration_without_verification
-    ):
-        if (
-            not config.captcha.enable_registration_captcha
-            and not config.registration.registrations_require_3pid
-            and not config.registration.registration_requires_token
-        ):
-            raise ConfigError(
-                "You have enabled open registration without any verification. This is a known vector for "
-                "spam and abuse. If you would like to allow public registration, please consider adding email, "
-                "captcha, or token-based verification. Otherwise this check can be removed by setting the "
-                "`enable_registration_without_verification` config option to `true`."
-            )
-
     hs = SynapseHomeServer(
-        config.server.server_name,
+        hostname=config.server.server_name,
         config=config,
-        version_string=f"Synapse/{VERSION}",
+        reactor=reactor,
     )
 
-    synapse.config.logger.setup_logging(hs, config, use_worker_options=False)
+    return hs
+
+
+def setup(
+    hs: SynapseHomeServer,
+    *,
+    freeze: bool = True,
+) -> None:
+    """
+    Setup a Synapse homeserver instance given a configuration.
 
+    Args:
+        hs: The homeserver to setup.
+        freeze: whether to freeze the homeserver base objects in the garbage collector.
+            May improve garbage collection performance by marking objects with an effectively
+            static lifetime as frozen so they don't need to be considered for cleanup.
+            If you ever want to `shutdown` the homeserver, this needs to be
+            False otherwise the homeserver cannot be garbage collected after `shutdown`.
+
+    Returns:
+        A homeserver instance.
+    """
+
+    setup_logging(hs, hs.config, use_worker_options=False)
+
+    # Log after we've configured logging.
     logger.info("Setting up server")
 
+    # Start the tracer
+    init_tracer(hs)  # noqa
+
     try:
         hs.setup()
     except Exception as e:
         handle_startup_exception(e)
 
-    async def start() -> None:
+    async def _start_when_reactor_running() -> None:
+        # TODO: Feels like this should be moved somewhere else.
+        #
         # Load the OIDC provider metadatas, if OIDC is enabled.
         if hs.config.oidc.oidc_enabled:
             oidc = hs.get_oidc_handler()
             # Loading the provider metadata also ensures the provider config is valid.
             await oidc.load_metadata()
 
-        await _base.start(hs)
+        await _base.start(hs, freeze)
 
+        # TODO: This should be moved to `SynapseHomeServer.start_background_tasks` (not
+        # `HomeServer.start_background_tasks`) (this way it matches the behavior of only
+        # running on `main`)
         hs.get_datastores().main.db_pool.updates.start_doing_background_updates()
 
-    register_start(hs, start)
+    # Register a callback to be invoked once the reactor is running
+    register_start(hs, _start_when_reactor_running)
 
-    return hs
 
+def start_reactor(
+    config: HomeServerConfig,
+) -> None:
+    """
+    Start the reactor (Twisted event-loop).
 
-def run(hs: HomeServer) -> None:
+    Args:
+        config: The configuration for the homeserver.
+    """
     _base.start_reactor(
         "synapse-homeserver",
-        soft_file_limit=hs.config.server.soft_file_limit,
-        gc_thresholds=hs.config.server.gc_thresholds,
-        pid_file=hs.config.server.pid_file,
-        daemonize=hs.config.server.daemonize,
-        print_pidfile=hs.config.server.print_pidfile,
+        soft_file_limit=config.server.soft_file_limit,
+        gc_thresholds=config.server.gc_thresholds,
+        pid_file=config.server.pid_file,
+        daemonize=config.server.daemonize,
+        print_pidfile=config.server.print_pidfile,
         logger=logger,
     )
 
@@ -425,16 +462,17 @@ def run(hs: HomeServer) -> None:
 def main() -> None:
     homeserver_config = load_or_generate_config(sys.argv[1:])
 
-    with LoggingContext("main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         # check base requirements
         check_requirements()
-        hs = setup(homeserver_config)
+        hs = create_homeserver(homeserver_config)
+        setup(hs)
 
         # redirect stdio to the logs, if configured.
         if not hs.config.logging.no_redirect_stdio:
             redirect_stdio_to_logs()
 
-        run(hs)
+        start_reactor(homeserver_config)
 
 
 if __name__ == "__main__":

synapse/app/media_repository.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/phone_stats_home.py

@@ -29,19 +29,18 @@ from prometheus_client import Gauge
 from twisted.internet import defer
 
 from synapse.metrics import SERVER_NAME_LABEL
-from synapse.metrics.background_process_metrics import (
-    run_as_background_process,
-)
 from synapse.types import JsonDict
-from synapse.util.constants import ONE_HOUR_SECONDS, ONE_MINUTE_SECONDS
+from synapse.util.constants import (
+    MILLISECONDS_PER_SECOND,
+    ONE_HOUR_SECONDS,
+    ONE_MINUTE_SECONDS,
+)
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
 
 logger = logging.getLogger("synapse.app.homeserver")
 
-MILLISECONDS_PER_SECOND = 1000
-
 INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS = 5 * ONE_MINUTE_SECONDS
 """
 We wait 5 minutes to send the first set of stats as the server can be quite busy the
@@ -85,8 +84,6 @@ def phone_stats_home(
     stats: JsonDict,
     stats_process: List[Tuple[int, "resource.struct_rusage"]] = _stats_process,
 ) -> "defer.Deferred[None]":
-    server_name = hs.hostname
-
     async def _phone_stats_home(
         hs: "HomeServer",
         stats: JsonDict,
@@ -200,8 +197,8 @@ def phone_stats_home(
         except Exception as e:
             logger.warning("Error reporting stats: %s", e)
 
-    return run_as_background_process(
-        "phone_stats_home", server_name, _phone_stats_home, hs, stats, stats_process
+    return hs.run_as_background_process(
+        "phone_stats_home", _phone_stats_home, hs, stats, stats_process
     )
 
 
@@ -263,9 +260,8 @@ def start_phone_stats_home(hs: "HomeServer") -> None:
                 float(hs.config.server.max_mau_value)
             )
 
-        return run_as_background_process(
+        return hs.run_as_background_process(
             "generate_monthly_active_users",
-            server_name,
             _generate_monthly_active_users,
         )
 
@@ -285,10 +281,16 @@ def start_phone_stats_home(hs: "HomeServer") -> None:
 
         # We need to defer this init for the cases that we daemonize
         # otherwise the process ID we get is that of the non-daemon process
-        clock.call_later(0, performance_stats_init)
+        clock.call_later(
+            0,
+            performance_stats_init,
+        )
 
         # We wait 5 minutes to send the first set of stats as the server can
         # be quite busy the first few minutes
         clock.call_later(
-            INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS, phone_stats_home, hs, stats
+            INITIAL_DELAY_BEFORE_FIRST_PHONE_HOME_SECONDS,
+            phone_stats_home,
+            hs,
+            stats,
         )

synapse/app/pusher.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/synchrotron.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/app/user_dir.py

@@ -27,7 +27,7 @@ from synapse.util.logcontext import LoggingContext
 
 def main() -> None:
     homeserver_config = load_config(sys.argv[1:])
-    with LoggingContext(name="main"):
+    with LoggingContext(name="main", server_name=homeserver_config.server.server_name):
         start(homeserver_config)
 
 

synapse/appservice/__init__.py

@@ -23,15 +23,33 @@
 import logging
 import re
 from enum import Enum
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Pattern, Sequence
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Pattern,
+    Sequence,
+    cast,
+)
 
 import attr
 from netaddr import IPSet
 
+from twisted.internet import reactor
+
 from synapse.api.constants import EventTypes
 from synapse.events import EventBase
-from synapse.types import DeviceListUpdates, JsonDict, JsonMapping, UserID
+from synapse.types import (
+    DeviceListUpdates,
+    ISynapseThreadlessReactor,
+    JsonDict,
+    JsonMapping,
+    UserID,
+)
 from synapse.util.caches.descriptors import _CacheContext, cached
+from synapse.util.clock import Clock
 
 if TYPE_CHECKING:
     from synapse.appservice.api import ApplicationServiceApi
@@ -98,6 +116,15 @@ class ApplicationService:
         self.sender = sender
         # The application service user should be part of the server's domain.
         self.server_name = sender.domain  # nb must be called this for @cached
+
+        # Ideally we would require passing in the `HomeServer` `Clock` instance.
+        # However this is not currently possible as there are places which use
+        # `@cached` that aren't aware of the `HomeServer` instance.
+        # nb must be called this for @cached
+        self.clock = Clock(
+            cast(ISynapseThreadlessReactor, reactor), server_name=self.server_name
+        )  # type: ignore[multiple-internal-clocks]
+
         self.namespaces = self._check_namespaces(namespaces)
         self.id = id
         self.ip_range_whitelist = ip_range_whitelist

synapse/appservice/scheduler.py

@@ -81,7 +81,6 @@ from synapse.appservice import (
 from synapse.appservice.api import ApplicationServiceApi
 from synapse.events import EventBase
 from synapse.logging.context import run_in_background
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.databases.main import DataStore
 from synapse.types import DeviceListUpdates, JsonMapping
 from synapse.util.clock import Clock
@@ -200,6 +199,7 @@ class _ServiceQueuer:
         )
         self.server_name = hs.hostname
         self.clock = hs.get_clock()
+        self.hs = hs
         self._store = hs.get_datastores().main
 
     def start_background_request(self, service: ApplicationService) -> None:
@@ -207,9 +207,7 @@ class _ServiceQueuer:
         if service.id in self.requests_in_flight:
             return
 
-        run_as_background_process(
-            "as-sender", self.server_name, self._send_request, service
-        )
+        self.hs.run_as_background_process("as-sender", self._send_request, service)
 
     async def _send_request(self, service: ApplicationService) -> None:
         # sanity-check: we shouldn't get here if this service already has a sender
@@ -361,6 +359,7 @@ class _TransactionController:
     def __init__(self, hs: "HomeServer"):
         self.server_name = hs.hostname
         self.clock = hs.get_clock()
+        self.hs = hs
         self.store = hs.get_datastores().main
         self.as_api = hs.get_application_service_api()
 
@@ -448,6 +447,7 @@ class _TransactionController:
         recoverer = self.RECOVERER_CLASS(
             self.server_name,
             self.clock,
+            self.hs,
             self.store,
             self.as_api,
             service,
@@ -494,6 +494,7 @@ class _Recoverer:
         self,
         server_name: str,
         clock: Clock,
+        hs: "HomeServer",
         store: DataStore,
         as_api: ApplicationServiceApi,
         service: ApplicationService,
@@ -501,6 +502,7 @@ class _Recoverer:
     ):
         self.server_name = server_name
         self.clock = clock
+        self.hs = hs
         self.store = store
         self.as_api = as_api
         self.service = service
@@ -513,9 +515,8 @@ class _Recoverer:
         logger.info("Scheduling retries on %s in %fs", self.service.id, delay)
         self.scheduled_recovery = self.clock.call_later(
             delay,
-            run_as_background_process,
+            self.hs.run_as_background_process,
             "as-recoverer",
-            self.server_name,
             self.retry,
         )
 
@@ -535,9 +536,8 @@ class _Recoverer:
         if self.scheduled_recovery:
             self.clock.cancel_call_later(self.scheduled_recovery)
         # Run a retry, which will resechedule a recovery if it fails.
-        run_as_background_process(
+        self.hs.run_as_background_process(
             "retry",
-            self.server_name,
             self.retry,
         )
 

synapse/config/_base.py

@@ -545,18 +545,22 @@ class RootConfig:
 
     @classmethod
     def load_config(
-        cls: Type[TRootConfig], description: str, argv: List[str]
+        cls: Type[TRootConfig], description: str, argv_options: List[str]
     ) -> TRootConfig:
         """Parse the commandline and config files
 
         Doesn't support config-file-generation: used by the worker apps.
 
+        Args:
+            description: TODO
+            argv_options: The options passed to Synapse. Usually `sys.argv[1:]`.
+
         Returns:
             Config object.
         """
         config_parser = argparse.ArgumentParser(description=description)
         cls.add_arguments_to_parser(config_parser)
-        obj, _ = cls.load_config_with_parser(config_parser, argv)
+        obj, _ = cls.load_config_with_parser(config_parser, argv_options)
 
         return obj
 
@@ -601,7 +605,7 @@ class RootConfig:
 
     @classmethod
     def load_config_with_parser(
-        cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv: List[str]
+        cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv_options: List[str]
     ) -> Tuple[TRootConfig, argparse.Namespace]:
         """Parse the commandline and config files with the given parser
 
@@ -609,16 +613,20 @@ class RootConfig:
 
         Used for workers where we want to add extra flags/subcommands.
 
+        Note: This is the common denominator for loading config and is also used by
+        `load_config` and `load_or_generate_config`. Which is why we call
+        `validate_config()` here.
+
         Args:
             parser
-            argv
+            argv_options: The options passed to Synapse. Usually `sys.argv[1:]`.
 
         Returns:
             Returns the parsed config object and the parsed argparse.Namespace
             object from parser.parse_args(..)`
         """
 
-        config_args = parser.parse_args(argv)
+        config_args = parser.parse_args(argv_options)
 
         config_files = find_config_files(search_paths=config_args.config_path)
         obj = cls(config_files)
@@ -642,6 +650,10 @@ class RootConfig:
 
         obj.invoke_all("read_arguments", config_args)
 
+        # Now that we finally have the full config sections parsed, allow subclasses to
+        # do some extra validation across the entire config.
+        obj.validate_config()
+
         return obj, config_args
 
     @classmethod
@@ -842,15 +854,7 @@ class RootConfig:
         ):
             return None
 
-        obj.parse_config_dict(
-            config_dict,
-            config_dir_path=config_dir_path,
-            data_dir_path=data_dir_path,
-            allow_secrets_in_config=config_args.secrets_in_config,
-        )
-        obj.invoke_all("read_arguments", config_args)
-
-        return obj
+        return cls.load_config(description, argv_options)
 
     def parse_config_dict(
         self,
@@ -911,6 +915,20 @@ class RootConfig:
         existing_config.root = None
         return existing_config
 
+    def validate_config(self) -> None:
+        """
+        Additional config validation across all config sections.
+
+        Override this in subclasses to add extra validation. This is called once all
+        config option values have been populated.
+
+        XXX: This should only validate, not modify the configuration, as the final
+        config state is required for proper validation across all config sections.
+
+        Raises:
+            ConfigError: if the config is invalid.
+        """
+
 
 def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]:
     """Read the config files and shallowly merge them into a dict.

synapse/config/_base.pyi

@@ -37,6 +37,7 @@ from synapse.config import (  # noqa: F401
     key,
     logger,
     mas,
+    matrixrtc,
     metrics,
     modules,
     oembed,
@@ -126,6 +127,7 @@ class RootConfig:
     auto_accept_invites: auto_accept_invites.AutoAcceptInvitesConfig
     user_types: user_types.UserTypesConfig
     mas: mas.MasConfig
+    matrix_rtc: matrixrtc.MatrixRtcConfig
 
     config_classes: List[Type["Config"]] = ...
     config_files: List[str]
@@ -156,11 +158,11 @@ class RootConfig:
     ) -> str: ...
     @classmethod
     def load_or_generate_config(
-        cls: Type[TRootConfig], description: str, argv: List[str]
+        cls: Type[TRootConfig], description: str, argv_options: List[str]
     ) -> Optional[TRootConfig]: ...
     @classmethod
     def load_config(
-        cls: Type[TRootConfig], description: str, argv: List[str]
+        cls: Type[TRootConfig], description: str, argv_options: List[str]
     ) -> TRootConfig: ...
     @classmethod
     def add_arguments_to_parser(
@@ -168,7 +170,7 @@ class RootConfig:
     ) -> None: ...
     @classmethod
     def load_config_with_parser(
-        cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv: List[str]
+        cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv_options: List[str]
     ) -> Tuple[TRootConfig, argparse.Namespace]: ...
     def generate_missing_files(
         self, config_dict: dict, config_dir_path: str

synapse/config/experimental.py

@@ -556,6 +556,9 @@ class ExperimentalConfig(Config):
         # MSC4133: Custom profile fields
         self.msc4133_enabled: bool = experimental.get("msc4133_enabled", False)
 
+        # MSC4143: Matrix RTC Transport using Livekit Backend
+        self.msc4143_enabled: bool = experimental.get("msc4143_enabled", False)
+
         # MSC4169: Backwards-compatible redaction sending using `/send`
         self.msc4169_enabled: bool = experimental.get("msc4169_enabled", False)