matrix-synapse 1.139.2__cp39-abi3-macosx_11_0_arm64.whl → 1.140.0rc1__cp39-abi3-macosx_11_0_arm64.whl

synapse/config/homeserver.py

@@ -18,7 +18,8 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
-from ._base import RootConfig
+
+from ._base import ConfigError, RootConfig
 from .account_validity import AccountValidityConfig
 from .api import ApiConfig
 from .appservice import AppServiceConfig
@@ -37,6 +38,7 @@ from .jwt import JWTConfig
 from .key import KeyConfig
 from .logger import LoggingConfig
 from .mas import MasConfig
+from .matrixrtc import MatrixRtcConfig
 from .metrics import MetricsConfig
 from .modules import ModulesConfig
 from .oembed import OembedConfig
@@ -66,6 +68,10 @@ from .workers import WorkerConfig
 
 
 class HomeServerConfig(RootConfig):
+    """
+    Top-level config object for Synapse homeserver (main process and workers).
+    """
+
     config_classes = [
         ModulesConfig,
         ServerConfig,
@@ -80,6 +86,7 @@ class HomeServerConfig(RootConfig):
         OembedConfig,
         CaptchaConfig,
         VoipConfig,
+        MatrixRtcConfig,
         RegistrationConfig,
         AccountValidityConfig,
         MetricsConfig,
@@ -113,3 +120,22 @@ class HomeServerConfig(RootConfig):
         # This must be last, as it checks for conflicts with other config options.
         MasConfig,
     ]
+
+    def validate_config(
+        self,
+    ) -> None:
+        if (
+            self.registration.enable_registration
+            and not self.registration.enable_registration_without_verification
+        ):
+            if (
+                not self.captcha.enable_registration_captcha
+                and not self.registration.registrations_require_3pid
+                and not self.registration.registration_requires_token
+            ):
+                raise ConfigError(
+                    "You have enabled open registration without any verification. This is a known vector for "
+                    "spam and abuse. If you would like to allow public registration, please consider adding email, "
+                    "captcha, or token-based verification. Otherwise this check can be removed by setting the "
+                    "`enable_registration_without_verification` config option to `true`."
+                )

synapse/config/logger.py

@@ -40,7 +40,6 @@ from twisted.logger import (
 )
 
 from synapse.logging.context import LoggingContextFilter
-from synapse.logging.filter import MetadataFilter
 from synapse.synapse_rust import reset_logging_config
 from synapse.types import JsonDict
 
@@ -213,13 +212,11 @@ def _setup_stdlib_logging(
     # writes.
 
     log_context_filter = LoggingContextFilter()
-    log_metadata_filter = MetadataFilter({"server_name": config.server.server_name})
     old_factory = logging.getLogRecordFactory()
 
     def factory(*args: Any, **kwargs: Any) -> logging.LogRecord:
         record = old_factory(*args, **kwargs)
         log_context_filter.filter(record)
-        log_metadata_filter.filter(record)
         return record
 
     logging.setLogRecordFactory(factory)
@@ -348,7 +345,9 @@ def setup_logging(
     # Add a SIGHUP handler to reload the logging configuration, if one is available.
     from synapse.app import _base as appbase
 
-    appbase.register_sighup(_reload_logging_config, log_config_path)
+    appbase.register_sighup(
+        hs.get_instance_id(), _reload_logging_config, log_config_path
+    )
 
     # Log immediately so we can grep backwards.
     logger.warning("***** STARTING SERVER *****")

synapse/config/matrixrtc.py

@@ -0,0 +1,67 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2025 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+from typing import Any, Optional
+
+from pydantic import ValidationError
+
+from synapse._pydantic_compat import Field, StrictStr, validator
+from synapse.types import JsonDict
+from synapse.util.pydantic_models import ParseModel
+
+from ._base import Config, ConfigError
+
+
+class TransportConfigModel(ParseModel):
+    type: StrictStr
+
+    livekit_service_url: Optional[StrictStr] = Field(default=None)
+    """An optional livekit service URL. Only required if type is "livekit"."""
+
+    @validator("livekit_service_url", always=True)
+    def validate_livekit_service_url(cls, v: Any, values: dict) -> Any:
+        if values.get("type") == "livekit" and not v:
+            raise ValueError(
+                "You must set a `livekit_service_url` when using the 'livekit' transport."
+            )
+
+        return v
+
+
+class MatrixRtcConfigModel(ParseModel):
+    transports: list = []
+
+
+class MatrixRtcConfig(Config):
+    section = "matrix_rtc"
+
+    def read_config(
+        self, config: JsonDict, allow_secrets_in_config: bool, **kwargs: Any
+    ) -> None:
+        matrix_rtc = config.get("matrix_rtc", {})
+        if matrix_rtc is None:
+            matrix_rtc = {}
+
+        try:
+            parsed = MatrixRtcConfigModel(**matrix_rtc)
+        except ValidationError as e:
+            raise ConfigError(
+                "Could not validate matrix_rtc config",
+                ("matrix_rtc",),
+            ) from e
+
+        self.transports = parsed.transports

synapse/crypto/keyring.py

@@ -172,7 +172,7 @@ class Keyring:
             _FetchKeyRequest, Dict[str, Dict[str, FetchKeyResult]]
         ] = BatchingQueue(
             name="keyring_server",
-            server_name=self.server_name,
+            hs=hs,
             clock=hs.get_clock(),
             # The method called to fetch each key
             process_batch_callback=self._inner_fetch_key_requests,
@@ -194,6 +194,14 @@ class Keyring:
             valid_until_ts=2**63,  # fake future timestamp
         )
 
+    def shutdown(self) -> None:
+        """
+        Prepares the KeyRing for garbage collection by shutting down it's queues.
+        """
+        self._fetch_keys_queue.shutdown()
+        for key_fetcher in self._key_fetchers:
+            key_fetcher.shutdown()
+
     async def verify_json_for_server(
         self,
         server_name: str,
@@ -316,7 +324,7 @@ class Keyring:
             if key_result.valid_until_ts < verify_request.minimum_valid_until_ts:
                 continue
 
-            await self._process_json(key_result.verify_key, verify_request)
+            await self.process_json(key_result.verify_key, verify_request)
             verified = True
 
         if not verified:
@@ -326,7 +334,7 @@ class Keyring:
                 Codes.UNAUTHORIZED,
             )
 
-    async def _process_json(
+    async def process_json(
         self, verify_key: VerifyKey, verify_request: VerifyJsonRequest
     ) -> None:
         """Processes the `VerifyJsonRequest`. Raises if the signature can't be
@@ -479,11 +487,17 @@ class KeyFetcher(metaclass=abc.ABCMeta):
         self.server_name = hs.hostname
         self._queue = BatchingQueue(
             name=self.__class__.__name__,
-            server_name=self.server_name,
+            hs=hs,
             clock=hs.get_clock(),
             process_batch_callback=self._fetch_keys,
         )
 
+    def shutdown(self) -> None:
+        """
+        Prepares the KeyFetcher for garbage collection by shutting down it's queue.
+        """
+        self._queue.shutdown()
+
     async def get_keys(
         self, server_name: str, key_ids: List[str], minimum_valid_until_ts: int
     ) -> Dict[str, FetchKeyResult]:

synapse/events/auto_accept_invites.py

@@ -119,7 +119,6 @@ class InviteAutoAccepter:
             event.state_key,
             event.room_id,
             "join",
-            bg_start_span=False,
         )
 
         if is_direct_message:

synapse/federation/federation_client.py

@@ -148,6 +148,7 @@ class FederationClient(FederationBase):
         self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache(
             cache_name="get_pdu_cache",
             server_name=self.server_name,
+            hs=self.hs,
             clock=self._clock,
             max_len=1000,
             expiry_ms=120 * 1000,
@@ -167,6 +168,7 @@ class FederationClient(FederationBase):
         ] = ExpiringCache(
             cache_name="get_room_hierarchy_cache",
             server_name=self.server_name,
+            hs=self.hs,
             clock=self._clock,
             max_len=1000,
             expiry_ms=5 * 60 * 1000,
@@ -495,6 +497,43 @@ class FederationClient(FederationBase):
             )
             return RECOMMENDATION_OK
 
+    @trace
+    @tag_args
+    async def ask_policy_server_to_sign_event(
+        self, destination: str, pdu: EventBase, timeout: Optional[int] = None
+    ) -> Optional[JsonDict]:
+        """Requests that the destination server (typically a policy server)
+        sign the event as not spam.
+
+        If the policy server could not be contacted or the policy server
+        returned an error, this returns no signature.
+
+        Args:
+            destination: The remote homeserver to ask (a policy server)
+            pdu: The event to sign
+            timeout: How long to try (in ms) the destination for before
+                giving up. None indicates no timeout.
+        Returns:
+            The signature from the policy server, structured in the same was as the 'signatures'
+            JSON in the event e.g { "$policy_server_via_domain" : { "ed25519:policy_server": "signature_base64" }}
+        """
+        logger.debug(
+            "ask_policy_server_to_sign_event for event_id=%s from %s",
+            pdu.event_id,
+            destination,
+        )
+        try:
+            return await self.transport_layer.ask_policy_server_to_sign_event(
+                destination, pdu, timeout=timeout
+            )
+        except Exception as e:
+            logger.warning(
+                "ask_policy_server_to_sign_event: server %s responded with error: %s",
+                destination,
+                e,
+            )
+        return None
+
     @trace
     @tag_args
     async def get_pdu(

synapse/federation/federation_server.py

@@ -159,7 +159,7 @@ class FederationServer(FederationBase):
         # with FederationHandlerRegistry.
         hs.get_directory_handler()
 
-        self._server_linearizer = Linearizer("fed_server")
+        self._server_linearizer = Linearizer(name="fed_server", clock=hs.get_clock())
 
         # origins that we are currently processing a transaction from.
         # a dict from origin to txn id.

synapse/federation/send_queue.py

@@ -144,6 +144,9 @@ class FederationRemoteSendQueue(AbstractFederationSender):
 
         self.clock.looping_call(self._clear_queue, 30 * 1000)
 
+    def shutdown(self) -> None:
+        """Stops this federation sender instance from sending further transactions."""
+
     def _next_pos(self) -> int:
         pos = self.pos
         self.pos += 1

synapse/federation/sender/__init__.py

@@ -168,7 +168,6 @@ from synapse.metrics import (
     events_processed_counter,
 )
 from synapse.metrics.background_process_metrics import (
-    run_as_background_process,
     wrap_as_background_process,
 )
 from synapse.types import (
@@ -232,6 +231,11 @@ WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC = 5
 
 
 class AbstractFederationSender(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def shutdown(self) -> None:
+        """Stops this federation sender instance from sending further transactions."""
+        raise NotImplementedError()
+
     @abc.abstractmethod
     def notify_new_events(self, max_token: RoomStreamToken) -> None:
         """This gets called when we have some new events we might want to
@@ -326,6 +330,7 @@ class _DestinationWakeupQueue:
     _MAX_TIME_IN_QUEUE = 30.0
 
     sender: "FederationSender" = attr.ib()
+    hs: "HomeServer" = attr.ib()
     server_name: str = attr.ib()
     """
     Our homeserver name (used to label metrics) (`hs.hostname`).
@@ -453,18 +458,30 @@ class FederationSender(AbstractFederationSender):
             1.0 / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second
         )
         self._destination_wakeup_queue = _DestinationWakeupQueue(
-            self, self.server_name, self.clock, max_delay_s=rr_txn_interval_per_room_s
+            self,
+            hs,
+            self.server_name,
+            self.clock,
+            max_delay_s=rr_txn_interval_per_room_s,
         )
 
+        # It is important for `_is_shutdown` to be instantiated before the looping call
+        # for `wake_destinations_needing_catchup`.
+        self._is_shutdown = False
+
         # Regularly wake up destinations that have outstanding PDUs to be caught up
         self.clock.looping_call_now(
-            run_as_background_process,
+            self.hs.run_as_background_process,
             WAKEUP_RETRY_PERIOD_SEC * 1000.0,
             "wake_destinations_needing_catchup",
-            self.server_name,
             self._wake_destinations_needing_catchup,
         )
 
+    def shutdown(self) -> None:
+        self._is_shutdown = True
+        for queue in self._per_destination_queues.values():
+            queue.shutdown()
+
     def _get_per_destination_queue(
         self, destination: str
     ) -> Optional[PerDestinationQueue]:
@@ -503,16 +520,15 @@ class FederationSender(AbstractFederationSender):
             return
 
         # fire off a processing loop in the background
-        run_as_background_process(
+        self.hs.run_as_background_process(
             "process_event_queue_for_federation",
-            self.server_name,
             self._process_event_queue_loop,
         )
 
     async def _process_event_queue_loop(self) -> None:
         try:
             self._is_processing = True
-            while True:
+            while not self._is_shutdown:
                 last_token = await self.store.get_federation_out_pos("events")
                 (
                     next_token,
@@ -1123,7 +1139,7 @@ class FederationSender(AbstractFederationSender):
 
         last_processed: Optional[str] = None
 
-        while True:
+        while not self._is_shutdown:
             destinations_to_wake = (
                 await self.store.get_catch_up_outstanding_destinations(last_processed)
             )

synapse/federation/sender/per_destination_queue.py

@@ -28,6 +28,8 @@ from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Optional, Tupl
 import attr
 from prometheus_client import Counter
 
+from twisted.internet import defer
+
 from synapse.api.constants import EduTypes
 from synapse.api.errors import (
     FederationDeniedError,
@@ -41,7 +43,6 @@ from synapse.handlers.presence import format_user_presence_state
 from synapse.logging import issue9533_logger
 from synapse.logging.opentracing import SynapseTags, set_tag
 from synapse.metrics import SERVER_NAME_LABEL, sent_transactions_counter
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import JsonDict, ReadReceipt
 from synapse.util.retryutils import NotRetryingDestination, get_retry_limiter
 from synapse.visibility import filter_events_for_server
@@ -79,6 +80,7 @@ MAX_PRESENCE_STATES_PER_EDU = 50
 class PerDestinationQueue:
     """
     Manages the per-destination transmission queues.
+    Runs until `shutdown()` is called on the queue.
 
     Args:
         hs
@@ -94,6 +96,7 @@ class PerDestinationQueue:
         destination: str,
     ):
         self.server_name = hs.hostname
+        self._hs = hs
         self._clock = hs.get_clock()
         self._storage_controllers = hs.get_storage_controllers()
         self._store = hs.get_datastores().main
@@ -117,6 +120,8 @@ class PerDestinationQueue:
 
         self._destination = destination
         self.transmission_loop_running = False
+        self._transmission_loop_enabled = True
+        self.active_transmission_loop: Optional[defer.Deferred] = None
 
         # Flag to signal to any running transmission loop that there is new data
         # queued up to be sent.
@@ -171,6 +176,20 @@ class PerDestinationQueue:
     def __str__(self) -> str:
         return "PerDestinationQueue[%s]" % self._destination
 
+    def shutdown(self) -> None:
+        """Instruct the queue to stop processing any further requests"""
+        self._transmission_loop_enabled = False
+        # The transaction manager must be shutdown before cancelling the active
+        # transmission loop. Otherwise the transmission loop can enter a new cycle of
+        # sleeping before retrying since the shutdown flag of the _transaction_manager
+        # hasn't been set yet.
+        self._transaction_manager.shutdown()
+        try:
+            if self.active_transmission_loop is not None:
+                self.active_transmission_loop.cancel()
+        except Exception:
+            pass
+
     def pending_pdu_count(self) -> int:
         return len(self._pending_pdus)
 
@@ -309,11 +328,14 @@ class PerDestinationQueue:
             )
             return
 
+        if not self._transmission_loop_enabled:
+            logger.warning("Shutdown has been requested. Not sending transaction")
+            return
+
         logger.debug("TX [%s] Starting transaction loop", self._destination)
 
-        run_as_background_process(
+        self.active_transmission_loop = self._hs.run_as_background_process(
             "federation_transaction_transmission_loop",
-            self.server_name,
             self._transaction_transmission_loop,
         )
 
@@ -321,13 +343,13 @@ class PerDestinationQueue:
         pending_pdus: List[EventBase] = []
         try:
             self.transmission_loop_running = True
-
             # This will throw if we wouldn't retry. We do this here so we fail
             # quickly, but we will later check this again in the http client,
             # hence why we throw the result away.
             await get_retry_limiter(
                 destination=self._destination,
                 our_server_name=self.server_name,
+                hs=self._hs,
                 clock=self._clock,
                 store=self._store,
             )
@@ -339,7 +361,7 @@ class PerDestinationQueue:
                     # not caught up yet
                     return
 
-            while True:
+            while self._transmission_loop_enabled:
                 self._new_data_to_send = False
 
                 async with _TransactionQueueManager(self) as (
@@ -352,8 +374,8 @@ class PerDestinationQueue:
                         # If we've gotten told about new things to send during
                         # checking for things to send, we try looking again.
                         # Otherwise new PDUs or EDUs might arrive in the meantime,
-                        # but not get sent because we hold the
-                        # `transmission_loop_running` flag.
+                        # but not get sent because we currently have an
+                        # `_active_transmission_loop` running.
                         if self._new_data_to_send:
                             continue
                         else:
@@ -442,6 +464,7 @@ class PerDestinationQueue:
                 )
         finally:
             # We want to be *very* sure we clear this after we stop processing
+            self.active_transmission_loop = None
             self.transmission_loop_running = False
 
     async def _catch_up_transmission_loop(self) -> None:
@@ -469,7 +492,7 @@ class PerDestinationQueue:
         last_successful_stream_ordering: int = _tmp_last_successful_stream_ordering
 
         # get at most 50 catchup room/PDUs
-        while True:
+        while self._transmission_loop_enabled:
             event_ids = await self._store.get_catch_up_room_event_ids(
                 self._destination, last_successful_stream_ordering
             )

synapse/federation/sender/transaction_manager.py

@@ -72,6 +72,12 @@ class TransactionManager:
         # HACK to get unique tx id
         self._next_txn_id = int(self.clock.time_msec())
 
+        self._is_shutdown = False
+
+    def shutdown(self) -> None:
+        self._is_shutdown = True
+        self._transport_layer.shutdown()
+
     @measure_func("_send_new_transaction")
     async def send_new_transaction(
         self,
@@ -86,6 +92,12 @@ class TransactionManager:
             edus: List of EDUs to send
         """
 
+        if self._is_shutdown:
+            logger.warning(
+                "TransactionManager has been shutdown, not sending transaction"
+            )
+            return
+
         # Make a transaction-sending opentracing span. This span follows on from
         # all the edus in that transaction. This needs to be done since there is
         # no active span here, so if the edus were not received by the remote the

synapse/federation/transport/client.py

@@ -70,6 +70,9 @@ class TransportLayerClient:
         self.client = hs.get_federation_http_client()
         self._is_mine_server_name = hs.is_mine_server_name
 
+    def shutdown(self) -> None:
+        self.client.shutdown()
+
     async def get_room_state_ids(
         self, destination: str, room_id: str, event_id: str
     ) -> JsonDict:
@@ -170,6 +173,32 @@ class TransportLayerClient:
             timeout=timeout,
         )
 
+    async def ask_policy_server_to_sign_event(
+        self, destination: str, event: EventBase, timeout: Optional[int] = None
+    ) -> JsonDict:
+        """Requests that the destination server (typically a policy server)
+        sign the event as not spam.
+
+        If the policy server could not be contacted or the policy server
+        returned an error, this raises that error.
+
+        Args:
+            destination: The host name of the policy server / homeserver.
+            event: The event to sign.
+            timeout: How long to try (in ms) the destination for before giving up.
+                None indicates no timeout.
+        Returns:
+            The signature from the policy server, structured in the same was as the 'signatures'
+            JSON in the event e.g { "$policy_server_via_domain" : { "ed25519:policy_server": "signature_base64" }}
+        """
+        return await self.client.post_json(
+            destination=destination,
+            path="/_matrix/policy/unstable/org.matrix.msc4284/sign",
+            data=event.get_pdu_json(),
+            ignore_backoff=True,
+            timeout=timeout,
+        )
+
     async def backfill(
         self, destination: str, room_id: str, event_tuples: Collection[str], limit: int
     ) -> Optional[Union[JsonDict, list]]:

synapse/handlers/account_validity.py

@@ -37,10 +37,8 @@ logger = logging.getLogger(__name__)
 
 class AccountValidityHandler:
     def __init__(self, hs: "HomeServer"):
-        self.hs = hs
-        self.server_name = (
-            hs.hostname
-        )  # nb must be called this for @wrap_as_background_process
+        self.hs = hs  # nb must be called this for @wrap_as_background_process
+        self.server_name = hs.hostname
         self.config = hs.config
         self.store = hs.get_datastores().main
         self.send_email_handler = hs.get_send_email_handler()

synapse/handlers/appservice.py

@@ -47,7 +47,6 @@ from synapse.metrics import (
     event_processing_loop_room_count,
 )
 from synapse.metrics.background_process_metrics import (
-    run_as_background_process,
     wrap_as_background_process,
 )
 from synapse.storage.databases.main.directory import RoomAliasMapping
@@ -76,9 +75,8 @@ events_processed_counter = Counter(
 
 class ApplicationServicesHandler:
     def __init__(self, hs: "HomeServer"):
-        self.server_name = (
-            hs.hostname
-        )  # nb must be called this for @wrap_as_background_process
+        self.server_name = hs.hostname
+        self.hs = hs  # nb must be called this for @wrap_as_background_process
         self.store = hs.get_datastores().main
         self.is_mine_id = hs.is_mine_id
         self.appservice_api = hs.get_application_service_api()
@@ -98,7 +96,7 @@ class ApplicationServicesHandler:
         self.is_processing = False
 
         self._ephemeral_events_linearizer = Linearizer(
-            name="appservice_ephemeral_events"
+            name="appservice_ephemeral_events", clock=hs.get_clock()
         )
 
     def notify_interested_services(self, max_token: RoomStreamToken) -> None:
@@ -171,8 +169,8 @@ class ApplicationServicesHandler:
                                 except Exception:
                                     logger.error("Application Services Failure")
 
-                            run_as_background_process(
-                                "as_scheduler", self.server_name, start_scheduler
+                            self.hs.run_as_background_process(
+                                "as_scheduler", start_scheduler
                             )
                             self.started_scheduler = True
 

synapse/handlers/deactivate_account.py

@@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Optional
 
 from synapse.api.constants import Membership
 from synapse.api.errors import SynapseError
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.deactivate_account import (
     ReplicationNotifyAccountDeactivatedServlet,
 )
@@ -272,8 +271,8 @@ class DeactivateAccountHandler:
         pending deactivation, if it isn't already running.
         """
         if not self._user_parter_running:
-            run_as_background_process(
-                "user_parter_loop", self.server_name, self._user_parter_loop
+            self.hs.run_as_background_process(
+                "user_parter_loop", self._user_parter_loop
             )
 
     async def _user_parter_loop(self) -> None: