matrix-synapse 1.139.2__cp39-abi3-macosx_11_0_arm64.whl → 1.140.0rc1__cp39-abi3-macosx_11_0_arm64.whl

synapse/replication/tcp/client.py

@@ -32,7 +32,6 @@ from synapse.api.constants import EventTypes, Membership, ReceiptTypes
 from synapse.federation import send_queue
 from synapse.federation.sender import FederationSender
 from synapse.logging.context import PreserveLoggingContext, make_deferred_yieldable
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.tcp.streams import (
     AccountDataStream,
     DeviceListsStream,
@@ -344,7 +343,9 @@ class ReplicationDataHandler:
         # to wedge here forever.
         deferred: "Deferred[None]" = Deferred()
         deferred = timeout_deferred(
-            deferred, _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS, self._reactor
+            deferred=deferred,
+            timeout=_WAIT_FOR_REPLICATION_TIMEOUT_SECONDS,
+            clock=self._clock,
         )
 
         waiting_list = self._streams_to_waiters.setdefault(
@@ -436,7 +437,9 @@ class FederationSenderHandler:
         # to. This is always set before we use it.
         self.federation_position: Optional[int] = None
 
-        self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer")
+        self._fed_position_linearizer = Linearizer(
+            name="_fed_position_linearizer", clock=hs.get_clock()
+        )
 
     async def process_replication_rows(
         self, stream_name: str, token: int, rows: list
@@ -511,8 +514,8 @@ class FederationSenderHandler:
             # no need to queue up another task.
             return
 
-        run_as_background_process(
-            "_save_and_send_ack", self.server_name, self._save_and_send_ack
+        self._hs.run_as_background_process(
+            "_save_and_send_ack", self._save_and_send_ack
         )
 
     async def _save_and_send_ack(self) -> None:

synapse/replication/tcp/handler.py

@@ -41,7 +41,6 @@ from prometheus_client import Counter
 from twisted.internet.protocol import ReconnectingClientFactory
 
 from synapse.metrics import SERVER_NAME_LABEL, LaterGauge
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.tcp.commands import (
     ClearUserSyncsCommand,
     Command,
@@ -132,6 +131,7 @@ class ReplicationCommandHandler:
 
     def __init__(self, hs: "HomeServer"):
         self.server_name = hs.hostname
+        self.hs = hs
         self._replication_data_handler = hs.get_replication_data_handler()
         self._presence_handler = hs.get_presence_handler()
         self._store = hs.get_datastores().main
@@ -361,9 +361,8 @@ class ReplicationCommandHandler:
             return
 
         # fire off a background process to start processing the queue.
-        run_as_background_process(
+        self.hs.run_as_background_process(
             "process-replication-data",
-            self.server_name,
             self._unsafe_process_queue,
             stream_name,
         )

synapse/replication/tcp/protocol.py

@@ -42,7 +42,6 @@ from synapse.logging.context import PreserveLoggingContext
 from synapse.metrics import SERVER_NAME_LABEL, LaterGauge
 from synapse.metrics.background_process_metrics import (
     BackgroundProcessLoggingContext,
-    run_as_background_process,
 )
 from synapse.replication.tcp.commands import (
     VALID_CLIENT_COMMANDS,
@@ -140,9 +139,14 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
     max_line_buffer = 10000
 
     def __init__(
-        self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler"
+        self,
+        hs: "HomeServer",
+        server_name: str,
+        clock: Clock,
+        handler: "ReplicationCommandHandler",
     ):
         self.server_name = server_name
+        self.hs = hs
         self.clock = clock
         self.command_handler = handler
 
@@ -290,9 +294,8 @@ class BaseReplicationStreamProtocol(LineOnlyReceiver):
             # if so.
 
             if isawaitable(res):
-                run_as_background_process(
+                self.hs.run_as_background_process(
                     "replication-" + cmd.get_logcontext_id(),
-                    self.server_name,
                     lambda: res,
                 )
 
@@ -470,9 +473,13 @@ class ServerReplicationStreamProtocol(BaseReplicationStreamProtocol):
     VALID_OUTBOUND_COMMANDS = VALID_SERVER_COMMANDS
 
     def __init__(
-        self, server_name: str, clock: Clock, handler: "ReplicationCommandHandler"
+        self,
+        hs: "HomeServer",
+        server_name: str,
+        clock: Clock,
+        handler: "ReplicationCommandHandler",
     ):
-        super().__init__(server_name, clock, handler)
+        super().__init__(hs, server_name, clock, handler)
 
         self.server_name = server_name
 
@@ -497,7 +504,7 @@ class ClientReplicationStreamProtocol(BaseReplicationStreamProtocol):
         clock: Clock,
         command_handler: "ReplicationCommandHandler",
     ):
-        super().__init__(server_name, clock, command_handler)
+        super().__init__(hs, server_name, clock, command_handler)
 
         self.client_name = client_name
         self.server_name = server_name

synapse/replication/tcp/redis.py

@@ -40,7 +40,6 @@ from synapse.logging.context import PreserveLoggingContext, make_deferred_yielda
 from synapse.metrics import SERVER_NAME_LABEL
 from synapse.metrics.background_process_metrics import (
     BackgroundProcessLoggingContext,
-    run_as_background_process,
     wrap_as_background_process,
 )
 from synapse.replication.tcp.commands import (
@@ -109,6 +108,7 @@ class RedisSubscriber(SubscriberProtocol):
     """
 
     server_name: str
+    hs: "HomeServer"
     synapse_handler: "ReplicationCommandHandler"
     synapse_stream_prefix: str
     synapse_channel_names: List[str]
@@ -146,9 +146,7 @@ class RedisSubscriber(SubscriberProtocol):
     def connectionMade(self) -> None:
         logger.info("Connected to redis")
         super().connectionMade()
-        run_as_background_process(
-            "subscribe-replication", self.server_name, self._send_subscribe
-        )
+        self.hs.run_as_background_process("subscribe-replication", self._send_subscribe)
 
     async def _send_subscribe(self) -> None:
         # it's important to make sure that we only send the REPLICATE command once we
@@ -223,8 +221,8 @@ class RedisSubscriber(SubscriberProtocol):
         # if so.
 
         if isawaitable(res):
-            run_as_background_process(
-                "replication-" + cmd.get_logcontext_id(), self.server_name, lambda: res
+            self.hs.run_as_background_process(
+                "replication-" + cmd.get_logcontext_id(), lambda: res
             )
 
     def connectionLost(self, reason: Failure) -> None:  # type: ignore[override]
@@ -245,11 +243,17 @@ class RedisSubscriber(SubscriberProtocol):
         Args:
             cmd: The command to send
         """
-        run_as_background_process(
+        self.hs.run_as_background_process(
             "send-cmd",
-            self.server_name,
             self._async_send_command,
             cmd,
+            # We originally started tracing background processes to avoid `There was no
+            # active span` errors but this change meant we started generating 15x the
+            # number of spans than before (this is one of the most heavily called
+            # instances of `run_as_background_process`).
+            #
+            # Since we don't log or tag a tracing span in the downstream
+            # code, we can safely disable this.
             bg_start_span=False,
         )
 
@@ -310,9 +314,8 @@ class SynapseRedisFactory(RedisFactory):
             convertNumbers=convertNumbers,
         )
 
-        self.server_name = (
-            hs.hostname
-        )  # nb must be called this for @wrap_as_background_process
+        self.hs = hs  # nb must be called this for @wrap_as_background_process
+        self.server_name = hs.hostname
 
         hs.get_clock().looping_call(self._send_ping, 30 * 1000)
 
@@ -390,6 +393,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory):
         )
 
         self.server_name = hs.hostname
+        self.hs = hs
         self.synapse_handler = hs.get_replication_command_handler()
         self.synapse_stream_prefix = hs.hostname
         self.synapse_channel_names = channel_names
@@ -405,6 +409,7 @@ class RedisDirectTcpReplicationClientFactory(SynapseRedisFactory):
         # the base method does some other things than just instantiating the
         # protocol.
         p.server_name = self.server_name
+        p.hs = self.hs
         p.synapse_handler = self.synapse_handler
         p.synapse_outbound_redis_connection = self.synapse_outbound_redis_connection
         p.synapse_stream_prefix = self.synapse_stream_prefix

synapse/replication/tcp/resource.py

@@ -30,7 +30,6 @@ from twisted.internet.interfaces import IAddress
 from twisted.internet.protocol import ServerFactory
 
 from synapse.metrics import SERVER_NAME_LABEL
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.tcp.commands import PositionCommand
 from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol
 from synapse.replication.tcp.streams import EventsStream
@@ -55,6 +54,7 @@ class ReplicationStreamProtocolFactory(ServerFactory):
     def __init__(self, hs: "HomeServer"):
         self.command_handler = hs.get_replication_command_handler()
         self.clock = hs.get_clock()
+        self.hs = hs
         self.server_name = hs.config.server.server_name
 
         # If we've created a `ReplicationStreamProtocolFactory` then we're
@@ -69,7 +69,7 @@ class ReplicationStreamProtocolFactory(ServerFactory):
 
     def buildProtocol(self, addr: IAddress) -> ServerReplicationStreamProtocol:
         return ServerReplicationStreamProtocol(
-            self.server_name, self.clock, self.command_handler
+            self.hs, self.server_name, self.clock, self.command_handler
         )
 
 
@@ -82,6 +82,7 @@ class ReplicationStreamer:
 
     def __init__(self, hs: "HomeServer"):
         self.server_name = hs.hostname
+        self.hs = hs
         self.store = hs.get_datastores().main
         self.clock = hs.get_clock()
         self.notifier = hs.get_notifier()
@@ -147,8 +148,8 @@ class ReplicationStreamer:
             logger.debug("Notifier poke loop already running")
             return
 
-        run_as_background_process(
-            "replication_notifier", self.server_name, self._run_notifier_loop
+        self.hs.run_as_background_process(
+            "replication_notifier", self._run_notifier_loop
         )
 
     async def _run_notifier_loop(self) -> None:

synapse/replication/tcp/streams/__init__.py

@@ -77,6 +77,7 @@ STREAMS_MAP = {
 __all__ = [
     "STREAMS_MAP",
     "Stream",
+    "EventsStream",
     "BackfillStream",
     "PresenceStream",
     "PresenceFederationStream",
@@ -87,6 +88,7 @@ __all__ = [
     "CachesStream",
     "DeviceListsStream",
     "ToDeviceStream",
+    "FederationStream",
     "AccountDataStream",
     "ThreadSubscriptionsStream",
     "UnPartialStatedRoomStream",

synapse/res/providers.json

@@ -1,14 +1,15 @@
 [
     {
-        "provider_name": "Twitter",
-        "provider_url": "http://www.twitter.com/",
+        "provider_name": "X",
+        "provider_url": "https://x.com/",
         "endpoints": [
             {
                 "schemes": [
-                    "https://twitter.com/*/moments/*",
-                    "https://*.twitter.com/*/moments/*"
+                    "https://x.com/*",
+                    "https://x.com/*/status/*",
+                    "https://*.x.com/*/status/*"
                 ],
-                "url": "https://publish.twitter.com/oembed"
+                "url": "https://publish.x.com/oembed"
             }
         ]
     },

synapse/rest/__init__.py

@@ -42,6 +42,7 @@ from synapse.rest.client import (
     login,
     login_token_request,
     logout,
+    matrixrtc,
     mutual_rooms,
     notifications,
     openid,
@@ -89,6 +90,7 @@ CLIENT_SERVLET_FUNCTIONS: Tuple[RegisterServletsFunc, ...] = (
     presence.register_servlets,
     directory.register_servlets,
     voip.register_servlets,
+    matrixrtc.register_servlets,
     pusher.register_servlets,
     push_rule.register_servlets,
     logout.register_servlets,

synapse/rest/admin/__init__.py

@@ -57,6 +57,9 @@ from synapse.rest.admin.event_reports import (
     EventReportDetailRestServlet,
     EventReportsRestServlet,
 )
+from synapse.rest.admin.events import (
+    EventRestServlet,
+)
 from synapse.rest.admin.experimental_features import ExperimentalFeaturesRestServlet
 from synapse.rest.admin.federation import (
     DestinationMembershipRestServlet,
@@ -339,6 +342,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
     ExperimentalFeaturesRestServlet(hs).register(http_server)
     SuspendAccountRestServlet(hs).register(http_server)
     ScheduledTasksRestServlet(hs).register(http_server)
+    EventRestServlet(hs).register(http_server)
 
 
 def register_servlets_for_client_rest_resource(

synapse/rest/admin/events.py

@@ -0,0 +1,69 @@
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Tuple
+
+from synapse.api.errors import NotFoundError
+from synapse.events.utils import (
+    SerializeEventConfig,
+    format_event_raw,
+    serialize_event,
+)
+from synapse.http.servlet import RestServlet
+from synapse.http.site import SynapseRequest
+from synapse.rest.admin import admin_patterns
+from synapse.rest.admin._base import assert_user_is_admin
+from synapse.storage.databases.main.events_worker import EventRedactBehaviour
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+class EventRestServlet(RestServlet):
+    """
+    Get an event that is known to the homeserver.
+    The requester must have administrator access in Synapse.
+
+    GET /_synapse/admin/v1/fetch_event/<event_id>
+    returns:
+        200 OK with event json if the event is known to the homeserver. Otherwise raises
+        a NotFound error.
+
+    Args:
+        event_id: the id of the requested event.
+    Returns:
+        JSON blob of the event
+    """
+
+    PATTERNS = admin_patterns("/fetch_event/(?P<event_id>[^/]*)$")
+
+    def __init__(self, hs: "HomeServer"):
+        self._auth = hs.get_auth()
+        self._store = hs.get_datastores().main
+        self._clock = hs.get_clock()
+
+    async def on_GET(
+        self, request: SynapseRequest, event_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self._auth.get_user_by_req(request)
+        await assert_user_is_admin(self._auth, requester)
+
+        event = await self._store.get_event(
+            event_id,
+            EventRedactBehaviour.as_is,
+            allow_none=True,
+        )
+
+        if event is None:
+            raise NotFoundError("Event not found")
+
+        config = SerializeEventConfig(
+            as_client_event=False,
+            event_format=format_event_raw,
+            requester=requester,
+            only_event_fields=None,
+            include_stripped_room_state=True,
+            include_admin_metadata=True,
+        )
+        res = {"event": serialize_event(event, self._clock.time_msec(), config=config)}
+
+        return HTTPStatus.OK, res

synapse/rest/admin/media.py

@@ -18,7 +18,6 @@
 # [This file includes modifications made by New Vector Limited]
 #
 #
-
 import logging
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Optional, Tuple
@@ -41,7 +40,9 @@ from synapse.rest.admin._base import (
     assert_requester_is_admin,
     assert_user_is_admin,
 )
-from synapse.storage.databases.main.media_repository import MediaSortOrder
+from synapse.storage.databases.main.media_repository import (
+    MediaSortOrder,
+)
 from synapse.types import JsonDict, UserID
 
 if TYPE_CHECKING:
@@ -50,6 +51,72 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class QueryMediaById(RestServlet):
+    """
+    Fetch info about a piece of local or cached remote media.
+    """
+
+    PATTERNS = admin_patterns("/media/(?P<server_name>[^/]*)/(?P<media_id>[^/]*)$")
+
+    def __init__(self, hs: "HomeServer"):
+        self.store = hs.get_datastores().main
+        self.auth = hs.get_auth()
+        self.server_name = hs.hostname
+        self.hs = hs
+        self.media_repo = hs.get_media_repository()
+
+    async def on_GET(
+        self, request: SynapseRequest, server_name: str, media_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self.auth.get_user_by_req(request)
+        await assert_user_is_admin(self.auth, requester)
+
+        if not self.hs.is_mine_server_name(server_name):
+            remote_media_info = await self.media_repo.get_cached_remote_media_info(
+                server_name, media_id
+            )
+            if remote_media_info is None:
+                raise NotFoundError("Unknown media")
+            resp = {
+                "media_origin": remote_media_info.media_origin,
+                "user_id": None,
+                "media_id": remote_media_info.media_id,
+                "media_type": remote_media_info.media_type,
+                "media_length": remote_media_info.media_length,
+                "upload_name": remote_media_info.upload_name,
+                "created_ts": remote_media_info.created_ts,
+                "filesystem_id": remote_media_info.filesystem_id,
+                "url_cache": None,
+                "last_access_ts": remote_media_info.last_access_ts,
+                "quarantined_by": remote_media_info.quarantined_by,
+                "authenticated": remote_media_info.authenticated,
+                "safe_from_quarantine": None,
+                "sha256": remote_media_info.sha256,
+            }
+        else:
+            local_media_info = await self.store.get_local_media(media_id)
+            if local_media_info is None:
+                raise NotFoundError("Unknown media")
+            resp = {
+                "media_origin": None,
+                "user_id": local_media_info.user_id,
+                "media_id": local_media_info.media_id,
+                "media_type": local_media_info.media_type,
+                "media_length": local_media_info.media_length,
+                "upload_name": local_media_info.upload_name,
+                "created_ts": local_media_info.created_ts,
+                "filesystem_id": None,
+                "url_cache": local_media_info.url_cache,
+                "last_access_ts": local_media_info.last_access_ts,
+                "quarantined_by": local_media_info.quarantined_by,
+                "authenticated": local_media_info.authenticated,
+                "safe_from_quarantine": local_media_info.safe_from_quarantine,
+                "sha256": local_media_info.sha256,
+            }
+
+        return HTTPStatus.OK, {"media_info": resp}
+
+
 class QuarantineMediaInRoom(RestServlet):
     """Quarantines all media in a room so that no one can download it via
     this server.
@@ -470,3 +537,4 @@ def register_servlets_for_media_repo(hs: "HomeServer", http_server: HttpServer)
     DeleteMediaByDateSize(hs).register(http_server)
     DeleteMediaByID(hs).register(http_server)
     UserMediaRestServlet(hs).register(http_server)
+    QueryMediaById(hs).register(http_server)

synapse/rest/client/matrixrtc.py

@@ -0,0 +1,52 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2025 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+from typing import TYPE_CHECKING, Tuple
+
+from synapse.http.server import HttpServer
+from synapse.http.servlet import RestServlet
+from synapse.http.site import SynapseRequest
+from synapse.rest.client._base import client_patterns
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+class MatrixRTCRestServlet(RestServlet):
+    PATTERNS = client_patterns(r"/org\.matrix\.msc4143/rtc/transports$", releases=())
+    CATEGORY = "Client API requests"
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        self._hs = hs
+        self._auth = hs.get_auth()
+        self._transports = hs.config.matrix_rtc.transports
+
+    async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
+        # Require authentication for this endpoint.
+        await self._auth.get_user_by_req(request)
+
+        if self._transports:
+            return 200, {"rtc_transports": self._transports}
+
+        return 200, {}
+
+
+def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
+    if hs.config.experimental.msc4143_enabled:
+        MatrixRTCRestServlet(hs).register(http_server)

synapse/rest/client/push_rule.py

@@ -65,7 +65,7 @@ class PushRuleRestServlet(RestServlet):
             hs.get_instance_name() in hs.config.worker.writers.push_rules
         )
         self._push_rules_handler = hs.get_push_rules_handler()
-        self._push_rule_linearizer = Linearizer(name="push_rules")
+        self._push_rule_linearizer = Linearizer(name="push_rules", clock=hs.get_clock())
 
     async def on_PUT(self, request: SynapseRequest, path: str) -> Tuple[int, JsonDict]:
         if not self._is_push_worker:

synapse/rest/client/room.py

@@ -66,7 +66,6 @@ from synapse.http.site import SynapseRequest
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.logging.opentracing import set_tag
 from synapse.metrics import SERVER_NAME_LABEL
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.rest.client._base import client_patterns
 from synapse.rest.client.transactions import HttpTransactionCache
 from synapse.state import CREATE_KEY, POWER_KEY
@@ -1225,6 +1224,7 @@ class RoomRedactEventRestServlet(TransactionRestServlet):
     def __init__(self, hs: "HomeServer"):
         super().__init__(hs)
         self.server_name = hs.hostname
+        self.hs = hs
         self.event_creation_handler = hs.get_event_creation_handler()
         self.auth = hs.get_auth()
         self._store = hs.get_datastores().main
@@ -1307,9 +1307,8 @@ class RoomRedactEventRestServlet(TransactionRestServlet):
                 )
 
                 if with_relations:
-                    run_as_background_process(
+                    self.hs.run_as_background_process(
                         "redact_related_events",
-                        self.server_name,
                         self._relation_handler.redact_events_related_to,
                         requester=requester,
                         event_id=event_id,

synapse/rest/client/sync.py

@@ -126,6 +126,7 @@ class SyncRestServlet(RestServlet):
 
         self._json_filter_cache: LruCache[str, bool] = LruCache(
             max_size=1000,
+            clock=self.clock,
             cache_name="sync_valid_filter",
             server_name=self.server_name,
         )

synapse/rest/client/transactions.py

@@ -56,7 +56,7 @@ class HttpTransactionCache:
         ] = {}
         # Try to clean entries every 30 mins. This means entries will exist
         # for at *LEAST* 30 mins, and at *MOST* 60 mins.
-        self.cleaner = self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS)
+        self.clock.looping_call(self._cleanup, CLEANUP_PERIOD_MS)
 
     def _get_transaction_key(self, request: IRequest, requester: Requester) -> Hashable:
         """A helper function which returns a transaction key that can be used