matrix-synapse 1.139.2__cp39-abi3-manylinux_2_28_aarch64.whl → 1.140.0rc1__cp39-abi3-manylinux_2_28_aarch64.whl

synapse/handlers/room_policy.py

@@ -17,6 +17,11 @@
 import logging
 from typing import TYPE_CHECKING
 
+from signedjson.key import decode_verify_key_bytes
+from unpaddedbase64 import decode_base64
+
+from synapse.api.errors import SynapseError
+from synapse.crypto.keyring import VerifyJsonRequest
 from synapse.events import EventBase
 from synapse.types.handlers.policy_server import RECOMMENDATION_OK
 from synapse.util.stringutils import parse_and_validate_server_name
@@ -26,6 +31,9 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+POLICY_SERVER_EVENT_TYPE = "org.matrix.msc4284.policy"
+POLICY_SERVER_KEY_ID = "ed25519:policy_server"
+
 
 class RoomPolicyHandler:
     def __init__(self, hs: "HomeServer"):
@@ -54,11 +62,11 @@ class RoomPolicyHandler:
         Returns:
             bool: True if the event is allowed in the room, False otherwise.
         """
-        if event.type == "org.matrix.msc4284.policy" and event.state_key is not None:
+        if event.type == POLICY_SERVER_EVENT_TYPE and event.state_key is not None:
             return True  # always allow policy server change events
 
         policy_event = await self._storage_controllers.state.get_current_state_event(
-            event.room_id, "org.matrix.msc4284.policy", ""
+            event.room_id, POLICY_SERVER_EVENT_TYPE, ""
         )
         if not policy_event:
             return True  # no policy server == default allow
@@ -81,6 +89,22 @@ class RoomPolicyHandler:
         if not is_in_room:
             return True  # policy server not in room == default allow
 
+        # Check if the event has been signed with the public key in the policy server state event.
+        # If it is, we can save an HTTP hit.
+        # We actually want to get the policy server state event BEFORE THE EVENT rather than
+        # the current state value, else changing the public key will cause all of these checks to fail.
+        # However, if we are checking outlier events (which we will due to is_event_allowed being called
+        # near the edges at _check_sigs_and_hash) we won't know the state before the event, so the
+        # only safe option is to use the current state
+        public_key = policy_event.content.get("public_key", None)
+        if public_key is not None and isinstance(public_key, str):
+            valid = await self._verify_policy_server_signature(
+                event, policy_server, public_key
+            )
+            if valid:
+                return True
+            # fallthrough to hit /check manually
+
         # At this point, the server appears valid and is in the room, so ask it to check
         # the event.
         recommendation = await self._federation_client.get_pdu_policy_recommendation(
@@ -90,3 +114,73 @@ class RoomPolicyHandler:
             return False
 
         return True  # default allow
+
+    async def _verify_policy_server_signature(
+        self, event: EventBase, policy_server: str, public_key: str
+    ) -> bool:
+        # check the event is signed with this (via, public_key).
+        verify_json_req = VerifyJsonRequest.from_event(policy_server, event, 0)
+        try:
+            key_bytes = decode_base64(public_key)
+            verify_key = decode_verify_key_bytes(POLICY_SERVER_KEY_ID, key_bytes)
+            # We would normally use KeyRing.verify_event_for_server but we can't here as we don't
+            # want to fetch the server key, and instead want to use the public key in the state event.
+            await self._hs.get_keyring().process_json(verify_key, verify_json_req)
+            # if the event is correctly signed by the public key in the policy server state event = Allow
+            return True
+        except Exception as ex:
+            logger.warning(
+                "failed to verify event using public key in policy server event: %s", ex
+            )
+        return False
+
+    async def ask_policy_server_to_sign_event(
+        self, event: EventBase, verify: bool = False
+    ) -> None:
+        """Ask the policy server to sign this event. The signature is added to the event signatures block.
+
+        Does nothing if there is no policy server state event in the room. If the policy server
+        refuses to sign the event (as it's marked as spam) does nothing.
+
+        Args:
+            event: The event to sign
+            verify: If True, verify that the signature is correctly signed by the public_key in the
+            policy server state event.
+        Raises:
+            if verify=True and the policy server signed the event with an invalid signature. Does
+            not raise if the policy server refuses to sign the event.
+        """
+        policy_event = await self._storage_controllers.state.get_current_state_event(
+            event.room_id, POLICY_SERVER_EVENT_TYPE, ""
+        )
+        if not policy_event:
+            return
+        policy_server = policy_event.content.get("via", None)
+        if policy_server is None or not isinstance(policy_server, str):
+            return
+        # Only ask to sign events if the policy state event has a public_key (so they can be subsequently verified)
+        public_key = policy_event.content.get("public_key", None)
+        if public_key is None or not isinstance(public_key, str):
+            return
+
+        # Ask the policy server to sign this event.
+        # We set a smallish timeout here as we don't want to block event sending too long.
+        signature = await self._federation_client.ask_policy_server_to_sign_event(
+            policy_server,
+            event,
+            timeout=3000,
+        )
+        if (
+            # the policy server returns {} if it refuses to sign the event.
+            signature and len(signature) > 0
+        ):
+            event.signatures.update(signature)
+            if verify:
+                is_valid = await self._verify_policy_server_signature(
+                    event, policy_server, public_key
+                )
+                if not is_valid:
+                    raise SynapseError(
+                        500,
+                        f"policy server {policy_server} failed to sign event correctly",
+                    )

synapse/handlers/sso.py

@@ -224,7 +224,7 @@ class SsoHandler:
         )
 
         # a lock on the mappings
-        self._mapping_lock = Linearizer(name="sso_user_mapping", clock=hs.get_clock())
+        self._mapping_lock = Linearizer(clock=hs.get_clock(), name="sso_user_mapping")
 
         # a map from session id to session data
         self._username_mapping_sessions: Dict[str, UsernameMappingSession] = {}

synapse/handlers/stats.py

@@ -33,7 +33,6 @@ from typing import (
 
 from synapse.api.constants import EventContentFields, EventTypes, Membership
 from synapse.metrics import SERVER_NAME_LABEL, event_processing_positions
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.databases.main.state_deltas import StateDelta
 from synapse.types import JsonDict
 from synapse.util.events import get_plain_text_topic_from_event_content
@@ -75,7 +74,10 @@ class StatsHandler:
 
             # We kick this off so that we don't have to wait for a change before
             # we start populating stats
-            self.clock.call_later(0, self.notify_new_event)
+            self.clock.call_later(
+                0,
+                self.notify_new_event,
+            )
 
     def notify_new_event(self) -> None:
         """Called when there may be more deltas to process"""
@@ -90,7 +92,7 @@ class StatsHandler:
             finally:
                 self._is_processing = False
 
-        run_as_background_process("stats.notify_new_event", self.server_name, process)
+        self.hs.run_as_background_process("stats.notify_new_event", process)
 
     async def _unsafe_process(self) -> None:
         # If self.pos is None then means we haven't fetched it from DB

synapse/handlers/sync.py

@@ -323,6 +323,7 @@ class SyncHandler:
         ] = ExpiringCache(
             cache_name="lazy_loaded_members_cache",
             server_name=self.server_name,
+            hs=hs,
             clock=self.clock,
             max_len=0,
             expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE,
@@ -552,7 +553,7 @@ class SyncHandler:
         Returns:
             A tuple of the now StreamToken, updated to reflect the which typing
             events are included, and a dict mapping from room_id to a list of
-            typing events for that room.
+            ephemeral events for that room.
         """
 
         sync_config = sync_result_builder.sync_config
@@ -577,12 +578,8 @@ class SyncHandler:
             ephemeral_by_room: JsonDict = {}
 
             for event in typing:
-                # we want to exclude the room_id from the event, but modifying the
-                # result returned by the event source is poor form (it might cache
-                # the object)
                 room_id = event["room_id"]
-                event_copy = {k: v for (k, v) in event.items() if k != "room_id"}
-                ephemeral_by_room.setdefault(room_id, []).append(event_copy)
+                ephemeral_by_room.setdefault(room_id, []).append(event)
 
             receipt_key = (
                 since_token.receipt_key
@@ -602,9 +599,7 @@ class SyncHandler:
 
             for event in receipts:
                 room_id = event["room_id"]
-                # exclude room id, as above
-                event_copy = {k: v for (k, v) in event.items() if k != "room_id"}
-                ephemeral_by_room.setdefault(room_id, []).append(event_copy)
+                ephemeral_by_room.setdefault(room_id, []).append(event)
 
         return now_token, ephemeral_by_room
 
@@ -980,7 +975,11 @@ class SyncHandler:
         )
         if cache is None:
             logger.debug("creating LruCache for %r", cache_key)
-            cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE)
+            cache = LruCache(
+                max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE,
+                clock=self.clock,
+                server_name=self.server_name,
+            )
             self.lazy_loaded_members_cache[cache_key] = cache
         else:
             logger.debug("found LruCache for %r", cache_key)
@@ -2729,9 +2728,17 @@ class SyncHandler:
                 )
             )
 
-            ephemeral = await sync_config.filter_collection.filter_room_ephemeral(
-                ephemeral
-            )
+            ephemeral = [
+                # per spec, ephemeral events (typing notifications and read receipts)
+                # should not have a `room_id` field when sent to clients
+                # refs:
+                # - https://spec.matrix.org/v1.16/client-server-api/#mtyping
+                # - https://spec.matrix.org/v1.16/client-server-api/#mreceipt
+                {k: v for (k, v) in event.items() if k != "room_id"}
+                for event in await sync_config.filter_collection.filter_room_ephemeral(
+                    ephemeral
+                )
+            ]
 
             if not (
                 always_include

synapse/handlers/typing.py

@@ -28,7 +28,6 @@ from synapse.api.constants import EduTypes
 from synapse.api.errors import AuthError, ShadowBanError, SynapseError
 from synapse.appservice import ApplicationService
 from synapse.metrics.background_process_metrics import (
-    run_as_background_process,
     wrap_as_background_process,
 )
 from synapse.replication.tcp.streams import TypingStream
@@ -78,11 +77,10 @@ class FollowerTypingHandler:
     """
 
     def __init__(self, hs: "HomeServer"):
+        self.hs = hs  # nb must be called this for @wrap_as_background_process
         self.store = hs.get_datastores().main
         self._storage_controllers = hs.get_storage_controllers()
-        self.server_name = (
-            hs.hostname
-        )  # nb must be called this for @wrap_as_background_process
+        self.server_name = hs.hostname
         self.clock = hs.get_clock()
         self.is_mine_id = hs.is_mine_id
         self.is_mine_server_name = hs.is_mine_server_name
@@ -144,9 +142,8 @@ class FollowerTypingHandler:
         if self.federation and self.is_mine_id(member.user_id):
             last_fed_poke = self._member_last_federation_poke.get(member, None)
             if not last_fed_poke or last_fed_poke + FEDERATION_PING_INTERVAL <= now:
-                run_as_background_process(
+                self.hs.run_as_background_process(
                     "typing._push_remote",
-                    self.server_name,
                     self._push_remote,
                     member=member,
                     typing=True,
@@ -220,9 +217,8 @@ class FollowerTypingHandler:
             self._rooms_updated.add(row.room_id)
 
             if self.federation:
-                run_as_background_process(
+                self.hs.run_as_background_process(
                     "_send_changes_in_typing_to_remotes",
-                    self.server_name,
                     self._send_changes_in_typing_to_remotes,
                     row.room_id,
                     prev_typing,
@@ -384,9 +380,8 @@ class TypingWriterHandler(FollowerTypingHandler):
     def _push_update(self, member: RoomMember, typing: bool) -> None:
         if self.hs.is_mine_id(member.user_id):
             # Only send updates for changes to our own users.
-            run_as_background_process(
+            self.hs.run_as_background_process(
                 "typing._push_remote",
-                self.server_name,
                 self._push_remote,
                 member,
                 typing,

synapse/handlers/user_directory.py

@@ -36,7 +36,6 @@ from synapse.api.constants import (
 from synapse.api.errors import Codes, SynapseError
 from synapse.handlers.state_deltas import MatchChange, StateDeltasHandler
 from synapse.metrics import SERVER_NAME_LABEL
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.databases.main.state_deltas import StateDelta
 from synapse.storage.databases.main.user_directory import SearchResult
 from synapse.storage.roommember import ProfileInfo
@@ -137,11 +136,15 @@ class UserDirectoryHandler(StateDeltasHandler):
 
             # We kick this off so that we don't have to wait for a change before
             # we start populating the user directory
-            self.clock.call_later(0, self.notify_new_event)
+            self.clock.call_later(
+                0,
+                self.notify_new_event,
+            )
 
             # Kick off the profile refresh process on startup
             self._refresh_remote_profiles_call_later = self.clock.call_later(
-                10, self.kick_off_remote_profile_refresh_process
+                10,
+                self.kick_off_remote_profile_refresh_process,
             )
 
     async def search_users(
@@ -193,9 +196,7 @@ class UserDirectoryHandler(StateDeltasHandler):
                 self._is_processing = False
 
         self._is_processing = True
-        run_as_background_process(
-            "user_directory.notify_new_event", self.server_name, process
-        )
+        self._hs.run_as_background_process("user_directory.notify_new_event", process)
 
     async def handle_local_profile_change(
         self, user_id: str, profile: ProfileInfo
@@ -609,8 +610,8 @@ class UserDirectoryHandler(StateDeltasHandler):
                 self._is_refreshing_remote_profiles = False
 
         self._is_refreshing_remote_profiles = True
-        run_as_background_process(
-            "user_directory.refresh_remote_profiles", self.server_name, process
+        self._hs.run_as_background_process(
+            "user_directory.refresh_remote_profiles", process
         )
 
     async def _unsafe_refresh_remote_profiles(self) -> None:
@@ -655,8 +656,9 @@ class UserDirectoryHandler(StateDeltasHandler):
                 if not users:
                     return
                 _, _, next_try_at_ts = users[0]
+                delay = ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2
                 self._refresh_remote_profiles_call_later = self.clock.call_later(
-                    ((next_try_at_ts - self.clock.time_msec()) // 1000) + 2,
+                    delay,
                     self.kick_off_remote_profile_refresh_process,
                 )
 
@@ -692,9 +694,8 @@ class UserDirectoryHandler(StateDeltasHandler):
                 self._is_refreshing_remote_profiles_for_servers.remove(server_name)
 
         self._is_refreshing_remote_profiles_for_servers.add(server_name)
-        run_as_background_process(
+        self._hs.run_as_background_process(
             "user_directory.refresh_remote_profiles_for_remote_server",
-            self.server_name,
             process,
         )
 

synapse/handlers/worker_lock.py

@@ -37,13 +37,13 @@ from weakref import WeakSet
 import attr
 
 from twisted.internet import defer
-from twisted.internet.interfaces import IReactorTime
 
 from synapse.logging.context import PreserveLoggingContext
 from synapse.logging.opentracing import start_active_span
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.storage.databases.main.lock import Lock, LockStore
 from synapse.util.async_helpers import timeout_deferred
+from synapse.util.clock import Clock
 from synapse.util.constants import ONE_MINUTE_SECONDS
 
 if TYPE_CHECKING:
@@ -66,10 +66,8 @@ class WorkerLocksHandler:
     """
 
     def __init__(self, hs: "HomeServer") -> None:
-        self.server_name = (
-            hs.hostname
-        )  # nb must be called this for @wrap_as_background_process
-        self._reactor = hs.get_reactor()
+        self.hs = hs  # nb must be called this for @wrap_as_background_process
+        self._clock = hs.get_clock()
         self._store = hs.get_datastores().main
         self._clock = hs.get_clock()
         self._notifier = hs.get_notifier()
@@ -98,7 +96,7 @@ class WorkerLocksHandler:
         """
 
         lock = WaitingLock(
-            reactor=self._reactor,
+            clock=self._clock,
             store=self._store,
             handler=self,
             lock_name=lock_name,
@@ -129,7 +127,7 @@ class WorkerLocksHandler:
         """
 
         lock = WaitingLock(
-            reactor=self._reactor,
+            clock=self._clock,
             store=self._store,
             handler=self,
             lock_name=lock_name,
@@ -160,7 +158,7 @@ class WorkerLocksHandler:
         lock = WaitingMultiLock(
             lock_names=lock_names,
             write=write,
-            reactor=self._reactor,
+            clock=self._clock,
             store=self._store,
             handler=self,
         )
@@ -197,7 +195,11 @@ class WorkerLocksHandler:
                 if not deferred.called:
                     deferred.callback(None)
 
-        self._clock.call_later(0, _wake_all_locks, locks)
+        self._clock.call_later(
+            0,
+            _wake_all_locks,
+            locks,
+        )
 
     @wrap_as_background_process("_cleanup_locks")
     async def _cleanup_locks(self) -> None:
@@ -207,7 +209,7 @@ class WorkerLocksHandler:
 
 @attr.s(auto_attribs=True, eq=False)
 class WaitingLock:
-    reactor: IReactorTime
+    clock: Clock
     store: LockStore
     handler: WorkerLocksHandler
     lock_name: str
@@ -246,10 +248,11 @@ class WaitingLock:
                     # periodically wake up in case the lock was released but we
                     # weren't notified.
                     with PreserveLoggingContext():
+                        timeout = self._get_next_retry_interval()
                         await timeout_deferred(
                             deferred=self.deferred,
-                            timeout=self._get_next_retry_interval(),
-                            reactor=self.reactor,
+                            timeout=timeout,
+                            clock=self.clock,
                         )
                 except Exception:
                     pass
@@ -290,7 +293,7 @@ class WaitingMultiLock:
 
     write: bool
 
-    reactor: IReactorTime
+    clock: Clock
     store: LockStore
     handler: WorkerLocksHandler
 
@@ -323,10 +326,11 @@ class WaitingMultiLock:
                     # periodically wake up in case the lock was released but we
                     # weren't notified.
                     with PreserveLoggingContext():
+                        timeout = self._get_next_retry_interval()
                         await timeout_deferred(
                             deferred=self.deferred,
-                            timeout=self._get_next_retry_interval(),
-                            reactor=self.reactor,
+                            timeout=timeout,
+                            clock=self.clock,
                         )
                 except Exception:
                     pass

synapse/http/client.py

@@ -54,7 +54,6 @@ from twisted.internet.interfaces import (
     IOpenSSLContextFactory,
     IReactorCore,
     IReactorPluggableNameResolver,
-    IReactorTime,
     IResolutionReceiver,
     ITCPTransport,
 )
@@ -88,6 +87,7 @@ from synapse.logging.opentracing import set_tag, start_active_span, tags
 from synapse.metrics import SERVER_NAME_LABEL
 from synapse.types import ISynapseReactor, StrSequence
 from synapse.util.async_helpers import timeout_deferred
+from synapse.util.clock import Clock
 from synapse.util.json import json_decoder
 
 if TYPE_CHECKING:
@@ -165,16 +165,17 @@ def _is_ip_blocked(
 _EPSILON = 0.00000001
 
 
-def _make_scheduler(
-    reactor: IReactorTime,
-) -> Callable[[Callable[[], object]], IDelayedCall]:
+def _make_scheduler(clock: Clock) -> Callable[[Callable[[], object]], IDelayedCall]:
     """Makes a schedular suitable for a Cooperator using the given reactor.
 
     (This is effectively just a copy from `twisted.internet.task`)
     """
 
     def _scheduler(x: Callable[[], object]) -> IDelayedCall:
-        return reactor.callLater(_EPSILON, x)
+        return clock.call_later(
+            _EPSILON,
+            x,
+        )
 
     return _scheduler
 
@@ -367,7 +368,7 @@ class BaseHttpClient:
 
         # We use this for our body producers to ensure that they use the correct
         # reactor.
-        self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_reactor()))
+        self._cooperator = Cooperator(scheduler=_make_scheduler(hs.get_clock()))
 
     async def request(
         self,
@@ -436,9 +437,9 @@ class BaseHttpClient:
                 # we use our own timeout mechanism rather than treq's as a workaround
                 # for https://twistedmatrix.com/trac/ticket/9534.
                 request_deferred = timeout_deferred(
-                    request_deferred,
-                    60,
-                    self.hs.get_reactor(),
+                    deferred=request_deferred,
+                    timeout=60,
+                    clock=self.hs.get_clock(),
                 )
 
                 # turn timeouts into RequestTimedOutErrors
@@ -763,7 +764,11 @@ class BaseHttpClient:
             d = read_body_with_max_size(response, output_stream, max_size)
 
             # Ensure that the body is not read forever.
-            d = timeout_deferred(d, 30, self.hs.get_reactor())
+            d = timeout_deferred(
+                deferred=d,
+                timeout=30,
+                clock=self.hs.get_clock(),
+            )
 
             length = await make_deferred_yieldable(d)
         except BodyExceededMaxSize:
@@ -957,9 +962,9 @@ class ReplicationClient(BaseHttpClient):
                 # for https://twistedmatrix.com/trac/ticket/9534.
                 # (Updated url https://github.com/twisted/twisted/issues/9534)
                 request_deferred = timeout_deferred(
-                    request_deferred,
-                    60,
-                    self.hs.get_reactor(),
+                    deferred=request_deferred,
+                    timeout=60,
+                    clock=self.hs.get_clock(),
                 )
 
                 # turn timeouts into RequestTimedOutErrors

synapse/http/federation/matrix_federation_agent.py

@@ -67,6 +67,9 @@ class MatrixFederationAgent:
     Args:
         reactor: twisted reactor to use for underlying requests
 
+        clock: Internal `HomeServer` clock used to track delayed and looping calls.
+            Should be obtained from `hs.get_clock()`.
+
         tls_client_options_factory:
             factory to use for fetching client tls options, or none to disable TLS.
 
@@ -97,6 +100,7 @@ class MatrixFederationAgent:
         *,
         server_name: str,
         reactor: ISynapseReactor,
+        clock: Clock,
         tls_client_options_factory: Optional[FederationPolicyForHTTPS],
         user_agent: bytes,
         ip_allowlist: Optional[IPSet],
@@ -109,6 +113,7 @@ class MatrixFederationAgent:
         Args:
             server_name: Our homeserver name (used to label metrics) (`hs.hostname`).
             reactor
+            clock: Should be the `hs` clock from `hs.get_clock()`
             tls_client_options_factory
             user_agent
             ip_allowlist
@@ -124,7 +129,6 @@ class MatrixFederationAgent:
         # addresses, to prevent DNS rebinding.
         reactor = BlocklistingReactorWrapper(reactor, ip_allowlist, ip_blocklist)
 
-        self._clock = Clock(reactor)
         self._pool = HTTPConnectionPool(reactor)
         self._pool.retryAutomatically = False
         self._pool.maxPersistentPerHost = 5
@@ -147,6 +151,7 @@ class MatrixFederationAgent:
             _well_known_resolver = WellKnownResolver(
                 server_name=server_name,
                 reactor=reactor,
+                clock=clock,
                 agent=BlocklistingAgentWrapper(
                     ProxyAgent(
                         reactor=reactor,

synapse/http/federation/well_known_resolver.py

@@ -90,6 +90,7 @@ class WellKnownResolver:
         self,
         server_name: str,
         reactor: ISynapseThreadlessReactor,
+        clock: Clock,
         agent: IAgent,
         user_agent: bytes,
         well_known_cache: Optional[TTLCache[bytes, Optional[bytes]]] = None,
@@ -99,6 +100,7 @@ class WellKnownResolver:
         Args:
             server_name: Our homeserver name (used to label metrics) (`hs.hostname`).
             reactor
+            clock: Should be the `hs` clock from `hs.get_clock()`
             agent
             user_agent
             well_known_cache
@@ -107,7 +109,7 @@ class WellKnownResolver:
 
         self.server_name = server_name
         self._reactor = reactor
-        self._clock = Clock(reactor)
+        self._clock = clock
 
         if well_known_cache is None:
             well_known_cache = TTLCache(