matrix-synapse 1.142.0rc3__cp314-abi3-musllinux_1_2_aarch64.whl

synapse/util/distributor.py

@@ -0,0 +1,159 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import logging
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Generic,
+    Optional,
+    TypeVar,
+    Union,
+)
+
+from typing_extensions import ParamSpec
+
+from twisted.internet import defer
+
+from synapse.logging.context import make_deferred_yieldable, run_in_background
+from synapse.types import UserID
+from synapse.util.async_helpers import maybe_awaitable
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+logger = logging.getLogger(__name__)
+
+
+def user_left_room(distributor: "Distributor", user: UserID, room_id: str) -> None:
+    distributor.fire("user_left_room", user=user, room_id=room_id)
+
+
+class Distributor:
+    """A central dispatch point for loosely-connected pieces of code to
+    register, observe, and fire signals.
+
+    Signals are named simply by strings.
+
+    TODO(paul): It would be nice to give signals stronger object identities,
+      so we can attach metadata, docstrings, detect typos, etc... But this
+      model will do for today.
+    """
+
+    def __init__(self, hs: "HomeServer") -> None:
+        """
+        Args:
+            server_name: The homeserver name of the server (used to label metrics)
+                (this should be `hs.hostname`).
+        """
+        self.hs = hs
+        self.signals: dict[str, Signal] = {}
+        self.pre_registration: dict[str, list[Callable]] = {}
+
+    def declare(self, name: str) -> None:
+        if name in self.signals:
+            raise KeyError("%r already has a signal named %s" % (self, name))
+
+        self.signals[name] = Signal(name)
+
+        if name in self.pre_registration:
+            signal = self.signals[name]
+            for observer in self.pre_registration[name]:
+                signal.observe(observer)
+
+    def observe(self, name: str, observer: Callable) -> None:
+        if name in self.signals:
+            self.signals[name].observe(observer)
+        else:
+            # TODO: Avoid strong ordering dependency by allowing people to
+            # pre-register observations on signals that don't exist yet.
+            if name not in self.pre_registration:
+                self.pre_registration[name] = []
+            self.pre_registration[name].append(observer)
+
+    def fire(self, name: str, *args: Any, **kwargs: Any) -> None:
+        """Dispatches the given signal to the registered observers.
+
+        Runs the observers as a background process. Does not return a deferred.
+        """
+        if name not in self.signals:
+            raise KeyError("%r does not have a signal named %s" % (self, name))
+
+        self.hs.run_as_background_process(
+            name, self.signals[name].fire, *args, **kwargs
+        )
+
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+class Signal(Generic[P]):
+    """A Signal is a dispatch point that stores a list of callables as
+    observers of it.
+
+    Signals can be "fired", meaning that every callable observing it is
+    invoked. Firing a signal does not change its state; it can be fired again
+    at any later point. Firing a signal passes any arguments from the fire
+    method into all of the observers.
+    """
+
+    def __init__(self, name: str):
+        self.name: str = name
+        self.observers: list[Callable[P, Any]] = []
+
+    def observe(self, observer: Callable[P, Any]) -> None:
+        """Adds a new callable to the observer list which will be invoked by
+        the 'fire' method.
+
+        Each observer callable may return a Deferred."""
+        self.observers.append(observer)
+
+    def fire(self, *args: P.args, **kwargs: P.kwargs) -> "defer.Deferred[list[Any]]":
+        """Invokes every callable in the observer list, passing in the args and
+        kwargs. Exceptions thrown by observers are logged but ignored. It is
+        not an error to fire a signal with no observers.
+
+        Returns a Deferred that will complete when all the observers have
+        completed."""
+
+        async def do(observer: Callable[P, Union[R, Awaitable[R]]]) -> Optional[R]:
+            try:
+                return await maybe_awaitable(observer(*args, **kwargs))
+            except Exception as e:
+                logger.warning(
+                    "%s signal observer %s failed: %r",
+                    self.name,
+                    observer,
+                    e,
+                )
+                return None
+
+        deferreds = [run_in_background(do, o) for o in self.observers]
+
+        return make_deferred_yieldable(
+            defer.gatherResults(deferreds, consumeErrors=True)
+        )
+
+    def __repr__(self) -> str:
+        return "<Signal name=%r>" % (self.name,)

synapse/util/events.py

@@ -0,0 +1,134 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2024 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+#
+
+from typing import Any, Optional
+
+from pydantic import Field, StrictStr, ValidationError, field_validator
+
+from synapse.types import JsonDict
+from synapse.util.pydantic_models import ParseModel
+from synapse.util.stringutils import random_string
+
+
+def generate_fake_event_id() -> str:
+    """
+    Generate an event ID from random ASCII characters.
+
+    This is primarily useful for generating fake event IDs in response to
+    requests from shadow-banned users.
+
+    Returns:
+        A string intended to look like an event ID, but with no actual meaning.
+    """
+    return "$" + random_string(43)
+
+
+class MTextRepresentation(ParseModel):
+    """
+    See `TextualRepresentation` in the Matrix specification.
+    """
+
+    body: StrictStr
+    mimetype: Optional[StrictStr] = None
+
+
+class MTopic(ParseModel):
+    """
+    `m.room.topic` -> `content` -> `m.topic`
+
+    Textual representation of the room topic in different mimetypes. Added in Matrix v1.15.
+
+    See `TopicContentBlock` in the Matrix specification.
+    """
+
+    m_text: Optional[list[MTextRepresentation]] = Field(None, alias="m.text")
+    """
+    An ordered array of textual representations in different mimetypes.
+    """
+
+    # Because "Receivers SHOULD use the first representation in the array that they
+    # understand.", we ignore invalid representations in the `m.text` field and use
+    # what we can.
+    @field_validator("m_text", mode="before")
+    @classmethod
+    def ignore_invalid_representations(
+        cls, m_text: Any
+    ) -> Optional[list[MTextRepresentation]]:
+        if not isinstance(m_text, (list, tuple)):
+            raise ValueError("m.text must be a list or a tuple")
+        representations = []
+        for element in m_text:
+            try:
+                representations.append(MTextRepresentation.model_validate(element))
+            except ValidationError:
+                continue
+        return representations
+
+
+class TopicContent(ParseModel):
+    """
+    Represents the `content` field of an `m.room.topic` event
+    """
+
+    topic: StrictStr
+    """
+    The topic in plain text.
+    """
+
+    m_topic: Optional[MTopic] = Field(None, alias="m.topic")
+    """
+    Textual representation of the room topic in different mimetypes.
+    """
+
+    # We ignore invalid `m.topic` fields as we can always fall back to the plain-text
+    # `topic` field.
+    @field_validator("m_topic", mode="before")
+    @classmethod
+    def ignore_invalid_m_topic(cls, m_topic: Any) -> Optional[MTopic]:
+        try:
+            return MTopic.model_validate(m_topic)
+        except ValidationError:
+            return None
+
+
+def get_plain_text_topic_from_event_content(content: JsonDict) -> Optional[str]:
+    """
+    Given the `content` of an `m.room.topic` event, returns the plain-text topic
+    representation. Prefers pulling plain-text from the newer `m.topic` field if
+    available with a fallback to `topic`.
+
+    Args:
+        content: The `content` field of an `m.room.topic` event.
+
+    Returns:
+        A string representing the plain text topic.
+    """
+
+    try:
+        topic_content = TopicContent.model_validate(content, strict=False)
+    except ValidationError:
+        return None
+
+    # Find the first `text/plain` topic ("Receivers SHOULD use the first
+    # representationin the array that they understand.")
+    if topic_content.m_topic and topic_content.m_topic.m_text:
+        for representation in topic_content.m_topic.m_text:
+            # The mimetype property defaults to `text/plain` if omitted.
+            if not representation.mimetype or representation.mimetype == "text/plain":
+                return representation.body
+
+    # Fallback to the plain-old `topic` field if there isn't any `text/plain` topic
+    # representation available.
+    return topic_content.topic

synapse/util/file_consumer.py

@@ -0,0 +1,164 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import queue
+from typing import Any, BinaryIO, Optional, Union, cast
+
+from twisted.internet import threads
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IPullProducer, IPushProducer
+
+from synapse.logging.context import make_deferred_yieldable, run_in_background
+from synapse.types import ISynapseReactor
+
+
+class BackgroundFileConsumer:
+    """A consumer that writes to a file like object. Supports both push
+    and pull producers
+
+    Args:
+        file_obj: The file like object to write to. Closed when
+            finished.
+        reactor: the Twisted reactor to use
+    """
+
+    # For PushProducers pause if we have this many unwritten slices
+    _PAUSE_ON_QUEUE_SIZE = 5
+    # And resume once the size of the queue is less than this
+    _RESUME_ON_QUEUE_SIZE = 2
+
+    def __init__(self, file_obj: BinaryIO, reactor: ISynapseReactor) -> None:
+        self._file_obj: BinaryIO = file_obj
+
+        self._reactor: ISynapseReactor = reactor
+
+        # Producer we're registered with
+        self._producer: Optional[Union[IPushProducer, IPullProducer]] = None
+
+        # True if PushProducer, false if PullProducer
+        self.streaming = False
+
+        # For PushProducers, indicates whether we've paused the producer and
+        # need to call resumeProducing before we get more data.
+        self._paused_producer = False
+
+        # Queue of slices of bytes to be written. When producer calls
+        # unregister a final None is sent.
+        self._bytes_queue: queue.Queue[Optional[bytes]] = queue.Queue()
+
+        # Deferred that is resolved when finished writing
+        #
+        # This is really Deferred[None], but mypy doesn't seem to like that.
+        self._finished_deferred: Optional[Deferred[Any]] = None
+
+        # If the _writer thread throws an exception it gets stored here.
+        self._write_exception: Optional[Exception] = None
+
+    def registerProducer(
+        self, producer: Union[IPushProducer, IPullProducer], streaming: bool
+    ) -> None:
+        """Part of IConsumer interface
+
+        Args:
+            producer
+            streaming: True if push based producer, False if pull
+                based.
+        """
+        if self._producer:
+            raise Exception("registerProducer called twice")
+
+        self._producer = producer
+        self.streaming = streaming
+        self._finished_deferred = run_in_background(
+            threads.deferToThreadPool,
+            # mypy seems to get confused with the chaining of ParamSpec from
+            # run_in_background to deferToThreadPool.
+            #
+            # For Twisted trunk, ignore arg-type; for Twisted release ignore unused-ignore.
+            self._reactor,  # type: ignore[arg-type,unused-ignore]
+            self._reactor.getThreadPool(),  # type: ignore[arg-type,unused-ignore]
+            self._writer,  # type: ignore[arg-type,unused-ignore]
+        )
+        if not streaming:
+            self._producer.resumeProducing()
+
+    def unregisterProducer(self) -> None:
+        """Part of IProducer interface"""
+        self._producer = None
+        assert self._finished_deferred is not None
+        if not self._finished_deferred.called:
+            self._bytes_queue.put_nowait(None)
+
+    def write(self, write_bytes: bytes) -> None:
+        """Part of IProducer interface"""
+        if self._write_exception:
+            raise self._write_exception
+
+        assert self._finished_deferred is not None
+        if self._finished_deferred.called:
+            raise Exception("consumer has closed")
+
+        self._bytes_queue.put_nowait(write_bytes)
+
+        # If this is a PushProducer and the queue is getting behind
+        # then we pause the producer.
+        if self.streaming and self._bytes_queue.qsize() >= self._PAUSE_ON_QUEUE_SIZE:
+            self._paused_producer = True
+            assert self._producer is not None
+            # cast safe because `streaming` means this is an IPushProducer
+            cast(IPushProducer, self._producer).pauseProducing()
+
+    def _writer(self) -> None:
+        """This is run in a background thread to write to the file."""
+        try:
+            while self._producer or not self._bytes_queue.empty():
+                # If we've paused the producer check if we should resume the
+                # producer.
+                if self._producer and self._paused_producer:
+                    if self._bytes_queue.qsize() <= self._RESUME_ON_QUEUE_SIZE:
+                        self._reactor.callFromThread(self._resume_paused_producer)
+
+                bytes = self._bytes_queue.get()
+
+                # If we get a None (or empty list) then that's a signal used
+                # to indicate we should check if we should stop.
+                if bytes:
+                    self._file_obj.write(bytes)
+
+                # If its a pull producer then we need to explicitly ask for
+                # more stuff.
+                if not self.streaming and self._producer:
+                    self._reactor.callFromThread(self._producer.resumeProducing)
+        except Exception as e:
+            self._write_exception = e
+            raise
+        finally:
+            self._file_obj.close()
+
+    def wait(self) -> "Deferred[None]":
+        """Returns a deferred that resolves when finished writing to file"""
+        assert self._finished_deferred is not None
+        return make_deferred_yieldable(self._finished_deferred)
+
+    def _resume_paused_producer(self) -> None:
+        """Gets called if we should resume producing after being paused"""
+        if self._paused_producer and self._producer:
+            self._paused_producer = False
+            self._producer.resumeProducing()

synapse/util/frozenutils.py

@@ -0,0 +1,57 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import collections.abc
+from typing import Any
+
+from immutabledict import immutabledict
+
+
+def freeze(o: Any) -> Any:
+    if isinstance(o, dict):
+        return immutabledict({k: freeze(v) for k, v in o.items()})
+
+    if isinstance(o, immutabledict):
+        return o
+
+    if isinstance(o, (bytes, str)):
+        return o
+
+    try:
+        return tuple(freeze(i) for i in o)
+    except TypeError:
+        pass
+
+    return o
+
+
+def unfreeze(o: Any) -> Any:
+    if isinstance(o, collections.abc.Mapping):
+        return {k: unfreeze(v) for k, v in o.items()}
+
+    if isinstance(o, (bytes, str)):
+        return o
+
+    try:
+        return [unfreeze(i) for i in o]
+    except TypeError:
+        pass
+
+    return o

synapse/util/gai_resolver.py

@@ -0,0 +1,180 @@
+# This is a direct lift from
+# https://github.com/twisted/twisted/blob/release-21.2.0-10091/src/twisted/internet/_resolver.py.
+# We copy it here as we need to instantiate `GAIResolver` manually, but it is a
+# private class.
+
+from socket import (
+    AF_INET,
+    AF_INET6,
+    AF_UNSPEC,
+    SOCK_DGRAM,
+    SOCK_STREAM,
+    AddressFamily,
+    SocketKind,
+    gaierror,
+    getaddrinfo,
+)
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    NoReturn,
+    Optional,
+    Sequence,
+    Union,
+)
+
+from zope.interface import implementer
+
+from twisted.internet.address import IPv4Address, IPv6Address
+from twisted.internet.interfaces import (
+    IAddress,
+    IHostnameResolver,
+    IHostResolution,
+    IReactorThreads,
+    IResolutionReceiver,
+)
+from twisted.internet.threads import deferToThreadPool
+
+if TYPE_CHECKING:
+    # The types below are copied from
+    # https://github.com/twisted/twisted/blob/release-21.2.0-10091/src/twisted/internet/interfaces.py
+    # so that the type hints can match the interfaces.
+    from twisted.python.runtime import platform
+
+    if platform.supportsThreads():
+        from twisted.python.threadpool import ThreadPool
+    else:
+        ThreadPool = object  # type: ignore[misc, assignment]
+
+
+@implementer(IHostResolution)
+class HostResolution:
+    """
+    The in-progress resolution of a given hostname.
+    """
+
+    def __init__(self, name: str):
+        """
+        Create a L{HostResolution} with the given name.
+        """
+        self.name = name
+
+    def cancel(self) -> NoReturn:
+        # IHostResolution.cancel
+        raise NotImplementedError()
+
+
+_any = frozenset([IPv4Address, IPv6Address])
+
+_typesToAF = {
+    frozenset([IPv4Address]): AF_INET,
+    frozenset([IPv6Address]): AF_INET6,
+    _any: AF_UNSPEC,
+}
+
+_afToType = {
+    AF_INET: IPv4Address,
+    AF_INET6: IPv6Address,
+}
+
+_transportToSocket = {
+    "TCP": SOCK_STREAM,
+    "UDP": SOCK_DGRAM,
+}
+
+_socktypeToType = {
+    SOCK_STREAM: "TCP",
+    SOCK_DGRAM: "UDP",
+}
+
+
+_GETADDRINFO_RESULT = list[
+    tuple[
+        AddressFamily,
+        SocketKind,
+        int,
+        str,
+        Union[tuple[str, int], tuple[str, int, int, int], tuple[int, bytes]],
+    ]
+]
+
+
+@implementer(IHostnameResolver)
+class GAIResolver:
+    """
+    L{IHostnameResolver} implementation that resolves hostnames by calling
+    L{getaddrinfo} in a thread.
+    """
+
+    def __init__(
+        self,
+        reactor: IReactorThreads,
+        getThreadPool: Optional[Callable[[], "ThreadPool"]] = None,
+        getaddrinfo: Callable[[str, int, int, int], _GETADDRINFO_RESULT] = getaddrinfo,
+    ):
+        """
+        Create a L{GAIResolver}.
+        @param reactor: the reactor to schedule result-delivery on
+        @type reactor: L{IReactorThreads}
+        @param getThreadPool: a function to retrieve the thread pool to use for
+            scheduling name resolutions.  If not supplied, the use the given
+            C{reactor}'s thread pool.
+        @type getThreadPool: 0-argument callable returning a
+            L{twisted.python.threadpool.ThreadPool}
+        @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly
+            parameterized for testing.
+        @type getaddrinfo: callable with the same signature as L{getaddrinfo}
+        """
+        self._reactor = reactor
+        self._getThreadPool = (
+            reactor.getThreadPool if getThreadPool is None else getThreadPool
+        )
+        self._getaddrinfo = getaddrinfo
+
+    # The types on IHostnameResolver is incorrect in Twisted, see
+    # https://twistedmatrix.com/trac/ticket/10276
+    def resolveHostName(
+        self,
+        resolutionReceiver: IResolutionReceiver,
+        hostName: str,
+        portNumber: int = 0,
+        addressTypes: Optional[Sequence[type[IAddress]]] = None,
+        transportSemantics: str = "TCP",
+    ) -> IHostResolution:
+        """
+        See L{IHostnameResolver.resolveHostName}
+        @param resolutionReceiver: see interface
+        @param hostName: see interface
+        @param portNumber: see interface
+        @param addressTypes: see interface
+        @param transportSemantics: see interface
+        @return: see interface
+        """
+        pool = self._getThreadPool()
+        addressFamily = _typesToAF[
+            _any if addressTypes is None else frozenset(addressTypes)
+        ]
+        socketType = _transportToSocket[transportSemantics]
+
+        def get() -> _GETADDRINFO_RESULT:
+            try:
+                return self._getaddrinfo(
+                    hostName, portNumber, addressFamily, socketType
+                )
+            except gaierror:
+                return []
+
+        d = deferToThreadPool(self._reactor, pool, get)
+        resolution = HostResolution(hostName)
+        resolutionReceiver.resolutionBegan(resolution)
+
+        @d.addCallback
+        def deliverResults(result: _GETADDRINFO_RESULT) -> None:
+            for family, socktype, _proto, _cannoname, sockaddr in result:
+                addrType = _afToType[family]
+                resolutionReceiver.addressResolved(
+                    addrType(_socktypeToType.get(socktype, "TCP"), *sockaddr)
+                )
+            resolutionReceiver.resolutionComplete()
+
+        return resolution