matrix-synapse 1.143.0rc2__cp310-abi3-manylinux_2_28_aarch64.whl

synapse/util/daemonize.py

@@ -0,0 +1,165 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+# Copyright (c) 2012, 2013, 2014 Ilya Otyutskiy <ilya.otyutskiy@icloud.com>
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import atexit
+import fcntl
+import logging
+import os
+import signal
+import sys
+from types import FrameType, TracebackType
+from typing import NoReturn
+
+from synapse.logging.context import (
+    LoggingContext,
+    PreserveLoggingContext,
+    current_context,
+)
+
+
+def daemonize_process(pid_file: str, logger: logging.Logger, chdir: str = "/") -> None:
+    """daemonize the current process
+
+    This calls fork(), and has the main process exit. When it returns we will be
+    running in the child process.
+    """
+
+    # If pidfile already exists, we should read pid from there; to overwrite it, if
+    # locking will fail, because locking attempt somehow purges the file contents.
+    if os.path.isfile(pid_file):
+        with open(pid_file) as pid_fh:
+            old_pid = pid_fh.read()
+
+    # Create a lockfile so that only one instance of this daemon is running at any time.
+    try:
+        lock_fh = open(pid_file, "w")
+    except OSError:
+        print("Unable to create the pidfile.")
+        sys.exit(1)
+
+    try:
+        # Try to get an exclusive lock on the file. This will fail if another process
+        # has the file locked.
+        fcntl.flock(lock_fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except OSError:
+        print("Unable to lock on the pidfile.")
+        # We need to overwrite the pidfile if we got here.
+        #
+        # XXX better to avoid overwriting it, surely. this looks racey as the pid file
+        # could be created between us trying to read it and us trying to lock it.
+        with open(pid_file, "w") as pid_fh:
+            pid_fh.write(old_pid)
+        sys.exit(1)
+
+    # Stop the existing context *before* we fork the process. Otherwise the cputime
+    # metrics get confused about the per-thread resource usage appearing to go backwards
+    # because we're comparing the resource usage from the original process to the forked
+    # process. `PreserveLoggingContext` already takes care of restarting the original
+    # context *after* the block.
+    with PreserveLoggingContext():
+        # Fork, creating a new process for the child.
+        process_id = os.fork()
+
+    if process_id != 0:
+        # parent process: exit.
+
+        # we use os._exit to avoid running the atexit handlers. In particular, that
+        # means we don't flush the logs. This is important because if we are using
+        # a MemoryHandler, we could have logs buffered which are now buffered in both
+        # the main and the child process, so if we let the main process flush the logs,
+        # we'll get two copies.
+        os._exit(0)
+
+    # This is the child process. Continue.
+
+    # Stop listening for signals that the parent process receives.
+    # This is done by getting a new process id.
+    # setpgrp() is an alternative to setsid().
+    # setsid puts the process in a new parent group and detaches its controlling
+    # terminal.
+
+    os.setsid()
+
+    # point stdin, stdout, stderr at /dev/null
+    devnull = "/dev/null"
+    if hasattr(os, "devnull"):
+        # Python has set os.devnull on this system, use it instead as it might be
+        # different than /dev/null.
+        devnull = os.devnull
+
+    devnull_fd = os.open(devnull, os.O_RDWR)
+    os.dup2(devnull_fd, 0)
+    os.dup2(devnull_fd, 1)
+    os.dup2(devnull_fd, 2)
+    os.close(devnull_fd)
+
+    # now that we have redirected stderr to /dev/null, any uncaught exceptions will
+    # get sent to /dev/null, so make sure we log them.
+    #
+    # (we don't normally expect reactor.run to raise any exceptions, but this will
+    # also catch any other uncaught exceptions before we get that far.)
+
+    def excepthook(
+        type_: type[BaseException],
+        value: BaseException,
+        traceback: TracebackType | None,
+    ) -> None:
+        logger.critical("Unhanded exception", exc_info=(type_, value, traceback))
+
+    sys.excepthook = excepthook
+
+    # Set umask to default to safe file permissions when running as a root daemon. 027
+    # is an octal number which we are typing as 0o27 for Python3 compatibility.
+    os.umask(0o27)
+
+    # Change to a known directory. If this isn't done, starting a daemon in a
+    # subdirectory that needs to be deleted results in "directory busy" errors.
+    os.chdir(chdir)
+
+    try:
+        lock_fh.write("%s" % (os.getpid()))
+        lock_fh.flush()
+    except OSError:
+        logger.error("Unable to write pid to the pidfile.")
+        print("Unable to write pid to the pidfile.")
+        sys.exit(1)
+
+    # write a log line on SIGTERM.
+    def sigterm(signum: int, frame: FrameType | None) -> NoReturn:
+        logger.warning("Caught signal %s. Stopping daemon.", signum)
+        sys.exit(0)
+
+    signal.signal(signal.SIGTERM, sigterm)
+
+    # Copy the `server_name` from the current logcontext
+    server_name = current_context().server_name
+
+    # Cleanup pid file at exit.
+    def exit() -> None:
+        with LoggingContext(name="atexit", server_name=server_name):
+            logger.warning("Stopping daemon.")
+            os.remove(pid_file)
+            sys.exit(0)
+
+    atexit.register(exit)
+
+    logger.warning("Starting daemon.")

synapse/util/distributor.py

@@ -0,0 +1,157 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import logging
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Callable,
+    Generic,
+    TypeVar,
+)
+
+from typing_extensions import ParamSpec
+
+from twisted.internet import defer
+
+from synapse.logging.context import make_deferred_yieldable, run_in_background
+from synapse.types import UserID
+from synapse.util.async_helpers import maybe_awaitable
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+
+logger = logging.getLogger(__name__)
+
+
+def user_left_room(distributor: "Distributor", user: UserID, room_id: str) -> None:
+    distributor.fire("user_left_room", user=user, room_id=room_id)
+
+
+class Distributor:
+    """A central dispatch point for loosely-connected pieces of code to
+    register, observe, and fire signals.
+
+    Signals are named simply by strings.
+
+    TODO(paul): It would be nice to give signals stronger object identities,
+      so we can attach metadata, docstrings, detect typos, etc... But this
+      model will do for today.
+    """
+
+    def __init__(self, hs: "HomeServer") -> None:
+        """
+        Args:
+            server_name: The homeserver name of the server (used to label metrics)
+                (this should be `hs.hostname`).
+        """
+        self.hs = hs
+        self.signals: dict[str, Signal] = {}
+        self.pre_registration: dict[str, list[Callable]] = {}
+
+    def declare(self, name: str) -> None:
+        if name in self.signals:
+            raise KeyError("%r already has a signal named %s" % (self, name))
+
+        self.signals[name] = Signal(name)
+
+        if name in self.pre_registration:
+            signal = self.signals[name]
+            for observer in self.pre_registration[name]:
+                signal.observe(observer)
+
+    def observe(self, name: str, observer: Callable) -> None:
+        if name in self.signals:
+            self.signals[name].observe(observer)
+        else:
+            # TODO: Avoid strong ordering dependency by allowing people to
+            # pre-register observations on signals that don't exist yet.
+            if name not in self.pre_registration:
+                self.pre_registration[name] = []
+            self.pre_registration[name].append(observer)
+
+    def fire(self, name: str, *args: Any, **kwargs: Any) -> None:
+        """Dispatches the given signal to the registered observers.
+
+        Runs the observers as a background process. Does not return a deferred.
+        """
+        if name not in self.signals:
+            raise KeyError("%r does not have a signal named %s" % (self, name))
+
+        self.hs.run_as_background_process(
+            name, self.signals[name].fire, *args, **kwargs
+        )
+
+
+P = ParamSpec("P")
+R = TypeVar("R")
+
+
+class Signal(Generic[P]):
+    """A Signal is a dispatch point that stores a list of callables as
+    observers of it.
+
+    Signals can be "fired", meaning that every callable observing it is
+    invoked. Firing a signal does not change its state; it can be fired again
+    at any later point. Firing a signal passes any arguments from the fire
+    method into all of the observers.
+    """
+
+    def __init__(self, name: str):
+        self.name: str = name
+        self.observers: list[Callable[P, Any]] = []
+
+    def observe(self, observer: Callable[P, Any]) -> None:
+        """Adds a new callable to the observer list which will be invoked by
+        the 'fire' method.
+
+        Each observer callable may return a Deferred."""
+        self.observers.append(observer)
+
+    def fire(self, *args: P.args, **kwargs: P.kwargs) -> "defer.Deferred[list[Any]]":
+        """Invokes every callable in the observer list, passing in the args and
+        kwargs. Exceptions thrown by observers are logged but ignored. It is
+        not an error to fire a signal with no observers.
+
+        Returns a Deferred that will complete when all the observers have
+        completed."""
+
+        async def do(observer: Callable[P, R | Awaitable[R]]) -> R | None:
+            try:
+                return await maybe_awaitable(observer(*args, **kwargs))
+            except Exception as e:
+                logger.warning(
+                    "%s signal observer %s failed: %r",
+                    self.name,
+                    observer,
+                    e,
+                )
+                return None
+
+        deferreds = [run_in_background(do, o) for o in self.observers]
+
+        return make_deferred_yieldable(
+            defer.gatherResults(deferreds, consumeErrors=True)
+        )
+
+    def __repr__(self) -> str:
+        return "<Signal name=%r>" % (self.name,)

synapse/util/events.py

@@ -0,0 +1,134 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2024 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+#
+
+from typing import Any
+
+from pydantic import Field, StrictStr, ValidationError, field_validator
+
+from synapse.types import JsonDict
+from synapse.util.pydantic_models import ParseModel
+from synapse.util.stringutils import random_string
+
+
+def generate_fake_event_id() -> str:
+    """
+    Generate an event ID from random ASCII characters.
+
+    This is primarily useful for generating fake event IDs in response to
+    requests from shadow-banned users.
+
+    Returns:
+        A string intended to look like an event ID, but with no actual meaning.
+    """
+    return "$" + random_string(43)
+
+
+class MTextRepresentation(ParseModel):
+    """
+    See `TextualRepresentation` in the Matrix specification.
+    """
+
+    body: StrictStr
+    mimetype: StrictStr | None = None
+
+
+class MTopic(ParseModel):
+    """
+    `m.room.topic` -> `content` -> `m.topic`
+
+    Textual representation of the room topic in different mimetypes. Added in Matrix v1.15.
+
+    See `TopicContentBlock` in the Matrix specification.
+    """
+
+    m_text: list[MTextRepresentation] | None = Field(None, alias="m.text")
+    """
+    An ordered array of textual representations in different mimetypes.
+    """
+
+    # Because "Receivers SHOULD use the first representation in the array that they
+    # understand.", we ignore invalid representations in the `m.text` field and use
+    # what we can.
+    @field_validator("m_text", mode="before")
+    @classmethod
+    def ignore_invalid_representations(
+        cls, m_text: Any
+    ) -> list[MTextRepresentation] | None:
+        if not isinstance(m_text, (list, tuple)):
+            raise ValueError("m.text must be a list or a tuple")
+        representations = []
+        for element in m_text:
+            try:
+                representations.append(MTextRepresentation.model_validate(element))
+            except ValidationError:
+                continue
+        return representations
+
+
+class TopicContent(ParseModel):
+    """
+    Represents the `content` field of an `m.room.topic` event
+    """
+
+    topic: StrictStr
+    """
+    The topic in plain text.
+    """
+
+    m_topic: MTopic | None = Field(None, alias="m.topic")
+    """
+    Textual representation of the room topic in different mimetypes.
+    """
+
+    # We ignore invalid `m.topic` fields as we can always fall back to the plain-text
+    # `topic` field.
+    @field_validator("m_topic", mode="before")
+    @classmethod
+    def ignore_invalid_m_topic(cls, m_topic: Any) -> MTopic | None:
+        try:
+            return MTopic.model_validate(m_topic)
+        except ValidationError:
+            return None
+
+
+def get_plain_text_topic_from_event_content(content: JsonDict) -> str | None:
+    """
+    Given the `content` of an `m.room.topic` event, returns the plain-text topic
+    representation. Prefers pulling plain-text from the newer `m.topic` field if
+    available with a fallback to `topic`.
+
+    Args:
+        content: The `content` field of an `m.room.topic` event.
+
+    Returns:
+        A string representing the plain text topic.
+    """
+
+    try:
+        topic_content = TopicContent.model_validate(content, strict=False)
+    except ValidationError:
+        return None
+
+    # Find the first `text/plain` topic ("Receivers SHOULD use the first
+    # representationin the array that they understand.")
+    if topic_content.m_topic and topic_content.m_topic.m_text:
+        for representation in topic_content.m_topic.m_text:
+            # The mimetype property defaults to `text/plain` if omitted.
+            if not representation.mimetype or representation.mimetype == "text/plain":
+                return representation.body
+
+    # Fallback to the plain-old `topic` field if there isn't any `text/plain` topic
+    # representation available.
+    return topic_content.topic

synapse/util/file_consumer.py

@@ -0,0 +1,164 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import queue
+from typing import Any, BinaryIO, cast
+
+from twisted.internet import threads
+from twisted.internet.defer import Deferred
+from twisted.internet.interfaces import IPullProducer, IPushProducer
+
+from synapse.logging.context import make_deferred_yieldable, run_in_background
+from synapse.types import ISynapseReactor
+
+
+class BackgroundFileConsumer:
+    """A consumer that writes to a file like object. Supports both push
+    and pull producers
+
+    Args:
+        file_obj: The file like object to write to. Closed when
+            finished.
+        reactor: the Twisted reactor to use
+    """
+
+    # For PushProducers pause if we have this many unwritten slices
+    _PAUSE_ON_QUEUE_SIZE = 5
+    # And resume once the size of the queue is less than this
+    _RESUME_ON_QUEUE_SIZE = 2
+
+    def __init__(self, file_obj: BinaryIO, reactor: ISynapseReactor) -> None:
+        self._file_obj: BinaryIO = file_obj
+
+        self._reactor: ISynapseReactor = reactor
+
+        # Producer we're registered with
+        self._producer: IPushProducer | IPullProducer | None = None
+
+        # True if PushProducer, false if PullProducer
+        self.streaming = False
+
+        # For PushProducers, indicates whether we've paused the producer and
+        # need to call resumeProducing before we get more data.
+        self._paused_producer = False
+
+        # Queue of slices of bytes to be written. When producer calls
+        # unregister a final None is sent.
+        self._bytes_queue: queue.Queue[bytes | None] = queue.Queue()
+
+        # Deferred that is resolved when finished writing
+        #
+        # This is really Deferred[None], but mypy doesn't seem to like that.
+        self._finished_deferred: Deferred[Any] | None = None
+
+        # If the _writer thread throws an exception it gets stored here.
+        self._write_exception: Exception | None = None
+
+    def registerProducer(
+        self, producer: IPushProducer | IPullProducer, streaming: bool
+    ) -> None:
+        """Part of IConsumer interface
+
+        Args:
+            producer
+            streaming: True if push based producer, False if pull
+                based.
+        """
+        if self._producer:
+            raise Exception("registerProducer called twice")
+
+        self._producer = producer
+        self.streaming = streaming
+        self._finished_deferred = run_in_background(
+            threads.deferToThreadPool,
+            # mypy seems to get confused with the chaining of ParamSpec from
+            # run_in_background to deferToThreadPool.
+            #
+            # For Twisted trunk, ignore arg-type; for Twisted release ignore unused-ignore.
+            self._reactor,  # type: ignore[arg-type,unused-ignore]
+            self._reactor.getThreadPool(),  # type: ignore[arg-type,unused-ignore]
+            self._writer,  # type: ignore[arg-type,unused-ignore]
+        )
+        if not streaming:
+            self._producer.resumeProducing()
+
+    def unregisterProducer(self) -> None:
+        """Part of IProducer interface"""
+        self._producer = None
+        assert self._finished_deferred is not None
+        if not self._finished_deferred.called:
+            self._bytes_queue.put_nowait(None)
+
+    def write(self, write_bytes: bytes) -> None:
+        """Part of IProducer interface"""
+        if self._write_exception:
+            raise self._write_exception
+
+        assert self._finished_deferred is not None
+        if self._finished_deferred.called:
+            raise Exception("consumer has closed")
+
+        self._bytes_queue.put_nowait(write_bytes)
+
+        # If this is a PushProducer and the queue is getting behind
+        # then we pause the producer.
+        if self.streaming and self._bytes_queue.qsize() >= self._PAUSE_ON_QUEUE_SIZE:
+            self._paused_producer = True
+            assert self._producer is not None
+            # cast safe because `streaming` means this is an IPushProducer
+            cast(IPushProducer, self._producer).pauseProducing()
+
+    def _writer(self) -> None:
+        """This is run in a background thread to write to the file."""
+        try:
+            while self._producer or not self._bytes_queue.empty():
+                # If we've paused the producer check if we should resume the
+                # producer.
+                if self._producer and self._paused_producer:
+                    if self._bytes_queue.qsize() <= self._RESUME_ON_QUEUE_SIZE:
+                        self._reactor.callFromThread(self._resume_paused_producer)
+
+                bytes = self._bytes_queue.get()
+
+                # If we get a None (or empty list) then that's a signal used
+                # to indicate we should check if we should stop.
+                if bytes:
+                    self._file_obj.write(bytes)
+
+                # If its a pull producer then we need to explicitly ask for
+                # more stuff.
+                if not self.streaming and self._producer:
+                    self._reactor.callFromThread(self._producer.resumeProducing)
+        except Exception as e:
+            self._write_exception = e
+            raise
+        finally:
+            self._file_obj.close()
+
+    def wait(self) -> "Deferred[None]":
+        """Returns a deferred that resolves when finished writing to file"""
+        assert self._finished_deferred is not None
+        return make_deferred_yieldable(self._finished_deferred)
+
+    def _resume_paused_producer(self) -> None:
+        """Gets called if we should resume producing after being paused"""
+        if self._paused_producer and self._producer:
+            self._paused_producer = False
+            self._producer.resumeProducing()

synapse/util/frozenutils.py

@@ -0,0 +1,57 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2014-2016 OpenMarket Ltd
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+import collections.abc
+from typing import Any
+
+from immutabledict import immutabledict
+
+
+def freeze(o: Any) -> Any:
+    if isinstance(o, dict):
+        return immutabledict({k: freeze(v) for k, v in o.items()})
+
+    if isinstance(o, immutabledict):
+        return o
+
+    if isinstance(o, (bytes, str)):
+        return o
+
+    try:
+        return tuple(freeze(i) for i in o)
+    except TypeError:
+        pass
+
+    return o
+
+
+def unfreeze(o: Any) -> Any:
+    if isinstance(o, collections.abc.Mapping):
+        return {k: unfreeze(v) for k, v in o.items()}
+
+    if isinstance(o, (bytes, str)):
+        return o
+
+    try:
+        return [unfreeze(i) for i in o]
+    except TypeError:
+        pass
+
+    return o