matrix-synapse 1.139.0rc2__cp39-abi3-musllinux_1_2_aarch64.whl → 1.140.0rc1__cp39-abi3-musllinux_1_2_aarch64.whl

synapse/logging/scopecontextmanager.py

@@ -0,0 +1,161 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+from typing import Optional
+
+from opentracing import Scope, ScopeManager, Span
+
+from synapse.logging.context import (
+    LoggingContext,
+    current_context,
+    nested_logging_context,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class LogContextScopeManager(ScopeManager):
+    """
+    The LogContextScopeManager tracks the active scope in opentracing
+    by using the log contexts which are native to synapse. This is so
+    that the basic opentracing api can be used across twisted defereds.
+
+    It would be nice just to use opentracing's ContextVarsScopeManager,
+    but currently that doesn't work due to https://twistedmatrix.com/trac/ticket/10301.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    @property
+    def active(self) -> Optional[Scope]:
+        """
+        Returns the currently active Scope which can be used to access the
+        currently active Scope.span.
+        If there is a non-null Scope, its wrapped Span
+        becomes an implicit parent of any newly-created Span at
+        Tracer.start_active_span() time.
+
+        Return:
+            The Scope that is active, or None if not available.
+        """
+        ctx = current_context()
+        return ctx.scope
+
+    def activate(self, span: Span, finish_on_close: bool) -> Scope:
+        """
+        Makes a Span active.
+        Args
+            span: the span that should become active.
+            finish_on_close: whether Span should be automatically finished when
+                Scope.close() is called.
+
+        Returns:
+            Scope to control the end of the active period for
+            *span*. It is a programming error to neglect to call
+            Scope.close() on the returned instance.
+        """
+
+        ctx = current_context()
+
+        if not ctx:
+            logger.error("Tried to activate scope outside of loggingcontext")
+            return Scope(None, span)  # type: ignore[arg-type]
+
+        if ctx.scope is not None:
+            # start a new logging context as a child of the existing one.
+            # Doing so -- rather than updating the existing logcontext -- means that
+            # creating several concurrent spans under the same logcontext works
+            # correctly.
+            ctx = nested_logging_context("")
+            enter_logcontext = True
+        else:
+            # if there is no span currently associated with the current logcontext, we
+            # just store the scope in it.
+            #
+            # This feels a bit dubious, but it does hack around a problem where a
+            # span outlasts its parent logcontext (which would otherwise lead to
+            # "Re-starting finished log context" errors).
+            enter_logcontext = False
+
+        scope = _LogContextScope(self, span, ctx, enter_logcontext, finish_on_close)
+        ctx.scope = scope
+        if enter_logcontext:
+            ctx.__enter__()
+
+        return scope
+
+
+class _LogContextScope(Scope):
+    """
+    A custom opentracing scope, associated with a LogContext
+
+      * When the scope is closed, the logcontext's active scope is reset to None.
+        and - if enter_logcontext was set - the logcontext is finished too.
+    """
+
+    def __init__(
+        self,
+        manager: LogContextScopeManager,
+        span: Span,
+        logcontext: LoggingContext,
+        enter_logcontext: bool,
+        finish_on_close: bool,
+    ):
+        """
+        Args:
+            manager:
+                the manager that is responsible for this scope.
+            span:
+                the opentracing span which this scope represents the local
+                lifetime for.
+            logcontext:
+                the log context to which this scope is attached.
+            enter_logcontext:
+                if True the log context will be exited when the scope is finished
+            finish_on_close:
+                if True finish the span when the scope is closed
+        """
+        super().__init__(manager, span)
+        self.logcontext = logcontext
+        self._finish_on_close = finish_on_close
+        self._enter_logcontext = enter_logcontext
+
+    def __str__(self) -> str:
+        return f"Scope<{self.span}>"
+
+    def close(self) -> None:
+        active_scope = self.manager.active
+        if active_scope is not self:
+            logger.error(
+                "Closing scope %s which is not the currently-active one %s",
+                self,
+                active_scope,
+            )
+
+        if self._finish_on_close:
+            self.span.finish()
+
+        self.logcontext.scope = None
+
+        if self._enter_logcontext:
+            self.logcontext.__exit__(None, None, None)

synapse/media/_base.py

@@ -704,6 +704,7 @@ class ThreadedFileSender:
 
     def __init__(self, hs: "HomeServer") -> None:
         self.reactor = hs.get_reactor()
+        self.clock = hs.get_clock()
         self.thread_pool = hs.get_media_sender_thread_pool()
 
         self.file: Optional[BinaryIO] = None
@@ -712,7 +713,7 @@ class ThreadedFileSender:
 
         # Signals if the thread should keep reading/sending data. Set means
         # continue, clear means pause.
-        self.wakeup_event = DeferredEvent(self.reactor)
+        self.wakeup_event = DeferredEvent(self.clock)
 
         # Signals if the thread should terminate, e.g. because the consumer has
         # gone away.

synapse/media/media_repository.py

@@ -67,7 +67,6 @@ from synapse.media.media_storage import (
 from synapse.media.storage_provider import StorageProviderWrapper
 from synapse.media.thumbnailer import Thumbnailer, ThumbnailError
 from synapse.media.url_previewer import UrlPreviewer
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia
 from synapse.types import UserID
 from synapse.util.async_helpers import Linearizer
@@ -108,7 +107,7 @@ class MediaRepository:
         self.dynamic_thumbnails = hs.config.media.dynamic_thumbnails
         self.thumbnail_requirements = hs.config.media.thumbnail_requirements
 
-        self.remote_media_linearizer = Linearizer(name="media_remote")
+        self.remote_media_linearizer = Linearizer(name="media_remote", clock=self.clock)
 
         self.recently_accessed_remotes: Set[Tuple[str, str]] = set()
         self.recently_accessed_locals: Set[str] = set()
@@ -187,16 +186,14 @@ class MediaRepository:
         self.media_repository_callbacks = hs.get_module_api_callbacks().media_repository
 
     def _start_update_recently_accessed(self) -> Deferred:
-        return run_as_background_process(
+        return self.hs.run_as_background_process(
             "update_recently_accessed_media",
-            self.server_name,
             self._update_recently_accessed,
         )
 
     def _start_apply_media_retention_rules(self) -> Deferred:
-        return run_as_background_process(
+        return self.hs.run_as_background_process(
             "apply_media_retention_rules",
-            self.server_name,
             self._apply_media_retention_rules,
         )
 
@@ -423,6 +420,23 @@ class MediaRepository:
             send_cors=True,
         )
 
+    async def get_cached_remote_media_info(
+        self, origin: str, media_id: str
+    ) -> Optional[RemoteMedia]:
+        """
+        Get cached remote media info for a given origin/media ID combo. If the requested
+        media is not found locally, it will not be requested over federation and the
+        call will return None.
+
+        Args:
+            origin: The origin of the remote media
+            media_id: The media ID of the requested content
+
+        Returns:
+            The info for the cached remote media or None if it was not found
+        """
+        return await self.store.get_cached_remote_media(origin, media_id)
+
     async def get_local_media_info(
         self, request: SynapseRequest, media_id: str, max_timeout_ms: int
     ) -> Optional[LocalMedia]:

synapse/media/url_previewer.py

@@ -44,7 +44,6 @@ from synapse.media._base import FileInfo, get_filename_from_headers
 from synapse.media.media_storage import MediaStorage, SHA256TransparentIOWriter
 from synapse.media.oembed import OEmbedProvider
 from synapse.media.preview_html import decode_body, parse_html_to_open_graph
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import JsonDict, UserID
 from synapse.util.async_helpers import ObservableDeferred
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -167,6 +166,7 @@ class UrlPreviewer:
         media_storage: MediaStorage,
     ):
         self.clock = hs.get_clock()
+        self.hs = hs
         self.filepaths = media_repo.filepaths
         self.max_spider_size = hs.config.media.max_spider_size
         self.server_name = hs.hostname
@@ -201,15 +201,14 @@ class UrlPreviewer:
         self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache(
             cache_name="url_previews",
             server_name=self.server_name,
+            hs=self.hs,
             clock=self.clock,
             # don't spider URLs more often than once an hour
             expiry_ms=ONE_HOUR,
         )
 
         if self._worker_run_media_background_jobs:
-            self._cleaner_loop = self.clock.looping_call(
-                self._start_expire_url_cache_data, 10 * 1000
-            )
+            self.clock.looping_call(self._start_expire_url_cache_data, 10 * 1000)
 
     async def preview(self, url: str, user: UserID, ts: int) -> bytes:
         # the in-memory cache:
@@ -739,8 +738,8 @@ class UrlPreviewer:
         return open_graph_result, oembed_response.author_name, expiration_ms
 
     def _start_expire_url_cache_data(self) -> Deferred:
-        return run_as_background_process(
-            "expire_url_cache_data", self.server_name, self._expire_url_cache_data
+        return self.hs.run_as_background_process(
+            "expire_url_cache_data", self._expire_url_cache_data
         )
 
     async def _expire_url_cache_data(self) -> None:

synapse/metrics/_gc.py

@@ -138,7 +138,9 @@ def install_gc_manager() -> None:
                 gc_time.labels(i).observe(end - start)
                 gc_unreachable.labels(i).set(unreachable)
 
-    gc_task = task.LoopingCall(_maybe_gc)
+    # We can ignore the lint here since this looping call does not hold a `HomeServer`
+    # reference so can be cleaned up by other means on shutdown.
+    gc_task = task.LoopingCall(_maybe_gc)  # type: ignore[prefer-synapse-clock-looping-call]
     gc_task.start(0.1)
 
 

synapse/metrics/background_process_metrics.py

@@ -20,7 +20,7 @@
 
 import logging
 import threading
-from contextlib import nullcontext
+from contextlib import contextmanager, nullcontext
 from functools import wraps
 from types import TracebackType
 from typing import (
@@ -28,7 +28,9 @@ from typing import (
     Any,
     Awaitable,
     Callable,
+    ContextManager,
     Dict,
+    Generator,
     Iterable,
     Optional,
     Protocol,
@@ -49,7 +51,12 @@ from synapse.logging.context import (
     LoggingContext,
     PreserveLoggingContext,
 )
-from synapse.logging.opentracing import SynapseTags, start_active_span
+from synapse.logging.opentracing import (
+    SynapseTags,
+    active_span,
+    start_active_span,
+    start_active_span_follows_from,
+)
 from synapse.metrics import SERVER_NAME_LABEL
 from synapse.metrics._types import Collector
 
@@ -59,6 +66,13 @@ if TYPE_CHECKING:
     # Old versions don't have `LiteralString`
     from typing_extensions import LiteralString
 
+    from synapse.server import HomeServer
+
+    try:
+        import opentracing
+    except ImportError:
+        opentracing = None  # type: ignore[assignment]
+
 
 logger = logging.getLogger(__name__)
 
@@ -216,6 +230,7 @@ def run_as_background_process(
     func: Callable[..., Awaitable[Optional[R]]],
     *args: Any,
     bg_start_span: bool = True,
+    test_only_tracer: Optional["opentracing.Tracer"] = None,
     **kwargs: Any,
 ) -> "defer.Deferred[Optional[R]]":
     """Run the given function in its own logcontext, with resource metrics
@@ -241,6 +256,8 @@ def run_as_background_process(
         bg_start_span: Whether to start an opentracing span. Defaults to True.
             Should only be disabled for processes that will not log to or tag
             a span.
+        test_only_tracer: Set the OpenTracing tracer to use. This is only useful for
+            tests.
         args: positional args for func
         kwargs: keyword args for func
 
@@ -250,6 +267,12 @@ def run_as_background_process(
         rules.
     """
 
+    # Since we track the tracing scope in the `LoggingContext`, before we move to the
+    # sentinel logcontext (or a new `LoggingContext`), grab the currently active
+    # tracing span (if any) so that we can create a cross-link to the background process
+    # trace.
+    original_active_tracing_span = active_span(tracer=test_only_tracer)
+
     async def run() -> Optional[R]:
         with _bg_metrics_lock:
             count = _background_process_counts.get(desc, 0)
@@ -264,15 +287,101 @@ def run_as_background_process(
 
         with BackgroundProcessLoggingContext(
             name=desc, server_name=server_name, instance_id=count
-        ) as context:
+        ) as logging_context:
             try:
                 if bg_start_span:
-                    ctx = start_active_span(
-                        f"bgproc.{desc}", tags={SynapseTags.REQUEST_ID: str(context)}
-                    )
+                    # If there is already an active span (e.g. because this background
+                    # process was started as part of handling a request for example),
+                    # because this is a long-running background task that may serve a
+                    # broader purpose than the request that kicked it off, we don't want
+                    # it to be a direct child of the currently active trace connected to
+                    # the request. We only want a loose reference to jump between the
+                    # traces.
+                    #
+                    # For example, when making a `/messages` request, when approaching a
+                    # gap, we may kick off a background process to fetch missing events
+                    # from federation. The `/messages` request trace should't include
+                    # the entire time taken and details around fetching the missing
+                    # events since the request doesn't rely on the result, it was just
+                    # part of the heuristic to initiate things.
+                    #
+                    # We don't care about the value from the context manager as it's not
+                    # used (so we just use `Any` for the type). Ideally, we'd be able to
+                    # mark this as unused like an `assert_never` of sorts.
+                    tracing_scope: ContextManager[Any]
+                    if original_active_tracing_span is not None:
+                        # With the OpenTracing client that we're using, it's impossible to
+                        # create a disconnected root span while also providing `references`
+                        # so we first create a bare root span, then create a child span that
+                        # includes the references that we want.
+                        root_tracing_scope = start_active_span(
+                            f"bgproc.{desc}",
+                            tags={SynapseTags.REQUEST_ID: str(logging_context)},
+                            # Create a root span for the background process (disconnected
+                            # from other spans)
+                            ignore_active_span=True,
+                            tracer=test_only_tracer,
+                        )
+
+                        # Also add a span in the original request trace that cross-links
+                        # to background process trace. We immediately finish the span as
+                        # this is just a marker to follow where the real work is being
+                        # done.
+                        #
+                        # In OpenTracing, `FOLLOWS_FROM` indicates parent-child
+                        # relationship whereas we just want a cross-link to the
+                        # downstream trace. This is a bit hacky, but the closest we
+                        # can get to in OpenTracing land. If we ever migrate to
+                        # OpenTelemetry, we should use a normal `Link` for this.
+                        with start_active_span_follows_from(
+                            f"start_bgproc.{desc}",
+                            child_of=original_active_tracing_span,
+                            ignore_active_span=True,
+                            # Create the `FOLLOWS_FROM` reference to the background
+                            # process span so there is a loose coupling between the two
+                            # traces and it's easy to jump between.
+                            contexts=[root_tracing_scope.span.context],
+                            tracer=test_only_tracer,
+                        ):
+                            pass
+
+                        # Then start the tracing scope that we're going to use for
+                        # the duration of the background process within the root
+                        # span we just created.
+                        child_tracing_scope = start_active_span_follows_from(
+                            f"bgproc_child.{desc}",
+                            child_of=root_tracing_scope.span,
+                            ignore_active_span=True,
+                            tags={SynapseTags.REQUEST_ID: str(logging_context)},
+                            # Create the `FOLLOWS_FROM` reference to the request's
+                            # span so there is a loose coupling between the two
+                            # traces and it's easy to jump between.
+                            contexts=[original_active_tracing_span.context],
+                            tracer=test_only_tracer,
+                        )
+
+                        # For easy usage down below, we create a context manager that
+                        # combines both scopes.
+                        @contextmanager
+                        def combined_context_manager() -> Generator[None, None, None]:
+                            with root_tracing_scope, child_tracing_scope:
+                                yield
+
+                        tracing_scope = combined_context_manager()
+
+                    else:
+                        # Otherwise, when there is no active span, we will be creating
+                        # a disconnected root span already and we don't have to
+                        # worry about cross-linking to anything.
+                        tracing_scope = start_active_span(
+                            f"bgproc.{desc}",
+                            tags={SynapseTags.REQUEST_ID: str(logging_context)},
+                            tracer=test_only_tracer,
+                        )
                 else:
-                    ctx = nullcontext()  # type: ignore[assignment]
-                with ctx:
+                    tracing_scope = nullcontext()
+
+                with tracing_scope:
                     return await func(*args, **kwargs)
             except Exception:
                 logger.exception(
@@ -308,11 +417,11 @@ def run_as_background_process(
 P = ParamSpec("P")
 
 
-class HasServerName(Protocol):
-    server_name: str
+class HasHomeServer(Protocol):
+    hs: "HomeServer"
     """
-    The homeserver name that this cache is associated with (used to label the metric)
-    (`hs.hostname`).
+    The homeserver that this cache is associated with (used to label the metric and
+    track backgroun processes for clean shutdown).
     """
 
 
@@ -342,27 +451,22 @@ def wrap_as_background_process(
     """
 
     def wrapper(
-        func: Callable[Concatenate[HasServerName, P], Awaitable[Optional[R]]],
+        func: Callable[Concatenate[HasHomeServer, P], Awaitable[Optional[R]]],
     ) -> Callable[P, "defer.Deferred[Optional[R]]"]:
         @wraps(func)
         def wrapped_func(
-            self: HasServerName, *args: P.args, **kwargs: P.kwargs
+            self: HasHomeServer, *args: P.args, **kwargs: P.kwargs
         ) -> "defer.Deferred[Optional[R]]":
-            assert self.server_name is not None, (
-                "The `server_name` attribute must be set on the object where `@wrap_as_background_process` decorator is used."
+            assert self.hs is not None, (
+                "The `hs` attribute must be set on the object where `@wrap_as_background_process` decorator is used."
             )
 
-            return run_as_background_process(
+            return self.hs.run_as_background_process(
                 desc,
-                self.server_name,
                 func,
                 self,
                 *args,
-                # type-ignore: mypy is confusing kwargs with the bg_start_span kwarg.
-                #     Argument 4 to "run_as_background_process" has incompatible type
-                #     "**P.kwargs"; expected "bool"
-                # See https://github.com/python/mypy/issues/8862
-                **kwargs,  # type: ignore[arg-type]
+                **kwargs,
             )
 
         # There are some shenanigans here, because we're decorating a method but
@@ -401,7 +505,7 @@ class BackgroundProcessLoggingContext(LoggingContext):
         """
         if instance_id is None:
             instance_id = id(self)
-        super().__init__("%s-%s" % (name, instance_id))
+        super().__init__(name="%s-%s" % (name, instance_id), server_name=server_name)
         self._proc: Optional[_BackgroundProcess] = _BackgroundProcess(
             desc=name, server_name=server_name, ctx=self
         )

synapse/metrics/common_usage_metrics.py

@@ -23,7 +23,6 @@ from typing import TYPE_CHECKING
 import attr
 
 from synapse.metrics import SERVER_NAME_LABEL
-from synapse.metrics.background_process_metrics import run_as_background_process
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -52,6 +51,7 @@ class CommonUsageMetricsManager:
         self.server_name = hs.hostname
         self._store = hs.get_datastores().main
         self._clock = hs.get_clock()
+        self._hs = hs
 
     async def get_metrics(self) -> CommonUsageMetrics:
         """Get the CommonUsageMetrics object. If no collection has happened yet, do it
@@ -64,16 +64,14 @@ class CommonUsageMetricsManager:
 
     async def setup(self) -> None:
         """Keep the gauges for common usage metrics up to date."""
-        run_as_background_process(
+        self._hs.run_as_background_process(
             desc="common_usage_metrics_update_gauges",
-            server_name=self.server_name,
             func=self._update_gauges,
         )
         self._clock.looping_call(
-            run_as_background_process,
+            self._hs.run_as_background_process,
             5 * 60 * 1000,
             desc="common_usage_metrics_update_gauges",
-            server_name=self.server_name,
             func=self._update_gauges,
         )
 

synapse/module_api/__init__.py

@@ -43,6 +43,7 @@ from typing_extensions import Concatenate, ParamSpec
 
 from twisted.internet import defer
 from twisted.internet.interfaces import IDelayedCall
+from twisted.python.threadpool import ThreadPool
 from twisted.web.resource import Resource
 
 from synapse.api import errors
@@ -79,6 +80,7 @@ from synapse.http.servlet import parse_json_object_from_request
 from synapse.http.site import SynapseRequest
 from synapse.logging.context import (
     defer_to_thread,
+    defer_to_threadpool,
     make_deferred_yieldable,
     run_in_background,
 )
@@ -275,7 +277,15 @@ def run_as_background_process(
     # function instead.
     stub_server_name = "synapse_module_running_from_unknown_server"
 
-    return _run_as_background_process(
+    # Ignore the linter error here. Since this is leveraging the
+    # `run_as_background_process` function directly and we don't want to break the
+    # module api, we need to keep the function signature the same. This means we don't
+    # have access to the running `HomeServer` and cannot track this background process
+    # for cleanup during shutdown.
+    # This is not an issue during runtime and is only potentially problematic if the
+    # application cares about being able to garbage collect `HomeServer` instances
+    # during runtime.
+    return _run_as_background_process(  # type: ignore[untracked-background-process]
         desc,
         stub_server_name,
         func,
@@ -1402,7 +1412,7 @@ class ModuleApi:
 
         if self._hs.config.worker.run_background_tasks or run_on_all_instances:
             self._clock.looping_call(
-                self.run_as_background_process,
+                self._hs.run_as_background_process,
                 msec,
                 desc,
                 lambda: maybe_awaitable(f(*args, **kwargs)),
@@ -1460,7 +1470,7 @@ class ModuleApi:
         return self._clock.call_later(
             # convert ms to seconds as needed by call_later.
             msec * 0.001,
-            self.run_as_background_process,
+            self._hs.run_as_background_process,
             desc,
             lambda: maybe_awaitable(f(*args, **kwargs)),
         )
@@ -1701,8 +1711,8 @@ class ModuleApi:
             Note that the returned Deferred does not follow the synapse logcontext
             rules.
         """
-        return _run_as_background_process(
-            desc, self.server_name, func, *args, bg_start_span=bg_start_span, **kwargs
+        return self._hs.run_as_background_process(
+            desc, func, *args, bg_start_span=bg_start_span, **kwargs
         )
 
     async def defer_to_thread(
@@ -1725,6 +1735,33 @@ class ModuleApi:
         """
         return await defer_to_thread(self._hs.get_reactor(), f, *args, **kwargs)
 
+    async def defer_to_threadpool(
+        self,
+        threadpool: ThreadPool,
+        f: Callable[P, T],
+        *args: P.args,
+        **kwargs: P.kwargs,
+    ) -> T:
+        """Runs the given function in a separate thread from the given thread pool.
+
+        Allows specifying a custom thread pool instead of using the default Synapse
+        one. To use the default Synapse threadpool, use `defer_to_thread` instead.
+
+        Added in Synapse v1.140.0.
+
+        Args:
+            threadpool: The thread pool to use.
+            f: The function to run.
+            args: The function's arguments.
+            kwargs: The function's keyword arguments.
+
+        Returns:
+            The return value of the function once ran in a thread.
+        """
+        return await defer_to_threadpool(
+            self._hs.get_reactor(), threadpool, f, *args, **kwargs
+        )
+
     async def check_username(self, username: str) -> None:
         """Checks if the provided username uses the grammar defined in the Matrix
         specification, and is already being used by an existing user.