matrix-synapse 1.139.2__cp39-abi3-macosx_11_0_arm64.whl → 1.140.0rc1__cp39-abi3-macosx_11_0_arm64.whl

synapse/logging/context.py

@@ -33,7 +33,6 @@ See doc/log_contexts.rst for details on how this works.
 import logging
 import threading
 import typing
-import warnings
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
@@ -55,11 +54,29 @@ from typing_extensions import ParamSpec
 from twisted.internet import defer, threads
 from twisted.python.threadpool import ThreadPool
 
+from synapse.logging.loggers import ExplicitlyConfiguredLogger
+from synapse.util.stringutils import random_string
+
 if TYPE_CHECKING:
+    from synapse.logging.scopecontextmanager import _LogContextScope
     from synapse.types import ISynapseReactor
 
 logger = logging.getLogger(__name__)
 
+original_logger_class = logging.getLoggerClass()
+logging.setLoggerClass(ExplicitlyConfiguredLogger)
+logcontext_debug_logger = logging.getLogger("synapse.logging.context.debug")
+"""
+A logger for debugging when the logcontext switches.
+
+Because this is very noisy and probably something only developers want to see when
+debugging logcontext problems, we want people to explictly opt-in before seeing anything
+in the logs. Requires explicitly setting `synapse.logging.context.debug` in the logging
+configuration and does not inherit the log level from the parent logger.
+"""
+# Restore the original logger class
+logging.setLoggerClass(original_logger_class)
+
 try:
     import resource
 
@@ -238,13 +255,22 @@ class _Sentinel:
     we should always know which server the logs are coming from.
     """
 
-    __slots__ = ["previous_context", "finished", "request", "tag"]
+    __slots__ = [
+        "previous_context",
+        "finished",
+        "scope",
+        "server_name",
+        "request",
+        "tag",
+    ]
 
     def __init__(self) -> None:
         # Minimal set for compatibility with LoggingContext
         self.previous_context = None
         self.finished = False
+        self.server_name = "unknown_server_from_sentinel_context"
         self.request = None
+        self.scope = None
         self.tag = None
 
     def __str__(self) -> str:
@@ -282,14 +308,19 @@ class LoggingContext:
           child to the parent
 
     Args:
-        name: Name for the context for logging. If this is omitted, it is
-           inherited from the parent context.
+        name: Name for the context for logging.
+        server_name: The name of the server this context is associated with
+            (`config.server.server_name` or `hs.hostname`)
         parent_context (LoggingContext|None): The parent of the new context
+        request: Synapse Request Context object. Useful to associate all the logs
+            happening to a given request.
+
     """
 
     __slots__ = [
         "previous_context",
         "name",
+        "server_name",
         "parent_context",
         "_resource_usage",
         "usage_start",
@@ -297,11 +328,14 @@ class LoggingContext:
         "finished",
         "request",
         "tag",
+        "scope",
     ]
 
     def __init__(
         self,
-        name: Optional[str] = None,
+        *,
+        name: str,
+        server_name: str,
         parent_context: "Optional[LoggingContext]" = None,
         request: Optional[ContextRequest] = None,
     ) -> None:
@@ -314,9 +348,12 @@ class LoggingContext:
         # if the context is not currently active.
         self.usage_start: Optional[resource.struct_rusage] = None
 
+        self.name = name
+        self.server_name = server_name
         self.main_thread = get_thread_id()
         self.request = None
         self.tag = ""
+        self.scope: Optional["_LogContextScope"] = None
 
         # keep track of whether we have hit the __exit__ block for this context
         # (suggesting that the the thing that created the context thinks it should
@@ -325,69 +362,24 @@ class LoggingContext:
 
         self.parent_context = parent_context
 
+        # Inherit some fields from the parent context
         if self.parent_context is not None:
-            # we track the current request_id
+            # which request this corresponds to
             self.request = self.parent_context.request
 
+            # we also track the current scope:
+            self.scope = self.parent_context.scope
+
         if request is not None:
             # the request param overrides the request from the parent context
             self.request = request
 
-        # if we don't have a `name`, but do have a parent context, use its name.
-        if self.parent_context and name is None:
-            name = str(self.parent_context)
-        if name is None:
-            raise ValueError(
-                "LoggingContext must be given either a name or a parent context"
-            )
-        self.name = name
-
     def __str__(self) -> str:
         return self.name
 
-    @classmethod
-    def current_context(cls) -> LoggingContextOrSentinel:
-        """Get the current logging context from thread local storage
-
-        This exists for backwards compatibility. ``current_context()`` should be
-        called directly.
-
-        Returns:
-            The current logging context
-        """
-        warnings.warn(
-            "synapse.logging.context.LoggingContext.current_context() is deprecated "
-            "in favor of synapse.logging.context.current_context().",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return current_context()
-
-    @classmethod
-    def set_current_context(
-        cls, context: LoggingContextOrSentinel
-    ) -> LoggingContextOrSentinel:
-        """Set the current logging context in thread local storage
-
-        This exists for backwards compatibility. ``set_current_context()`` should be
-        called directly.
-
-        Args:
-            context: The context to activate.
-
-        Returns:
-            The context that was previously active
-        """
-        warnings.warn(
-            "synapse.logging.context.LoggingContext.set_current_context() is deprecated "
-            "in favor of synapse.logging.context.set_current_context().",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-        return set_current_context(context)
-
     def __enter__(self) -> "LoggingContext":
         """Enters this logging context into thread local storage"""
+        logcontext_debug_logger.debug("LoggingContext(%s).__enter__", self.name)
         old_context = set_current_context(self)
         if self.previous_context != old_context:
             logcontext_error(
@@ -410,6 +402,9 @@ class LoggingContext:
         Returns:
             None to avoid suppressing any exceptions that were thrown.
         """
+        logcontext_debug_logger.debug(
+            "LoggingContext(%s).__exit__ --> %s", self.name, self.previous_context
+        )
         current = set_current_context(self.previous_context)
         if current is not self:
             if current is SENTINEL_CONTEXT:
@@ -588,7 +583,26 @@ class LoggingContextFilter(logging.Filter):
     record.
     """
 
-    def __init__(self, request: str = ""):
+    def __init__(
+        self,
+        # `request` is here for backwards compatibility since we previously recommended
+        # people manually configure `LoggingContextFilter` like the following.
+        #
+        # ```yaml
+        # filters:
+        #   context:
+        #       (): synapse.logging.context.LoggingContextFilter
+        #       request: ""
+        # ```
+        #
+        # TODO: Since we now configure `LoggingContextFilter` automatically since #8051
+        # (2020-08-11), we could consider removing this useless parameter. This would
+        # require people to remove their own manual configuration of
+        # `LoggingContextFilter` as it would cause `TypeError: Filter.__init__() got an
+        # unexpected keyword argument 'request'` -> `ValueError: Unable to configure
+        # filter 'context'`
+        request: str = "",
+    ):
         self._default_request = request
 
     def filter(self, record: logging.LogRecord) -> Literal[True]:
@@ -598,11 +612,13 @@ class LoggingContextFilter(logging.Filter):
         """
         context = current_context()
         record.request = self._default_request
+        record.server_name = "unknown_server_from_no_context"
 
         # context should never be None, but if it somehow ends up being, then
         # we end up in a death spiral of infinite loops, so let's check, for
         # robustness' sake.
         if context is not None:
+            record.server_name = context.server_name
             # Logging is interested in the request ID. Note that for backwards
             # compatibility this is stored as the "request" on the record.
             record.request = str(context)
@@ -637,14 +653,21 @@ class PreserveLoggingContext:
     reactor back to the code).
     """
 
-    __slots__ = ["_old_context", "_new_context"]
+    __slots__ = ["_old_context", "_new_context", "_instance_id"]
 
     def __init__(
         self, new_context: LoggingContextOrSentinel = SENTINEL_CONTEXT
     ) -> None:
         self._new_context = new_context
+        self._instance_id = random_string(5)
 
     def __enter__(self) -> None:
+        logcontext_debug_logger.debug(
+            "PreserveLoggingContext(%s).__enter__ %s --> %s",
+            self._instance_id,
+            current_context(),
+            self._new_context,
+        )
         self._old_context = set_current_context(self._new_context)
 
     def __exit__(
@@ -653,6 +676,12 @@ class PreserveLoggingContext:
         value: Optional[BaseException],
         traceback: Optional[TracebackType],
     ) -> None:
+        logcontext_debug_logger.debug(
+            "PreserveLoggingContext(%s).__exit %s --> %s",
+            self._instance_id,
+            current_context(),
+            self._old_context,
+        )
         context = set_current_context(self._old_context)
 
         if context != self._new_context:
@@ -728,12 +757,15 @@ def nested_logging_context(suffix: str) -> LoggingContext:
             "Starting nested logging context from sentinel context: metrics will be lost"
         )
         parent_context = None
+        server_name = "unknown_server_from_sentinel_context"
     else:
         assert isinstance(curr_context, LoggingContext)
         parent_context = curr_context
+        server_name = parent_context.server_name
     prefix = str(curr_context)
     return LoggingContext(
-        prefix + "-" + suffix,
+        name=prefix + "-" + suffix,
+        server_name=server_name,
         parent_context=parent_context,
     )
 
@@ -829,7 +861,11 @@ def run_in_background(
         Note that the returned Deferred does not follow the synapse logcontext
         rules.
     """
+    instance_id = random_string(5)
     calling_context = current_context()
+    logcontext_debug_logger.debug(
+        "run_in_background(%s): called with logcontext=%s", instance_id, calling_context
+    )
     try:
         # (kick off the task in the current context)
         res = f(*args, **kwargs)
@@ -871,6 +907,11 @@ def run_in_background(
         # to reset the logcontext to the sentinel logcontext as that would run
         # immediately (remember our goal is to maintain the calling logcontext when we
         # return).
+        logcontext_debug_logger.debug(
+            "run_in_background(%s): deferred already completed and the function should have maintained the logcontext %s",
+            instance_id,
+            calling_context,
+        )
         return d
 
     # Since the function we called may follow the Synapse logcontext rules (Rules for
@@ -881,6 +922,11 @@ def run_in_background(
     #
     # Our goal is to have the caller logcontext unchanged after firing off the
     # background task and returning.
+    logcontext_debug_logger.debug(
+        "run_in_background(%s): restoring calling logcontext %s",
+        instance_id,
+        calling_context,
+    )
     set_current_context(calling_context)
 
     # If the function we called is playing nice and following the Synapse logcontext
@@ -896,7 +942,23 @@ def run_in_background(
     # which is supposed to have a single entry and exit point. But
     # by spawning off another deferred, we are effectively
     # adding a new exit point.)
-    d.addBoth(_set_context_cb, SENTINEL_CONTEXT)
+    if logcontext_debug_logger.isEnabledFor(logging.DEBUG):
+
+        def _log_set_context_cb(
+            result: ResultT, context: LoggingContextOrSentinel
+        ) -> ResultT:
+            logcontext_debug_logger.debug(
+                "run_in_background(%s): resetting logcontext to %s",
+                instance_id,
+                context,
+            )
+            set_current_context(context)
+            return result
+
+        d.addBoth(_log_set_context_cb, SENTINEL_CONTEXT)
+    else:
+        d.addBoth(_set_context_cb, SENTINEL_CONTEXT)
+
     return d
 
 
@@ -952,10 +1014,21 @@ def make_deferred_yieldable(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]
     restores the old context once the awaitable completes (execution passes from the
     reactor back to the code).
     """
+    instance_id = random_string(5)
+    logcontext_debug_logger.debug(
+        "make_deferred_yieldable(%s): called with logcontext=%s",
+        instance_id,
+        current_context(),
+    )
+
     # The deferred has already completed
     if deferred.called and not deferred.paused:
         # it looks like this deferred is ready to run any callbacks we give it
         # immediately. We may as well optimise out the logcontext faffery.
+        logcontext_debug_logger.debug(
+            "make_deferred_yieldable(%s): deferred already completed and the function should have maintained the logcontext",
+            instance_id,
+        )
         return deferred
 
     # Our goal is to have the caller logcontext unchanged after they yield/await the
@@ -967,8 +1040,31 @@ def make_deferred_yieldable(deferred: "defer.Deferred[T]") -> "defer.Deferred[T]
     # does) while the deferred runs in the reactor event loop, we reset the logcontext
     # and add a callback to the deferred to restore it so the caller's logcontext is
     # active when the deferred completes.
-    prev_context = set_current_context(SENTINEL_CONTEXT)
-    deferred.addBoth(_set_context_cb, prev_context)
+
+    logcontext_debug_logger.debug(
+        "make_deferred_yieldable(%s): resetting logcontext to %s",
+        instance_id,
+        SENTINEL_CONTEXT,
+    )
+    calling_context = set_current_context(SENTINEL_CONTEXT)
+
+    if logcontext_debug_logger.isEnabledFor(logging.DEBUG):
+
+        def _log_set_context_cb(
+            result: ResultT, context: LoggingContextOrSentinel
+        ) -> ResultT:
+            logcontext_debug_logger.debug(
+                "make_deferred_yieldable(%s): restoring calling logcontext to %s",
+                instance_id,
+                context,
+            )
+            set_current_context(context)
+            return result
+
+        deferred.addBoth(_log_set_context_cb, calling_context)
+    else:
+        deferred.addBoth(_set_context_cb, calling_context)
+
     return deferred
 
 
@@ -1058,12 +1154,18 @@ def defer_to_threadpool(
             "Calling defer_to_threadpool from sentinel context: metrics will be lost"
         )
         parent_context = None
+        server_name = "unknown_server_from_sentinel_context"
     else:
         assert isinstance(curr_context, LoggingContext)
         parent_context = curr_context
+        server_name = parent_context.server_name
 
     def g() -> R:
-        with LoggingContext(str(curr_context), parent_context=parent_context):
+        with LoggingContext(
+            name=str(curr_context),
+            server_name=server_name,
+            parent_context=parent_context,
+        ):
             return f(*args, **kwargs)
 
     return make_deferred_yieldable(threads.deferToThreadPool(reactor, threadpool, g))

synapse/logging/opentracing.py

@@ -251,17 +251,18 @@ class _DummyTagNames:
 try:
     import opentracing
     import opentracing.tags
-    from opentracing.scope_managers.contextvars import ContextVarsScopeManager
 
     tags = opentracing.tags
 except ImportError:
     opentracing = None  # type: ignore[assignment]
     tags = _DummyTagNames  # type: ignore[assignment]
-    ContextVarsScopeManager = None  # type: ignore
 try:
     from jaeger_client import Config as JaegerConfig
+
+    from synapse.logging.scopecontextmanager import LogContextScopeManager
 except ImportError:
     JaegerConfig = None  # type: ignore
+    LogContextScopeManager = None  # type: ignore
 
 
 try:
@@ -483,7 +484,7 @@ def init_tracer(hs: "HomeServer") -> None:
     config = JaegerConfig(
         config=jaeger_config,
         service_name=f"{hs.config.server.server_name} {instance_name_by_type}",
-        scope_manager=ContextVarsScopeManager(),
+        scope_manager=LogContextScopeManager(),
         metrics_factory=PrometheusMetricsFactory(),
     )
 
@@ -576,7 +577,9 @@ def start_active_span_follows_from(
     operation_name: str,
     contexts: Collection,
     child_of: Optional[Union["opentracing.Span", "opentracing.SpanContext"]] = None,
+    tags: Optional[Dict[str, str]] = None,
     start_time: Optional[float] = None,
+    ignore_active_span: bool = False,
     *,
     inherit_force_tracing: bool = False,
     tracer: Optional["opentracing.Tracer"] = None,
@@ -591,9 +594,16 @@ def start_active_span_follows_from(
            span will be the parent. (If there is no currently active span, the first
            span in `contexts` will be the parent.)
 
+        tags: an optional dictionary of span tags. The caller gives up ownership of that
+            dictionary, because the :class:`Tracer` may use it as-is to avoid extra data
+            copying.
+
         start_time: optional override for the start time of the created span. Seconds
             since the epoch.
 
+        ignore_active_span: an explicit flag that ignores the current active
+            scope and creates a root span.
+
         inherit_force_tracing: if set, and any of the previous contexts have had tracing
            forced, the new span will also have tracing forced.
         tracer: override the opentracing tracer. By default the global tracer is used.
@@ -606,7 +616,9 @@ def start_active_span_follows_from(
         operation_name,
         child_of=child_of,
         references=references,
+        tags=tags,
         start_time=start_time,
+        ignore_active_span=ignore_active_span,
         tracer=tracer,
     )
 
@@ -672,9 +684,21 @@ def start_active_span_from_edu(
 
 # Opentracing setters for tags, logs, etc
 @only_if_tracing
-def active_span() -> Optional["opentracing.Span"]:
-    """Get the currently active span, if any"""
-    return opentracing.tracer.active_span
+def active_span(
+    *,
+    tracer: Optional["opentracing.Tracer"] = None,
+) -> Optional["opentracing.Span"]:
+    """
+    Get the currently active span, if any
+
+    Args:
+        tracer: override the opentracing tracer. By default the global tracer is used.
+    """
+    if tracer is None:
+        # use the global tracer by default
+        tracer = opentracing.tracer
+
+    return tracer.active_span
 
 
 @ensure_active_span("set a tag")

synapse/logging/scopecontextmanager.py

@@ -0,0 +1,161 @@
+#
+# This file is licensed under the Affero General Public License (AGPL) version 3.
+#
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright (C) 2023 New Vector, Ltd
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# See the GNU Affero General Public License for more details:
+# <https://www.gnu.org/licenses/agpl-3.0.html>.
+#
+# Originally licensed under the Apache License, Version 2.0:
+# <http://www.apache.org/licenses/LICENSE-2.0>.
+#
+# [This file includes modifications made by New Vector Limited]
+#
+#
+
+import logging
+from typing import Optional
+
+from opentracing import Scope, ScopeManager, Span
+
+from synapse.logging.context import (
+    LoggingContext,
+    current_context,
+    nested_logging_context,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class LogContextScopeManager(ScopeManager):
+    """
+    The LogContextScopeManager tracks the active scope in opentracing
+    by using the log contexts which are native to synapse. This is so
+    that the basic opentracing api can be used across twisted defereds.
+
+    It would be nice just to use opentracing's ContextVarsScopeManager,
+    but currently that doesn't work due to https://twistedmatrix.com/trac/ticket/10301.
+    """
+
+    def __init__(self) -> None:
+        pass
+
+    @property
+    def active(self) -> Optional[Scope]:
+        """
+        Returns the currently active Scope which can be used to access the
+        currently active Scope.span.
+        If there is a non-null Scope, its wrapped Span
+        becomes an implicit parent of any newly-created Span at
+        Tracer.start_active_span() time.
+
+        Return:
+            The Scope that is active, or None if not available.
+        """
+        ctx = current_context()
+        return ctx.scope
+
+    def activate(self, span: Span, finish_on_close: bool) -> Scope:
+        """
+        Makes a Span active.
+        Args
+            span: the span that should become active.
+            finish_on_close: whether Span should be automatically finished when
+                Scope.close() is called.
+
+        Returns:
+            Scope to control the end of the active period for
+            *span*. It is a programming error to neglect to call
+            Scope.close() on the returned instance.
+        """
+
+        ctx = current_context()
+
+        if not ctx:
+            logger.error("Tried to activate scope outside of loggingcontext")
+            return Scope(None, span)  # type: ignore[arg-type]
+
+        if ctx.scope is not None:
+            # start a new logging context as a child of the existing one.
+            # Doing so -- rather than updating the existing logcontext -- means that
+            # creating several concurrent spans under the same logcontext works
+            # correctly.
+            ctx = nested_logging_context("")
+            enter_logcontext = True
+        else:
+            # if there is no span currently associated with the current logcontext, we
+            # just store the scope in it.
+            #
+            # This feels a bit dubious, but it does hack around a problem where a
+            # span outlasts its parent logcontext (which would otherwise lead to
+            # "Re-starting finished log context" errors).
+            enter_logcontext = False
+
+        scope = _LogContextScope(self, span, ctx, enter_logcontext, finish_on_close)
+        ctx.scope = scope
+        if enter_logcontext:
+            ctx.__enter__()
+
+        return scope
+
+
+class _LogContextScope(Scope):
+    """
+    A custom opentracing scope, associated with a LogContext
+
+      * When the scope is closed, the logcontext's active scope is reset to None.
+        and - if enter_logcontext was set - the logcontext is finished too.
+    """
+
+    def __init__(
+        self,
+        manager: LogContextScopeManager,
+        span: Span,
+        logcontext: LoggingContext,
+        enter_logcontext: bool,
+        finish_on_close: bool,
+    ):
+        """
+        Args:
+            manager:
+                the manager that is responsible for this scope.
+            span:
+                the opentracing span which this scope represents the local
+                lifetime for.
+            logcontext:
+                the log context to which this scope is attached.
+            enter_logcontext:
+                if True the log context will be exited when the scope is finished
+            finish_on_close:
+                if True finish the span when the scope is closed
+        """
+        super().__init__(manager, span)
+        self.logcontext = logcontext
+        self._finish_on_close = finish_on_close
+        self._enter_logcontext = enter_logcontext
+
+    def __str__(self) -> str:
+        return f"Scope<{self.span}>"
+
+    def close(self) -> None:
+        active_scope = self.manager.active
+        if active_scope is not self:
+            logger.error(
+                "Closing scope %s which is not the currently-active one %s",
+                self,
+                active_scope,
+            )
+
+        if self._finish_on_close:
+            self.span.finish()
+
+        self.logcontext.scope = None
+
+        if self._enter_logcontext:
+            self.logcontext.__exit__(None, None, None)

synapse/media/_base.py

@@ -704,6 +704,7 @@ class ThreadedFileSender:
 
     def __init__(self, hs: "HomeServer") -> None:
         self.reactor = hs.get_reactor()
+        self.clock = hs.get_clock()
         self.thread_pool = hs.get_media_sender_thread_pool()
 
         self.file: Optional[BinaryIO] = None
@@ -712,7 +713,7 @@ class ThreadedFileSender:
 
         # Signals if the thread should keep reading/sending data. Set means
         # continue, clear means pause.
-        self.wakeup_event = DeferredEvent(self.reactor)
+        self.wakeup_event = DeferredEvent(self.clock)
 
         # Signals if the thread should terminate, e.g. because the consumer has
         # gone away.