fastapi-observer 1.9.0__tar.gz → 1.11.0__tar.gz

{fastapi_observer-1.9.0/src/fastapi_observer.egg-info → fastapi_observer-1.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fastapi-observer
-Version: 1.9.0
+Version: 1.11.0
 Summary: Zero-glue FastAPI observability with security presets and runtime controls
 Author-email: Vitaee <opensource@vitaee.dev>
 License-Expression: MIT
@@ -152,6 +152,7 @@ Explicit variables and Python arguments always win over profile defaults. Every
 - **Prometheus metrics** labelled by route template (`/orders/{order_id}`), and 404s for paths no route matches share one `<unmatched>` label, so neither IDs nor scanners explode cardinality.
 - **OpenTelemetry traces** with `trace_id` and `span_id` in every log line, and optional log export over OTLP.
 - **Non-blocking log writes.** Lines are written from a background thread, so a slow stdout never stalls the event loop.
+- **Help finding what slows a request down.** Opt-in: a warning with the stack of any sync call that blocks the event loop, and `DEBUG` lines written only for requests that fail. With tracing on, each request line counts its queries and outbound HTTP calls, and a query run once per item (N+1) gets a warning. With metrics on, `/metrics` shows how many bytes of logs each route writes, and `LOG_SAMPLE_RATE` keeps a share of successful requests, chosen by trace ID, while failed and slow requests are always written in full.
 
 Optional extras: runtime log-level control, a tamper-evident audit log, SQLAlchemy, `httpx` and `requests` tracing, Strawberry GraphQL and a Loguru bridge. See the [documentation](https://vitaee.github.io/FastapiObserver/).
 

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/README.md

@@ -76,6 +76,7 @@ Explicit variables and Python arguments always win over profile defaults. Every
 - **Prometheus metrics** labelled by route template (`/orders/{order_id}`), and 404s for paths no route matches share one `<unmatched>` label, so neither IDs nor scanners explode cardinality.
 - **OpenTelemetry traces** with `trace_id` and `span_id` in every log line, and optional log export over OTLP.
 - **Non-blocking log writes.** Lines are written from a background thread, so a slow stdout never stalls the event loop.
+- **Help finding what slows a request down.** Opt-in: a warning with the stack of any sync call that blocks the event loop, and `DEBUG` lines written only for requests that fail. With tracing on, each request line counts its queries and outbound HTTP calls, and a query run once per item (N+1) gets a warning. With metrics on, `/metrics` shows how many bytes of logs each route writes, and `LOG_SAMPLE_RATE` keeps a share of successful requests, chosen by trace ID, while failed and slow requests are always written in full.
 
 Optional extras: runtime log-level control, a tamper-evident audit log, SQLAlchemy, `httpx` and `requests` tracing, Strawberry GraphQL and a Loguru bridge. See the [documentation](https://vitaee.github.io/FastapiObserver/).
 

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0/src/fastapi_observer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fastapi-observer
-Version: 1.9.0
+Version: 1.11.0
 Summary: Zero-glue FastAPI observability with security presets and runtime controls
 Author-email: Vitaee <opensource@vitaee.dev>
 License-Expression: MIT
@@ -152,6 +152,7 @@ Explicit variables and Python arguments always win over profile defaults. Every
 - **Prometheus metrics** labelled by route template (`/orders/{order_id}`), and 404s for paths no route matches share one `<unmatched>` label, so neither IDs nor scanners explode cardinality.
 - **OpenTelemetry traces** with `trace_id` and `span_id` in every log line, and optional log export over OTLP.
 - **Non-blocking log writes.** Lines are written from a background thread, so a slow stdout never stalls the event loop.
+- **Help finding what slows a request down.** Opt-in: a warning with the stack of any sync call that blocks the event loop, and `DEBUG` lines written only for requests that fail. With tracing on, each request line counts its queries and outbound HTTP calls, and a query run once per item (N+1) gets a warning. With metrics on, `/metrics` shows how many bytes of logs each route writes, and `LOG_SAMPLE_RATE` keeps a share of successful requests, chosen by trace ID, while failed and slow requests are always written in full.
 
 Optional extras: runtime log-level control, a tamper-evident audit log, SQLAlchemy, `httpx` and `requests` tracing, Strawberry GraphQL and a Loguru bridge. See the [documentation](https://vitaee.github.io/FastapiObserver/).
 

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapi_observer.egg-info/SOURCES.txt

@@ -13,6 +13,7 @@ src/fastapiobserver/control_plane.py
 src/fastapiobserver/db_tracing.py
 src/fastapiobserver/fastapi.py
 src/fastapiobserver/loguru.py
+src/fastapiobserver/loop_watchdog.py
 src/fastapiobserver/plugins.py
 src/fastapiobserver/profiles.py
 src/fastapiobserver/propagation.py
@@ -29,9 +30,11 @@ src/fastapiobserver/logging/__init__.py
 src/fastapiobserver/logging/circuit_breaker.py
 src/fastapiobserver/logging/filters.py
 src/fastapiobserver/logging/formatter.py
+src/fastapiobserver/logging/held.py
 src/fastapiobserver/logging/queueing.py
 src/fastapiobserver/logging/setup.py
 src/fastapiobserver/logging/state.py
+src/fastapiobserver/logging/volume.py
 src/fastapiobserver/metrics/__init__.py
 src/fastapiobserver/metrics/prometheus.py
 src/fastapiobserver/middleware/__init__.py
@@ -80,6 +83,7 @@ tests/test_log_queue_controls.py
 tests/test_logging_schema_contract.py
 tests/test_logtail_dlq.py
 tests/test_loguru_bridge.py
+tests/test_loop_watchdog.py
 tests/test_metrics_contracts.py
 tests/test_metrics_optional_dependency.py
 tests/test_middleware.py
@@ -95,6 +99,7 @@ tests/test_plugin_isolation.py
 tests/test_profiles.py
 tests/test_propagation.py
 tests/test_redaction_presets.py
+tests/test_request_insights.py
 tests/test_route_template_cardinality.py
 tests/test_runtime_control_auth.py
 tests/test_security_audit_fixes.py

fastapi_observer-1.11.0/src/fastapiobserver/_version.py

@@ -0,0 +1 @@
+__version__ = "1.11.0"

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/config.py

@@ -32,6 +32,15 @@ class ObservabilitySettings(InternalSettingsBase):
 
     # --- logging ---
     log_level: str = Field(default="INFO", validation_alias="LOG_LEVEL")
+    # Records from this level up to log_level are held per request and written
+    # only if the request fails.
+    log_level_on_error: str | None = Field(default=None, validation_alias="LOG_LEVEL_ON_ERROR")
+    # Share of requests whose INFO and lower lines are written, chosen by trace ID.
+    log_sample_rate: float = Field(default=1.0, ge=0.0, le=1.0, validation_alias="LOG_SAMPLE_RATE")
+    # Requests at least this slow are written in full, sampled or not.
+    log_slow_request_ms: float = Field(
+        default=1000.0, gt=0, validation_alias="LOG_SLOW_REQUEST_MS"
+    )
     log_dir: str | None = Field(default=None, validation_alias="LOG_DIR")
     log_queue_max_size: int = Field(default=10_000, gt=0, validation_alias="LOG_QUEUE_MAX_SIZE")
     log_queue_overflow_policy: LogQueueOverflowPolicy = Field(
@@ -81,6 +90,12 @@ class ObservabilitySettings(InternalSettingsBase):
         default="negotiate", validation_alias="METRICS_FORMAT"
     )
 
+    # --- event loop ---
+    # Log the blocking call's stack when the loop doesn't run for this long.
+    loop_block_threshold_ms: float | None = Field(
+        default=None, gt=0, validation_alias="OBS_LOOP_BLOCK_THRESHOLD_MS"
+    )
+
     # --- otel trace noise control ---
     otel_excluded_urls: tuple[str, ...] | None = Field(
         default=None, validation_alias="OTEL_EXCLUDED_URLS"
@@ -135,9 +150,11 @@ class ObservabilitySettings(InternalSettingsBase):
 
     # ---------- validators ----------
 
-    @field_validator("log_level")
+    @field_validator("log_level", "log_level_on_error")
     @classmethod
-    def _validate_log_level(cls, value: str) -> str:
+    def _validate_log_level(cls, value: str | None) -> str | None:
+        if value is None:
+            return None
         normalized_log_level = value.upper().strip()
         level = logging.getLevelName(normalized_log_level)
         if not isinstance(level, int):

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/control_plane.py

@@ -10,6 +10,7 @@ from fastapi import APIRouter, Depends, FastAPI, Header, HTTPException, Request,
 from pydantic import BaseModel, ConfigDict, Field, field_validator
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+from .logging import get_log_level, set_log_level
 from .otel import get_trace_sampling_ratio, set_trace_sampling_ratio
 from .security import TrustedProxyPolicy, is_trusted_client_ip, resolve_client_ip
 from .utils import EnvLoadable, normalize_path
@@ -137,7 +138,7 @@ def _is_trusted_caller(request: Request, policy: TrustedProxyPolicy) -> bool:
 
 def _current_runtime_settings() -> dict[str, object]:
     return {
-        "log_level": logging.getLevelName(logging.getLogger().getEffectiveLevel()),
+        "log_level": get_log_level(),
         "trace_sampling_ratio": get_trace_sampling_ratio(),
     }
 
@@ -151,9 +152,7 @@ def _set_log_level(value: str) -> None:
             detail=f"Invalid log level: {value}",
         )
 
-    logging.getLogger().setLevel(level)
-    for logger_name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
-        logging.getLogger(logger_name).setLevel(level)
+    set_log_level(level)
 
 
 class _RuntimeControlEnvSettings(BaseSettings):

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/fastapi.py

@@ -16,6 +16,7 @@ from fastapi import FastAPI
 from .config import ObservabilitySettings
 from .control_plane import RuntimeControlSettings, mount_control_plane
 from .logging import setup_logging, shutdown_logging
+from .loop_watchdog import EventLoopWatchdog
 from .profiles import apply_profile_context
 from .db_tracing import uninstrument_sqlalchemy
 from .metrics import (
@@ -148,6 +149,10 @@ def install_observability(
             "event": {
                 "profile": os.getenv("OBS_PROFILE", "").strip().lower() or None,
                 "log_level": settings.log_level,
+                "log_level_on_error": settings.log_level_on_error,
+                "log_sample_rate": settings.log_sample_rate,
+                "log_slow_request_ms": settings.log_slow_request_ms,
+                "loop_block_threshold_ms": settings.loop_block_threshold_ms,
                 "metrics_path": (
                     settings.metrics_path
                     if (settings.metrics_enabled if metrics_enabled is None else metrics_enabled)
@@ -389,12 +394,18 @@ async def observability_lifespan(app: FastAPI) -> typing.AsyncGenerator[typing.A
             yield
     """
     state = _get_runtime_state(app)
+    watchdog = None
     if state:
         _auto_discover_excluded_routes(app, state.settings)
+        if state.settings.loop_block_threshold_ms:
+            watchdog = EventLoopWatchdog(state.settings.loop_block_threshold_ms / 1000)
+            watchdog.start()
 
     try:
         yield {}
     finally:
+        if watchdog:
+            watchdog.stop()
         _teardown_observability(app)
 
 

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/logging/__init__.py

@@ -14,7 +14,14 @@ from .queueing import (
     OverflowPolicyQueueHandler,
     get_log_queue_stats,
 )
-from .setup import get_managed_output_handlers, setup_logging, shutdown_logging
+from .setup import (
+    get_log_level,
+    get_managed_output_handlers,
+    set_log_level,
+    setup_logging,
+    shutdown_logging,
+)
+from .volume import get_log_bytes_by_route
 
 __all__ = [
     "LOG_SCHEMA_VERSION",
@@ -28,9 +35,12 @@ __all__ = [
     "SinkCircuitBreakerSnapshot",
     "StructuredJsonFormatter",
     "TraceContextFilter",
+    "get_log_bytes_by_route",
+    "get_log_level",
     "get_log_queue_stats",
     "get_managed_output_handlers",
     "get_sink_circuit_breaker_stats",
+    "set_log_level",
     "setup_logging",
     "shutdown_logging",
 ]

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/logging/filters.py

@@ -5,12 +5,13 @@ import sys
 
 from ..plugins import apply_log_filters
 from ..request_context import get_request_id, get_span_id, get_trace_id, get_user_context
+from . import volume
 
 _LOGGER = logging.getLogger("fastapiobserver.logging")
 
 
 class RequestIdFilter(logging.Filter):
-    """Attach request ID and user context from context vars to every log record.
+    """Attach request ID, user context and route from context vars to every log record.
 
     Records are formatted on the queue listener thread, where the request's
     context vars are not visible, so they are captured here.
@@ -21,6 +22,8 @@ class RequestIdFilter(logging.Filter):
             record.request_id = get_request_id()
         if not getattr(record, "user_context", None):
             record.user_context = get_user_context()
+        if volume.COUNTING:
+            record._obs_route = volume.current_route()
         return True
 
 

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/logging/formatter.py

@@ -13,6 +13,7 @@ from ..plugins import apply_log_enrichers
 from ..request_context import get_request_id, get_span_id, get_trace_id, get_user_context
 from ..security import SecurityPolicy, sanitize_event
 from ..utils import lazy_import
+from .volume import count_log_bytes
 
 orjson: Any
 try:
@@ -112,7 +113,9 @@ class StructuredJsonFormatter(logging.Formatter):
             sanitized_payload = _as_dict(sanitized_candidate)
         else:
             sanitized_payload = enriched_payload
-        return _json_dumps(sanitized_payload)
+        line = _json_dumps(sanitized_payload)
+        count_log_bytes(record, line)
+        return line
 
 
 def _as_dict(value: Mapping[str, Any]) -> dict[str, Any]:

fastapi_observer-1.11.0/src/fastapiobserver/logging/held.py

@@ -0,0 +1,111 @@
+"""A request's log records, held until it ends and then written or dropped.
+
+Two settings hold records:
+
+- ``LOG_LEVEL_ON_ERROR``: a request's records below ``LOG_LEVEL`` are written
+  only if it ends in a 5xx or an unhandled exception.
+- ``LOG_SAMPLE_RATE``: a request that isn't sampled has its ``INFO`` and lower
+  records written only if it fails or takes ``LOG_SLOW_REQUEST_MS`` or longer.
+  ``WARNING`` and higher are always written.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections import deque
+
+from ..request_context import RequestState, get_request_state
+from . import state
+
+# ponytail: fixed cap per request, newest kept; make it a setting if 200 isn't enough.
+_MAX_HELD_RECORDS = 200
+_LOW_64_BITS = (1 << 64) - 1
+
+
+class HeldLogFilter(logging.Filter):
+    """Holds a request's records instead of passing them on, until it ends.
+
+    It is the queue handler's last filter, so held records already carry their
+    request ID and trace context. Outside a request, records below *level* are
+    dropped, as the log level would have dropped them. *lowest_level* is
+    ``LOG_LEVEL_ON_ERROR``, or *level* without it.
+    """
+
+    def __init__(
+        self,
+        handler: logging.Handler,
+        level: int,
+        lowest_level: int,
+        sample_rate: float = 1.0,
+    ) -> None:
+        super().__init__()
+        self.handler = handler
+        self.level = level
+        self.lowest_level = lowest_level
+        # The bound OpenTelemetry's TraceIdRatioBased sampler uses for the same ratio.
+        self.sample_bound = round(sample_rate * (_LOW_64_BITS + 1))
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        request_state = get_request_state()
+        if record.levelno >= self.level and (
+            request_state is None
+            or record.levelno >= logging.WARNING
+            or self._sampled(request_state, record)
+        ):
+            return True
+        if request_state is not None:
+            if request_state.held_logs is None:
+                request_state.held_logs = deque(maxlen=_MAX_HELD_RECORDS)
+            request_state.held_logs.append(record)
+        return False
+
+    def _sampled(self, request_state: RequestState, record: logging.LogRecord) -> bool:
+        if self.sample_bound > _LOW_64_BITS:
+            return True
+        if request_state.sampled is None:
+            bits = _sampling_bits(record)
+            request_state.sampled = bits is None or bits < self.sample_bound
+        return request_state.sampled
+
+
+def _sampling_bits(record: logging.LogRecord) -> int | None:
+    """The 64 bits that decide whether a record's request is sampled.
+
+    The low half of its trace ID, which OpenTelemetry's TraceIdRatioBased sampler
+    compares, so every service that samples logs and traces at the same ratio
+    keeps the same requests. Without a trace ID, a hash of the request ID, which
+    services forward to each other.
+    """
+    trace_id = getattr(record, "trace_id", None)
+    if isinstance(trace_id, str):
+        try:
+            return int(trace_id, 16) & _LOW_64_BITS
+        except ValueError:
+            pass
+    request_id = getattr(record, "request_id", None)
+    if not request_id:
+        return None
+    digest = hashlib.blake2b(str(request_id).encode(), digest_size=8).digest()
+    return int.from_bytes(digest, "big")
+
+
+def keep_request_logs(*, failed: bool) -> None:
+    """Write the current request's held records and pass its later ones.
+
+    Called when a request fails or is slow. Records held only because they are
+    below the log level are written only if it *failed*.
+    """
+    held_filter = state._HELD_LOG_FILTER
+    request_state = get_request_state()
+    if held_filter is None or request_state is None:
+        return
+    request_state.sampled = True
+    held, request_state.held_logs = request_state.held_logs, None
+    for record in held or ():
+        if failed or record.levelno >= held_filter.level:
+            # emit(), not handle(): the filters already ran when the record was held.
+            held_filter.handler.emit(record)
+
+
+__all__ = ["HeldLogFilter", "keep_request_logs"]

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/logging/setup.py

@@ -12,9 +12,11 @@ from ..sinks import build_sink_handlers
 from .circuit_breaker import SinkCircuitBreakerHandler
 from .filters import PluginLogFilter, RequestIdFilter, TraceContextFilter
 from .formatter import StructuredJsonFormatter
+from .held import HeldLogFilter
 from .queueing import OverflowPolicyQueueHandler, _LOG_QUEUE_TELEMETRY
 
 _LOGGER = logging.getLogger("fastapiobserver.logging")
+_UVICORN_LOGGERS = ("uvicorn", "uvicorn.error", "uvicorn.access")
 
 
 def get_managed_output_handlers() -> list[logging.Handler]:
@@ -94,6 +96,7 @@ def _teardown_handlers_locked(suppress_errors: bool = True) -> None:
             
     state._MANAGED_OUTPUT_HANDLERS = []
     state._SINK_CIRCUIT_BREAKERS = []
+    state._HELD_LOG_FILTER = None
     state._MANAGED_HANDLERS.clear()
 
     if errors and not suppress_errors:
@@ -150,8 +153,6 @@ def setup_logging(
                 delegate=formatter, key_provider=cast(AuditKeyProvider, provider),
             )
 
-        root_logger.setLevel(settings.log_level.upper())
-
         # Build output handlers based on logs_mode
         output_handlers: list[tuple[logging.Handler, str]] = []
         if logs_mode in ("local_json", "both"):
@@ -195,6 +196,17 @@ def setup_logging(
             telemetry=_LOG_QUEUE_TELEMETRY,
         )
         _configure_queue_handler(queue_handler)
+        level = logging.getLevelName(settings.log_level)
+        lowest_level = level
+        if settings.log_level_on_error is not None:
+            lowest_level = min(level, logging.getLevelName(settings.log_level_on_error))
+        if lowest_level < level or settings.log_sample_rate < 1:
+            held_filter = HeldLogFilter(
+                queue_handler, level, lowest_level, settings.log_sample_rate
+            )
+            queue_handler.addFilter(held_filter)
+            state._HELD_LOG_FILTER = held_filter
+        set_log_level(level)
         root_logger.addHandler(queue_handler)
 
         listener = logging.handlers.QueueListener(
@@ -208,11 +220,42 @@ def setup_logging(
         _ensure_atexit_shutdown_hook_locked()
         _ensure_fork_reinitialization_locked()
 
-        for name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
+        for name in _UVICORN_LOGGERS:
             uvicorn_logger = logging.getLogger(name)
             uvicorn_logger.handlers = []
             uvicorn_logger.propagate = True
-            uvicorn_logger.setLevel(settings.log_level.upper())
+
+
+def set_log_level(level: int) -> None:
+    """Set the log level, at startup and from the runtime control plane.
+
+    With ``LOG_LEVEL_ON_ERROR`` below *level*, the root logger stays at that
+    lower level so the records are created, and the held-log filter applies
+    *level* instead.
+    """
+    import fastapiobserver.logging.state as state
+
+    root_level = level
+    held_filter = state._HELD_LOG_FILTER
+    if held_filter is not None:
+        held_filter.level = level
+        root_level = min(level, held_filter.lowest_level)
+    logging.getLogger().setLevel(root_level)
+    for name in _UVICORN_LOGGERS:
+        logging.getLogger(name).setLevel(level)
+
+
+def get_log_level() -> str:
+    """The level records are written at, as set by :func:`set_log_level`."""
+    import fastapiobserver.logging.state as state
+
+    held_filter = state._HELD_LOG_FILTER
+    level = (
+        held_filter.level
+        if held_filter is not None
+        else logging.getLogger().getEffectiveLevel()
+    )
+    return logging.getLevelName(level)
 
 
 def _configure_queue_handler(handler: logging.Handler) -> None:

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/logging/state.py

@@ -8,10 +8,12 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .circuit_breaker import SinkCircuitBreakerHandler
+    from .held import HeldLogFilter
 
 _LOGGING_LOCK = threading.Lock()
 _QUEUE_LISTENER: logging.handlers.QueueListener | None = None
 _MANAGED_OUTPUT_HANDLERS: list[logging.Handler] = []
 _SINK_CIRCUIT_BREAKERS: list["SinkCircuitBreakerHandler"] = []
 _MANAGED_HANDLERS: weakref.WeakSet[logging.Handler] = weakref.WeakSet()
+_HELD_LOG_FILTER: "HeldLogFilter | None" = None
 _ATEXIT_SHUTDOWN_REGISTERED = False

fastapi_observer-1.11.0/src/fastapiobserver/logging/volume.py

@@ -0,0 +1,55 @@
+"""JSON log volume by route, exported as ``fastapiobserver_log_bytes_total``."""
+
+from __future__ import annotations
+
+import logging
+import threading
+
+from ..request_context import UNMATCHED_ROUTE, get_request_state, route_template
+
+# Label for lines written outside a request, such as startup and background work.
+NO_REQUEST = "<none>"
+
+_LOCK = threading.Lock()
+_BYTES_BY_ROUTE: dict[str, int] = {}
+# Off until the Prometheus collector that exports the counts is registered.
+COUNTING = False
+
+
+def start_counting() -> None:
+    global COUNTING
+    COUNTING = True
+
+
+def current_route() -> str:
+    """The route label for a record created now, read on the caller's thread."""
+    request_state = get_request_state()
+    if request_state is None:
+        return NO_REQUEST
+    return route_template(request_state.scope) or UNMATCHED_ROUTE
+
+
+def count_log_bytes(record: logging.LogRecord, line: str) -> None:
+    route = getattr(record, "_obs_route", None)
+    # Not counting, or every sink formats the same record and it's counted once.
+    if route is None or getattr(record, "_obs_bytes_counted", False):
+        return
+    record._obs_bytes_counted = True
+    size = len(line) if line.isascii() else len(line.encode())
+    with _LOCK:
+        _BYTES_BY_ROUTE[route] = _BYTES_BY_ROUTE.get(route, 0) + size
+
+
+def get_log_bytes_by_route() -> dict[str, int]:
+    """Bytes of JSON log lines written so far, by the route that wrote them."""
+    with _LOCK:
+        return dict(_BYTES_BY_ROUTE)
+
+
+__all__ = [
+    "NO_REQUEST",
+    "count_log_bytes",
+    "current_route",
+    "get_log_bytes_by_route",
+    "start_counting",
+]

fastapi_observer-1.11.0/src/fastapiobserver/loop_watchdog.py

@@ -0,0 +1,77 @@
+"""Log the blocking call when something stops the event loop.
+
+A sync call inside ``async def``, such as ``requests.get``, ``time.sleep`` or a
+sync database driver, stops every request on the worker until it returns.
+asyncio's debug mode reports the slow callback afterwards, without the line that
+blocked. This thread reads the loop thread's stack while it is still blocked.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import sys
+import threading
+import time
+import traceback
+
+_LOGGER = logging.getLogger("fastapiobserver.event_loop")
+# Innermost frames only; the outer ones are the server and the middleware.
+_STACK_LIMIT = 30
+
+
+class EventLoopWatchdog:
+    """Sends the running loop a callback and logs ``event_loop.blocked`` if it
+    hasn't run after *threshold_seconds*, once per block.
+
+    Create it on the loop's thread, as the app's lifespan does.
+    """
+
+    def __init__(self, threshold_seconds: float) -> None:
+        self._loop = asyncio.get_running_loop()
+        self._loop_thread_id = threading.get_ident()
+        self._threshold = threshold_seconds
+        self._sent_at: float | None = None  # when the unanswered callback was sent
+        self._stopped = threading.Event()
+        self._thread = threading.Thread(
+            target=self._watch, name="fastapiobserver-loop-watchdog", daemon=True
+        )
+
+    def start(self) -> None:
+        self._thread.start()
+
+    def stop(self) -> None:
+        self._stopped.set()
+        self._thread.join(timeout=1.0)
+
+    def _answer(self) -> None:
+        self._sent_at = None
+
+    def _watch(self) -> None:
+        reported = False
+        while not self._stopped.wait(self._threshold / 2):
+            sent_at = self._sent_at
+            if sent_at is None:
+                reported = False
+                self._sent_at = time.monotonic()
+                try:
+                    self._loop.call_soon_threadsafe(self._answer)
+                except RuntimeError:  # the loop is closed
+                    return
+            elif not reported and time.monotonic() - sent_at >= self._threshold:
+                reported = True
+                self._report(time.monotonic() - sent_at)
+
+    def _report(self, blocked_seconds: float) -> None:
+        frame = sys._current_frames().get(self._loop_thread_id)
+        stack = "".join(traceback.format_stack(frame, limit=_STACK_LIMIT)) if frame else None
+        _LOGGER.warning(
+            "event_loop.blocked",
+            extra={
+                "event": {"blocked_ms": round(blocked_seconds * 1000), "stack": stack},
+                "_skip_enrichers": True,
+            },
+        )
+
+
+__all__ = ["EventLoopWatchdog"]

{fastapi_observer-1.9.0 → fastapi_observer-1.11.0}/src/fastapiobserver/metrics/prometheus.py

@@ -82,12 +82,17 @@ def _get_trace_exemplar() -> dict[str, str] | None:
 
 
 class _LogQueueMetricsCollector:
-    """Exposes log-queue, Logtail DLQ and sink circuit-breaker stats at scrape time."""
+    """Exposes log-queue, log volume, Logtail DLQ and sink circuit-breaker stats at
+    scrape time."""
 
     def collect(self) -> Iterator[Any]:
         from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
 
-        from ..logging import get_log_queue_stats, get_sink_circuit_breaker_stats
+        from ..logging import (
+            get_log_bytes_by_route,
+            get_log_queue_stats,
+            get_sink_circuit_breaker_stats,
+        )
         from ..sinks import get_logtail_dlq_stats
 
         queue = get_log_queue_stats()
@@ -132,6 +137,15 @@ class _LogQueueMetricsCollector:
             value=float(queue["block_timeout_total"]),
         )
 
+        log_bytes = CounterMetricFamily(
+            "fastapiobserver_log_bytes_total",
+            "Bytes of JSON log lines written, by the route of the request that wrote them.",
+            labels=["route"],
+        )
+        for route, size in sorted(get_log_bytes_by_route().items()):
+            log_bytes.add_metric([route], float(size))
+        yield log_bytes
+
         dlq = get_logtail_dlq_stats()
         written = CounterMetricFamily(
             "fastapiobserver_dlq_written_total",
@@ -176,6 +190,9 @@ def _register_log_queue_metrics_collector() -> None:
     # Per-worker queue stats would be meaningless once aggregated across processes.
     if _multiprocess_dir():
         return
+    from ..logging.volume import start_counting
+
+    start_counting()
     try:
         _import_prometheus_client().REGISTRY.register(_LogQueueMetricsCollector())
     except ValueError: