codexLB-up 1.25.0b10__py3-none-any.whl

app/core/config/background_jobs.py

@@ -0,0 +1,73 @@
+"""Dashboard-managed background job toggles (M2, slop-removal 0908).
+
+``auth_guardian_enabled``, ``automations_scheduler_enabled`` and
+``rate_limit_reset_credits_refresh_enabled`` are nullable ``dashboard_settings``
+columns that inherit the deprecated ``CODEX_LB_*`` environment alias and then
+the code default, resolved through :func:`resolve_inheritable`.
+
+The schedulers always start their loop; each cycle reads the ``SettingsCache``
+snapshot at its entry and skips the pass while the effective value is False, so
+a dashboard change takes effect on the next tick without a restart. Nothing
+here reads the database inside a runtime lock or per-item loop: the snapshot is
+taken once per cycle, before the scheduler's lock.
+"""
+
+from __future__ import annotations
+
+from typing import Final
+
+from app.core.config.inheritable import resolve_inheritable
+from app.core.config.settings import Settings, get_settings
+
+# ``dashboard_settings`` column, ``Settings`` field and provenance key share
+# one name (configuration-tiers).
+BACKGROUND_JOB_SETTINGS: Final[tuple[str, ...]] = (
+    "auth_guardian_enabled",
+    "automations_scheduler_enabled",
+    "rate_limit_reset_credits_refresh_enabled",
+)
+
+
+def resolve_background_job_toggle(
+    dashboard_settings: object | None,
+    name: str,
+    *,
+    startup_settings: object | None = None,
+) -> bool:
+    """Effective value of one background job toggle from a dashboard-settings snapshot.
+
+    ``dashboard_settings`` is the cached ``DashboardSettings`` row (or any object
+    exposing the same attribute; a missing attribute reads as NULL).
+    ``startup_settings`` supplies the environment layer when the caller already
+    holds it; it defaults to ``get_settings()``.
+    """
+    default = bool(Settings.model_fields[name].default)
+    startup = get_settings() if startup_settings is None else startup_settings
+    return bool(
+        resolve_inheritable(
+            getattr(dashboard_settings, name, None),
+            bool(getattr(startup, name, default)),
+            default,
+        ).value
+    )
+
+
+async def background_job_enabled(name: str, *, startup_settings: object | None = None) -> bool:
+    """Effective toggle from the shared ``SettingsCache`` snapshot (one read per scheduler cycle)."""
+    from app.core.config.settings_cache import get_settings_cache
+
+    return resolve_background_job_toggle(await get_settings_cache().get(), name, startup_settings=startup_settings)
+
+
+def auth_guardian_blocked_by_topology(settings: Settings | None = None) -> bool:
+    """True when the static topology forbids the Auth Guardian regardless of its toggle.
+
+    Concurrent force token refreshes across replicas can invalidate rotated
+    refresh tokens, so without leader election the guardian must not run in a
+    multi-replica ring at all (usage-refresh-policy "Multi-replica leader
+    guard"). The dashboard toggle cannot override this; the settings API
+    reports it so the UI can say why the guardian is idle.
+    """
+    startup = get_settings() if settings is None else settings
+    multi_replica = len(startup.http_responses_session_bridge_instance_ring) > 1
+    return multi_replica and not startup.leader_election_enabled

app/core/config/context_window_overrides.py

@@ -0,0 +1,166 @@
+"""Per-model context window overrides: dashboard rows over the environment dict.
+
+``CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES`` (``slug -> window``) is a T3
+setting whose dashboard home is the ``model_context_window_overrides`` table
+(one row per slug) rather than a ``dashboard_settings`` column, because the
+value is a mapping, not a scalar. The precedence of ``configuration-tiers``
+still applies per slug: a dashboard row wins, a slug without a row inherits
+the environment entry, and a slug with neither has no override (the catalog
+reports the upstream window). There is no code default for a slug, so this
+module carries its own per-slug resolver instead of ``resolve_inheritable``.
+
+The catalog endpoints (``GET /v1/models``, ``GET /backend-api/codex/models``)
+read the dashboard rows through ``ModelContextWindowOverridesCache`` (TTL +
+the cross-replica ``settings`` invalidation namespace, like
+``SettingsCache``) — never per model entry and never inside a lock.
+"""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Literal
+
+import anyio
+
+from app.core.cache.invalidation import NAMESPACE_SETTINGS, get_cache_invalidation_poller
+
+type ContextWindowOverrideSource = Literal["dashboard", "env"]
+
+
+@dataclass(frozen=True, slots=True)
+class ContextWindowOverride:
+    """Effective override for one slug and where it came from.
+
+    ``env_value`` is the environment entry for the slug (``None`` when the
+    environment has none), reported even while a dashboard row wins so the
+    dashboard can show what clearing the row returns to.
+    """
+
+    slug: str
+    context_window: int
+    source: ContextWindowOverrideSource
+    env_value: int | None
+
+
+def resolve_context_window_overrides(
+    dashboard: Mapping[str, int], environment: Mapping[str, int]
+) -> dict[str, ContextWindowOverride]:
+    """Combine dashboard rows and the environment dict, per slug, dashboard first.
+
+    This is the only place that combines the two layers; the catalog and the
+    settings API both derive their view from it.
+    """
+    resolved: dict[str, ContextWindowOverride] = {}
+    for slug in sorted(set(dashboard) | set(environment)):
+        env_value = environment.get(slug)
+        if slug in dashboard:
+            resolved[slug] = ContextWindowOverride(slug, dashboard[slug], "dashboard", env_value)
+        else:
+            assert env_value is not None
+            resolved[slug] = ContextWindowOverride(slug, env_value, "env", env_value)
+    return resolved
+
+
+def effective_context_window_overrides(dashboard: Mapping[str, int], environment: Mapping[str, int]) -> dict[str, int]:
+    """``slug -> effective window`` (dashboard row, else environment entry)."""
+    return {
+        slug: override.context_window
+        for slug, override in resolve_context_window_overrides(dashboard, environment).items()
+    }
+
+
+class ModelContextWindowOverridesCache:
+    """Process-wide snapshot of the dashboard override rows (``slug -> window``).
+
+    Mirrors ``SettingsCache``: a short TTL bounds staleness for out-of-band
+    database edits, and the settings API invalidates durably through the
+    ``settings`` namespace so every replica drops its snapshot at once.
+    """
+
+    # An expired snapshot is never fresh, whatever ``time.monotonic()`` returns
+    # this early after boot; 0.0 would read as fresh for the first TTL seconds
+    # of process uptime.
+    _EXPIRED = float("-inf")
+
+    def __init__(self, *, ttl_seconds: float = 5.0) -> None:
+        if ttl_seconds <= 0:
+            raise ValueError("ttl_seconds must be positive")
+        self._ttl_seconds = ttl_seconds
+        self._cached: dict[str, int] | None = None
+        self._cached_at = self._EXPIRED
+        self._lock = anyio.Lock()
+
+    async def get(self) -> Mapping[str, int]:
+        now = time.monotonic()
+        if self._cached is not None and now - self._cached_at < self._ttl_seconds:
+            return self._cached
+
+        async with self._lock:
+            now = time.monotonic()
+            if self._cached is not None and now - self._cached_at < self._ttl_seconds:
+                return self._cached
+            # Imported lazily: ``app.db.session`` builds the engine from the
+            # process settings on import, which unit tests of the resolver
+            # must not trigger.
+            from app.db.session import SessionLocal
+            from app.modules.settings.repository import ModelContextWindowOverridesRepository
+
+            try:
+                async with SessionLocal() as session:
+                    rows = await ModelContextWindowOverridesRepository(session).by_slug()
+            except Exception:
+                # The catalog must not 500 because the database is briefly
+                # unreachable: keep serving the last known rows (like
+                # ``SettingsCache.cached_row``) and retry on the next call. The
+                # timestamp is deliberately not refreshed.
+                if self._cached is None:
+                    raise
+                return self._cached
+            self._cached = rows
+            self._cached_at = now
+            return rows
+
+    def clear(self) -> None:
+        """Forget the snapshot entirely, without taking the lock.
+
+        Only for process-local resets with no concurrent load (test teardown);
+        every runtime path goes through ``invalidate``, which keeps the last
+        known rows as a fallback.
+        """
+        self._cached = None
+        self._cached_at = self._EXPIRED
+
+    async def invalidate(self, *, propagate: bool = True) -> None:
+        """Drop the snapshot and, unless ``propagate`` is False, durably bump the
+        cross-replica ``settings`` namespace before returning.
+
+        Only the snapshot's *freshness* is dropped, not the rows: the next
+        ``get`` always reloads, but a load that fails while the database is
+        briefly unreachable can still fall back to the last known rows instead
+        of failing the catalog request. Settings-namespace bumps are frequent
+        and mostly unrelated to this table, so forgetting the rows on every one
+        of them would make the fallback almost never available.
+
+        The expiry happens under the lock, like ``SettingsCache``: a catalog
+        build already inside ``get`` and awaiting the database must finish
+        before the snapshot is expired, otherwise it would install its pre-write
+        rows afterwards and keep serving the old window until the TTL expires.
+        The poller callback registered for the namespace passes
+        ``propagate=False`` so a remote bump never re-bumps (feedback-loop
+        prevention).
+        """
+        async with self._lock:
+            self._cached_at = self._EXPIRED
+        if propagate:
+            poller = get_cache_invalidation_poller()
+            if poller is not None:
+                await poller.bump(NAMESPACE_SETTINGS)
+
+
+_cache = ModelContextWindowOverridesCache()
+
+
+def get_model_context_window_overrides_cache() -> ModelContextWindowOverridesCache:
+    return _cache

app/core/config/dashboard_overrides.py

@@ -0,0 +1,145 @@
+"""Request-scoped dashboard overrides for ``Settings`` fields that moved to the dashboard.
+
+Precedence is fixed by ``configuration-tiers``: code default < environment <
+dashboard. ``Settings`` already folds the first two layers; this module adds the
+third for the fields that have a nullable ``dashboard_settings`` column of the
+same name (a non-NULL column wins, NULL inherits the environment value).
+
+Hot paths must not read the database: the request entry point (the
+``DashboardOverridesMiddleware``) reads the ``SettingsCache`` snapshot once and
+binds the overrides to a ``ContextVar``; consumers then call
+``with_dashboard_overrides(get_settings())`` (the proxy service facade does this
+for every ``_service_get_settings()`` caller) and read fields as before. Outside
+a bound context — startup, schedulers, unit tests — the base ``Settings`` is
+returned unchanged, so the environment fallback keeps applying there.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import contextvars
+from collections.abc import Iterator, Mapping
+from typing import TYPE_CHECKING, Final
+
+from app.core.config.settings import Settings
+
+if TYPE_CHECKING:
+    from app.db.models import DashboardSettings
+
+# C2-1 timeouts: upstream timeouts and request budgets managed from the
+# dashboard (Settings -> Advanced -> Upstream timeouts). The ``Settings`` field
+# is the deprecated env alias; the ``dashboard_settings`` column of the same
+# name wins when it is non-NULL.
+DASHBOARD_TIMEOUT_SETTINGS: Final[tuple[str, ...]] = (
+    "upstream_connect_timeout_seconds",
+    "proxy_request_budget_seconds",
+    "compact_request_budget_seconds",
+    "transcription_request_budget_seconds",
+    "stream_idle_timeout_seconds",
+    "proxy_downstream_websocket_idle_timeout_seconds",
+    "sse_keepalive_interval_seconds",
+    # M1 stream/bridge budgets: the Responses stream request budget and the
+    # HTTP session bridge request budget share the Upstream timeouts card.
+    "http_responses_stream_request_budget_seconds",
+    "http_responses_session_bridge_request_budget_seconds",
+    # end M1 stream/bridge budgets
+)
+
+# M3 codex prewarm: dashboard-managed behaviour switches, overlaid exactly like
+# the timeouts. Keeping them here means their single consumer reads the field
+# off ``Settings`` as before -- no extra snapshot read, and in particular no
+# ``await`` in the hot path that reads them.
+DASHBOARD_SWITCH_SETTINGS: Final[tuple[str, ...]] = ("http_responses_session_bridge_codex_prewarm_enabled",)
+
+# Dashboard-managed *enumerated* values, overlaid the same way. Unlike the
+# switches these are strings, which is why ``DashboardOverrideValue`` admits
+# ``str``: the thread cache identity mode has to be overlaid here and not only
+# folded into the settings API response, or the proxy would keep serving
+# ``shared`` while ``GET /api/settings`` reported the operator's choice as the
+# effective value.
+DASHBOARD_MODE_SETTINGS: Final[tuple[str, ...]] = ("thread_cache_identity_mode",)
+
+# Every ``Settings`` field whose dashboard column overrides the environment
+# value at runtime (the account-capacity caps have their own consumer path
+# through ``SettingsService``, and the resilience toggles their own task-bound
+# ``ResilienceToggles``; neither is overlaid here).
+DASHBOARD_OVERRIDE_SETTINGS: Final[tuple[str, ...]] = (
+    DASHBOARD_TIMEOUT_SETTINGS + DASHBOARD_SWITCH_SETTINGS + DASHBOARD_MODE_SETTINGS
+)
+
+type DashboardOverrideValue = float | bool | str
+
+
+def dashboard_overrides(row: DashboardSettings) -> dict[str, DashboardOverrideValue]:
+    """Non-NULL dashboard values keyed by ``Settings`` field name."""
+    overrides: dict[str, DashboardOverrideValue] = {}
+    for name in DASHBOARD_OVERRIDE_SETTINGS:
+        value = getattr(row, name, None)
+        if value is not None:
+            overrides[name] = value
+    return overrides
+
+
+class DashboardOverlay:
+    """Overrides bound to one request, with the overlaid ``Settings`` memoised per base object.
+
+    ``Settings`` is a process-wide cached singleton, so the copy is built once
+    per request; a test that swaps ``get_settings`` gets its own overlay.
+    """
+
+    __slots__ = ("_applied", "_base", "overrides")
+
+    def __init__(self, overrides: Mapping[str, DashboardOverrideValue]) -> None:
+        self.overrides: Mapping[str, DashboardOverrideValue] = dict(overrides)
+        self._base: Settings | None = None
+        self._applied: Settings | None = None
+
+    def apply(self, base: Settings) -> Settings:
+        if not self.overrides:
+            return base
+        if self._applied is not None and self._base is base:
+            return self._applied
+        applied = base.model_copy(update=dict(self.overrides))
+        self._base = base
+        self._applied = applied
+        return applied
+
+
+_OVERLAY: contextvars.ContextVar[DashboardOverlay | None] = contextvars.ContextVar(
+    "dashboard_settings_overlay", default=None
+)
+
+
+def with_dashboard_overrides(base: Settings) -> Settings:
+    """``base`` with the request-bound dashboard values applied (``base`` itself when none are bound)."""
+    overlay = _OVERLAY.get()
+    if overlay is None:
+        return base
+    return overlay.apply(base)
+
+
+def effective_settings(row: DashboardSettings, base: Settings) -> Settings:
+    """``base`` with ``row``'s non-NULL dashboard values applied, independent of the bound context.
+
+    For code that already holds a dashboard snapshot outside a request (startup
+    hooks, background loops) and must honour the dashboard the same way the
+    request path does.
+    """
+    return DashboardOverlay(dashboard_overrides(row)).apply(base)
+
+
+def bind_dashboard_overrides(row: DashboardSettings) -> contextvars.Token[DashboardOverlay | None]:
+    return _OVERLAY.set(DashboardOverlay(dashboard_overrides(row)))
+
+
+def reset_dashboard_overrides(token: contextvars.Token[DashboardOverlay | None]) -> None:
+    _OVERLAY.reset(token)
+
+
+@contextlib.contextmanager
+def dashboard_overrides_bound(row: DashboardSettings) -> Iterator[None]:
+    token = bind_dashboard_overrides(row)
+    try:
+        yield
+    finally:
+        reset_dashboard_overrides(token)

app/core/config/inheritable.py

@@ -0,0 +1,47 @@
+"""Resolver for dashboard-tier settings that inherit an environment or code default.
+
+One place combines the three layers (``configuration-tiers``): code default <
+process environment < dashboard column. ``SettingsService`` uses it to build
+the settings API's ``provenance`` map, and hot paths that read a dashboard
+snapshot use it to derive the effective value without re-implementing the
+precedence.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+type SettingSource = Literal["dashboard", "env", "default"]
+type SettingScalar = int | float | str | bool
+
+
+@dataclass(frozen=True, slots=True)
+class InheritableValue[T: SettingScalar]:
+    """Effective value of a dashboard-tier setting and where it came from.
+
+    ``source`` is ``"dashboard"`` when the dashboard column is non-NULL,
+    ``"env"`` when the column is NULL and the environment value differs from
+    the code default, and ``"default"`` otherwise. ``env_value`` is ``None``
+    for database-only settings that have no environment fallback.
+    """
+
+    value: T
+    source: SettingSource
+    env_value: T | None
+    default: T
+
+
+def resolve_inheritable[T: SettingScalar](
+    column_value: T | None, env_value: T | None, default: T
+) -> InheritableValue[T]:
+    """Resolve one inheritable setting as code default < environment < dashboard.
+
+    This is the only place that combines the three layers; call sites must not
+    re-implement the precedence (``configuration-tiers``).
+    """
+    if column_value is not None:
+        return InheritableValue(column_value, "dashboard", env_value, default)
+    if env_value is not None and env_value != default:
+        return InheritableValue(env_value, "env", env_value, default)
+    return InheritableValue(default, "default", env_value, default)

app/core/config/key_fingerprint.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from collections.abc import Callable
+from pathlib import Path
+
+from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+from sqlalchemy.dialects.sqlite import insert as sqlite_insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config.settings import get_settings
+from app.core.crypto import get_or_create_key
+from app.db.models import RuntimeSentinel
+from app.db.sqlite_lock_retry import retry_on_sqlite_lock
+
+logger = logging.getLogger(__name__)
+
+ENCRYPTION_KEY_FINGERPRINT_SENTINEL = "encryption_key_fingerprint"
+_FINGERPRINT_PREFIX_CHARS = 12
+
+
+class EncryptionKeyFingerprintMismatchError(RuntimeError):
+    """Raised when this replica's encryption key differs from the shared sentinel."""
+
+
+def compute_encryption_key_fingerprint(key_file: Path | None = None) -> str:
+    """Return ``sha256:<hex>`` over the raw encryption key bytes."""
+    key = get_or_create_key(key_file)
+    return f"sha256:{hashlib.sha256(key).hexdigest()}"
+
+
+def _fingerprint_prefix(fingerprint: str) -> str:
+    return fingerprint[: len("sha256:") + _FINGERPRINT_PREFIX_CHARS]
+
+
+async def _stamp_if_absent(session: AsyncSession, fingerprint: str) -> None:
+    values = {"name": ENCRYPTION_KEY_FINGERPRINT_SENTINEL, "value": fingerprint}
+    dialect = session.get_bind().dialect.name
+    if dialect == "postgresql":
+        stmt = pg_insert(RuntimeSentinel).values(**values).on_conflict_do_nothing(index_elements=[RuntimeSentinel.name])
+        await session.execute(stmt)
+    elif dialect == "sqlite":
+        stmt = (
+            sqlite_insert(RuntimeSentinel)
+            .values(**values)
+            .on_conflict_do_nothing(index_elements=[RuntimeSentinel.name])
+        )
+        await session.execute(stmt)
+    else:
+        existing = await session.scalar(
+            select(RuntimeSentinel).where(RuntimeSentinel.name == ENCRYPTION_KEY_FINGERPRINT_SENTINEL)
+        )
+        if existing is None:
+            session.add(RuntimeSentinel(**values))
+    await session.commit()
+
+
+async def verify_encryption_key_fingerprint(
+    session_factory: Callable[[], AsyncSession] | None = None,
+    *,
+    key_file: Path | None = None,
+    mode: str | None = None,
+) -> None:
+    """Stamp or verify the encryption-key fingerprint sentinel in the shared database.
+
+    The first replica to boot atomically stamps ``sha256(key)`` into
+    ``runtime_sentinels`` (insert-if-absent); every replica then compares its
+    local fingerprint against the stored value. A mismatch means the replicas
+    do not share the same encryption key material, which fails
+    replica-dependently at use time (dashboard cookie 401s, undecryptable
+    proxy passwords, ``bridge_forward_invalid`` HMAC rejections), so the
+    default mode refuses startup with a remediation message.
+    """
+    settings = get_settings()
+    effective_mode = mode if mode is not None else settings.encryption_key_fingerprint_mode
+    if effective_mode == "off":
+        return
+
+    if session_factory is None:
+        from app.db.session import SessionLocal
+
+        make_session: Callable[[], AsyncSession] = SessionLocal
+    else:
+        make_session = session_factory
+
+    local_fingerprint = compute_encryption_key_fingerprint(key_file)
+
+    async def stamp_and_read() -> str | None:
+        async with make_session() as session:
+            await _stamp_if_absent(session, local_fingerprint)
+            return await session.scalar(
+                select(RuntimeSentinel.value).where(RuntimeSentinel.name == ENCRYPTION_KEY_FINGERPRINT_SENTINEL)
+            )
+
+    # A lock failure that outlives the budget still propagates and fails
+    # startup, exactly as it did before the retry was factored out.
+    stored = await retry_on_sqlite_lock(stamp_and_read, what="encryption-key fingerprint stamp")
+
+    if stored is None or stored == local_fingerprint:
+        return
+
+    message = (
+        "Encryption key mismatch across replicas: this replica's key fingerprint "
+        f"{_fingerprint_prefix(local_fingerprint)}... does not match the fingerprint "
+        f"{_fingerprint_prefix(stored)}... stamped in the shared database. Every replica must "
+        "mount the same encryption.key file. If you rotated the key intentionally, delete the "
+        f"'{ENCRYPTION_KEY_FINGERPRINT_SENTINEL}' row from runtime_sentinels, or set "
+        "CODEX_LB_ENCRYPTION_KEY_FINGERPRINT_MODE=warn to bypass this check."
+    )
+    if effective_mode == "warn":
+        logger.error(message)
+        return
+    raise EncryptionKeyFingerprintMismatchError(message)