codexLB-up 1.25.0b10__py3-none-any.whl

app/core/config/settings_cache.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import time
+
+import anyio
+
+from app.core.cache.invalidation import NAMESPACE_SETTINGS, get_cache_invalidation_poller
+from app.db.models import DashboardSettings
+from app.db.session import SessionLocal
+from app.modules.settings.repository import SettingsRepository
+
+
+class SettingsCache:
+    def __init__(self, *, ttl_seconds: float = 5.0) -> None:
+        if ttl_seconds <= 0:
+            raise ValueError("ttl_seconds must be positive")
+        self._ttl_seconds = ttl_seconds
+        self._cached_settings: DashboardSettings | None = None
+        self._cached_at = 0.0
+        # The last row ever loaded, kept across ``invalidate`` for the
+        # fallback readers of ``cached_row`` (see its docstring).
+        self._last_loaded_settings: DashboardSettings | None = None
+        self._lock = anyio.Lock()
+
+    async def get(self) -> DashboardSettings:
+        now = time.monotonic()
+        if self._cached_settings is not None and now - self._cached_at < self._ttl_seconds:
+            return self._cached_settings
+
+        async with self._lock:
+            now = time.monotonic()
+            if self._cached_settings is not None and now - self._cached_at < self._ttl_seconds:
+                return self._cached_settings
+
+            async with SessionLocal() as session:
+                settings = await SettingsRepository(session).get_or_create()
+                self._cached_settings = settings
+                self._last_loaded_settings = settings
+                self._cached_at = now
+                return settings
+
+    async def refresh(self) -> DashboardSettings:
+        """Load the row now and make it the current snapshot.
+
+        Registered as a cache-invalidation callback for the ``settings``
+        namespace so that a dashboard change reaches every replica on the bus
+        instead of waiting for the next request to pull it in. That bound
+        matters for readers that cannot await — the conversation archive gate
+        runs per archived frame, and a replica carrying only already-open
+        streams would otherwise keep the previous value indefinitely.
+        """
+        async with self._lock:
+            async with SessionLocal() as session:
+                settings = await SettingsRepository(session).get_or_create()
+            self._cached_settings = settings
+            self._last_loaded_settings = settings
+            self._cached_at = time.monotonic()
+            return settings
+
+    def cached_row(self) -> DashboardSettings | None:
+        """The last row this cache loaded, even if past its TTL or invalidated.
+
+        ``None`` only before the first successful load. For callers that must
+        not fail when a refresh is impossible or has not happened yet (the
+        request-scoped dashboard overrides, the conversation archive gate) and
+        prefer the last known dashboard values over silently reverting to the
+        environment: an invalidation means "reload before trusting this as
+        current", not "the operator never configured anything".
+        """
+        return self._last_loaded_settings
+
+    async def invalidate(self, *, propagate: bool = True) -> None:
+        """Drop the cached settings row and, unless ``propagate`` is False, durably
+        bump the cross-replica ``settings`` namespace before returning.
+
+        Settings mutations are security-bearing (password hash, guest access, TOTP,
+        API-key auth toggle), so the bump is awaited rather than coalesced. The
+        cache-invalidation poller callback registers ``propagate=False`` so a remote
+        bump never re-bumps (feedback-loop prevention).
+
+        ``cached_row`` keeps returning the last loaded row: dropping it here
+        would make the fallback readers revert to the environment values in the
+        window between an invalidation and the next load, which for a toggle
+        such as the conversation archive means the deprecated environment alias
+        could resume recording that an operator just turned off.
+        """
+        async with self._lock:
+            self._cached_settings = None
+            self._cached_at = 0.0
+        if propagate:
+            poller = get_cache_invalidation_poller()
+            if poller is not None:
+                await poller.bump(NAMESPACE_SETTINGS)
+
+
+_settings_cache = SettingsCache()
+
+
+def get_settings_cache() -> SettingsCache:
+    return _settings_cache

app/core/config/spool_retention.py

@@ -0,0 +1,217 @@
+"""Dashboard-managed retention of the durable HTTP-bridge operation spool (R2, slop-removal 0908).
+
+``http_responses_session_bridge_operation_spool_retention_seconds`` is a
+nullable ``dashboard_settings`` column that inherits the deprecated
+``CODEX_LB_*`` environment alias and then the code default (7 days), resolved
+through :func:`resolve_inheritable`. Its home is the dashboard's data retention
+card because the spool holds raw request payloads: a shorter window deletes
+prompt material sooner.
+
+The window has a floor. The spool is the replay source for durable bridge
+recovery, so it must outlive every window in which something may still read a
+spooled operation (:func:`operation_spool_retention_floor_seconds`). Purging
+earlier deletes the transcript out from under a live reader -- which is why
+abandoned durable bridge rows are already retained for the session reuse window
+(``responses-api-compat`` "Retain completed recovery transcripts").
+
+Neither consumer is on a request path: startup runs one bounded purge and the
+leader-gated cleanup scheduler runs one pass per tick, each resolving the value
+once per pass from the dashboard snapshot it already holds -- never inside a
+runtime lock and never per operation.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Final
+
+from app.core.config.inheritable import resolve_inheritable
+from app.core.config.settings import Settings, get_settings
+
+logger = logging.getLogger(__name__)
+
+# ``dashboard_settings`` column, ``Settings`` field and provenance key share
+# one name (configuration-tiers).
+OPERATION_SPOOL_RETENTION_SETTING: Final = "http_responses_session_bridge_operation_spool_retention_seconds"
+
+# NOT NULL ``dashboard_settings`` columns; no environment layer.
+_DASHBOARD_REUSE_WINDOW_COLUMNS: Final[tuple[str, ...]] = (
+    "openai_cache_affinity_max_age_seconds",
+    "http_responses_session_bridge_prompt_cache_idle_ttl_seconds",
+)
+_BRIDGE_REQUEST_BUDGET_SETTING: Final = "http_responses_session_bridge_request_budget_seconds"
+
+# Settings whose value moves the floor, so a PUT touching any of them is
+# re-checked against it.
+SPOOL_RETENTION_FLOOR_INPUTS: Final[tuple[str, ...]] = (
+    *_DASHBOARD_REUSE_WINDOW_COLUMNS,
+    _BRIDGE_REQUEST_BUDGET_SETTING,
+)
+
+_OPERATION_SPOOL_RETENTION_DEFAULT: Final[float] = float(
+    Settings.model_fields[OPERATION_SPOOL_RETENTION_SETTING].default
+)
+_BRIDGE_REQUEST_BUDGET_DEFAULT: Final[float] = float(Settings.model_fields[_BRIDGE_REQUEST_BUDGET_SETTING].default)
+# Floor of the stale-operation abandonment sweep's inactivity window
+# (``session_registry.abandon_stale_http_bridge_operations``).
+_STALE_OPERATION_ABANDONMENT_FLOOR_SECONDS: Final[float] = 30.0 * 60.0
+# An ever-claimed retry circuit gets one extra TTL of grace in both
+# ``retry_circuit`` and ``purge_retry_circuits_before``, because a claim leaves
+# ``updated_at_epoch`` untouched.
+_CLAIMED_RETRY_CIRCUIT_TTL_MULTIPLIER: Final[float] = 2.0
+
+
+def _effective_seconds(dashboard_settings: object | None, name: str, environment_value: float, default: float) -> float:
+    """One inheritable duration from either snapshot shape.
+
+    Accepts the ``DashboardSettings`` row (a NULL column inherits) and the
+    ``DashboardSettingsData`` the settings API builds (already effective, so
+    the value resolves to itself).
+    """
+    column_value = getattr(dashboard_settings, name, None)
+    return float(
+        resolve_inheritable(None if column_value is None else float(column_value), environment_value, default).value
+    )
+
+
+def bridge_session_reuse_window_seconds(dashboard_settings: object) -> float:
+    """Longest window in which an idle durable bridge session stays reusable.
+
+    An idle local bridge session stays reusable until its effective idle TTL --
+    up to the prompt-cache reuse TTL for prompt-cache sessions -- which can
+    exceed the prompt-cache affinity max age. The two idle TTLs are fixed
+    constants (#2256) read through their module at call time, so a test
+    monkeypatching the attribute is honoured here too; the import is lazy
+    because this module is configuration and that module is the request-path
+    implementation owning them.
+
+    Both column terms are NOT NULL on ``dashboard_settings``, so a real row
+    always carries them; a term is skipped only for the partial snapshot fakes
+    the startup and scheduler tests build, the same tolerance
+    ``warn_environment_shadowed_by_dashboard`` and ``effective_settings`` apply
+    to that row.
+    """
+    from app.modules.proxy._service.http_bridge import helpers as _http_bridge_helpers
+
+    terms = [
+        float(_http_bridge_helpers.HTTP_BRIDGE_IDLE_TTL_SECONDS),
+        float(_http_bridge_helpers.HTTP_BRIDGE_CODEX_IDLE_TTL_SECONDS),
+    ]
+    for name in _DASHBOARD_REUSE_WINDOW_COLUMNS:
+        value = getattr(dashboard_settings, name, None)
+        if value is not None:
+            terms.append(float(value))
+    return max(terms)
+
+
+def spool_retention_floor_terms_seconds(
+    dashboard_settings: object,
+    *,
+    startup_settings: object | None = None,
+) -> dict[str, float]:
+    """Every window that may still read a spooled operation, by term name.
+
+    - ``bridge_session_reuse_window``: while the owning session is still
+      reusable, the next request on it resolves the operation ledger and may
+      replay a spooled transcript.
+    - ``stale_operation_abandonment_window``: until the abandonment sweep
+      fences an ownerless nonterminal operation, that operation is still
+      reading and appending its own spool.
+    - ``claimed_retry_circuit_lifetime``: a surviving retry circuit can admit a
+      replay of an operation whose spool must therefore still exist. A claim
+      advances only the admission generation and leaves the row timestamp
+      alone, so an ever-claimed circuit is honoured -- by both the loader and
+      the scheduled purge -- for two TTLs, not one.
+
+    The terms are read from the configuration in force, so this is a
+    steady-state bound. A live bridge session keeps the idle TTL it captured
+    (``session_registry`` only ever max-promotes it), so lowering a reuse
+    window does not shorten sessions that are already open; the abandoned-row
+    retention derived from the same windows is non-retroactive in exactly the
+    same way.
+    """
+    from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
+
+    startup = get_settings() if startup_settings is None else startup_settings
+    bridge_request_budget = _effective_seconds(
+        dashboard_settings,
+        _BRIDGE_REQUEST_BUDGET_SETTING,
+        float(getattr(startup, _BRIDGE_REQUEST_BUDGET_SETTING, _BRIDGE_REQUEST_BUDGET_DEFAULT)),
+        _BRIDGE_REQUEST_BUDGET_DEFAULT,
+    )
+    return {
+        "bridge_session_reuse_window": bridge_session_reuse_window_seconds(dashboard_settings),
+        "stale_operation_abandonment_window": max(_STALE_OPERATION_ABANDONMENT_FLOOR_SECONDS, bridge_request_budget),
+        "claimed_retry_circuit_lifetime": _CLAIMED_RETRY_CIRCUIT_TTL_MULTIPLIER
+        * float(DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS),
+    }
+
+
+def operation_spool_retention_floor_seconds(
+    dashboard_settings: object,
+    *,
+    startup_settings: object | None = None,
+) -> float:
+    """Lowest spool retention that still covers every replay window."""
+    return max(spool_retention_floor_terms_seconds(dashboard_settings, startup_settings=startup_settings).values())
+
+
+def binding_spool_retention_floor_term(
+    dashboard_settings: object,
+    *,
+    startup_settings: object | None = None,
+) -> tuple[str, float]:
+    """``(term name, seconds)`` of the window that binds the floor, for error messages."""
+    terms = spool_retention_floor_terms_seconds(dashboard_settings, startup_settings=startup_settings)
+    name = max(terms, key=lambda key: terms[key])
+    return name, terms[name]
+
+
+def resolve_operation_spool_retention_seconds(
+    dashboard_settings: object | None,
+    *,
+    startup_settings: object | None = None,
+) -> float:
+    """Effective spool retention from a dashboard-settings snapshot.
+
+    ``dashboard_settings`` is a ``DashboardSettings`` row (or any object
+    exposing the same attribute; a missing attribute reads as NULL).
+    ``startup_settings`` supplies the environment layer when the caller already
+    holds it; it defaults to ``get_settings()``.
+    """
+    startup = get_settings() if startup_settings is None else startup_settings
+    return _effective_seconds(
+        dashboard_settings,
+        OPERATION_SPOOL_RETENTION_SETTING,
+        float(getattr(startup, OPERATION_SPOOL_RETENTION_SETTING, _OPERATION_SPOOL_RETENTION_DEFAULT)),
+        _OPERATION_SPOOL_RETENTION_DEFAULT,
+    )
+
+
+def warn_spool_retention_below_floor(
+    dashboard_settings: object,
+    startup_settings: object | None = None,
+) -> tuple[str, float, float] | None:
+    """Warn once at startup when the effective spool retention is below its floor.
+
+    The environment alias alone can put a deployment below the floor while the
+    dashboard column is NULL, and the settings API never saw that value. Warning
+    here means the operator learns about it before a dashboard edit is refused.
+    Returns ``(binding term, effective retention, floor)`` when it warned, so the
+    caller can assert on it; ``None`` otherwise. Only these two numbers and the
+    term name are logged -- never the value of any other setting.
+    """
+    retention = resolve_operation_spool_retention_seconds(dashboard_settings, startup_settings=startup_settings)
+    term, floor = binding_spool_retention_floor_term(dashboard_settings, startup_settings=startup_settings)
+    if retention >= floor:
+        return None
+    logger.warning(
+        "HTTP bridge operation spool retention is below its replay floor: %s=%gs < %gs (bound by %s). "
+        "Expired transcripts can be deleted while a recovery can still replay them; "
+        "raise the value in Settings -> Data retention. Dashboard updates that lower it further are refused.",
+        OPERATION_SPOOL_RETENTION_SETTING,
+        retention,
+        floor,
+        term,
+    )
+    return term, retention, floor

app/core/config/tiers.py

@@ -0,0 +1,174 @@
+"""Configuration tiers for every ``Settings`` field (configuration-tiers capability).
+
+Each ``CODEX_LB_*`` field belongs to exactly one tier:
+
+- ``T0`` bootstrap: needed before the database is reachable (data dir, DB URL,
+  encryption key, migration policy, first-login token). env only.
+- ``T1`` instance topology: legitimately differs per replica or per deployment
+  (bridge instance id/ring, bind hosts, trusted proxies, pool/worker sizes,
+  leader election, observability endpoints). env only.
+- ``T2`` secret: encrypted in the database, env is at most a seed.
+- ``T3`` behaviour tunable / feature flag: the dashboard (``dashboard_settings``)
+  is the management surface. A T3 field that has no ``dashboard_settings``
+  column of the same name MUST either name its existing database home in
+  ``DASHBOARD_HOMES`` (``table.column``) or be listed in ``MIGRATING`` until
+  it gets one.
+- ``T4`` incident debug: env allowed, dashboard toggle recommended.
+
+``scripts/check_settings_tiers.py`` (run by ``make lint``) fails when a field
+is missing here, when a T3 field has none of a same-name dashboard column, a
+``DASHBOARD_HOMES`` mapping or a ``MIGRATING`` entry, when a ``DASHBOARD_HOMES``
+target names a column that does not exist, or when ``.env.example`` mentions a
+T2-T4 field. Entries for fields that no longer exist only warn, so removals can
+land in either order.
+"""
+
+from __future__ import annotations
+
+from typing import Final, Literal
+
+Tier = Literal["T0", "T1", "T2", "T3", "T4"]
+
+TIERS: Final[tuple[Tier, ...]] = ("T0", "T1", "T2", "T3", "T4")
+
+# Declaration order follows ``Settings``; the tier is the policy answer to
+# "may this value differ between two replicas / must it exist before the DB?".
+SETTING_TIERS: Final[dict[str, Tier]] = {
+    "data_dir": "T0",
+    "database_url": "T0",
+    "database_pool_size": "T1",
+    "database_max_overflow": "T1",
+    "database_migrate_on_startup": "T0",
+    "database_sqlite_pre_migrate_backup_enabled": "T0",
+    "database_sqlite_pre_migrate_backup_max_files": "T0",
+    "database_sqlite_startup_check_mode": "T0",
+    "database_alembic_auto_remap_enabled": "T0",
+    "database_migration_lock_timeout_seconds": "T0",
+    "upstream_base_url": "T1",
+    "upstream_connect_timeout_seconds": "T3",
+    "upstream_websocket_trust_env": "T1",
+    "proxy_request_budget_seconds": "T3",
+    "http_responses_stream_request_budget_seconds": "T3",
+    "compact_request_budget_seconds": "T3",
+    "stream_idle_timeout_seconds": "T3",
+    "sse_keepalive_interval_seconds": "T3",
+    "proxy_downstream_websocket_idle_timeout_seconds": "T3",
+    # bind host of the OAuth callback listener; policy §2 T1 example
+    "oauth_callback_host": "T1",
+    "auth_guardian_enabled": "T3",
+    "transcription_request_budget_seconds": "T3",
+    # path to a replacement quota-key registry; deployment artefact, not behaviour
+    "additional_quota_registry_file": "T1",
+    "rate_limit_reset_credits_refresh_enabled": "T3",
+    # K2 bridge: T4 kill switch for the request-path bridge, not a tunable.
+    "http_responses_session_bridge_enabled": "T4",
+    "http_responses_session_bridge_request_budget_seconds": "T3",
+    "http_responses_session_bridge_codex_prewarm_enabled": "T3",
+    "http_responses_session_bridge_max_sessions": "T1",
+    "http_responses_session_bridge_queue_limit": "T1",
+    "http_responses_session_bridge_operation_event_spool_max_bytes": "T1",
+    "http_responses_session_bridge_operation_spool_format": "T1",
+    "http_responses_session_bridge_operation_event_spool_batch_size": "T1",
+    "http_responses_session_bridge_operation_event_spool_flush_interval_seconds": "T1",
+    "http_responses_session_bridge_operation_event_spool_max_pending_events": "T1",
+    "http_responses_session_bridge_operation_event_spool_max_pending_bytes": "T1",
+    "http_responses_session_bridge_operation_spool_retention_seconds": "T3",
+    "http_responses_session_bridge_instance_id": "T1",
+    "http_responses_session_bridge_instance_ring": "T1",
+    "http_responses_session_bridge_advertise_base_url": "T1",
+    "upstream_route_cache_ttl_seconds": "T1",
+    "automations_scheduler_enabled": "T3",
+    "telemetry_enabled": "T3",
+    "telemetry_endpoint": "T1",
+    "encryption_key_file": "T0",
+    "encryption_key_fingerprint_mode": "T0",
+    "database_migrations_fail_fast": "T0",
+    "trace": "T4",
+    "conversation_archive_enabled": "T3",
+    "conversation_archive_dir": "T1",
+    "conversation_archive_queue_max_bytes": "T1",
+    "model_registry_client_version": "T1",
+    "model_registry_snapshot_max_age_seconds": "T1",
+    "model_context_window_overrides": "T3",
+    # raw socket-peer CIDRs of this replica's network namespace; sibling of the
+    # trusted-proxy CIDRs below, not of the firewall allowlist (projected IPs)
+    "proxy_unauthenticated_client_cidrs": "T1",
+    # trusted-proxy topology; policy §2 T1 example
+    "firewall_trust_proxy_headers": "T1",
+    # trusted-proxy topology; policy §2 T1 example
+    "firewall_trusted_proxy_cidrs": "T1",
+    "firewall_ip_cache_ttl_seconds": "T1",
+    # reverse-proxy trust list for scope["client"] projection (Uvicorn semantics)
+    "forwarded_allow_ips": "T1",
+    # reverse-proxy deployment dependent, self-lockout risk from the dashboard (policy D2)
+    "dashboard_auth_mode": "T1",
+    # last link of the dashboard_auth_mode trust chain: whether a loopback Host
+    # header is believed is a per-deployment reverse-proxy fact (policy D2)
+    "dashboard_trust_loopback_host_header_for_long_sessions": "T1",
+    # header name is fixed by the reverse-proxy deployment (policy D2)
+    "dashboard_auth_proxy_header": "T1",
+    # the group header the same reverse proxy sets (policy D2)
+    "dashboard_auth_proxy_groups_header": "T1",
+    "metrics_enabled": "T1",
+    "metrics_port": "T1",
+    "log_format": "T1",
+    "leader_election_enabled": "T1",
+    "leader_election_ttl_seconds": "T1",
+    "circuit_breaker_enabled": "T3",
+    "soft_drain_enabled": "T3",
+    "deterministic_failover_enabled": "T3",
+    "thread_cache_identity_mode": "T3",
+    "backpressure_max_concurrent_requests": "T1",
+    "bulkhead_proxy_limit": "T1",
+    "bulkhead_dashboard_limit": "T1",
+    # first remote login token; policy §2 lists it under T0 bootstrap, not T2
+    "dashboard_bootstrap_token": "T0",
+    # advertised client-facing address; differs per deployment
+    "connect_address": "T1",
+    # per-process asyncio.Semaphore capacity, same class as bulkhead_proxy_limit
+    "proxy_response_create_limit": "T1",
+    "proxy_account_response_create_limit": "T3",
+    "proxy_account_stream_limit": "T3",
+    "proxy_account_stream_recovery_reserve": "T3",
+    "proxy_api_key_fair_share_congestion_threshold_pct": "T3",
+    "proxy_account_inflight_penalty_pct": "T3",
+    "proxy_overload_isolation_seconds": "T3",
+    "proxy_account_error_rate_weighting_enabled": "T3",
+    "proxy_account_lease_token_weight": "T3",
+    "proxy_account_lease_ttl_seconds": "T3",
+    "proxy_account_caps_scope": "T1",
+    "proxy_account_cap_partition_scale_down_seconds": "T1",
+    "timeout_invariant_validation_strict": "T4",
+    "memory_reject_threshold_mb": "T1",
+    "event_loop_lag_warn_threshold_seconds": "T1",
+    "otel_enabled": "T1",
+    "otel_exporter_endpoint": "T1",
+    "shutdown_drain_timeout_seconds": "T1",
+    "http_connector_limit": "T1",
+    "http_connector_limit_per_host": "T1",
+}
+
+# T3 fields that still live only in env. Value = target dashboard home or
+# "backlog" while none has been designed. Remove the entry in the PR that adds
+# the ``dashboard_settings`` column (the checker warns once it is redundant).
+#
+# Empty since ``constantize-token-refresh-interval``: every T3 field now has a
+# dashboard home (a same-name ``dashboard_settings`` column or a
+# ``DASHBOARD_HOMES`` mapping), and the last backlog entry
+# (``token_refresh_interval_days``) became the fixed
+# ``app/core/auth/refresh.TOKEN_REFRESH_INTERVAL_DAYS`` instead of getting a
+# card nobody would flip. An empty registry is the intended terminal state and
+# is not an error; the registry stays as the declared landing spot for a T3
+# field that must ship one release ahead of its column.
+MIGRATING: Final[dict[str, str]] = {}
+
+# T3 fields whose database home already exists under a different column name
+# (or in another configuration table). Value = ``table.column``; the checker
+# fails when the column does not exist. The field's environment variable is the
+# fallback while that column holds no decision, per the precedence rule.
+DASHBOARD_HOMES: Final[dict[str, str]] = {
+    # persisted decision > CODEX_LB_TELEMETRY_ENABLED > default (telemetry spec)
+    "telemetry_enabled": "dashboard_settings.telemetry_consent",
+    # M4 model catalogue: one row per slug; the env dict is the per-slug fallback
+    "model_context_window_overrides": "model_context_window_overrides.context_window",
+}