codexLB-up 1.25.0b10__py3-none-any.whl

app/modules/accounts/usage_time_rollup.py

@@ -0,0 +1,1037 @@
+"""Time-axis request-usage rollups: schema primitives and repository.
+
+Four permanent aggregate tables serve the dashboard/statistics read paths
+without scanning raw ``request_logs``:
+
+- ``request_usage_hourly_rollups`` — UTC hour buckets x (account_id,
+  api_key_id, model, service_tier, request_kind, is_deleted).
+- ``request_usage_hourly_error_rollups`` — UTC hour buckets x (account_id,
+  error_code); the top-error satellite (unbounded cardinality isolated).
+- ``request_demand_quarter_rollups`` — 900s slots x (account_id, api_key_id,
+  model, reasoning_effort, request_kind, status, is_deleted) for the quota
+  planner. The full legacy demand grain is preserved on purpose: the
+  planner's ``_bin_demand_units`` applies ``max()`` PER BIN before summing
+  (nonlinear), so folding to a coarser grain would change forecasts.
+- ``request_conversation_hourly_rollups`` — UTC hour buckets x (normalized
+  conversation_id, account_id, is_deleted); the distinct-conversation
+  presence satellite. Folded on its OWN watermark and fold pass with the
+  shared read-side filter (normalized conversation id present, warmup kinds
+  excluded); the deleted split stays a dimension because the dashboard
+  conversation reads exclude soft-deleted rows while the reports reads
+  include them.
+
+Watermark contract: ``account_usage_rollup_state.hourly_folded_through`` and
+``conversation_folded_through`` (the same single state row as the lifetime
+fold, so one ``FOR UPDATE`` row lock serializes every fold and lifecycle
+mutation) are ALWAYS aligned to a whole UTC hour. Raw rows with
+``requested_at < watermark`` are fully folded by that watermark's tables;
+rows at or above it are the live tail. Buckets are half-open ``[start,
+end)``.
+
+NULL-dimension sentinel: nullable raw dimensions (account_id, api_key_id,
+service_tier, reasoning_effort) are stored as ``'\\x1f'`` so they can
+participate in the primary key on both dialects (UNIQUE/PK treat NULLs as
+distinct rows on PostgreSQL and SQLite). The encoding is collision-free:
+raw values that themselves start with the sentinel are escaped with one
+more sentinel character, so the empty string — a legitimate value the
+request models accept for service_tier/reasoning_effort, and a GROUP BY
+group distinct from NULL in the legacy raw queries — round-trips intact.
+Use :func:`to_dimension` / :func:`from_dimension` (SQL: ``_dimension_expr``)
+at the write and read boundaries — a missed mapping silently diverges
+rollups from raw.
+
+History-rewrite discipline (MUST): any code path that mutates a folded
+dimension (``requested_at``, ``deleted_at``, ``account_id``, ``api_key_id``,
+``model``, ``service_tier``, ``reasoning_effort``, ``request_kind``,
+``status``, ``error_code``, ``conversation_id``) or an aggregated measure
+column of ``request_logs`` rows BELOW the relevant watermark must take
+``lock_fold_state()`` in the same transaction and either mirror the mutation
+into every rollup table that folded it or skip the pre-watermark rows
+(folded buckets are never recomputed from raw — raw may already be pruned by
+retention). ``RequestLogsRepository.update_model_for_request`` takes the
+skip route (and needs no conversation-satellite bound at all: neither
+``model`` nor ``cost_usd`` is folded there).
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Iterable, Sequence
+from dataclasses import astuple, dataclass, replace
+from datetime import datetime, timedelta
+
+from sqlalchemy import BigInteger, ColumnElement, Integer, and_, case, cast, delete, func, insert, select, true, update
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.sql.elements import SQLCoreOperations
+
+from app.core.usage.logs import CANCELLED_STATUS, NON_ERROR_STATUSES
+from app.core.utils.time import utcnow
+from app.db.models import (
+    AccountUsageRollupState,
+    RequestConversationHourlyRollup,
+    RequestDemandQuarterRollup,
+    RequestLog,
+    RequestUsageHourlyErrorRollup,
+    RequestUsageHourlyRollup,
+)
+from app.db.session import get_background_session, sqlite_writer_section
+from app.modules.accounts.usage_rollup import (
+    _STATE_ROW_ID,
+    FOLD_LAG,
+    _FoldStatus,
+    _insert_fn,
+    _locked_state,
+    _state_bootstrap_stmt,
+)
+
+logger = logging.getLogger(__name__)
+
+HOURLY_BUCKET_SECONDS = 3600
+QUARTER_SLOT_SECONDS = 900
+
+# Historical backfill folds at most this much history per slice transaction
+# and at most TS_MAX_SLICES_PER_PASS slices per pass, so the initial backfill
+# (millions of raw rows) spreads its I/O bursts and fold-state row-lock
+# occupancy across scheduler ticks instead of one giant catch-up.
+TS_FOLD_SLICE = timedelta(hours=48)
+TS_MAX_SLICES_PER_PASS = 20
+
+# Rolling-upgrade repair (#1552): the cancelled_count migration runs before
+# old replicas drain, so a legacy leader can still fold hours with the old
+# error fold (cancelled terminals counted as errors, cancelled_count 0,
+# client_disconnected folded into the error satellite) and advance the shared
+# watermark — buckets new code would otherwise never revisit. Old writers run
+# old code and cannot be fenced, so the NEW code repairs instead: it refolds
+# the legacy-suspect range `[upgrade_repair_from, hourly_folded_through)`
+# from raw. The marker is stamped by the migration (existing rows get their
+# then-current watermark; a state row bootstrapped by an OLD replica after
+# the migration gets the epoch server default, marking its entire backfill
+# suspect — a legacy leader can advance TS_MAX_SLICES_PER_PASS x
+# TS_FOLD_SLICE per pass, so no fixed trailing window could cover it) and
+# cleared to NULL only by new code once the range is refolded. Repair
+# progress persists chunk-by-chunk through the marker, so a crash resumes
+# instead of restarting.
+#
+# With the marker already NULL, each process's first fold pass still refolds
+# this trailing window as flip-flop defense: an old replica that regains
+# fold leadership AFTER the marker was cleared (mid-rollout) writes legacy
+# buckets the marker no longer tracks. A steady-state legacy interlude
+# advances the watermark by minutes-to-hours — far below this span — and the
+# rollout that makes interludes possible also guarantees another new-code
+# process (the old replica's replacement) starts afterwards and runs this
+# defense. Cost: one backfill-slice-equivalent per process start.
+#
+# Both paths clamp to hours fully covered by surviving raw rows (retention
+# prunes oldest-first with a contiguous frontier, so every row at or above
+# the earliest surviving one is intact) — the repair can never erase folded
+# statistics it cannot recompute; clamped-out buckets keep the disclosed
+# legacy fold. Neither path is a historical backfill.
+UPGRADE_REPAIR_WINDOW = timedelta(hours=48)
+_upgrade_repair_done = False
+
+_EPOCH = datetime(1970, 1, 1)
+
+# Synthetic request kinds every statistics read path filters out. Folded
+# verbatim as a dimension; readers exclude them bucket-side exactly as the
+# legacy raw queries do row-side.
+WARMUP_REQUEST_KINDS = ("warmup", "limit_warmup")
+_EXCLUDED_REQUEST_KINDS = WARMUP_REQUEST_KINDS
+
+# The whitespace set the conversation readers strip before comparing/counting
+# conversation ids (str.strip()'s ASCII whitespace).
+CONVERSATION_WHITESPACE = " \t\n\v\f\r"
+
+
+def normalized_thread_key_expr(column: SQLCoreOperations[str | None]) -> ColumnElement:
+    """Trim a thread-identifying text column, collapsing blank to NULL.
+
+    The single SQL definition shared by the conversation fold and every
+    conversation reader (dashboard and reports repositories) — a drifted
+    variant would silently split one conversation into two. It is reused for
+    ``session_id`` so a session-keyed thread normalizes identically.
+    """
+    trimmed = func.ltrim(func.rtrim(column, CONVERSATION_WHITESPACE), CONVERSATION_WHITESPACE)
+    return func.nullif(trimmed, "")
+
+
+def conversation_id_expr() -> ColumnElement:
+    """Normalized conversation id."""
+    return normalized_thread_key_expr(RequestLog.conversation_id)
+
+
+# Stored stand-in for NULL account_id / api_key_id / service_tier /
+# reasoning_effort (PK columns cannot be NULL, and NULLs would be distinct
+# under a unique constraint). U+001F (unit separator) rather than '' because
+# '' is a legitimate raw value (the request models accept empty-string
+# service_tier/reasoning_effort) that the legacy GROUP BY treats as a group
+# distinct from NULL; raw values that start with the sentinel are escaped
+# with one more sentinel character so the mapping stays injective.
+DIMENSION_SENTINEL = "\x1f"
+
+
+def to_dimension(value: str | None) -> str:
+    """Map a nullable raw dimension value to its stored PK representation."""
+    if value is None:
+        return DIMENSION_SENTINEL
+    if value.startswith(DIMENSION_SENTINEL):
+        return DIMENSION_SENTINEL + value
+    return value
+
+
+def from_dimension(value: str) -> str | None:
+    """Map a stored dimension value back to the raw (nullable) domain."""
+    if value == DIMENSION_SENTINEL:
+        return None
+    if value.startswith(DIMENSION_SENTINEL):
+        return value[len(DIMENSION_SENTINEL) :]
+    return value
+
+
+def _dimension_expr(column) -> ColumnElement[str]:
+    """SQL mirror of :func:`to_dimension` for the fold INSERT..SELECTs."""
+    return case(
+        (column.is_(None), DIMENSION_SENTINEL),
+        (func.substr(column, 1, 1) == DIMENSION_SENTINEL, DIMENSION_SENTINEL + column),
+        else_=column,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class HourlyUsageRollupRow:
+    bucket_epoch: int
+    account_id: str
+    api_key_id: str
+    model: str
+    service_tier: str
+    request_kind: str
+    is_deleted: bool
+    request_count: int = 0
+    error_count: int = 0
+    cancelled_count: int = 0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    reasoning_tokens: int = 0
+    output_or_reasoning_tokens: int = 0
+    cached_input_tokens: int = 0
+    cached_input_tokens_clamped: int = 0
+    cost_usd: float = 0.0
+    cost_count: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class HourlyErrorRollupRow:
+    bucket_epoch: int
+    account_id: str
+    error_code: str
+    error_count: int = 0
+
+
+@dataclass(frozen=True, slots=True)
+class QuarterDemandRollupRow:
+    slot_epoch: int
+    account_id: str
+    api_key_id: str
+    model: str
+    reasoning_effort: str
+    request_kind: str
+    status: str
+    is_deleted: bool
+    request_count: int = 0
+    input_tokens: int = 0
+    output_or_reasoning_tokens: int = 0
+    cached_input_tokens: int = 0
+    cost_usd: float = 0.0
+
+
+@dataclass(frozen=True, slots=True)
+class ConversationHourlyRollupRow:
+    bucket_epoch: int
+    conversation_id: str
+    account_id: str
+    is_deleted: bool
+    request_count: int = 0
+
+
+_HOURLY_KEY_COLUMNS = (
+    "bucket_epoch",
+    "account_id",
+    "api_key_id",
+    "model",
+    "service_tier",
+    "request_kind",
+    "is_deleted",
+)
+_HOURLY_MEASURE_COLUMNS = (
+    "request_count",
+    "error_count",
+    "cancelled_count",
+    "input_tokens",
+    "output_tokens",
+    "reasoning_tokens",
+    "output_or_reasoning_tokens",
+    "cached_input_tokens",
+    "cached_input_tokens_clamped",
+    "cost_usd",
+    "cost_count",
+)
+_ERROR_KEY_COLUMNS = ("bucket_epoch", "account_id", "error_code")
+_ERROR_MEASURE_COLUMNS = ("error_count",)
+_QUARTER_KEY_COLUMNS = (
+    "slot_epoch",
+    "account_id",
+    "api_key_id",
+    "model",
+    "reasoning_effort",
+    "request_kind",
+    "status",
+    "is_deleted",
+)
+_QUARTER_MEASURE_COLUMNS = (
+    "request_count",
+    "input_tokens",
+    "output_or_reasoning_tokens",
+    "cached_input_tokens",
+    "cost_usd",
+)
+_CONVERSATION_KEY_COLUMNS = ("bucket_epoch", "conversation_id", "account_id", "is_deleted")
+_CONVERSATION_MEASURE_COLUMNS = ("request_count",)
+
+
+def _merge_rows(rows: Iterable, key_width: int, columns: tuple[str, ...], row_type):
+    """Pre-merge rows sharing a PK so one multi-row INSERT never touches the
+    same conflict target twice (PostgreSQL rejects that outright)."""
+    merged: dict[tuple, list] = {}
+    for row in rows:
+        values = list(astuple(row))
+        key = tuple(values[:key_width])
+        existing = merged.get(key)
+        if existing is None:
+            merged[key] = values
+        else:
+            for index in range(key_width, len(columns)):
+                existing[index] += values[index]
+    return [row_type(*values) for values in merged.values()]
+
+
+# Rows per multi-VALUES upsert statement. asyncpg rejects statements with
+# more than 32,767 bind parameters; at 18 columns (the widest table) 1,000
+# rows binds 18,000 — comfortable margin on both dialects. Lifecycle mirrors
+# rekey an account's ENTIRE folded history in one call (thousands of rows
+# for a long-lived account), so unchunked upserts would abort the whole
+# lifecycle transaction.
+_UPSERT_CHUNK_ROWS = 1_000
+
+
+def _add_rows_stmt(
+    session: AsyncSession, model, rows: Sequence, key_columns: tuple[str, ...], columns: tuple[str, ...]
+):
+    measure_columns = columns[len(key_columns) :]
+    stmt = _insert_fn(session)(model).values([dict(zip(columns, astuple(row), strict=True)) for row in rows])
+    return stmt.on_conflict_do_update(
+        index_elements=[getattr(model, column) for column in key_columns],
+        set_={column: getattr(model, column) + getattr(stmt.excluded, column) for column in measure_columns},
+    )
+
+
+async def _merge_add(
+    session: AsyncSession, model, rows: Sequence, key_columns: tuple[str, ...], columns: tuple[str, ...], row_type
+) -> None:
+    merged = _merge_rows(rows, len(key_columns), columns, row_type)
+    for start in range(0, len(merged), _UPSERT_CHUNK_ROWS):
+        chunk = merged[start : start + _UPSERT_CHUNK_ROWS]
+        await session.execute(_add_rows_stmt(session, model, chunk, key_columns, columns))
+
+
+class RequestUsageTimeRollupRepository:
+    """Upsert (merge-add) and range reads for the three time-axis rollups.
+
+    Every read returns ``(rows, hourly watermark)`` from ONE statement
+    (state LEFT JOIN rollup): a single statement sees a single snapshot even
+    under READ COMMITTED, so a fold slice committing concurrently can never
+    yield rollup rows from one watermark generation and a watermark from
+    another. Callers derive the raw-tail window from the returned watermark.
+    Range bounds are half-open ``[since_epoch, until_epoch)``.
+    """
+
+    def __init__(self, session: AsyncSession) -> None:
+        self._session = session
+
+    async def add_hourly(self, rows: Sequence[HourlyUsageRollupRow]) -> None:
+        """Merge-add hourly rows: inserts new (bucket, dimensions) rows and
+        adds measures onto existing ones, in bounded statement chunks. Used
+        by lifecycle mirrors and tests; the fold pass itself writes via
+        DELETE-then-INSERT..SELECT."""
+        await _merge_add(
+            self._session,
+            RequestUsageHourlyRollup,
+            rows,
+            _HOURLY_KEY_COLUMNS,
+            _HOURLY_KEY_COLUMNS + _HOURLY_MEASURE_COLUMNS,
+            HourlyUsageRollupRow,
+        )
+
+    async def add_errors(self, rows: Sequence[HourlyErrorRollupRow]) -> None:
+        await _merge_add(
+            self._session,
+            RequestUsageHourlyErrorRollup,
+            rows,
+            _ERROR_KEY_COLUMNS,
+            _ERROR_KEY_COLUMNS + _ERROR_MEASURE_COLUMNS,
+            HourlyErrorRollupRow,
+        )
+
+    async def add_demand(self, rows: Sequence[QuarterDemandRollupRow]) -> None:
+        await _merge_add(
+            self._session,
+            RequestDemandQuarterRollup,
+            rows,
+            _QUARTER_KEY_COLUMNS,
+            _QUARTER_KEY_COLUMNS + _QUARTER_MEASURE_COLUMNS,
+            QuarterDemandRollupRow,
+        )
+
+    async def add_conversations(self, rows: Sequence[ConversationHourlyRollupRow]) -> None:
+        await _merge_add(
+            self._session,
+            RequestConversationHourlyRollup,
+            rows,
+            _CONVERSATION_KEY_COLUMNS,
+            _CONVERSATION_KEY_COLUMNS + _CONVERSATION_MEASURE_COLUMNS,
+            ConversationHourlyRollupRow,
+        )
+
+    async def read_hourly(
+        self,
+        *,
+        since_epoch: int | None = None,
+        until_epoch: int | None = None,
+        filters: Sequence[ColumnElement[bool]] = (),
+    ) -> tuple[list[HourlyUsageRollupRow], datetime | None]:
+        return await self._read(
+            RequestUsageHourlyRollup,
+            RequestUsageHourlyRollup.bucket_epoch,
+            _HOURLY_KEY_COLUMNS + _HOURLY_MEASURE_COLUMNS,
+            HourlyUsageRollupRow,
+            since_epoch,
+            until_epoch,
+            filters,
+        )
+
+    async def read_errors(
+        self,
+        *,
+        since_epoch: int | None = None,
+        until_epoch: int | None = None,
+        filters: Sequence[ColumnElement[bool]] = (),
+    ) -> tuple[list[HourlyErrorRollupRow], datetime | None]:
+        return await self._read(
+            RequestUsageHourlyErrorRollup,
+            RequestUsageHourlyErrorRollup.bucket_epoch,
+            _ERROR_KEY_COLUMNS + _ERROR_MEASURE_COLUMNS,
+            HourlyErrorRollupRow,
+            since_epoch,
+            until_epoch,
+            filters,
+        )
+
+    async def read_demand(
+        self,
+        *,
+        since_epoch: int | None = None,
+        until_epoch: int | None = None,
+        filters: Sequence[ColumnElement[bool]] = (),
+    ) -> tuple[list[QuarterDemandRollupRow], datetime | None]:
+        return await self._read(
+            RequestDemandQuarterRollup,
+            RequestDemandQuarterRollup.slot_epoch,
+            _QUARTER_KEY_COLUMNS + _QUARTER_MEASURE_COLUMNS,
+            QuarterDemandRollupRow,
+            since_epoch,
+            until_epoch,
+            filters,
+        )
+
+    async def _read(
+        self, model, epoch_column, columns: tuple[str, ...], row_type, since_epoch, until_epoch, filters=()
+    ):
+        join_conditions = list(filters)
+        if since_epoch is not None:
+            join_conditions.append(epoch_column >= since_epoch)
+        if until_epoch is not None:
+            join_conditions.append(epoch_column < until_epoch)
+        stmt = (
+            select(
+                AccountUsageRollupState.hourly_folded_through,
+                *(getattr(model, column) for column in columns),
+            )
+            .select_from(AccountUsageRollupState)
+            .outerjoin(model, and_(*join_conditions) if join_conditions else true())
+            .where(AccountUsageRollupState.id == _STATE_ROW_ID)
+        )
+        rows = (await self._session.execute(stmt)).all()
+        if not rows:
+            return [], None
+        watermark = rows[0][0]
+        return [row_type(*row[1:]) for row in rows if row[1] is not None], watermark
+
+
+def epoch_seconds(value: datetime) -> int:
+    """Naive-UTC datetime to Unix epoch seconds (truncating sub-seconds)."""
+    return int((value - _EPOCH).total_seconds())
+
+
+def floor_to_hour(value: datetime) -> datetime:
+    """Floor a naive-UTC datetime to its whole UTC hour."""
+    return _EPOCH + timedelta(seconds=(epoch_seconds(value) // HOURLY_BUCKET_SECONDS) * HOURLY_BUCKET_SECONDS)
+
+
+def _requested_at_epoch_bucket_expr(session: AsyncSession, bucket_seconds: int) -> ColumnElement:
+    """Dialect-split bucket expression, identical arithmetic to the runtime
+    bucketing the read paths use (`RequestLogsRepository._bucket_epoch_expr`),
+    so folded buckets and legacy raw bucketing can never disagree."""
+    dialect = session.get_bind().dialect.name
+    if dialect == "postgresql":
+        return cast(
+            func.floor(func.extract("epoch", RequestLog.requested_at) / bucket_seconds) * bucket_seconds,
+            BigInteger,
+        )
+    epoch_col = cast(func.strftime("%s", RequestLog.requested_at), Integer)
+    return cast(epoch_col / bucket_seconds, Integer) * bucket_seconds
+
+
+def _hourly_fold_insert(session: AsyncSession, window: tuple[ColumnElement, ...]):
+    bucket = _requested_at_epoch_bucket_expr(session, HOURLY_BUCKET_SECONDS).label("bucket_epoch")
+    account_id = _dimension_expr(RequestLog.account_id).label("account_id")
+    api_key_id = _dimension_expr(RequestLog.api_key_id).label("api_key_id")
+    service_tier = _dimension_expr(RequestLog.service_tier).label("service_tier")
+    is_deleted = RequestLog.deleted_at.is_not(None).label("is_deleted")
+    output_or_reasoning = func.coalesce(RequestLog.output_tokens, RequestLog.reasoning_tokens, 0)
+    # Exact mirror of the usage-summary reader's per-row clamp
+    # (`aggregate_usage_metrics_since` / `cached_input_tokens_from_log`):
+    # NULL cached counts 0, a NULL input keeps the (non-negative) cached
+    # value UNclamped, otherwise clamp to [0, input]. SQLite's two-argument
+    # min()/max() scalar functions are its least()/greatest().
+    dialect = session.get_bind().dialect.name
+    least = func.least if dialect == "postgresql" else func.min
+    greatest = func.greatest if dialect == "postgresql" else func.max
+    cached_clamped = case(
+        (RequestLog.cached_input_tokens.is_(None), 0),
+        (RequestLog.input_tokens.is_(None), greatest(0, RequestLog.cached_input_tokens)),
+        else_=greatest(0, least(RequestLog.cached_input_tokens, RequestLog.input_tokens)),
+    )
+    stmt = (
+        select(
+            bucket,
+            account_id,
+            api_key_id,
+            RequestLog.model,
+            service_tier,
+            RequestLog.request_kind,
+            is_deleted,
+            func.count(RequestLog.id),
+            func.coalesce(func.sum(case((RequestLog.status.not_in(NON_ERROR_STATUSES), 1), else_=0)), 0),
+            func.coalesce(func.sum(case((RequestLog.status == CANCELLED_STATUS, 1), else_=0)), 0),
+            func.coalesce(func.sum(RequestLog.input_tokens), 0),
+            func.coalesce(func.sum(RequestLog.output_tokens), 0),
+            func.coalesce(func.sum(RequestLog.reasoning_tokens), 0),
+            func.coalesce(func.sum(output_or_reasoning), 0),
+            func.coalesce(func.sum(RequestLog.cached_input_tokens), 0),
+            func.coalesce(func.sum(cached_clamped), 0),
+            func.coalesce(func.sum(RequestLog.cost_usd), 0.0),
+            func.coalesce(func.sum(case((RequestLog.cost_usd.is_not(None), 1), else_=0)), 0),
+        )
+        .where(*window)
+        .group_by(bucket, account_id, api_key_id, RequestLog.model, service_tier, RequestLog.request_kind, is_deleted)
+    )
+    return insert(RequestUsageHourlyRollup).from_select(list(_HOURLY_KEY_COLUMNS + _HOURLY_MEASURE_COLUMNS), stmt)
+
+
+def _error_fold_insert(session: AsyncSession, window: tuple[ColumnElement, ...]):
+    bucket = _requested_at_epoch_bucket_expr(session, HOURLY_BUCKET_SECONDS).label("bucket_epoch")
+    account_id = _dimension_expr(RequestLog.account_id).label("account_id")
+    # Exact reproduction of the top-error read filter: warmup kinds excluded,
+    # soft-deleted rows INCLUDED, cancelled terminals excluded (they are not
+    # errors; buckets folded before that exclusion still carry
+    # client_disconnected counts, which the reads drop by error_code).
+    stmt = (
+        select(bucket, account_id, RequestLog.error_code, func.count(RequestLog.id))
+        .where(
+            *window,
+            RequestLog.request_kind.not_in(_EXCLUDED_REQUEST_KINDS),
+            RequestLog.status.not_in(NON_ERROR_STATUSES),
+            RequestLog.error_code.is_not(None),
+        )
+        .group_by(bucket, account_id, RequestLog.error_code)
+    )
+    return insert(RequestUsageHourlyErrorRollup).from_select(list(_ERROR_KEY_COLUMNS + _ERROR_MEASURE_COLUMNS), stmt)
+
+
+def _demand_fold_insert(session: AsyncSession, window: tuple[ColumnElement, ...]):
+    # Full legacy demand grain (slot, account, api_key, model,
+    # reasoning_effort, kind, status): the planner's `_bin_demand_units`
+    # takes max(token, cost, request units) PER BIN before summing, so
+    # folding to a coarser grain would shrink forecasts wherever one slot
+    # mixes groups with different dominant components.
+    slot = _requested_at_epoch_bucket_expr(session, QUARTER_SLOT_SECONDS).label("slot_epoch")
+    account_id = _dimension_expr(RequestLog.account_id).label("account_id")
+    api_key_id = _dimension_expr(RequestLog.api_key_id).label("api_key_id")
+    reasoning_effort = _dimension_expr(RequestLog.reasoning_effort).label("reasoning_effort")
+    is_deleted = RequestLog.deleted_at.is_not(None).label("is_deleted")
+    output_or_reasoning = func.coalesce(RequestLog.output_tokens, RequestLog.reasoning_tokens, 0)
+    stmt = (
+        select(
+            slot,
+            account_id,
+            api_key_id,
+            RequestLog.model,
+            reasoning_effort,
+            RequestLog.request_kind,
+            RequestLog.status,
+            is_deleted,
+            func.count(RequestLog.id),
+            func.coalesce(func.sum(RequestLog.input_tokens), 0),
+            func.coalesce(func.sum(output_or_reasoning), 0),
+            func.coalesce(func.sum(RequestLog.cached_input_tokens), 0),
+            func.coalesce(func.sum(RequestLog.cost_usd), 0.0),
+        )
+        .where(*window)
+        .group_by(
+            slot,
+            account_id,
+            api_key_id,
+            RequestLog.model,
+            reasoning_effort,
+            RequestLog.request_kind,
+            RequestLog.status,
+            is_deleted,
+        )
+    )
+    return insert(RequestDemandQuarterRollup).from_select(list(_QUARTER_KEY_COLUMNS + _QUARTER_MEASURE_COLUMNS), stmt)
+
+
+# Shared fold filter of the conversation satellite: exactly the row set every
+# conversation reader counts. The dashboard readers additionally require
+# `deleted_at IS NULL` and the reports readers additionally exclude
+# `source = 'limit_warmup'`; the former is the `is_deleted` dimension, the
+# latter is subsumed by the request_kind filter for every row the system
+# writes (limit-warmup traffic always carries a warmup request_kind).
+def _conversation_fold_conditions() -> tuple[ColumnElement[bool], ...]:
+    return (
+        conversation_id_expr().is_not(None),
+        RequestLog.request_kind.not_in(_EXCLUDED_REQUEST_KINDS),
+    )
+
+
+def _conversation_fold_insert(session: AsyncSession, window: tuple[ColumnElement, ...]):
+    bucket = _requested_at_epoch_bucket_expr(session, HOURLY_BUCKET_SECONDS).label("bucket_epoch")
+    conversation_id = conversation_id_expr().label("conversation_id")
+    account_id = _dimension_expr(RequestLog.account_id).label("account_id")
+    is_deleted = RequestLog.deleted_at.is_not(None).label("is_deleted")
+    stmt = (
+        select(bucket, conversation_id, account_id, is_deleted, func.count(RequestLog.id))
+        .where(*window, *_conversation_fold_conditions())
+        .group_by(bucket, conversation_id, account_id, is_deleted)
+    )
+    return insert(RequestConversationHourlyRollup).from_select(
+        list(_CONVERSATION_KEY_COLUMNS + _CONVERSATION_MEASURE_COLUMNS), stmt
+    )
+
+
+async def run_hourly_fold_pass(*, now: datetime | None = None) -> int:
+    """Advance the hourly watermark toward `floor_hour(now - FOLD_LAG)`.
+
+    Bounded slices, each committed in its own transaction; at most
+    `TS_MAX_SLICES_PER_PASS` slices per pass, so the initial backfill resumes
+    across scheduler ticks instead of monopolizing one. Returns the number of
+    committed slices. Crash-safe: the defensive DELETE, the three
+    INSERT..SELECTs and the watermark advance commit atomically, so a crash
+    rolls the whole slice back and the retry recomputes it from scratch —
+    re-folding always converges to the same values (no add-fold double
+    counting is possible).
+
+    The first pass of each process additionally repairs the legacy-suspect
+    range `[upgrade_repair_from, watermark)` — buckets a legacy replica may
+    have folded after the cancelled_count migration during a rolling
+    upgrade — falling back to the trailing `UPGRADE_REPAIR_WINDOW` as
+    flip-flop defense once the marker is cleared (see the constant's note).
+    An incomplete (pass-bounded) repair re-arms itself for the next pass.
+    """
+    global _upgrade_repair_done
+    if not _upgrade_repair_done:
+        _upgrade_repair_done = await _run_upgrade_repair()
+    target = floor_to_hour((now or utcnow()) - FOLD_LAG)
+    committed = 0
+    while committed < TS_MAX_SLICES_PER_PASS:
+        async with get_background_session() as session:
+            status, wrote = await _fold_next_hourly_slice(session, target)
+        if wrote:
+            committed += 1
+        if status is _FoldStatus.DONE:
+            break
+    return committed
+
+
+def _ceil_to_hour(value: datetime) -> datetime:
+    floored = floor_to_hour(value)
+    return floored if floored == value else floored + timedelta(hours=1)
+
+
+async def _run_upgrade_repair() -> bool:
+    """Drive the upgrade repair in pass-bounded chunks; True when complete.
+
+    Same chunk/transaction discipline as the fold backfill: at most
+    `TS_MAX_SLICES_PER_PASS` chunks of `TS_FOLD_SLICE` per pass, each in its
+    own transaction, so repairing a legacy-folded backlog never monopolizes
+    a scheduler tick. An incomplete repair returns False and resumes from
+    the persisted marker on the next pass.
+    """
+    for _ in range(TS_MAX_SLICES_PER_PASS):
+        async with get_background_session() as session:
+            if await _repair_next_upgrade_chunk(session):
+                return True
+    return False
+
+
+async def _repair_next_upgrade_chunk(session: AsyncSession) -> bool:
+    """Refold the next chunk of the legacy-suspect range; True when none left.
+
+    The suspect range is `[upgrade_repair_from, watermark)` while the marker
+    is set (stamped by the migration, or the epoch server default for a
+    state row an old replica bootstrapped after it), and the trailing
+    `UPGRADE_REPAIR_WINDOW` flip-flop defense once it is NULL. Both are
+    clamped to `ceil_hour(earliest surviving raw row)` — an unfiltered min:
+    retention's oldest-first contiguous frontier guarantees every row
+    (soft-deleted included) at or above it survives, so the repair never
+    deletes folded statistics it cannot recompute from raw.
+
+    Runs under the fold-state row lock (the caller already holds the leader
+    gate), so it can never interleave with a fold slice or lifecycle mirror,
+    and never moves the watermark. The same DELETE-then-INSERT statements as
+    the fold make every chunk converge on any input state — repairing a
+    legacy-folded bucket, re-running after a crash (the marker advances only
+    with the chunk's commit), or recomputing an already-correct window are
+    the same recomputation. The conversation satellite is untouched: its
+    fold never involved status.
+    """
+    async with sqlite_writer_section():
+        state = await _locked_state(session)
+        if state is None:
+            # Nothing folded yet, so nothing a legacy writer could have left.
+            return True
+        watermark = state.hourly_folded_through
+        marker = state.upgrade_repair_from
+        suspect_from = marker if marker is not None else watermark - UPGRADE_REPAIR_WINDOW
+        earliest_raw = (await session.execute(select(func.min(RequestLog.requested_at)))).scalar_one_or_none()
+        start = watermark if earliest_raw is None else max(suspect_from, _ceil_to_hour(earliest_raw))
+        if start >= watermark:
+            if marker is not None:
+                await _set_upgrade_repair_marker(session, None)
+                await session.commit()
+            return True
+        chunk_end = min(start + TS_FOLD_SLICE, watermark)
+        start_epoch, end_epoch = epoch_seconds(start), epoch_seconds(chunk_end)
+        await session.execute(
+            delete(RequestUsageHourlyRollup).where(
+                RequestUsageHourlyRollup.bucket_epoch >= start_epoch,
+                RequestUsageHourlyRollup.bucket_epoch < end_epoch,
+            )
+        )
+        await session.execute(
+            delete(RequestUsageHourlyErrorRollup).where(
+                RequestUsageHourlyErrorRollup.bucket_epoch >= start_epoch,
+                RequestUsageHourlyErrorRollup.bucket_epoch < end_epoch,
+            )
+        )
+        await session.execute(
+            delete(RequestDemandQuarterRollup).where(
+                RequestDemandQuarterRollup.slot_epoch >= start_epoch,
+                RequestDemandQuarterRollup.slot_epoch < end_epoch,
+            )
+        )
+        window = (RequestLog.requested_at >= start, RequestLog.requested_at < chunk_end)
+        await session.execute(_hourly_fold_insert(session, window))
+        await session.execute(_error_fold_insert(session, window))
+        await session.execute(_demand_fold_insert(session, window))
+        done = chunk_end >= watermark
+        if marker is not None:
+            await _set_upgrade_repair_marker(session, None if done else chunk_end)
+        await session.commit()
+        logger.info(
+            "Refolded hourly usage rollups in [%s, %s) (post-upgrade repair)",
+            start.isoformat(),
+            chunk_end.isoformat(),
+        )
+        return done
+
+
+async def _set_upgrade_repair_marker(session: AsyncSession, value: datetime | None) -> None:
+    await session.execute(
+        update(AccountUsageRollupState)
+        .where(AccountUsageRollupState.id == _STATE_ROW_ID)
+        .values(upgrade_repair_from=value)
+    )
+
+
+async def _fold_next_hourly_slice(session: AsyncSession, target: datetime) -> tuple[_FoldStatus, bool]:
+    async with sqlite_writer_section():
+        # Same state row (id=1) as the lifetime fold: one FOR UPDATE row lock
+        # serializes concurrent hourly passes, lifetime passes, and lifecycle
+        # mirrors. Re-read the watermark AFTER taking the lock — a concurrent
+        # pass may have advanced it while we waited.
+        state = await _locked_state(session)
+        if state is None:
+            await session.execute(_state_bootstrap_stmt(session))
+            await session.commit()
+            state = await _locked_state(session)
+        if state is None:
+            logger.warning("account_usage_rollup_state row missing; skipping hourly fold pass")
+            return _FoldStatus.DONE, False
+        watermark = state.hourly_folded_through
+        if watermark >= target:
+            return _FoldStatus.DONE, False
+
+        # Next populated instant in [watermark, target): jumps the empty
+        # prefix on first backfill and any mid-history gap on later slices,
+        # and guarantees the slice's first hour actually holds rows. Every
+        # row counts for at least one of the three aggregates (the hourly
+        # fold has no filter), so no filtered variant is needed.
+        next_populated = (
+            await session.execute(
+                select(func.min(RequestLog.requested_at)).where(
+                    RequestLog.requested_at >= watermark,
+                    RequestLog.requested_at < target,
+                )
+            )
+        ).scalar_one_or_none()
+        if next_populated is None:
+            # Nothing left below the target. Advancing (rather than leaving
+            # the watermark behind) keeps the readers' raw-tail window and
+            # the retention min-gate current; FOLD_LAG guarantees no insert
+            # can land below `now - FOLD_LAG`, so nothing can appear behind
+            # the advanced watermark later.
+            await _advance_hourly_watermark(session, target)
+            await session.commit()
+            logger.info("Folded hourly usage rollups through %s (no rows below target)", target.isoformat())
+            return _FoldStatus.DONE, True
+
+        start = max(watermark, floor_to_hour(next_populated))
+        slice_end = min(start + TS_FOLD_SLICE, target)
+        start_epoch, end_epoch = epoch_seconds(start), epoch_seconds(slice_end)
+
+        # Defensive DELETE: zero rows on the normal path (the watermark only
+        # moves forward), but makes an operator watermark reset (escape
+        # hatch) converge — a re-fold can never double-count or leave rows
+        # from a previous fold generation behind. Convergence is guaranteed
+        # under the escape hatch's documented precondition (raw below the
+        # target still present, or the reset truncated the rollups in the
+        # same transaction): every previously-folded hour then either has
+        # raw rows (re-covered by a slice window and its DELETE) or no
+        # rollup rows. Hours the min()-jump skips are deliberately NOT
+        # cleared — after a rewind-only reset over retention-pruned history
+        # (the documented forbidden state) the skipped rollup rows are the
+        # ONLY surviving copy of those statistics, and deleting them would
+        # turn an operator mistake into permanent data loss.
+        await session.execute(
+            delete(RequestUsageHourlyRollup).where(
+                RequestUsageHourlyRollup.bucket_epoch >= start_epoch,
+                RequestUsageHourlyRollup.bucket_epoch < end_epoch,
+            )
+        )
+        await session.execute(
+            delete(RequestUsageHourlyErrorRollup).where(
+                RequestUsageHourlyErrorRollup.bucket_epoch >= start_epoch,
+                RequestUsageHourlyErrorRollup.bucket_epoch < end_epoch,
+            )
+        )
+        await session.execute(
+            delete(RequestDemandQuarterRollup).where(
+                RequestDemandQuarterRollup.slot_epoch >= start_epoch,
+                RequestDemandQuarterRollup.slot_epoch < end_epoch,
+            )
+        )
+
+        # Half-open [start, slice_end): hour-aligned bounds, so a display
+        # bucket is never split between the folded side and the raw tail.
+        window = (RequestLog.requested_at >= start, RequestLog.requested_at < slice_end)
+        await session.execute(_hourly_fold_insert(session, window))
+        await session.execute(_error_fold_insert(session, window))
+        await session.execute(_demand_fold_insert(session, window))
+        await _advance_hourly_watermark(session, slice_end)
+        await session.commit()
+        logger.info("Folded hourly usage rollups through %s", slice_end.isoformat())
+        return (_FoldStatus.DONE if slice_end >= target else _FoldStatus.CONTINUE), True
+
+
+async def _advance_hourly_watermark(session: AsyncSession, value: datetime) -> None:
+    await session.execute(
+        update(AccountUsageRollupState)
+        .where(AccountUsageRollupState.id == _STATE_ROW_ID)
+        .values(hourly_folded_through=value)
+    )
+
+
+async def run_conversation_fold_pass(*, now: datetime | None = None) -> int:
+    """Advance the conversation watermark toward `floor_hour(now - FOLD_LAG)`.
+
+    Same slice/transaction/crash-safety contract as `run_hourly_fold_pass`
+    (DELETE-then-INSERT slices, bounded per pass, watermark advance committed
+    atomically with the slice), on the satellite's own watermark so its
+    from-epoch backfill neither rewinds nor stalls the other rollups.
+    """
+    target = floor_to_hour((now or utcnow()) - FOLD_LAG)
+    committed = 0
+    while committed < TS_MAX_SLICES_PER_PASS:
+        async with get_background_session() as session:
+            status, wrote = await _fold_next_conversation_slice(session, target)
+        if wrote:
+            committed += 1
+        if status is _FoldStatus.DONE:
+            break
+    return committed
+
+
+async def _fold_next_conversation_slice(session: AsyncSession, target: datetime) -> tuple[_FoldStatus, bool]:
+    async with sqlite_writer_section():
+        # Same state row lock as every other fold and lifecycle mirror.
+        state = await _locked_state(session)
+        if state is None:
+            await session.execute(_state_bootstrap_stmt(session))
+            await session.commit()
+            state = await _locked_state(session)
+        if state is None:
+            logger.warning("account_usage_rollup_state row missing; skipping conversation fold pass")
+            return _FoldStatus.DONE, False
+        watermark = state.conversation_folded_through
+        if watermark >= target:
+            return _FoldStatus.DONE, False
+
+        # Unlike the hourly fold (which folds every row), only rows matching
+        # the fold filter contribute here, so the empty-prefix/gap jump scans
+        # for the next COUNTABLE row — hours holding only conversation-less
+        # rows are skipped exactly like empty ones.
+        next_populated = (
+            await session.execute(
+                select(func.min(RequestLog.requested_at)).where(
+                    RequestLog.requested_at >= watermark,
+                    RequestLog.requested_at < target,
+                    *_conversation_fold_conditions(),
+                )
+            )
+        ).scalar_one_or_none()
+        if next_populated is None:
+            await _advance_conversation_watermark(session, target)
+            await session.commit()
+            logger.info("Folded conversation rollups through %s (no rows below target)", target.isoformat())
+            return _FoldStatus.DONE, True
+
+        start = max(watermark, floor_to_hour(next_populated))
+        slice_end = min(start + TS_FOLD_SLICE, target)
+        start_epoch, end_epoch = epoch_seconds(start), epoch_seconds(slice_end)
+
+        # Defensive DELETE with the same convergence/preservation trade-off
+        # as the hourly slice: skipped hours are deliberately NOT cleared.
+        await session.execute(
+            delete(RequestConversationHourlyRollup).where(
+                RequestConversationHourlyRollup.bucket_epoch >= start_epoch,
+                RequestConversationHourlyRollup.bucket_epoch < end_epoch,
+            )
+        )
+        window = (RequestLog.requested_at >= start, RequestLog.requested_at < slice_end)
+        await session.execute(_conversation_fold_insert(session, window))
+        await _advance_conversation_watermark(session, slice_end)
+        await session.commit()
+        logger.info("Folded conversation rollups through %s", slice_end.isoformat())
+        return (_FoldStatus.DONE if slice_end >= target else _FoldStatus.CONTINUE), True
+
+
+async def _advance_conversation_watermark(session: AsyncSession, value: datetime) -> None:
+    await session.execute(
+        update(AccountUsageRollupState)
+        .where(AccountUsageRollupState.id == _STATE_ROW_ID)
+        .values(conversation_folded_through=value)
+    )
+
+
+# --- Account lifecycle mirrors -------------------------------------------
+#
+# The ONLY code paths allowed to touch folded buckets after the watermark
+# passed them. They mirror the raw request_logs mutation exactly (a dimension
+# move, never a recompute — raw below the watermark may already be pruned).
+# Callers MUST hold the fold-state lock (`lock_fold_state`) in the same
+# transaction, so a mirror can never interleave with an in-flight fold slice.
+
+_ROLLUP_TABLES = (
+    (RequestUsageHourlyRollup, _HOURLY_KEY_COLUMNS + _HOURLY_MEASURE_COLUMNS, HourlyUsageRollupRow, "add_hourly"),
+    (RequestUsageHourlyErrorRollup, _ERROR_KEY_COLUMNS + _ERROR_MEASURE_COLUMNS, HourlyErrorRollupRow, "add_errors"),
+    (RequestDemandQuarterRollup, _QUARTER_KEY_COLUMNS + _QUARTER_MEASURE_COLUMNS, QuarterDemandRollupRow, "add_demand"),
+    (
+        RequestConversationHourlyRollup,
+        _CONVERSATION_KEY_COLUMNS + _CONVERSATION_MEASURE_COLUMNS,
+        ConversationHourlyRollupRow,
+        "add_conversations",
+    ),
+)
+
+
+async def _rekey_account_rows(session: AsyncSession, account_ids: list[str], rekey) -> None:
+    repo = RequestUsageTimeRollupRepository(session)
+    stored_ids = [to_dimension(account_id) for account_id in account_ids]
+    for model, columns, row_type, adder in _ROLLUP_TABLES:
+        stmt = select(*(getattr(model, column) for column in columns)).where(model.account_id.in_(stored_ids))
+        rows = [row_type(*row) for row in (await session.execute(stmt)).all()]
+        if not rows:
+            continue
+        await session.execute(delete(model).where(model.account_id.in_(stored_ids)))
+        await getattr(repo, adder)([rekey(row) for row in rows])
+
+
+async def mirror_account_soft_delete_into_time_rollups(session: AsyncSession, account_id: str) -> None:
+    """Mirror `AccountsRepository.delete()`'s soft path, which retroactively
+    detaches the account's ENTIRE raw history (`account_id=NULL,
+    deleted_at=now`): folded buckets move to the `(NULL-sentinel,
+    is_deleted=true)` dimension (merge-added — an orphaned-deleted bucket
+    may already exist).
+    The error satellite has no `is_deleted` dimension (its read includes
+    soft-deleted rows), so only `account_id` is re-keyed there. The
+    conversation satellite carries `account_id` precisely so this mirror can
+    re-attribute presence the way the raw UPDATE does — the dashboard
+    conversation reads (which exclude soft-deleted rows) then stop counting
+    it while the reports reads (which include them) keep it.
+    """
+
+    def _rekey(row):
+        if isinstance(row, HourlyErrorRollupRow):
+            return replace(row, account_id=DIMENSION_SENTINEL)
+        return replace(row, account_id=DIMENSION_SENTINEL, is_deleted=True)
+
+    await _rekey_account_rows(session, [account_id], _rekey)
+    from app.modules.reports.rollup import rekey_report_accounts
+
+    await rekey_report_accounts(session, [account_id], None)
+
+
+async def mirror_account_hard_delete_into_time_rollups(session: AsyncSession, account_id: str) -> None:
+    """Mirror the history-deleting path (raw rows physically removed)."""
+    from app.db.models import RequestReportHourlyRollup
+
+    await session.execute(
+        delete(RequestReportHourlyRollup).where(RequestReportHourlyRollup.account_id == to_dimension(account_id))
+    )
+    for model, *_rest in _ROLLUP_TABLES:
+        await session.execute(delete(model).where(model.account_id == to_dimension(account_id)))
+
+
+async def merge_time_rollups_into(session: AsyncSession, canonical_account_id: str, duplicate_ids: list[str]) -> None:
+    """Mirror duplicate-account consolidation, which reassigns the
+    duplicates' raw logs to the canonical account: folded buckets follow
+    bucket-wise (merge-add onto the canonical dimension, then the duplicate
+    rows are removed). Must run in the consolidation transaction, under the
+    fold-state lock the caller already holds.
+    """
+    if not duplicate_ids:
+        return
+    from app.modules.reports.rollup import rekey_report_accounts
+
+    await rekey_report_accounts(session, duplicate_ids, canonical_account_id)
+    canonical_dimension = to_dimension(canonical_account_id)
+    await _rekey_account_rows(session, duplicate_ids, lambda row: replace(row, account_id=canonical_dimension))