tumwater 0.1.2 → 0.2.0

package/dist/src/report/report-fold.js

@@ -0,0 +1,353 @@
+// vocabulary: plain
+/** Event-log folding for the usage report: turn raw harness events (run_end/merged/…) into
+ * per-day and whole-window `UsageFold` accumulators, memoizing the day-keyed read per
+ * (root, fromKey) so a dashboard polling /api/report only folds appended lines instead of
+ * re-reading the window. The two window collectors (collectReport / collectReportSince) in
+ * src/report/report-data.ts fold through this one shared layer, so the aggregation rules the
+ * --since and day windows present cannot drift apart; it lives here matching its own test
+ * (test/report-fold-cache.test.ts). Zero incidental runtime dependencies: node built-ins and
+ * the app's own modules only. */
+import { sameStatIdentity, statOrNull } from "../files/files.js";
+import { readWindowEvents } from "../events/event-window.js";
+import { eventDayKey, eventRole, eventUsage, parseEventLine } from "../events/event-read.js";
+import { readCompleteLines } from "../files/tail.js";
+import { eventsLogPath } from "../paths.js";
+import { isPassingFullCheck, isRedFullCheck } from "../build/build-check-events.js";
+import { addTo, memoSetLru } from "../collections.js";
+import { finiteNumber } from "../files/json-object.js";
+import { OUTCOME_CLASS } from "../failure/time-spend.js";
+import { baseRoleOf } from "../roles/loop-ids.js";
+import { commitTier } from "../roles/roles.js";
+/** Fold one event's `providerCost` map (`Record<provider, {costUsd, notionalCostUsd}>` — the
+ * shape tick-finalize.ts's run_end and merge-slot.ts's merge_done/merge_failed emit) into the
+ * per-provider tallies, one addTo per provider per nonzero figure so zero stays absent. A
+ * missing, foreign, or empty map folds nothing, so old logs render byte-identically. */
+function foldProviderCost(target, providerCost) {
+    if (typeof providerCost !== "object" || providerCost === null)
+        return;
+    for (const [provider, c] of Object.entries(providerCost)) {
+        if (typeof c !== "object" || c === null)
+            continue;
+        const entry = c;
+        const costUsd = finiteNumber(entry.costUsd, 0);
+        const notionalCostUsd = finiteNumber(entry.notionalCostUsd, 0);
+        if (costUsd !== 0)
+            addTo(target.costByProvider, provider, costUsd);
+        if (notionalCostUsd !== 0)
+            addTo(target.notionalByProvider, provider, notionalCostUsd);
+    }
+}
+/** Push a provider- or workspace-slot wait's ms into the accumulator list a median reads,
+ * when the event carried a positive wait: a wait is only event-logged at or above the 30 s
+ * threshold, so a 0 ms wait (a probe seam) or a pre-waitedMs event adds nothing and the two
+ * wait medians reflect real blocked admissions only. */
+function pushPositiveWaitedMs(list, ev) {
+    const waited = finiteNumber(ev.waitedMs, 0);
+    if (waited > 0)
+        list.push(waited);
+}
+/** Fold one event into a usage accumulator: run_end events add a tick (per role and, when
+ * the target counts ticks, in total), their output tokens, and their cost — zero-cost ticks
+ * contribute nothing, so they leave no costByRole key, keeping the render's "$0 roles are
+ * omitted" rule true on the aggregation itself, not just at display time; merged events add
+ * a commit; merge_done/merge_failed events add a merge run and
+ * accumulate their tokens/cost into the merge fields only — costByRole stays a ticks-only
+ * breakdown, and the merge line is how the reviewer's share is named. Both collectors fold
+ * through this one place, so the two windows' aggregation rules cannot drift apart. */
+export function foldUsageEvent(target, ev) {
+    if (ev.type === "run_end") {
+        const role = eventRole(ev);
+        addTo(target.runsByRole, role, 1);
+        // Error-class ticks get their own parallel tally — the day report's ``Failed ticks by role``
+        // line — reusing the digest's OUTCOME_CLASS so "failed" cannot drift from the failure
+        // report's meaning. Never subtracted from runsByRole; the two are independent counts.
+        if (OUTCOME_CLASS[ev.result] === "error")
+            addTo(target.failedRunsByRole, role, 1);
+        // A refusal is its own deliberate outcome (src/verdict/refusal.ts), not error-class, so
+        // it needs an explicit result check: the one per-role rejection-without-review count a
+        // day report should surface beside rejections, revisions and falls. Never subtracted from
+        // runsByRole; a refused tick still counts as a run.
+        if (ev.result === "refused")
+            addTo(target.refusalsByRole, role, 1);
+        if (target.runs !== undefined)
+            target.runs++;
+        const { tokens, costUsd, notionalCostUsd } = eventUsage(ev);
+        addTo(target.tokensByRole, role, tokens);
+        target.tokensOut += tokens;
+        target.costUsd += costUsd;
+        target.notionalCostUsd += notionalCostUsd;
+        if (costUsd !== 0)
+            addTo(target.costByRole, role, costUsd);
+        if (notionalCostUsd !== 0)
+            addTo(target.notionalByRole, role, notionalCostUsd);
+        // The tick's own runs' per-provider spend (tick-finalize.ts emits providerCost on run_end)
+        // — folded per provider beside the per-role maps, so a mixed team's `Cost by provider`
+        // line names the expensive seam.
+        foldProviderCost(target, ev.providerCost);
+    }
+    else if (ev.type === "review_rejected") {
+        // One rejection per event, on the day it happened, per loop. Zero-means-absent: a role
+        // that never rejected stays absent, like costByRole.
+        addTo(target.rejectionsByRole, eventRole(ev), 1);
+    }
+    else if (ev.type === "revision") {
+        // One rework round per event, on the day it happened, per loop. Zero-means-absent: a role
+        // that never reworked a rejected change stays absent, like rejectionsByRole.
+        addTo(target.revisionsByRole, eventRole(ev), 1);
+    }
+    else if (isPassingFullCheck(ev)) {
+        // The merge slot's passing full-suite checks, for the day report's check-time median.
+        const durationMs = finiteNumber(ev.durationMs, 0);
+        if (durationMs > 0)
+            target.checkDurationsMs.push(durationMs);
+    }
+    else if (isRedFullCheck(ev)) {
+        // The merge slot's red full-suite checks, for the day report's red-check count.
+        target.checkFailures++;
+    }
+    else if (ev.type === "model_fallback_started") {
+        // One episode per start event; durations are not summed (an episode can span the window edge).
+        addTo(target.fallbacksByRole, eventRole(ev), 1);
+    }
+    else if (ev.type === "image_reroute") {
+        // One reroute per event, preflight or after a blind read.
+        addTo(target.imageReroutesByRole, eventRole(ev), 1);
+    }
+    else if (ev.type === "run_deferred") {
+        // One deferral per event, a due maintenance tick that stood down. Bucket by the raw
+        // reason code ("backlog", "plans-full", else "no-work" for a missing/foreign/torn
+        // reason — the same fallback deferredReasonText applies in the feed), so the collector
+        // stays free of a presentation import and the render prints the code the event carried.
+        // Team-only: a reason is not a role.
+        const reason = ev.reason === "backlog" || ev.reason === "plans-full" ? ev.reason : "no-work";
+        addTo(target.deferredByReason, reason, 1);
+    }
+    else if (ev.type === "merged") {
+        target.commits++;
+        // Work ratio 4/4: split merged commits by base role and by tier. Instance suffixes fold
+        // into their role (`feature-2` counts as feature) so the split does not fragment.
+        const role = baseRoleOf(eventRole(ev));
+        addTo(target.commitsByRole, role, 1);
+        const tier = commitTier(role);
+        if (tier === "work")
+            target.workCommits++;
+        else if (tier === "maintenance")
+            target.maintenanceCommits++;
+    }
+    else if (ev.type === "run_start") {
+        // One tally per run start on a configured model: the same `model` field the CLI's
+        // `run #N started on <model>` line prints, and the only event that carries a model
+        // (run_end has none), so a run that started with no configured model contributes no key.
+        const model = typeof ev.model === "string" && ev.model.length > 0 ? ev.model : null;
+        if (model !== null)
+            addTo(target.runsByModel, model, 1);
+    }
+    else if (ev.type === "provider_wait") {
+        // The day report's provider-slot wait median, mirroring the slot_wait push: a wait only
+        // event-logged at or above the 30 s threshold counts, so a 0 ms wait (a probe seam) adds
+        // nothing and the median reflects real blocked admissions only.
+        pushPositiveWaitedMs(target.providerWaitsMs, ev);
+    }
+    else if (ev.type === "slot_wait") {
+        // The day report's workspace-slot wait median, mirroring the provider_wait push: a wait
+        // only event-logged at or above the 30 s threshold counts, so a 0 ms wait (a probe seam)
+        // or a pre-waitedMs event adds nothing.
+        pushPositiveWaitedMs(target.slotWaitsMs, ev);
+    }
+    else if (ev.type === "merge_done" || ev.type === "merge_failed") {
+        target.mergeRuns = (target.mergeRuns ?? 0) + 1;
+        if (ev.type === "merge_failed")
+            target.mergeFailed = (target.mergeFailed ?? 0) + 1;
+        // Attribute the outcome to the authoring role (writeMergeOutcome logs loop: entry.role)
+        // folded to base role, the same normalization the merged branch uses, so `feature-2`
+        // aggregates into `feature`. A parallel tally beside the team run counts, never
+        // subtracted from mergeRuns/mergeFailed.
+        const role = baseRoleOf(eventRole(ev));
+        if (ev.type === "merge_done") {
+            addTo(target.mergedByRole, role, 1);
+            // Events logged before queuedMs existed carry none: skipped, not counted as zero.
+            const waited = finiteNumber(ev.queuedMs, 0);
+            if (waited > 0)
+                target.mergeWaitsMs.push(waited);
+        }
+        else {
+            addTo(target.mergeFailedByRole, role, 1);
+            // The event's result (writeMergeOutcome's TickResult) names WHY the merge bounced
+            // off the gate — rejected/merge_conflict/merge_blocked/aborted. A legacy event with
+            // no result folds under `unknown`, so the reason buckets always sum to mergeFailed.
+            const reason = typeof ev.result === "string" && ev.result.length > 0 ? ev.result : "unknown";
+            addTo(target.mergeFailedByReason, reason, 1);
+        }
+        const { tokens, costUsd, notionalCostUsd } = eventUsage(ev);
+        target.mergeTokens = (target.mergeTokens ?? 0) + tokens;
+        target.mergeCostUsd = (target.mergeCostUsd ?? 0) + costUsd;
+        // A subscription merge's API-equivalent spend joins the notional line, never the
+        // budgeted cost totals (plans/agent-backends.md part 6a).
+        target.notionalCostUsd += notionalCostUsd;
+        // The merge's per-provider spend (merge-slot.ts keys it to its reviewer seam), so a
+        // merge's costs appear under the provider that ran the reviewer, matching the author
+        // side's foldMergeUsage charge.
+        foldProviderCost(target, ev.providerCost);
+    }
+}
+/** Bounded memo of window folds. Eight entries cover a dashboard flipping among a few day
+ * counts plus the midnight rollover's new fromKey; oldest-insertion eviction (Map preserves
+ * insertion order) keeps the memory bound to a handful of per-day accumulator maps. */
+const reportFoldCache = new Map();
+const REPORT_FOLD_CACHE_MAX = 8;
+/** The empty commit-family counters every accumulator starts from: the total and per-base-role
+ * merged-commit count plus the work/maintenance split (Work ratio 4/4). Four shapes carry them —
+ * DayFold, UsageFold, the ReportDay series slots, and the window totals — so the empty quartet
+ * lives once here and a new commit counter is added in one place. */
+export function zeroCommitCounts() {
+    return { commits: 0, commitsByRole: {}, workCommits: 0, maintenanceCommits: 0 };
+}
+/** The empty merge-run counters every accumulator starts from: the merge slot's run count
+ * and its own token/cost share (foldUsageEvent's merge_done/merge_failed branch). Four shapes carry
+ * them — DayFold, UsageFold, the scoped window totals, and the window totals — so the empty
+ * triple lives once here and a new merge counter is added in one place. */
+export function zeroMergeCounts() {
+    return { mergeRuns: 0, mergeTokens: 0, mergeCostUsd: 0 };
+}
+/** Sum a per-loop-id count/cost map over one base role: instance keys (`feature-2`) fold into
+ * their base (`feature`), so a scoped report aggregates a whole role group exactly as the
+ * per-role cap's group does. Ticks/tokens/cost are keyed by exact loop id while commits are
+ * already base-role keyed; running every key through `baseRoleOf` reads both shapes the same
+ * and returns 0 when nothing matches. */
+export function roleSum(record, role) {
+    let sum = 0;
+    for (const [key, n] of Object.entries(record)) {
+        if (baseRoleOf(key) === role)
+            sum += n;
+    }
+    return sum;
+}
+/** Narrow a per-loop-id count/cost map to one base role, keyed by the base role; a role absent
+ * from the map (no ticks, or no nonzero cost) yields an empty map, never a zero entry — the
+ * same zero-means-absent rule the team render applies. Used by the role-scoped collectors so a
+ * scoped series derives from the fold's per-role maps without inventing entries. */
+export function narrowToRole(record, role) {
+    const n = roleSum(record, role);
+    return n === 0 ? {} : { [role]: n };
+}
+/** A per-loop map scoped to one base role for a report: the full map as-is when no scope is
+ * set (a team report), else the same map narrowed to `base` — narrowToRole's zero-means-absent
+ * rule, so a scoped report never invents a zero entry. The one home of the "keep everything,
+ * or keep only the named base role" rule: the 13 per-role/pair fields of the `--since` report's
+ * output all shape through it, so the scoping rule cannot drift between them. */
+export function scopedByRole(record, base) {
+    return base === undefined ? record : narrowToRole(record, base);
+}
+function dayFoldFor(entry, dayKey) {
+    let fold = entry.days.get(dayKey);
+    if (!fold) {
+        fold = {
+            tokensOut: 0,
+            ...zeroCommitCounts(),
+            costUsd: 0,
+            notionalCostUsd: 0,
+            runsByRole: {},
+            rejectionsByRole: {},
+            revisionsByRole: {},
+            refusalsByRole: {},
+            checkDurationsMs: [],
+            checkFailures: 0,
+            mergeWaitsMs: [],
+            providerWaitsMs: [],
+            slotWaitsMs: [],
+            failedRunsByRole: {},
+            fallbacksByRole: {},
+            imageReroutesByRole: {},
+            tokensByRole: {},
+            costByRole: {},
+            notionalByRole: {},
+            ...zeroMergeCounts(),
+            mergedByRole: {},
+            mergeFailedByRole: {},
+            mergeFailedByReason: {},
+            costByProvider: {},
+            notionalByProvider: {},
+            runsByModel: {},
+            deferredByReason: {},
+        };
+        entry.days.set(dayKey, fold);
+    }
+    return fold;
+}
+/** Fold one already-parsed event into a cache entry: only in-window events (a non-null day
+ * on or after fromKey) count, matching readWindowEvents's event list; the first such event
+ * seen takes firstEventTs. The one home of the fold body, shared by the incremental append
+ * path (foldLineInto) and the full re-read path, so the two can never drift. */
+function foldEventInto(entry, ev) {
+    const dayKey = eventDayKey(ev);
+    if (dayKey === null || dayKey < entry.fromKey)
+        return;
+    if (!entry.hadEvents) {
+        entry.hadEvents = true;
+        entry.firstEventTs = typeof ev.ts === "number" ? ev.ts : undefined;
+    }
+    foldUsageEvent(dayFoldFor(entry, dayKey), ev);
+}
+/** Parse one appended log line and fold it; a torn or non-event line folds nothing. */
+function foldLineInto(entry, line) {
+    const ev = parseEventLine(line);
+    if (ev)
+        foldEventInto(entry, ev);
+}
+/** The event-log half of collectReport, memoized: a dashboard re-fetching /api/report every
+ * poll re-read and re-parsed the whole window's log bytes each time (~6.8 MB and tens of ms
+ * on this repo's own day log, growing toward the 16 MB rotation cap). Now a request whose
+ * log has not changed costs one stat; growth folds only the appended complete lines
+ * (readCompleteLines holds back a torn trailing write, so it is folded whole next call).
+ * Same size with a changed mtime, a shrunken file (rotation or rewrite), or a new fromKey
+ * falls back to the full readWindowEvents pass — the fallback re-derives everything, so a
+ * mis-fold can only come from the append assumption, and rotation's whole-file replace
+ * always hits the shrink arm. The entry's coverage parts are pinned by the cache key:
+ * fromKey and file bytes decide them, so an advancing window re-derives fresh instead of
+ * carrying yesterday's proof forward. */
+export function foldWindowEvents(root, fromKey) {
+    const key = `${root}\0${fromKey}`;
+    const liveStat = statOrNull(eventsLogPath(root));
+    const size = liveStat?.size ?? 0;
+    const mtimeMs = liveStat?.mtimeMs ?? 0;
+    const dev = liveStat?.dev ?? 0;
+    const ino = liveStat?.ino ?? 0;
+    const cached = reportFoldCache.get(key);
+    if (cached &&
+        sameStatIdentity({ dev, ino, mtimeMs, size }, { dev: cached.dev, ino: cached.ino, mtimeMs: cached.mtimeMs, size: cached.offset })) {
+        memoSetLru(reportFoldCache, key, cached, REPORT_FOLD_CACHE_MAX);
+        return cached;
+    }
+    if (cached && size > cached.offset && dev === cached.dev && ino === cached.ino) {
+        const { lines, end } = readCompleteLines(eventsLogPath(root), cached.offset, size);
+        if (end > cached.offset) {
+            for (const line of lines)
+                foldLineInto(cached, line);
+            cached.offset = end;
+            cached.mtimeMs = mtimeMs;
+            return cached;
+        }
+        // Grown but no complete line yet (a torn write in flight): reuse the folds; the next
+        // request sees the completed line and folds it then.
+        return cached;
+    }
+    const raw = readWindowEvents(root, fromKey);
+    const entry = {
+        root,
+        fromKey,
+        // The scan's own end, not the outer `size`: an event appended between that stat and the
+        // read is already folded, and seeding the offset from the stale size would fold it again
+        // on the next append (readTailTextWithEnd's coveredEnd is exactly where the scan stopped).
+        offset: raw.liveEnd,
+        mtimeMs,
+        dev,
+        ino,
+        fileCovers: raw.coversFullWindow,
+        hadEvents: false,
+        days: new Map(),
+    };
+    for (const ev of raw.events)
+        foldEventInto(entry, ev);
+    memoSetLru(reportFoldCache, key, entry, REPORT_FOLD_CACHE_MAX);
+    return entry;
+}