tumwater 0.2.0 → 0.4.0

package/dist/src/orchestrator/orchestrator-poll.js

@@ -0,0 +1,352 @@
+import { publishInfoFields } from "../team/orchestrator-info.js";
+import { baseRoleOf, runnerEnabled } from "../roles/runner-ids.js";
+import { DIRECTOR_ROLE } from "../roles/roles.js";
+import { reviewHoldKey, pollDeferralInputs, pollRunnerReasons } from "./orchestrator-scheduling.js";
+import { seamConcurrencyKey } from "../config/config-views.js";
+import { launchDueRuns } from "./orchestrator-launch.js";
+import { branchHead, currentBranch } from "../git/git.js";
+import { queuedMergeFiles, queuedMergeRoles } from "../merge/merge-queue.js";
+import { requeueMainRedPin } from "../merge/merge-requeue.js";
+import { drainMergeQueue, settleAbortedApproved } from "../merge/merge-drain.js";
+import { abortableMerges, mergeTasks } from "../merge/merge-pipeline.js";
+import { TUMWATER_SOURCE } from "../events/events.js";
+import { consumeAbortRequests, consumeReclaimRequest, consumeRestartRequest, consumeResetRequest, consumeWakeRequest, } from "../operator/operator-requests.js";
+import { checkPermitSnapshot } from "../concurrency/check-permit.js";
+import { slotPoolSnapshot } from "../git/worktree-pool.js";
+import { redeployTarget } from "../baseline/main-baseline.js";
+import { p75RunDurationMs } from "../run/run-timing.js";
+import { pollTeamGates } from "../gates/gate-polls.js";
+import { heldProviders, windowHeldProviders } from "../team/team-hold.js";
+import { workReserveCount } from "../scheduling/claims.js";
+export function newPollState() {
+    return { restart: false, holdForRestart: false, diskHeld: false, warnedBranchDivergence: false, lastReserve: 0 };
+}
+/** The start gate every reserved run passes the moment its permit is granted
+ * (runTimedRoleRun's `held`): closed while a restart is pending, and after one is decided so
+ * no waiter the shutdown hands a permit to starts on the build being replaced. One predicate,
+ * so another team-wide hold on new runs can close the same gate point. */
+function runStartHeld(state) {
+    return state.holdForRestart || state.restart || state.diskHeld;
+}
+/** The one start-gate rule for merges: a restart/disk hold's `held` counts against every role,
+ * while a hold on the reviewer's provider (`reviewHeld`) parks only the entries a pre-merge
+ * review gates — a role on `review.afterMerge` keeps draining (it merges without a reviewer),
+ * and no role at all (the conservative team-wide verdict a caller that cannot name a role
+ * reads) still holds. Shared by pollMergeDrain's startHeld and the merge tests, so the
+ * production wiring and its coverage cannot drift. */
+export function mergeStartHeld(held, reviewHeld, afterMergeRoles, role) {
+    return held || (reviewHeld && (role === undefined || !afterMergeRoles.has(baseRoleOf(role))));
+}
+/** The base roles whose review runs after merging — the config's `review.afterMerge` list, mapped
+ * through `baseRoleOf` so a listed base id stands for all its parallel instances. The one build of
+ * the set pollMergeDrain's start gate and pollSchedule's launch gate both exempt from a
+ * reviewer-hold, so the two passes cannot read the standing differently. */
+function afterMergeRoleIds(config) {
+    return new Set((config.review.afterMerge ?? []).map(baseRoleOf));
+}
+/** Live-reload, notify/retention/launchservices, CLI request markers, abort settling and the
+ * branch-divergence watch. Writes state.warnedBranchDivergence. */
+export async function pollRequestsAndWatch(ctx, state) {
+    const { root, mainBranch, runners } = ctx;
+    // Live-reload tumwater.json — the single reload point shared by all roles.
+    // src/config/config-live.ts owns the last-known-good retention and the edge-triggered
+    // warnings/events around it.
+    const liveConfig = ctx.liveReload.poll();
+    // The notify command rides the same last-known-good reload (src/events/notify.ts): a live
+    // `config set notify` edit takes effect on the next poll, no restart.
+    ctx.notifier.update(liveConfig);
+    // Live session retention (the last restart-only setting): the edge-triggered
+    // retention_changed event, the once-per-day gate, and the prune itself live in
+    // src/orchestrator/retention.ts. The live config (last-known-good while the file is broken or
+    // missing) drives the check, like the budget gate.
+    ctx.retentionPruner.poll(root, liveConfig.sessionRetentionDays);
+    // A missing or stale opencode catalog is refetched at most once an hour while opencode is routed.
+    ctx.opencodeCatalogRefresh.poll(liveConfig);
+    // launchservicesd's Mach-port budget (src/process/launch-services.ts owns it):
+    // a background sample at most every 15 minutes, and a warning days before the kernel
+    // kills the daemon and wedges the Mac's GUI session. Never awaited — a poll does not
+    // wait on `top`.
+    void ctx.launchServicesWatch?.poll();
+    // Consume CLI request markers: a reset-counters request, a wake request, a forced-restart
+    // request, and per-role abort requests.
+    consumeResetRequest(root, runners);
+    consumeWakeRequest(root, runners);
+    consumeRestartRequest(root, ctx.redeploy);
+    consumeReclaimRequest(root, ctx.reclaim);
+    consumeAbortRequests(root, runners, abortableMerges(ctx.merges));
+    await settleAbortedApproved(root, ctx.merges);
+    // branchHead reads the ref files first (microsecond-scale; this runs every poll) and
+    // spawns `git rev-parse` only when they cannot resolve it. "" means main does not exist
+    // yet — isEligible treats an empty head as "no wake".
+    const mainHead = (await branchHead(root, mainBranch)) ?? "";
+    // Branch-divergence watch, once per poll: the team keeps fast-forwarding the branch it
+    // resolved at startup — the safe behavior, since role worktrees are based on it — and one
+    // warning names the divergence instead of leaving it mysterious. Back on the target branch
+    // re-arms the check.
+    const liveBranch = await currentBranch(root);
+    if (liveBranch !== null && liveBranch !== mainBranch) {
+        if (!state.warnedBranchDivergence) {
+            state.warnedBranchDivergence = true;
+            ctx.warn(`your checkout moved to ${liveBranch} — the team keeps merging into ${mainBranch}`);
+        }
+    }
+    else {
+        state.warnedBranchDivergence = false;
+    }
+    return { liveConfig, mainHead, now: Date.now() };
+}
+/** The team's gates and alarms, one poll of the family (src/gates/gate-polls.ts): the daily
+ * budget gate — with its derived-state publish and the fallback handback to the primary — the
+ * operator and per-role pause gates, quiet hours, the team-wide failure hold, and the two
+ * observational storm alarms. The family's cross-poll memory lives in gateStates, advanced in
+ * place. The failure hold is deliberately not among the returned verdicts — the start gate and
+ * the merges' startHeld read the LATEST poll's verdict at permit time from gateStates.teamHold,
+ * so a waiter granted its permit after later polls ran meets the hold as it stands then.
+ * Writes state.diskHeld. */
+export function pollGatesAndHolds(ctx, state, { liveConfig, now }) {
+    const { root, runners, gateStates } = ctx;
+    const gates = pollTeamGates(gateStates, {
+        root,
+        runners,
+        liveConfig,
+        modelsPath: ctx.modelsPath,
+        now,
+        info: ctx.info,
+        reclaim: ctx.reclaim,
+    });
+    // The latest poll's disk verdict, hoisted for runStartHeld's permit-time reads
+    // (plans/disk-floor.md, part 1/4).
+    state.diskHeld = gates.diskHeld;
+    // Publish the permit pool's live state (plans/provider-concurrency.md §8): observers
+    // (status, TUI, GUI) read one file instead of coupling to this Semaphore, and the team
+    // line shows per-provider counts only while a cap map is configured. Written only when
+    // it changes, like budget and disk; a map-less team publishes `byKey` empty, which
+    // renders exactly as before. The check-permit pool rides along, so status can show checks
+    // running and waiting while merges stall. The workspace pool's live state rides along too:
+    // one small slots.json read per poll, and publishInfoFields rewrites the info file only
+    // when a value changed.
+    publishInfoFields(root, ctx.info, [
+        ["permits", ctx.semaphore.snapshot()],
+        ["checkPermits", checkPermitSnapshot()],
+        ["slotPool", slotPoolSnapshot(root)],
+    ]);
+    // The failure hold, keyed per provider (PLANS.md 2026-10-05): a role is held when ITS
+    // run model's provider stands under a hold, and the roles a pre-merge review gates are held
+    // when the reviewer's provider is (with the review gate on their approved change could not
+    // merge without a reviewer — while a role on `review.afterMerge` merges without one, so it is
+    // exempt, and the merge queue reads the same scoped reviewHeld verdict). Key presence in the
+    // map is relapse memory, not a hold: heldProviders() reads only standing holds (`until`
+    // non-null), so a lifted hold never keeps blocking.
+    const teamHeldProviders = heldProviders(gateStates.teamHold);
+    const teamHeldNow = () => heldProviders(gateStates.teamHold);
+    // The reviewer's seam is read from the config merges actually review under — the poll's
+    // roleConfig, which a used-up claude window or the cap has already moved to a fallback —
+    // so a reviewer that left claude no longer holds the roles a pre-merge review gates on the
+    // claude hold.
+    const reviewSeamKey = reviewHoldKey(gates.roleConfig);
+    return {
+        ...gates,
+        teamHeldProviders,
+        teamHeldNow,
+        reviewHeldNow: () => liveConfig.review.enabled && teamHeldNow().has(reviewSeamKey),
+        reviewHeld: liveConfig.review.enabled && teamHeldProviders.has(reviewSeamKey),
+        roleProviders: new Map(runners.map((r) => [r.role, r.runProvider(now)])),
+    };
+}
+/** Self-redeploy (src/redeploy/redeployer.ts): with main's head in hand, let the state
+ * machine observe it. `hold` starts no new runs at all — director included; a restart
+ * takes effect within the drain's window plus a bounded merge hand-off, and its prompt waits
+ * in the inbox for the new build — while the green check/compile/drain run in the
+ * background. That covers runs reserved before the hold too: a waiter parked in the
+ * semaphore queue that is granted a permit mid-drain meets the closed start gate
+ * (runStartHeld), releases the permit, and hands its reservation back so it re-schedules
+ * once the hold lifts or on the new build. `restart` means dist/ already holds the new
+ * build: stop scheduling, abort whatever role runs the drain gave up waiting for — the
+ * permit holders; they resume on the new build — and return true. A director run can never
+ * be in flight here — poll only returns `restart` once it has finished. (holdForRestart keeps
+ * the previous poll's verdict until this one's is in: resetting it before the await
+ * would open the gate for a waiter granted a permit while poll runs.)
+ * Main watch (src/baseline/main-watch.ts): records the full check's verdict on main's new
+ * tip in the background; nothing here waits for it. Writes state.restart and
+ * state.holdForRestart; returns whether the loop should stop. */
+export async function pollRedeploy(ctx, state, { liveConfig, mainHead, now }) {
+    const { redeploy } = ctx;
+    if (!ctx.onceMode)
+        ctx.mainWatch.poll(mainHead, state.holdForRestart || state.diskHeld, liveConfig);
+    if (!redeploy)
+        return false;
+    const action = await redeploy.poll(await redeployTarget(ctx.root, mainHead, redeploy.build.sha), {
+        roleInFlight: ctx.rolePermitHolders.size,
+        directorInFlight: ctx.directorInFlight.size,
+        roleRunP75Ms: p75RunDurationMs(ctx.roleRunDurationsMs),
+    }, liveConfig.autoRestart, now);
+    publishInfoFields(ctx.root, ctx.info, [["build", redeploy.status(now)]]);
+    if (action === "restart") {
+        state.restart = true;
+        // Only permit holders have a pi run to cut off (the director is guaranteed finished by
+        // then): a parked waiter meets the closed start gate whenever the shutdown hands it a
+        // permit, so it needs no abort — and aborting for it alone would also cut off an
+        // in-flight merge that no permit-holding run put at stake. The role-only controller
+        // keeps that promise: the merge, wired to the full signal, runs on into the hand-off
+        // window the wait announces (BUGS.md 2026-09-30).
+        if (ctx.rolePermitHolders.size > 0)
+            ctx.internalRoleStop.abort();
+        return true;
+    }
+    state.holdForRestart = action === "hold";
+    return false;
+}
+/** Merge queue 3/5 — drain the durable merge queue while neither a restart nor a disk hold is
+ * pending (the scheduler's WHEN; merge-drain.ts owns the HOW — the verification stage, its
+ * merge slot, and the dedupe against main). A restart or disk hold stops the whole drain, as it
+ * starts no run; what is already in flight carries on, and a verify parked for its permit
+ * meets the same start gate as a parked run when the permit comes. A hold on the reviewer's
+ * provider parks only the entries a pre-merge review gates, per the role-scoped mergeStartHeld:
+ * a `review.afterMerge` role's approved/queued changes keep draining. */
+export async function pollMergeDrain(ctx, state, { liveConfig, mainHead }, gates) {
+    if (state.holdForRestart || state.diskHeld)
+        return;
+    const { root, mainBranch } = ctx;
+    // The roles whose review runs after merging (plans post-merge-review role): while the
+    // reviewer is held, their approved/queued changes keep draining, so the per-entry start gate
+    // exempts them from the reviewer-hold leg below.
+    const afterMergeRoles = afterMergeRoleIds(liveConfig);
+    // A `main_red` pin the pipeline kept goes back on the queue once main's head has moved.
+    for (const r of ctx.runners)
+        await requeueMainRedPin(root, mainBranch, r.state, mainHead);
+    await drainMergeQueue({
+        root,
+        mainBranch,
+        signal: ctx.signal,
+        semaphore: ctx.semaphore,
+        runners: ctx.runners,
+        liveConfig,
+        roleConfig: gates.roleConfig,
+        directorConfig: gates.directorConfig,
+        piDefault: ctx.piDefault,
+        startHeld: (role) => mergeStartHeld(runStartHeld(state), gates.reviewHeldNow(), afterMergeRoles, role),
+    }, ctx.merges);
+}
+/** The scheduling pass: the deferral inputs feed pollRunnerReasons, whose reasons map
+ * launchDueRuns starts runs from. */
+export async function pollSchedule(ctx, state, { liveConfig, mainHead, now }, gates) {
+    const { root, runners, gateStates } = ctx;
+    const inputs = pollDeferralInputs(root, runners, liveConfig, gateStates, gates, ctx.breakerNow());
+    // The roles whose review runs after merging: while the reviewer is held their runs stay
+    // launchable (the pre-merge gate skips their review), so the permit-time start gate exempts
+    // them from the reviewer-hold leg below, exactly as the scheduling pass and merge drain do.
+    const afterMergeRoles = afterMergeRoleIds(liveConfig);
+    const reasons = await pollRunnerReasons({
+        root,
+        runners,
+        mainBranch: ctx.mainBranch,
+        merges: ctx.merges,
+        once: ctx.once,
+        now,
+        mainHead,
+        userPaused: gates.userPaused,
+        quietNow: gates.quietNow,
+        pausedRoles: gates.pausedRoles,
+        capPaused: gates.capPaused,
+        maintenanceQuota: gates.maintenanceQuota,
+        bootstrapHeld: gates.bootstrapHeld,
+        budgetPausedRoles: gates.budgetPausedRoles,
+        probeRoles: inputs.probeRoles,
+        reviewStrongPaused: inputs.reviewStrongPaused,
+        roleQuietHeld: gates.roleQuietHeld,
+        holdForRestart: state.holdForRestart,
+        diskHeld: state.diskHeld,
+        heldProviders: gates.teamHeldProviders,
+        windowHeldProviders: windowHeldProviders(gateStates.teamHold),
+        reviewHeld: gates.reviewHeld,
+        afterMergeRoles,
+        roleProviders: gates.roleProviders,
+        openBugsNow: inputs.openBugsNow,
+        workBacklogOpen: inputs.workBacklogOpen,
+        planBacklogFull: inputs.plansFull,
+        deferredDue: ctx.deferredDue,
+        workMergedSince: ctx.workMergedSince,
+    });
+    launchDueRuns({
+        root,
+        reasons,
+        // roleSignal, not signal: the timed run wrapper watches the full signal PLUS the
+        // role-only stop, exactly as the pre-extraction inline pass did — the restart drain's
+        // give-up abort must also turn away a waiter granted its permit at that moment.
+        signal: ctx.roleSignal,
+        gateStates,
+        breakerPolicy: ctx.breakerPolicy,
+        breakerNow: ctx.breakerNow,
+        probePair: inputs.probePair,
+        probeRoles: inputs.probeRoles,
+        budgetActive: gates.budgetActive,
+        windowMovedRoles: gates.windowMovedRoles,
+        semaphore: ctx.semaphore,
+        piDefault: ctx.piDefault,
+        rolePermitHolders: ctx.rolePermitHolders,
+        roleInFlight: ctx.roleInFlight,
+        directorInFlight: ctx.directorInFlight,
+        roleRunDurationsMs: ctx.roleRunDurationsMs,
+        startHeld: (role) => runStartHeld(state) ||
+            (role !== DIRECTOR_ROLE
+                ? (!afterMergeRoles.has(baseRoleOf(role)) && gates.reviewHeldNow()) ||
+                    gates.teamHeldNow().has(gates.roleProviders.get(role))
+                : windowHeldProviders(gateStates.teamHold).has(gates.roleProviders.get(role))),
+    });
+}
+/** Priority headroom, then once mode's exit; returns whether the loop should stop. Writes
+ * state.lastReserve and, in once mode, fires internalStop. */
+export function pollReserveAndExit(ctx, state, { liveConfig, now }) {
+    const { root, runners, semaphore } = ctx;
+    // Priority headroom (plans/parallel-work-instances.md part 7/7): withhold a few permits
+    // from maintenance while a work role has work to take, so a due work run never waits out
+    // a whole maintenance run. Sized after the launch pass, so a runner launched this poll
+    // already reads as running and reserves nothing for itself.
+    const poolSnapshot = semaphore.snapshot();
+    // The provider-headroom predicate receives the structural ClaimRunner; the config that
+    // names the provider lives on the real RoleRunners, so look the runner up by role.
+    const runnerByRole = new Map(runners.map((r) => [r.role, r]));
+    const reserve = workReserveCount(root, runners, {
+        maxConcurrent: liveConfig.maxConcurrent,
+        enabled: new Set(runners.filter((r) => runnerEnabled(r.config, r.role)).map((r) => r.role)),
+        queuedMergeRoles: queuedMergeRoles(root),
+        instancesOf: (base) => liveConfig.roles[base]?.instances ?? 1,
+        // Per-provider headroom (plans/provider-concurrency.md §6): an idle work role reserves a
+        // permit only when its own provider could use it — capped and full, it would only idle
+        // the slot another provider's maintenance run could take. Read from the pool's live
+        // snapshot; an uncapped key (or the team-total-only case) always has headroom.
+        providerHasHeadroom: (r) => {
+            const runner = runnerByRole.get(r.role);
+            const cfg = runner && runner.role !== DIRECTOR_ROLE ? runner.runConfig(now) : null;
+            const key = cfg ? seamConcurrencyKey(cfg, ctx.piDefault) : undefined;
+            if (key === undefined)
+                return true;
+            const k = poolSnapshot.byKey[key];
+            if (k?.limit === undefined)
+                return true;
+            return k.inUse < k.limit;
+        },
+    });
+    semaphore.setReserve(reserve);
+    if ((state.lastReserve === 0) !== (reserve === 0)) {
+        ctx.emit({ source: TUMWATER_SOURCE, type: "permit_reserve", reserve });
+    }
+    state.lastReserve = reserve;
+    // Once mode's exit: every enabled role settled, no run in flight, and the merge queue
+    // empty with no merge in flight for one full poll cycle — then fire the internal
+    // stop, the same graceful shutdown the restart drain uses (the `finally` awaits
+    // in-flight merge tasks to the end). The one-poll-cycle requirement is the guard
+    // against dropping a queued-but-not-yet-started merge: a stop that arrives during the
+    // shutdown drain drops the queue entry, so the slot gets its poll to pick the entry up
+    // before the stop is trusted.
+    if (ctx.once.exitReady({
+        roleRuns: ctx.roleInFlight.size,
+        directorRuns: ctx.directorInFlight.size,
+        merges: mergeTasks(ctx.merges).length,
+        queuedMerges: queuedMergeFiles(root).length,
+    })) {
+        ctx.internalStop.abort();
+        return true;
+    }
+    return false;
+}