pi-amq 0.1.1

package/workflow/run-cleanup.ts

@@ -0,0 +1,315 @@
+import { randomUUID as nodeRandomUUID } from "node:crypto";
+import { lstat, mkdir, readdir, rm } from "node:fs/promises";
+import { isAbsolute, join, normalize, relative, sep } from "node:path";
+
+import { writeWorkflowJson } from "./atomic-storage.js";
+import { runDirectoryForRequest } from "./run-store.js";
+import { WorkflowLeaseManager } from "./workflow-lease.js";
+import type { RunProjectionV2, WorkerAttemptProjection } from "./workflow-types.js";
+
+export interface CleanupItem {
+  readonly kind: "run" | "worker-session" | "artifact";
+  readonly path: string;
+  readonly bytes: number;
+}
+
+export interface CleanupCandidate {
+  readonly requestKey: string;
+  readonly runId: string;
+  readonly updatedAt: string;
+  readonly ageMs: number;
+  readonly items: readonly CleanupItem[];
+  readonly reclaimableBytes: number;
+}
+
+export interface CleanupSkip {
+  readonly requestKey: string;
+  readonly runId: string;
+  readonly reason:
+    | "not-old-enough"
+    | "active-run"
+    | "ambiguous-run"
+    | "active-worker"
+    | "held-session-lease"
+    | "corrupt-session-lease";
+  readonly diagnostic: string;
+}
+
+export interface RunCleanupPlan {
+  readonly dryRun: true;
+  readonly olderThanMs: number;
+  readonly candidates: readonly CleanupCandidate[];
+  readonly skipped: readonly CleanupSkip[];
+  readonly reclaimableBytes: number;
+}
+
+export interface RunCleanupResult extends Omit<RunCleanupPlan, "dryRun"> {
+  readonly dryRun: false;
+  readonly auditEventPath: string | null;
+  readonly deletedRunIds: readonly string[];
+}
+
+interface CleanupRunCatalog {
+  listRuns(): Promise<readonly RunProjectionV2[]>;
+}
+
+interface CleanupLeaseAuthority {
+  withCurrentSupervisor<T>(generation: string, operation: () => Promise<T>): Promise<T>;
+  inspectWorkerSession(sessionPath: string): ReturnType<WorkflowLeaseManager["inspectWorkerSession"]>;
+}
+
+export interface WorkflowRunCleanupOptions {
+  readonly namespaceDirectory: string;
+  readonly store: CleanupRunCatalog;
+  readonly leaseManager: CleanupLeaseAuthority;
+  readonly supervisorGeneration: string;
+  readonly now?: () => Date;
+  readonly randomUUID?: () => string;
+  readonly removePath?: (path: string) => Promise<void>;
+}
+
+export class CleanupConfirmationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "CleanupConfirmationError";
+  }
+}
+
+export class WorkflowRunCleanup {
+  readonly #namespaceDirectory: string;
+  readonly #store: CleanupRunCatalog;
+  readonly #leaseManager: CleanupLeaseAuthority;
+  readonly #supervisorGeneration: string;
+  readonly #now: () => Date;
+  readonly #randomUUID: () => string;
+  readonly #removePath: (path: string) => Promise<void>;
+
+  constructor(options: WorkflowRunCleanupOptions) {
+    if (!isAbsolute(options.namespaceDirectory)) {
+      throw new TypeError("Workflow cleanup namespace must be absolute");
+    }
+    this.#namespaceDirectory = normalize(options.namespaceDirectory);
+    this.#store = options.store;
+    this.#leaseManager = options.leaseManager;
+    this.#supervisorGeneration = requireNonEmpty(
+      options.supervisorGeneration,
+      "workflow supervisor generation",
+    );
+    this.#now = options.now ?? (() => new Date());
+    this.#randomUUID = options.randomUUID ?? nodeRandomUUID;
+    this.#removePath = options.removePath ?? (async (path) => {
+      assertOwnedPath(this.#namespaceDirectory, path);
+      await rm(path, { recursive: true, force: true });
+    });
+  }
+
+  async plan(olderThanMs: number): Promise<RunCleanupPlan> {
+    const age = positiveInteger(olderThanMs, "cleanup age");
+    const nowMs = this.#now().getTime();
+    const candidates: CleanupCandidate[] = [];
+    const skipped: CleanupSkip[] = [];
+    for (const run of await this.#store.listRuns()) {
+      const evaluated = await this.#evaluate(run, age, nowMs);
+      if ("reason" in evaluated) skipped.push(evaluated);
+      else candidates.push(evaluated);
+    }
+    candidates.sort((left, right) => left.runId.localeCompare(right.runId));
+    skipped.sort((left, right) => left.runId.localeCompare(right.runId));
+    return {
+      dryRun: true,
+      olderThanMs: age,
+      candidates,
+      skipped,
+      reclaimableBytes: candidates.reduce((sum, candidate) => sum + candidate.reclaimableBytes, 0),
+    };
+  }
+
+  async execute(input: {
+    readonly olderThanMs: number;
+    readonly confirmed: boolean;
+  }): Promise<RunCleanupResult> {
+    if (!input.confirmed) {
+      throw new CleanupConfirmationError("Workflow cleanup deletion requires explicit confirmation");
+    }
+    return this.#leaseManager.withCurrentSupervisor(this.#supervisorGeneration, async () => {
+      const plan = await this.plan(input.olderThanMs);
+      if (plan.candidates.length === 0) {
+        return { ...plan, dryRun: false, auditEventPath: null, deletedRunIds: [] };
+      }
+      const auditEventPath = await this.#recordAudit(plan);
+      const deletedRunIds: string[] = [];
+      for (const candidate of plan.candidates) {
+        for (const session of candidate.items.filter((item) => item.kind === "worker-session")) {
+          await this.#removePath(session.path);
+        }
+        const run = candidate.items.find((item) => item.kind === "run");
+        if (!run) throw new Error(`Cleanup candidate ${candidate.runId} has no run directory`);
+        await this.#removePath(run.path);
+        deletedRunIds.push(candidate.runId);
+      }
+      return { ...plan, dryRun: false, auditEventPath, deletedRunIds };
+    });
+  }
+
+  async #evaluate(
+    run: RunProjectionV2,
+    olderThanMs: number,
+    nowMs: number,
+  ): Promise<CleanupCandidate | CleanupSkip> {
+    const ageMs = Math.max(0, nowMs - Date.parse(run.updatedAt));
+    if (ageMs < olderThanMs) {
+      return skip(run, "not-old-enough", `run age ${ageMs}ms is below ${olderThanMs}ms`);
+    }
+    if (run.state === "ambiguous" || run.state === "needs-attention"
+      || run.unresolvedAmbiguities.length > 0) {
+      return skip(run, "ambiguous-run", `run state is ${run.state}`);
+    }
+    if (run.state !== "done" && run.state !== "cancelled") {
+      return skip(run, "active-run", `run state is ${run.state}`);
+    }
+    const attempts = Object.values(run.workers).flatMap((worker) => worker.attempts);
+    if (attempts.some(isActiveAttempt)) {
+      return skip(run, "active-worker", "at least one worker attempt is active or lacks exit evidence");
+    }
+    const sessionPaths = [...new Set(attempts.map((attempt) => attempt.session.path))].sort();
+    for (const sessionPath of sessionPaths) {
+      const lease = await this.#leaseManager.inspectWorkerSession(sessionPath);
+      if (lease.kind === "owned") {
+        return skip(
+          run,
+          "held-session-lease",
+          `session ${sessionPath} is held by generation ${lease.owner.generation}`,
+        );
+      }
+      if (lease.kind === "corrupt") {
+        return skip(run, "corrupt-session-lease", lease.diagnostic);
+      }
+    }
+    const runPath = runDirectoryForRequest(this.#namespaceDirectory, run.requestKey);
+    const artifactPath = join(runPath, "artifacts");
+    const runBytes = await pathBytes(runPath);
+    const items: CleanupItem[] = [{ kind: "run", path: runPath, bytes: runBytes }];
+    for (const sessionPath of sessionPaths) {
+      assertOwnedPath(this.#namespaceDirectory, sessionPath);
+      items.push({ kind: "worker-session", path: sessionPath, bytes: await pathBytes(sessionPath) });
+    }
+    const artifactBytes = await pathBytes(artifactPath);
+    if (artifactBytes > 0) items.push({ kind: "artifact", path: artifactPath, bytes: artifactBytes });
+    return {
+      requestKey: run.requestKey,
+      runId: run.runId,
+      updatedAt: run.updatedAt,
+      ageMs,
+      items,
+      reclaimableBytes: runBytes + items
+        .filter((item) => item.kind === "worker-session")
+        .reduce((sum, item) => sum + item.bytes, 0),
+    };
+  }
+
+  async #recordAudit(plan: RunCleanupPlan): Promise<string> {
+    const directory = join(this.#namespaceDirectory, "cleanup-ledger", "events");
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    const eventId = `cleanup_${requireNonEmpty(this.#randomUUID(), "cleanup event UUID")}`;
+    const path = join(directory, `${this.#now().getTime()}.${safePathPart(eventId)}.json`);
+    await writeWorkflowJson(path, {
+      schema: 1,
+      eventId,
+      type: "cleanup-started",
+      occurredAt: this.#now().toISOString(),
+      supervisorGeneration: this.#supervisorGeneration,
+      olderThanMs: plan.olderThanMs,
+      candidates: plan.candidates,
+      reclaimableBytes: plan.reclaimableBytes,
+    }, { createOnly: true });
+    return path;
+  }
+}
+
+export async function confirmCleanupDeletion(input: {
+  readonly dryRun: boolean;
+  readonly yes: boolean;
+  readonly tui: boolean;
+  readonly confirm?: () => Promise<boolean>;
+}): Promise<boolean> {
+  if (input.dryRun) return false;
+  if (input.tui) {
+    if (!input.confirm || !await input.confirm()) {
+      throw new CleanupConfirmationError("Workflow cleanup was not confirmed; no state changed");
+    }
+    return true;
+  }
+  if (!input.yes) {
+    throw new CleanupConfirmationError("Headless workflow cleanup requires explicit yes=true");
+  }
+  return true;
+}
+
+export function parseCleanupAge(value: string): number {
+  const match = /^(\d+)(ms|s|m|h|d|w)$/.exec(value.trim());
+  if (!match) throw new TypeError("Cleanup age must use <integer><ms|s|m|h|d|w>");
+  const amount = Number(match[1]);
+  const multiplier = { ms: 1, s: 1_000, m: 60_000, h: 3_600_000, d: 86_400_000, w: 604_800_000 }[
+    match[2] as "ms" | "s" | "m" | "h" | "d" | "w"
+  ];
+  const result = amount * multiplier;
+  return positiveInteger(result, "cleanup age");
+}
+
+function isActiveAttempt(attempt: WorkerAttemptProjection): boolean {
+  return attempt.pidStartEvidence.observedExitAt === null
+    || attempt.pidStartEvidence.exitStatus === null
+    || attempt.state === "pending"
+    || attempt.state === "running"
+    || attempt.state === "blocked"
+    || attempt.state === "ambiguous";
+}
+
+function skip(
+  run: RunProjectionV2,
+  reason: CleanupSkip["reason"],
+  diagnostic: string,
+): CleanupSkip {
+  return { requestKey: run.requestKey, runId: run.runId, reason, diagnostic };
+}
+
+async function pathBytes(path: string): Promise<number> {
+  let info;
+  try {
+    info = await lstat(path);
+  } catch (error) {
+    if (isNodeError(error) && error.code === "ENOENT") return 0;
+    throw error;
+  }
+  if (!info.isDirectory()) return info.size;
+  let total = info.size;
+  for (const entry of await readdir(path, { withFileTypes: true })) {
+    total += await pathBytes(join(path, entry.name));
+  }
+  return total;
+}
+
+function assertOwnedPath(namespaceDirectory: string, path: string): void {
+  const child = relative(namespaceDirectory, normalize(path));
+  if (child.length === 0 || child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child)) {
+    throw new Error(`Workflow cleanup path escapes its namespace: ${path}`);
+  }
+}
+
+function positiveInteger(value: number, name: string): number {
+  if (!Number.isSafeInteger(value) || value <= 0) throw new TypeError(`${name} must be positive`);
+  return value;
+}
+
+function requireNonEmpty(value: string, name: string): string {
+  if (value.length === 0) throw new TypeError(`${name} is required`);
+  return value;
+}
+
+function safePathPart(value: string): string {
+  return value.replace(/[^A-Za-z0-9._-]/g, "_");
+}
+
+function isNodeError(error: unknown): error is NodeJS.ErrnoException {
+  return error instanceof Error && "code" in error;
+}

package/workflow/run-gate.ts

@@ -0,0 +1,122 @@
+import { RUN_STATES, type RunState } from "./workflow-types.js";
+
+export { RUN_STATES, type RunState };
+
+export interface RunGateFacts {
+  readonly objective: {
+    readonly evidenceComplete: boolean;
+    readonly verifierPassed: boolean;
+    readonly prerequisitesComplete: boolean;
+  };
+  readonly workers: {
+    readonly active: boolean;
+    readonly uncertain: boolean;
+    readonly detached: boolean;
+  };
+  readonly cancellation: {
+    readonly requested: boolean;
+    readonly settled: boolean;
+  };
+  readonly input: { readonly delivered: boolean };
+  readonly reconciliation: { readonly recorded: boolean; readonly uncertaintyCleared: boolean };
+  readonly attention: { readonly cleared: boolean };
+}
+
+export type RunGateGuard =
+  | "objective.evidence.complete"
+  | "objective.verifier.passed"
+  | "objective.prerequisites.complete"
+  | "workers.none_active"
+  | "workers.none_uncertain"
+  | "workers.none_detached"
+  | "cancellation.requested"
+  | "cancellation.settled"
+  | "input.delivered"
+  | "reconciliation.recorded"
+  | "reconciliation.uncertainty_cleared"
+  | "attention.cleared";
+
+export interface RunTransitionRule {
+  readonly from: RunState;
+  readonly to: RunState;
+  readonly guards: readonly RunGateGuard[];
+}
+
+const DONE_GUARDS: readonly RunGateGuard[] = [
+  "objective.evidence.complete",
+  "objective.verifier.passed",
+  "objective.prerequisites.complete",
+  "workers.none_active",
+  "workers.none_uncertain",
+  "workers.none_detached",
+];
+
+const RECONCILED: readonly RunGateGuard[] = [
+  "reconciliation.recorded",
+  "reconciliation.uncertainty_cleared",
+];
+
+export const RUN_LEGALITY_TABLE: readonly RunTransitionRule[] = [
+  { from: "open", to: "verifying", guards: [] },
+  { from: "open", to: "blocked", guards: [] },
+  { from: "open", to: "cancel-requested", guards: ["cancellation.requested"] },
+  { from: "open", to: "ambiguous", guards: [] },
+  { from: "open", to: "needs-attention", guards: [] },
+  { from: "verifying", to: "open", guards: [] },
+  { from: "verifying", to: "blocked", guards: [] },
+  { from: "verifying", to: "done", guards: DONE_GUARDS },
+  { from: "verifying", to: "cancel-requested", guards: ["cancellation.requested"] },
+  { from: "verifying", to: "ambiguous", guards: [] },
+  { from: "verifying", to: "needs-attention", guards: [] },
+  { from: "blocked", to: "open", guards: ["input.delivered"] },
+  { from: "blocked", to: "cancel-requested", guards: ["cancellation.requested"] },
+  { from: "blocked", to: "ambiguous", guards: [] },
+  { from: "blocked", to: "needs-attention", guards: [] },
+  { from: "cancel-requested", to: "cancelled", guards: ["cancellation.settled", "workers.none_uncertain"] },
+  { from: "cancel-requested", to: "ambiguous", guards: [] },
+  { from: "cancel-requested", to: "needs-attention", guards: [] },
+  { from: "ambiguous", to: "open", guards: RECONCILED },
+  { from: "ambiguous", to: "verifying", guards: RECONCILED },
+  { from: "ambiguous", to: "blocked", guards: RECONCILED },
+  { from: "ambiguous", to: "cancel-requested", guards: [...RECONCILED, "cancellation.requested"] },
+  { from: "ambiguous", to: "cancelled", guards: [...RECONCILED, "cancellation.settled"] },
+  { from: "ambiguous", to: "done", guards: [...RECONCILED, ...DONE_GUARDS] },
+  { from: "ambiguous", to: "needs-attention", guards: RECONCILED },
+  { from: "needs-attention", to: "open", guards: ["attention.cleared", "workers.none_detached"] },
+  { from: "needs-attention", to: "verifying", guards: ["attention.cleared", "workers.none_detached"] },
+  { from: "needs-attention", to: "blocked", guards: ["attention.cleared"] },
+  { from: "needs-attention", to: "cancel-requested", guards: ["cancellation.requested"] },
+  { from: "needs-attention", to: "ambiguous", guards: [] },
+] as const;
+
+export function evaluateRunTransition(
+  from: RunState,
+  to: RunState,
+  facts: RunGateFacts,
+): { readonly allowed: true } | {
+  readonly allowed: false;
+  readonly reason: "undeclared-transition" | "guard-failed";
+  readonly blockers: readonly RunGateGuard[];
+} {
+  const rule = RUN_LEGALITY_TABLE.find((candidate) => candidate.from === from && candidate.to === to);
+  if (!rule) return { allowed: false, reason: "undeclared-transition", blockers: [] };
+  const blockers = rule.guards.filter((guard) => !guardSatisfied(guard, facts));
+  return blockers.length === 0 ? { allowed: true } : { allowed: false, reason: "guard-failed", blockers };
+}
+
+function guardSatisfied(guard: RunGateGuard, facts: RunGateFacts): boolean {
+  switch (guard) {
+    case "objective.evidence.complete": return facts.objective.evidenceComplete;
+    case "objective.verifier.passed": return facts.objective.verifierPassed;
+    case "objective.prerequisites.complete": return facts.objective.prerequisitesComplete;
+    case "workers.none_active": return !facts.workers.active;
+    case "workers.none_uncertain": return !facts.workers.uncertain;
+    case "workers.none_detached": return !facts.workers.detached;
+    case "cancellation.requested": return facts.cancellation.requested;
+    case "cancellation.settled": return facts.cancellation.settled;
+    case "input.delivered": return facts.input.delivered;
+    case "reconciliation.recorded": return facts.reconciliation.recorded;
+    case "reconciliation.uncertainty_cleared": return facts.reconciliation.uncertaintyCleared;
+    case "attention.cleared": return facts.attention.cleared;
+  }
+}