sequant 2.7.0 → 2.9.0

package/dist/src/lib/workflow/phase-executor.js

@@ -12,7 +12,7 @@ import { execSync, execFileSync } from "child_process";
 import { readAgentsMd } from "../agents-md.js";
 import { getDriver } from "./drivers/index.js";
 import { classifyError } from "./error-classifier.js";
-import { ApiError } from "../errors.js";
+import { ApiError, BillingError, RateLimitError, resetsAtToMs, } from "../errors.js";
 import { phaseRegistry } from "./phase-registry.js";
 import { bracketedConsoleLog } from "./notice.js";
 /**
@@ -98,6 +98,44 @@ const SPEC_RETRY_STRATEGY = phaseRegistry.get("spec").retryStrategy;
 export const SPEC_RETRY_BACKOFF_MS = SPEC_RETRY_STRATEGY?.backoffMs ?? 5000;
 /** @internal Exported for testing only */
 export const SPEC_EXTRA_RETRIES = SPEC_RETRY_STRATEGY?.extraRetries ?? 1;
+/**
+ * A rate limit whose window resets further out than this is treated as
+ * exhausted rather than transient (#761 AC-2): no retry can succeed inside a
+ * closed window, so consuming cold-start retries (each burning up to a full
+ * `phaseTimeout`) only delays the labeled halt. Five minutes comfortably
+ * exceeds any backoff this executor performs while staying far below the
+ * five-hour/seven-day windows the check exists to catch.
+ *
+ * @internal Exported for testing only
+ */
+export const RATE_LIMIT_WINDOW_SKIP_THRESHOLD_MS = 5 * 60 * 1000;
+/**
+ * Base backoff for transient rate-limit retries (#761 AC-4), doubled per
+ * attempt (5s, 10s). Same scale as `SPEC_RETRY_BACKOFF_MS` — long enough to
+ * outlive a momentary throttle, short enough to be negligible next to a
+ * phase's runtime.
+ *
+ * @internal Exported for testing only
+ */
+export const RATE_LIMIT_RETRY_BACKOFF_MS = 5000;
+/**
+ * True when a failure is a rate limit whose reset lies beyond
+ * {@link RATE_LIMIT_WINDOW_SKIP_THRESHOLD_MS} — i.e. window exhaustion, not a
+ * transient throttle. Metadata-absent rate limits (the assistant-error channel
+ * carries no `resetsAt`, see #761 AC-9) return false and fall through to the
+ * transient path: with no timing signal, retry-with-backoff is the safe
+ * default, skipping all retries is not.
+ *
+ * @internal Exported for testing only
+ */
+export function isWindowExhaustedRateLimit(error, now = Date.now()) {
+    if (!(error instanceof RateLimitError))
+        return false;
+    const resetsAt = error.metadata.resetsAt;
+    if (typeof resetsAt !== "number")
+        return false;
+    return resetsAtToMs(resetsAt) - now > RATE_LIMIT_WINDOW_SKIP_THRESHOLD_MS;
+}
 export function parseQaVerdict(output) {
     if (!output)
         return null;
@@ -348,9 +386,12 @@ export function mapAgentSuccessToPhaseResult(phase, agentResult, durationSeconds
         const summary = agentResult.output
             ? (parseQaSummary(agentResult.output) ?? undefined)
             : undefined;
-        if (verdict &&
-            verdict !== "READY_FOR_MERGE" &&
-            verdict !== "NEEDS_VERIFICATION") {
+        if (verdict === "AC_NOT_MET") {
+            // #749: only AC_NOT_MET (and the null branch below, #534) hard-fails.
+            // AC_MET_BUT_NOT_A_PLUS is a stopping/ready state — it must break to PR,
+            // not feed the quality loop (mirrors ready-gate.ts's `ac` policy). The
+            // verdict is retained on the success result so the PR/log surfaces the
+            // "not A+" note.
             return {
                 phase,
                 success: false,
@@ -408,6 +449,43 @@ export function mapAgentSuccessToPhaseResult(phase, agentResult, durationSeconds
         ...tails,
     };
 }
+/**
+ * Map a failed driver result to a `PhaseResult`.
+ *
+ * Symmetric to {@link mapAgentSuccessToPhaseResult}; extracted so the
+ * failure-path mapping (notably the #739 capped/output gating) is unit-testable
+ * without spawning a driver.
+ *
+ * `output` is propagated **only** for a capped phase (#739): a capped result is
+ * incomplete-but-not-hard-failed, so its partial work must survive downstream.
+ * A genuine (non-capped) failure keeps the historical behaviour of dropping
+ * `output`, leaving the `/loop` fix-context (`formatFailureContext`) unchanged.
+ *
+ * @internal Exported for testing only
+ */
+export function mapAgentFailureToPhaseResult(phase, agentResult, durationSeconds) {
+    return {
+        phase,
+        success: false,
+        durationSeconds,
+        error: agentResult.error,
+        // Propagate the driver's typed cause (#732) so the retry logic can prefer
+        // it over stderr-regex classification and gate the MCP fallback.
+        structuredError: agentResult.structuredError,
+        // Propagate the turn-cap flag and the partial output (#739). On the failure
+        // path `output` was previously dropped entirely — for a capped phase the
+        // partial work is usable and must be preserved, mirroring the driver/skill
+        // slice from #733. Gating `output` on `capped` keeps non-capped failures
+        // byte-for-byte identical to pre-#739 behaviour.
+        capped: agentResult.capped,
+        output: agentResult.capped ? agentResult.output : undefined,
+        sessionId: agentResult.sessionId,
+        resumeHandle: agentResult.resumeHandle,
+        stderrTail: agentResult.stderrTail,
+        stdoutTail: agentResult.stdoutTail,
+        exitCode: agentResult.exitCode,
+    };
+}
 /**
  * Get the prompt for a phase with the issue number substituted.
  * Selects self-contained prompts for non-Claude agents.
@@ -642,17 +720,7 @@ async function executePhase(issueNumber, phase, config, resumeHandle, worktreePa
     if (agentResult.success) {
         return mapAgentSuccessToPhaseResult(phase, agentResult, durationSeconds, cwd);
     }
-    return {
-        phase,
-        success: false,
-        durationSeconds,
-        error: agentResult.error,
-        sessionId: agentResult.sessionId,
-        resumeHandle: agentResult.resumeHandle,
-        stderrTail: agentResult.stderrTail,
-        stdoutTail: agentResult.stdoutTail,
-        exitCode: agentResult.exitCode,
-    };
+    return mapAgentFailureToPhaseResult(phase, agentResult, durationSeconds);
 }
 /**
  * Execute a phase with automatic retry for cold-start failures and MCP fallback.
@@ -693,6 +761,14 @@ delayFn = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
         if (lastResult.success) {
             return lastResult;
         }
+        // Turn-capped phase (#739): incomplete-but-not-hard-failed. A retry cannot
+        // un-cap a turn limit, so short-circuit before any fallback — same rationale
+        // as the billing skip (#732), but capped must skip *all* retries (incl.
+        // cold-start), so an explicit early return is required, not just a guard
+        // flag at the MCP gate.
+        if (lastResult.capped) {
+            return lastResult;
+        }
     }
     else {
         // Phase 1: Cold-start retry attempts (with MCP enabled if configured)
@@ -703,11 +779,49 @@ delayFn = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
             if (lastResult.success) {
                 return lastResult;
             }
+            // Turn-capped phase (#739): short-circuit before cold-start retries, the
+            // MCP fallback, and the spec-extra retry — a retry cannot un-cap a turn
+            // limit. The early return here (rather than a guard at the MCP gate alone)
+            // is what skips the cold-start re-spawns, unlike the billing case which
+            // still cold-start-retries in the <60s window.
+            if (lastResult.capped) {
+                return lastResult;
+            }
+            // Window-exhausted rate limit (#761 AC-2): the reset is hours away, so
+            // every retry re-spawns into the same closed window — worst case
+            // ~4 × phaseTimeout (≈2h) of doomed attempts before the run halts.
+            // Modelled on the `capped` early return above: skip all remaining
+            // cold-start retries and (via the return) the MCP fallback. Checked
+            // before the duration branch because a rate-limit rejection typically
+            // fails fast and would otherwise be mistaken for a cold-start failure.
+            if (isWindowExhaustedRateLimit(lastResult.structuredError)) {
+                if (config.verbose) {
+                    bracketedConsoleLog(spinner, chalk.yellow(`\n    ✕ ${lastResult.error ?? "Rate limited"} — window exhausted, skipping retries`));
+                }
+                return lastResult;
+            }
+            // Transient rate limit (#761 AC-4): retry, but with real backoff — the
+            // bare `continue` this replaces re-spawned immediately into the same
+            // throttle. Reuses the injected `delayFn`; delay doubles per attempt.
+            // Metadata-absent rate limits land here by design (AC-9 fallback rule).
+            if (lastResult.structuredError instanceof RateLimitError &&
+                attempt < COLD_START_MAX_RETRIES) {
+                const backoffMs = RATE_LIMIT_RETRY_BACKOFF_MS * 2 ** attempt;
+                if (config.verbose) {
+                    bracketedConsoleLog(spinner, chalk.yellow(`\n    ⟳ ${lastResult.error ?? "Rate limited"} — backing off ${backoffMs}ms before retry... (attempt ${attempt + 2}/${COLD_START_MAX_RETRIES + 1})`));
+                }
+                await delayFn(backoffMs);
+                continue;
+            }
             // Genuine failure (took long enough to be real work) → skip cold-start retries.
             // Use error classification (AC-9): if the error is retryable (e.g., API
             // rate limit, transient 503), allow one more attempt even for genuine failures.
             if (duration >= COLD_START_THRESHOLD_SECONDS) {
-                const typedError = classifyError(lastResult.stderrTail ?? [], lastResult.exitCode);
+                // Prefer the driver's structured cause (#732) — it reflects the real
+                // SDK rate-limit/billing signal — over stderr-regex classification,
+                // which only sees text and never the structured data.
+                const typedError = lastResult.structuredError ??
+                    classifyError(lastResult.stderrTail ?? [], lastResult.exitCode);
                 if (typedError.isRetryable && attempt < COLD_START_MAX_RETRIES) {
                     if (config.verbose) {
                         const label = typedError instanceof ApiError
@@ -735,7 +849,27 @@ delayFn = (ms) => new Promise((resolve) => setTimeout(resolve, ms))) {
     // Phase 2: MCP fallback - if MCP is enabled and we're still failing, try without MCP
     // This handles npx-based MCP servers that fail on first run due to cold-cache issues.
     // Skip for `loop` phase — MCP is never the cause of loop failures (#488).
-    if (config.mcp && !lastResult.success && !skipColdStartRetry) {
+    //
+    // Also skip when the failure is a billing/credits error (#732): a no-MCP
+    // retry cannot refill credits, so the misleading "retrying without MCP"
+    // noise (#592) would only mask the real cause. The accurate structured
+    // message (e.g. "Out of credits") is surfaced instead.
+    const failureIsBilling = lastResult.structuredError instanceof BillingError;
+    // Belt-and-suspenders (#739): the capped early-returns above already exit
+    // before reaching here, but gate the MCP fallback on `!failureIsCapped` too so
+    // intent is documented and future code paths can't accidentally re-spawn a
+    // capped phase without MCP.
+    const failureIsCapped = lastResult.capped === true;
+    // A throttle must not trigger "retrying without MCP" (#761 AC-3): MCP was
+    // never the cause, and the re-spawn burns up to another full phaseTimeout
+    // against the same limit while mislabeling the failure as MCP-related.
+    const failureIsRateLimited = lastResult.structuredError instanceof RateLimitError;
+    if (config.mcp &&
+        !lastResult.success &&
+        !skipColdStartRetry &&
+        !failureIsBilling &&
+        !failureIsCapped &&
+        !failureIsRateLimited) {
         bracketedConsoleLog(spinner, chalk.yellow(`\n    ! Phase failed with MCP enabled, retrying without MCP...`));
         // Create config copy with MCP disabled
         const configWithoutMcp = {

package/dist/src/lib/workflow/run-log-schema.d.ts

@@ -89,6 +89,8 @@ export declare const ErrorContextSchema: z.ZodObject<{
         api_error: "api_error";
         hook_failure: "hook_failure";
         build_error: "build_error";
+        rate_limit: "rate_limit";
+        billing: "billing";
     }>;
     errorType: z.ZodOptional<z.ZodString>;
     errorMetadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
@@ -125,6 +127,7 @@ export declare const PhaseLogSchema: z.ZodObject<{
         timeout: "timeout";
     }>;
     error: z.ZodOptional<z.ZodString>;
+    capped: z.ZodOptional<z.ZodBoolean>;
     iterations: z.ZodOptional<z.ZodNumber>;
     filesModified: z.ZodOptional<z.ZodArray<z.ZodString>>;
     testsRun: z.ZodOptional<z.ZodNumber>;
@@ -169,6 +172,8 @@ export declare const PhaseLogSchema: z.ZodObject<{
             api_error: "api_error";
             hook_failure: "hook_failure";
             build_error: "build_error";
+            rate_limit: "rate_limit";
+            billing: "billing";
         }>;
         errorType: z.ZodOptional<z.ZodString>;
         errorMetadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
@@ -201,6 +206,7 @@ export declare const IssueLogSchema: z.ZodObject<{
             timeout: "timeout";
         }>;
         error: z.ZodOptional<z.ZodString>;
+        capped: z.ZodOptional<z.ZodBoolean>;
         iterations: z.ZodOptional<z.ZodNumber>;
         filesModified: z.ZodOptional<z.ZodArray<z.ZodString>>;
         testsRun: z.ZodOptional<z.ZodNumber>;
@@ -245,6 +251,8 @@ export declare const IssueLogSchema: z.ZodObject<{
                 api_error: "api_error";
                 hook_failure: "hook_failure";
                 build_error: "build_error";
+                rate_limit: "rate_limit";
+                billing: "billing";
             }>;
             errorType: z.ZodOptional<z.ZodString>;
             errorMetadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
@@ -275,6 +283,7 @@ export declare const RunSummarySchema: z.ZodObject<{
     totalIssues: z.ZodNumber;
     passed: z.ZodNumber;
     failed: z.ZodNumber;
+    partial: z.ZodDefault<z.ZodNumber>;
     totalDurationSeconds: z.ZodNumber;
 }, z.core.$strip>;
 export type RunSummary = z.infer<typeof RunSummarySchema>;
@@ -318,6 +327,7 @@ export declare const RunLogSchema: z.ZodObject<{
                 timeout: "timeout";
             }>;
             error: z.ZodOptional<z.ZodString>;
+            capped: z.ZodOptional<z.ZodBoolean>;
             iterations: z.ZodOptional<z.ZodNumber>;
             filesModified: z.ZodOptional<z.ZodArray<z.ZodString>>;
             testsRun: z.ZodOptional<z.ZodNumber>;
@@ -362,6 +372,8 @@ export declare const RunLogSchema: z.ZodObject<{
                     api_error: "api_error";
                     hook_failure: "hook_failure";
                     build_error: "build_error";
+                    rate_limit: "rate_limit";
+                    billing: "billing";
                 }>;
                 errorType: z.ZodOptional<z.ZodString>;
                 errorMetadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
@@ -376,6 +388,7 @@ export declare const RunLogSchema: z.ZodObject<{
         totalIssues: z.ZodNumber;
         passed: z.ZodNumber;
         failed: z.ZodNumber;
+        partial: z.ZodDefault<z.ZodNumber>;
         totalDurationSeconds: z.ZodNumber;
     }, z.core.$strip>;
     startCommit: z.ZodOptional<z.ZodString>;
@@ -436,3 +449,16 @@ export declare function completePhaseLog(phaseLog: Omit<PhaseLog, "endTime" | "d
 export declare function finalizeRunLog(runLog: Omit<RunLog, "endTime">, options?: {
     endCommit?: string;
 }): RunLog;
+/**
+ * Derive an issue's overall log status from its phase log entries (#766).
+ *
+ * Phases are appended in execution order, so the last entry for a given phase
+ * name is its latest attempt — that attempt wins. This lets a `timeout`/
+ * `failure` that a later quality-loop iteration recovers from de-escalate to
+ * `success`, keeping the JSON log consistent with the live card and summary
+ * table (AC-3/AC-5). `loop` is auxiliary recovery and never determines the
+ * verdict (mirrors the live-card rule); an unrecovered failure still leaves a
+ * non-loop phase failed. Priority among latest attempts: failure > timeout >
+ * success.
+ */
+export declare function deriveIssueLogStatus(phases: PhaseLog[]): IssueStatus;

package/dist/src/lib/workflow/run-log-schema.js

@@ -79,13 +79,19 @@ export const ErrorContextSchema = z.object({
     stdoutTail: z.array(z.string()),
     /** Process exit code */
     exitCode: z.number().int().optional(),
-    /** Classified error category (legacy, kept for backwards compatibility) */
+    /**
+     * Classified error category (legacy, kept for backwards compatibility).
+     * Keep in sync with `ERROR_CATEGORIES` in `error-classifier.ts` —
+     * `rate_limit` / `billing` added by #761 AC-6.
+     */
     category: z.enum([
         "context_overflow",
         "api_error",
         "hook_failure",
         "build_error",
         "timeout",
+        "rate_limit",
+        "billing",
         "unknown",
     ]),
     /** Typed error class name (AC-8), e.g. "ApiError", "BuildError" */
@@ -130,6 +136,13 @@ export const PhaseLogSchema = z.object({
     status: PhaseStatusSchema,
     /** Error message if failed */
     error: z.string().optional(),
+    /**
+     * Set when the phase hit its turn cap (`error_max_turns`) (#739). Distinguishes
+     * an incomplete-but-not-hard-failed phase (partial output preserved) from a
+     * genuine failure. Reuses the `"failure"` status — additive boolean rather than
+     * a new `PhaseStatus` enum value, to keep the persisted-log schema stable.
+     */
+    capped: z.boolean().optional(),
     /** Number of iterations (for loop phase) */
     iterations: z.number().int().nonnegative().optional(),
     /** Files modified during this phase */
@@ -199,6 +212,13 @@ export const RunSummarySchema = z.object({
     passed: z.number().int().nonnegative(),
     /** Number of issues that failed */
     failed: z.number().int().nonnegative(),
+    /**
+     * Number of issues that ended `partial` — timed out with no genuine failure
+     * and no recovery (#766). Given its own bucket so an all-partial run no longer
+     * vanishes from both `passed` and `failed` (the `0 passed · 0 failed` bug).
+     * `.default(0)` keeps logs written before this field parseable.
+     */
+    partial: z.number().int().nonnegative().default(0),
     /** Total execution time in seconds */
     totalDurationSeconds: z.number().nonnegative(),
 });
@@ -267,6 +287,7 @@ export function createEmptyRunLog(config, options) {
             totalIssues: 0,
             passed: 0,
             failed: 0,
+            partial: 0,
             totalDurationSeconds: 0,
         },
         startCommit: options?.startCommit,
@@ -319,6 +340,9 @@ export function finalizeRunLog(runLog, options) {
     const totalDurationSeconds = (endTime.getTime() - startTime.getTime()) / 1000;
     const passed = runLog.issues.filter((i) => i.status === "success").length;
     const failed = runLog.issues.filter((i) => i.status === "failure").length;
+    // #766: `partial` gets its own bucket so an all-partial run isn't counted as
+    // `0 passed · 0 failed` — it landed in neither before.
+    const partial = runLog.issues.filter((i) => i.status === "partial").length;
     return {
         ...runLog,
         endTime: endTime.toISOString(),
@@ -326,8 +350,35 @@ export function finalizeRunLog(runLog, options) {
             totalIssues: runLog.issues.length,
             passed,
             failed,
+            partial,
             totalDurationSeconds,
         },
         endCommit: options?.endCommit ?? runLog.endCommit,
     };
 }
+/**
+ * Derive an issue's overall log status from its phase log entries (#766).
+ *
+ * Phases are appended in execution order, so the last entry for a given phase
+ * name is its latest attempt — that attempt wins. This lets a `timeout`/
+ * `failure` that a later quality-loop iteration recovers from de-escalate to
+ * `success`, keeping the JSON log consistent with the live card and summary
+ * table (AC-3/AC-5). `loop` is auxiliary recovery and never determines the
+ * verdict (mirrors the live-card rule); an unrecovered failure still leaves a
+ * non-loop phase failed. Priority among latest attempts: failure > timeout >
+ * success.
+ */
+export function deriveIssueLogStatus(phases) {
+    const latest = new Map();
+    for (const p of phases) {
+        if (p.phase === "loop")
+            continue;
+        latest.set(p.phase, p.status);
+    }
+    const statuses = [...latest.values()];
+    if (statuses.some((s) => s === "failure"))
+        return "failure";
+    if (statuses.some((s) => s === "timeout"))
+        return "partial";
+    return "success";
+}

package/dist/src/lib/workflow/run-orchestrator.d.ts

@@ -13,6 +13,7 @@ import { LogWriter } from "./log-writer.js";
 import { StateManager } from "./state-manager.js";
 import { ShutdownManager } from "../shutdown.js";
 import type { LockFile } from "../locks/index.js";
+import { type ChainResumePlan } from "./chain-resume.js";
 import { WorkflowEventEmitter } from "./event-emitter.js";
 import type { SequantSettings } from "../settings.js";
 /**
@@ -55,6 +56,12 @@ export interface OrchestratorConfig {
     packageManager?: string;
     /** Base branch for rebase/PR targets */
     baseBranch?: string;
+    /**
+     * Chain resume plan (#760). Present only when re-running a `--chain` batch
+     * whose completed prefix is being skipped. Drives the first active link's
+     * rebase onto the last completed link's committed tip in `executeSequential`.
+     */
+    chainResume?: ChainResumePlan;
     /** Per-phase progress callback (parallel mode) */
     onProgress?: ProgressCallback;
     /** #672 AC-2: phase-plan callback forwarded into per-issue contexts. */
@@ -112,6 +119,13 @@ export interface ResolvedRun {
     config: ExecutionConfig;
     /** Parsed + dep-sorted issue numbers (pre-state-guard) */
     issueNumbers: number[];
+    /**
+     * Raw CLI issue order BEFORE `sortByDependencies` reorders it (#762).
+     * The chain content pre-flight must compare against this order — comparing
+     * against the dep-sorted `issueNumbers` would make the dependency-order and
+     * file-overlap-order checks dead code, since the sorter already fixed them.
+     */
+    rawIssueOrder: number[];
     /** Resolved batches if --batch specified, else null */
     batches: number[][] | null;
     /** Resolved base branch (CLI → settings → auto-detect → "main") */