steroids-cli 0.6.5 → 0.8.0

package/dist/health/stuck-task-detector.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Core stuck-task detection logic (detection only, no recovery).
+ *
+ * This implementation intentionally uses the project's current schemas:
+ * - Project DB: tasks, audit, task_invocations
+ * - Global DB: runners
+ *
+ * The design doc (docs/stuck-task-detection.md) describes additional fields/tables
+ * (incidents, invocation started/completed timestamps, last_tool_execution, etc.).
+ * Those are not present in the current repo schema, so we derive signals from
+ * existing timestamps (tasks.updated_at, task_invocations.created_at, runners.heartbeat_at).
+ */
+import type Database from 'better-sqlite3';
+export type FailureMode = 'orphaned_task' | 'hanging_invocation' | 'zombie_runner' | 'dead_runner' | 'db_inconsistency';
+export interface StuckTaskDetectionConfig {
+    /**
+     * How long a task may remain `in_progress` without a recent invocation and without
+     * an active runner assigned before being considered orphaned.
+     */
+    orphanedTaskTimeoutSec?: number;
+    /**
+     * Maximum allowed time a task may remain `in_progress` while an active runner is
+     * actively working on it (approximates "hanging coder invocation" without
+     * started/completed timestamps).
+     */
+    maxCoderDurationSec?: number;
+    /**
+     * Maximum allowed time a task may remain `review` while an active runner is
+     * actively working on it (approximates "hanging reviewer invocation").
+     */
+    maxReviewerDurationSec?: number;
+    /** Runner heartbeat staleness threshold. */
+    runnerHeartbeatTimeoutSec?: number;
+    /**
+     * How long since last task invocation to consider a task "inactive" for orphan checks.
+     * This is compared against task_invocations.created_at.
+     */
+    invocationStalenessSec?: number;
+    /** Recent-update window used for "DB inconsistency" transient detection. */
+    dbInconsistencyRecentUpdateSec?: number;
+}
+export interface OrphanedTaskSignal {
+    failureMode: 'orphaned_task';
+    taskId: string;
+    title: string;
+    status: 'in_progress';
+    updatedAt: Date;
+    secondsSinceUpdate: number;
+    invocationCount: number;
+    lastInvocationAt: Date | null;
+    hasActiveRunner: boolean;
+}
+export interface HangingTaskSignal {
+    failureMode: 'hanging_invocation';
+    phase: 'coder' | 'reviewer';
+    taskId: string;
+    title: string;
+    status: 'in_progress' | 'review';
+    updatedAt: Date;
+    secondsSinceUpdate: number;
+    runnerId: string;
+    runnerPid: number | null;
+    runnerHeartbeatAt: Date;
+}
+export interface ZombieRunnerSignal {
+    failureMode: 'zombie_runner';
+    runnerId: string;
+    pid: number | null;
+    status: string;
+    projectPath: string | null;
+    currentTaskId: string | null;
+    heartbeatAt: Date;
+    secondsSinceHeartbeat: number;
+}
+export interface DeadRunnerSignal {
+    failureMode: 'dead_runner';
+    runnerId: string;
+    pid: number | null;
+    status: string;
+    projectPath: string | null;
+    currentTaskId: string | null;
+    heartbeatAt: Date;
+    secondsSinceHeartbeat: number;
+}
+export interface DbInconsistencySignal {
+    failureMode: 'db_inconsistency';
+    taskId: string;
+    title: string;
+    status: 'in_progress';
+    updatedAt: Date;
+    secondsSinceUpdate: number;
+    invocationCount: 0;
+}
+export interface StuckTaskDetectionReport {
+    timestamp: Date;
+    orphanedTasks: OrphanedTaskSignal[];
+    hangingInvocations: HangingTaskSignal[];
+    zombieRunners: ZombieRunnerSignal[];
+    deadRunners: DeadRunnerSignal[];
+    dbInconsistencies: DbInconsistencySignal[];
+}
+export interface DetectStuckTasksOptions {
+    /** Absolute project path as stored in global runners DB. */
+    projectPath: string;
+    /** Project-local database connection (tasks, task_invocations). */
+    projectDb: Database.Database;
+    /** Global database connection (runners). */
+    globalDb: Database.Database;
+    /** Optional config overrides. */
+    config?: StuckTaskDetectionConfig;
+    /**
+     * PID liveness check override for testing.
+     * If omitted, a best-effort `process.kill(pid, 0)` check is used.
+     */
+    isPidAlive?: (pid: number) => boolean;
+    /** Override current time for deterministic tests. */
+    now?: Date;
+}
+/**
+ * Parse SQLite datetime('now') strings (YYYY-MM-DD HH:MM:SS) as UTC.
+ * Node's Date parser can treat "YYYY-MM-DD HH:MM:SS" as local time depending on runtime,
+ * so we normalize to ISO 8601 Zulu.
+ */
+export declare function parseSqliteDateTimeUtc(value: string): Date;
+/**
+ * Format a Date as SQLite datetime('now')-compatible UTC string: YYYY-MM-DD HH:MM:SS
+ * This keeps comparisons lexicographically safe against stored SQLite timestamps.
+ */
+export declare function formatSqliteDateTimeUtc(date: Date): string;
+export declare function detectStuckTasks(options: DetectStuckTasksOptions): StuckTaskDetectionReport;
+//# sourceMappingURL=stuck-task-detector.d.ts.map

package/dist/health/stuck-task-detector.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stuck-task-detector.d.ts","sourceRoot":"","sources":["../../src/health/stuck-task-detector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C,MAAM,MAAM,WAAW,GACnB,eAAe,GACf,oBAAoB,GACpB,eAAe,GACf,aAAa,GACb,kBAAkB,CAAC;AAEvB,MAAM,WAAW,wBAAwB;IACvC;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC,4CAA4C;IAC5C,yBAAyB,CAAC,EAAE,MAAM,CAAC;IAEnC;;;OAGG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAEhC,4EAA4E;IAC5E,8BAA8B,CAAC,EAAE,MAAM,CAAC;CACzC;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,eAAe,CAAC;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,SAAS,EAAE,IAAI,CAAC;IAChB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,IAAI,GAAG,IAAI,CAAC;IAC9B,eAAe,EAAE,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,oBAAoB,CAAC;IAClC,KAAK,EAAE,OAAO,GAAG,UAAU,CAAC;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,GAAG,QAAQ,CAAC;IACjC,SAAS,EAAE,IAAI,CAAC;IAChB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,iBAAiB,EAAE,IAAI,CAAC;CACzB;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,eAAe,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,IAAI,CAAC;IAClB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,aAAa,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,WAAW,EAAE,IAAI,CAAC;IAClB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,qBAAqB;IACpC,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,aAAa,CAAC;IACtB,SAAS,EAAE,IAAI,CAAC;IAChB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,CAAC,CAAC;CACpB;AAED,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,IAAI,CAAC;IAChB,aAAa,EAAE,kBAAkB,EAAE,CAAC;IACpC,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;IACxC,aAAa,EAAE,kBAAkB,EAAE,CAAC;IACpC,WAAW,EAAE,gBAAgB,EAAE,CAAC;IAChC,iBAAiB,EAAE,qBAAqB,EAAE,CAAC;CAC5C;AAED,MAAM,WAAW,uBAAuB;IACtC,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC7B,4CAA4C;IAC5C,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC5B,iCAAiC;IACjC,MAAM,CAAC,EAAE,wBAAwB,CAAC;IAClC;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACtC,qDAAqD;IACrD,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ;AAqBD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAK1D;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM,CAG1D;AAUD,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,uBAAuB,GAAG,wBAAwB,CAwB3F"}

package/dist/health/stuck-task-detector.js

@@ -0,0 +1,238 @@
+"use strict";
+/**
+ * Core stuck-task detection logic (detection only, no recovery).
+ *
+ * This implementation intentionally uses the project's current schemas:
+ * - Project DB: tasks, audit, task_invocations
+ * - Global DB: runners
+ *
+ * The design doc (docs/stuck-task-detection.md) describes additional fields/tables
+ * (incidents, invocation started/completed timestamps, last_tool_execution, etc.).
+ * Those are not present in the current repo schema, so we derive signals from
+ * existing timestamps (tasks.updated_at, task_invocations.created_at, runners.heartbeat_at).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseSqliteDateTimeUtc = parseSqliteDateTimeUtc;
+exports.formatSqliteDateTimeUtc = formatSqliteDateTimeUtc;
+exports.detectStuckTasks = detectStuckTasks;
+const DEFAULTS = {
+    orphanedTaskTimeoutSec: 600,
+    maxCoderDurationSec: 1800,
+    maxReviewerDurationSec: 900,
+    runnerHeartbeatTimeoutSec: 300,
+    invocationStalenessSec: 600,
+    dbInconsistencyRecentUpdateSec: 60,
+};
+function isProcessAliveBestEffort(pid) {
+    try {
+        // Signal 0: existence check only.
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Parse SQLite datetime('now') strings (YYYY-MM-DD HH:MM:SS) as UTC.
+ * Node's Date parser can treat "YYYY-MM-DD HH:MM:SS" as local time depending on runtime,
+ * so we normalize to ISO 8601 Zulu.
+ */
+function parseSqliteDateTimeUtc(value) {
+    // Already ISO? (e.g., 2026-02-10T13:45:00.000Z)
+    if (value.includes('T'))
+        return new Date(value);
+    // SQLite "YYYY-MM-DD HH:MM:SS" (UTC) => "YYYY-MM-DDTHH:MM:SSZ"
+    return new Date(value.replace(' ', 'T') + 'Z');
+}
+/**
+ * Format a Date as SQLite datetime('now')-compatible UTC string: YYYY-MM-DD HH:MM:SS
+ * This keeps comparisons lexicographically safe against stored SQLite timestamps.
+ */
+function formatSqliteDateTimeUtc(date) {
+    // Date#toISOString is always UTC; trim milliseconds and replace T with space.
+    return date.toISOString().slice(0, 19).replace('T', ' ');
+}
+function secondsBetween(a, b) {
+    return Math.max(0, Math.floor((a.getTime() - b.getTime()) / 1000));
+}
+function mergeConfig(config) {
+    return { ...DEFAULTS, ...(config ?? {}) };
+}
+function detectStuckTasks(options) {
+    const now = options.now ?? new Date();
+    const cfg = mergeConfig(options.config);
+    const isPidAlive = options.isPidAlive ?? isProcessAliveBestEffort;
+    const zombieRunners = detectZombieRunnersInternal(options.globalDb, options.projectPath, cfg, now, isPidAlive);
+    const deadRunners = detectDeadRunnersInternal(options.globalDb, options.projectPath, cfg, now, isPidAlive);
+    const { orphanedTasks, hangingInvocations, dbInconsistencies } = detectTaskSignalsInternal(options.projectDb, options.globalDb, options.projectPath, cfg, now, isPidAlive);
+    return {
+        timestamp: now,
+        orphanedTasks,
+        hangingInvocations,
+        zombieRunners,
+        deadRunners,
+        dbInconsistencies,
+    };
+}
+function detectZombieRunnersInternal(globalDb, projectPath, cfg, now, isPidAlive) {
+    const cutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.runnerHeartbeatTimeoutSec * 1000));
+    const rows = globalDb.prepare(`SELECT id, status, pid, project_path, current_task_id, heartbeat_at
+     FROM runners
+     WHERE status = 'running'
+       AND project_path = ?
+       AND heartbeat_at < ?`).all(projectPath, cutoff);
+    const result = [];
+    for (const row of rows) {
+        if (row.pid !== null && isPidAlive(row.pid)) {
+            const hb = parseSqliteDateTimeUtc(row.heartbeat_at);
+            result.push({
+                failureMode: 'zombie_runner',
+                runnerId: row.id,
+                pid: row.pid,
+                status: row.status,
+                projectPath: row.project_path,
+                currentTaskId: row.current_task_id,
+                heartbeatAt: hb,
+                secondsSinceHeartbeat: secondsBetween(now, hb),
+            });
+        }
+    }
+    return result;
+}
+function detectDeadRunnersInternal(globalDb, projectPath, cfg, now, isPidAlive) {
+    const rows = globalDb.prepare(`SELECT id, status, pid, project_path, current_task_id, heartbeat_at
+     FROM runners
+     WHERE status = 'running'
+       AND project_path = ?`).all(projectPath);
+    const result = [];
+    for (const row of rows) {
+        const alive = row.pid !== null && isPidAlive(row.pid);
+        if (!alive) {
+            const hb = parseSqliteDateTimeUtc(row.heartbeat_at);
+            result.push({
+                failureMode: 'dead_runner',
+                runnerId: row.id,
+                pid: row.pid,
+                status: row.status,
+                projectPath: row.project_path,
+                currentTaskId: row.current_task_id,
+                heartbeatAt: hb,
+                secondsSinceHeartbeat: secondsBetween(now, hb),
+            });
+        }
+    }
+    return result;
+}
+function detectTaskSignalsInternal(projectDb, globalDb, projectPath, cfg, now, isPidAlive) {
+    const orphanedTasks = [];
+    const hangingInvocations = [];
+    const dbInconsistencies = [];
+    // DB inconsistency (transient): in_progress, no invocations, very recently updated.
+    {
+        const cutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.dbInconsistencyRecentUpdateSec * 1000));
+        const rows = projectDb.prepare(`SELECT t.id, t.title, t.status, t.updated_at
+       FROM tasks t
+       LEFT JOIN task_invocations i ON i.task_id = t.id AND i.role = 'coder'
+       WHERE t.status = 'in_progress'
+       GROUP BY t.id
+       HAVING COUNT(i.id) = 0
+          AND t.updated_at >= ?`).all(cutoff);
+        for (const row of rows) {
+            const updatedAt = parseSqliteDateTimeUtc(row.updated_at);
+            dbInconsistencies.push({
+                failureMode: 'db_inconsistency',
+                taskId: row.id,
+                title: row.title,
+                status: 'in_progress',
+                updatedAt,
+                secondsSinceUpdate: secondsBetween(now, updatedAt),
+                invocationCount: 0,
+            });
+        }
+    }
+    // Orphaned tasks: in_progress, stale updated_at, and no recent invocations, and no active runner assigned.
+    {
+        const taskCutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.orphanedTaskTimeoutSec * 1000));
+        const invocationCutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.invocationStalenessSec * 1000));
+        const rows = projectDb.prepare(`SELECT
+         t.id,
+         t.title,
+         t.status,
+         t.updated_at,
+         COUNT(i.id) as invocation_count,
+         MAX(i.created_at) as last_invocation_at
+       FROM tasks t
+       LEFT JOIN task_invocations i ON i.task_id = t.id AND i.role = 'coder'
+       WHERE t.status = 'in_progress'
+         AND t.updated_at < ?
+       GROUP BY t.id
+       HAVING COUNT(i.id) = 0
+           OR MAX(i.created_at) < ?`).all(taskCutoff, invocationCutoff);
+        for (const row of rows) {
+            const updatedAt = parseSqliteDateTimeUtc(row.updated_at);
+            const lastInvocationAt = row.last_invocation_at ? parseSqliteDateTimeUtc(row.last_invocation_at) : null;
+            const activeRunner = getActiveRunnerForTask(globalDb, projectPath, row.id, cfg, now, isPidAlive);
+            if (!activeRunner) {
+                orphanedTasks.push({
+                    failureMode: 'orphaned_task',
+                    taskId: row.id,
+                    title: row.title,
+                    status: 'in_progress',
+                    updatedAt,
+                    secondsSinceUpdate: secondsBetween(now, updatedAt),
+                    invocationCount: row.invocation_count,
+                    lastInvocationAt,
+                    hasActiveRunner: false,
+                });
+            }
+        }
+    }
+    // Hanging invocations: in_progress/review for too long with an active runner currently executing the task.
+    // Approximates "invocation started but not completed" using task age + runner current_task_id.
+    {
+        const coderCutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.maxCoderDurationSec * 1000));
+        const reviewerCutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.maxReviewerDurationSec * 1000));
+        const rows = projectDb.prepare(`SELECT t.id, t.title, t.status, t.updated_at
+       FROM tasks t
+       WHERE (t.status = 'in_progress' AND t.updated_at < ?)
+          OR (t.status = 'review' AND t.updated_at < ?)`).all(coderCutoff, reviewerCutoff);
+        for (const row of rows) {
+            const activeRunner = getActiveRunnerForTask(globalDb, projectPath, row.id, cfg, now, isPidAlive);
+            if (!activeRunner)
+                continue;
+            const updatedAt = parseSqliteDateTimeUtc(row.updated_at);
+            const hbAt = parseSqliteDateTimeUtc(activeRunner.heartbeat_at);
+            hangingInvocations.push({
+                failureMode: 'hanging_invocation',
+                phase: row.status === 'review' ? 'reviewer' : 'coder',
+                taskId: row.id,
+                title: row.title,
+                status: row.status,
+                updatedAt,
+                secondsSinceUpdate: secondsBetween(now, updatedAt),
+                runnerId: activeRunner.id,
+                runnerPid: activeRunner.pid,
+                runnerHeartbeatAt: hbAt,
+            });
+        }
+    }
+    return { orphanedTasks, hangingInvocations, dbInconsistencies };
+}
+function getActiveRunnerForTask(globalDb, projectPath, taskId, cfg, now, isPidAlive) {
+    const cutoff = formatSqliteDateTimeUtc(new Date(now.getTime() - cfg.runnerHeartbeatTimeoutSec * 1000));
+    const row = globalDb.prepare(`SELECT id, pid, heartbeat_at
+     FROM runners
+     WHERE project_path = ?
+       AND current_task_id = ?
+       AND status = 'running'
+       AND heartbeat_at >= ?
+     ORDER BY heartbeat_at DESC
+     LIMIT 1`).get(projectPath, taskId, cutoff);
+    if (!row)
+        return null;
+    if (row.pid !== null && !isPidAlive(row.pid))
+        return null;
+    return row;
+}
+//# sourceMappingURL=stuck-task-detector.js.map

package/dist/health/stuck-task-detector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stuck-task-detector.js","sourceRoot":"","sources":["../../src/health/stuck-task-detector.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;AAwJH,wDAKC;AAMD,0DAGC;AAUD,4CAwBC;AAxED,MAAM,QAAQ,GAAuC;IACnD,sBAAsB,EAAE,GAAG;IAC3B,mBAAmB,EAAE,IAAI;IACzB,sBAAsB,EAAE,GAAG;IAC3B,yBAAyB,EAAE,GAAG;IAC9B,sBAAsB,EAAE,GAAG;IAC3B,8BAA8B,EAAE,EAAE;CACnC,CAAC;AAEF,SAAS,wBAAwB,CAAC,GAAW;IAC3C,IAAI,CAAC;QACH,kCAAkC;QAClC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAgB,sBAAsB,CAAC,KAAa;IAClD,gDAAgD;IAChD,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,+DAA+D;IAC/D,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;AACjD,CAAC;AAED;;;GAGG;AACH,SAAgB,uBAAuB,CAAC,IAAU;IAChD,8EAA8E;IAC9E,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,cAAc,CAAC,CAAO,EAAE,CAAO;IACtC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,WAAW,CAAC,MAAiC;IACpD,OAAO,EAAE,GAAG,QAAQ,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;AAC5C,CAAC;AAED,SAAgB,gBAAgB,CAAC,OAAgC;IAC/D,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IACtC,MAAM,GAAG,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,wBAAwB,CAAC;IAElE,MAAM,aAAa,GAAG,2BAA2B,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;IAC/G,MAAM,WAAW,GAAG,yBAAyB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;IAC3G,MAAM,EAAE,aAAa,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,GAAG,yBAAyB,CACxF,OAAO,CAAC,SAAS,EACjB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,WAAW,EACnB,GAAG,EACH,GAAG,EACH,UAAU,CACX,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,GAAG;QACd,aAAa;QACb,kBAAkB;QAClB,aAAa;QACb,WAAW;QACX,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAClC,QAA2B,EAC3B,WAAmB,EACnB,GAAuC,EACvC,GAAS,EACT,UAAoC;IAEpC,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC,CAAC;IACvG,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAC3B;;;;4BAIwB,CACzB,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAOvB,CAAC;IAEH,MAAM,MAAM,GAAyB,EAAE,CAAC;IACxC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,GAAG,KAAK,IAAI,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5C,MAAM,EAAE,GAAG,sBAAsB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC;gBACV,WAAW,EAAE,eAAe;gBAC5B,QAAQ,EAAE,GAAG,CAAC,EAAE;gBAChB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,WAAW,EAAE,GAAG,CAAC,YAAY;gBAC7B,aAAa,EAAE,GAAG,CAAC,eAAe;gBAClC,WAAW,EAAE,EAAE;gBACf,qBAAqB,EAAE,cAAc,CAAC,GAAG,EAAE,EAAE,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,yBAAyB,CAChC,QAA2B,EAC3B,WAAmB,EACnB,GAAuC,EACvC,GAAS,EACT,UAAoC;IAEpC,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAC3B;;;4BAGwB,CACzB,CAAC,GAAG,CAAC,WAAW,CAOf,CAAC;IAEH,MAAM,MAAM,GAAuB,EAAE,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,KAAK,IAAI,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,EAAE,GAAG,sBAAsB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC;gBACV,WAAW,EAAE,aAAa;gBAC1B,QAAQ,EAAE,GAAG,CAAC,EAAE;gBAChB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,WAAW,EAAE,GAAG,CAAC,YAAY;gBAC7B,aAAa,EAAE,GAAG,CAAC,eAAe;gBAClC,WAAW,EAAE,EAAE;gBACf,qBAAqB,EAAE,cAAc,CAAC,GAAG,EAAE,EAAE,CAAC;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,yBAAyB,CAChC,SAA4B,EAC5B,QAA2B,EAC3B,WAAmB,EACnB,GAAuC,EACvC,GAAS,EACT,UAAoC;IAMpC,MAAM,aAAa,GAAyB,EAAE,CAAC;IAC/C,MAAM,kBAAkB,GAAwB,EAAE,CAAC;IACnD,MAAM,iBAAiB,GAA4B,EAAE,CAAC;IAEtD,oFAAoF;IACpF,CAAC;QACC,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,8BAA8B,GAAG,IAAI,CAAC,CAAC,CAAC;QAC5G,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAC5B;;;;;;gCAM0B,CAC3B,CAAC,GAAG,CAAC,MAAM,CAAoF,CAAC;QAEjG,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACzD,iBAAiB,CAAC,IAAI,CAAC;gBACrB,WAAW,EAAE,kBAAkB;gBAC/B,MAAM,EAAE,GAAG,CAAC,EAAE;gBACd,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,aAAa;gBACrB,SAAS;gBACT,kBAAkB,EAAE,cAAc,CAAC,GAAG,EAAE,SAAS,CAAC;gBAClD,eAAe,EAAE,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,2GAA2G;IAC3G,CAAC;QACC,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC,CAAC;QACxG,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC,CAAC;QAE9G,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAC5B;;;;;;;;;;;;;oCAa8B,CAC/B,CAAC,GAAG,CAAC,UAAU,EAAE,gBAAgB,CAOhC,CAAC;QAEH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACzD,MAAM,gBAAgB,GAAG,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,sBAAsB,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACxG,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YAEjG,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,aAAa,CAAC,IAAI,CAAC;oBACjB,WAAW,EAAE,eAAe;oBAC5B,MAAM,EAAE,GAAG,CAAC,EAAE;oBACd,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,MAAM,EAAE,aAAa;oBACrB,SAAS;oBACT,kBAAkB,EAAE,cAAc,CAAC,GAAG,EAAE,SAAS,CAAC;oBAClD,eAAe,EAAE,GAAG,CAAC,gBAAgB;oBACrC,gBAAgB;oBAChB,eAAe,EAAE,KAAK;iBACvB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,2GAA2G;IAC3G,+FAA+F;IAC/F,CAAC;QACC,MAAM,WAAW,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC,CAAC;QACtG,MAAM,cAAc,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,sBAAsB,GAAG,IAAI,CAAC,CAAC,CAAC;QAE5G,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAC5B;;;wDAGkD,CACnD,CAAC,GAAG,CAAC,WAAW,EAAE,cAAc,CAA+F,CAAC;QAEjI,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,YAAY,GAAG,sBAAsB,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;YACjG,IAAI,CAAC,YAAY;gBAAE,SAAS;YAE5B,MAAM,SAAS,GAAG,sBAAsB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,sBAAsB,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC;YAE/D,kBAAkB,CAAC,IAAI,CAAC;gBACtB,WAAW,EAAE,oBAAoB;gBACjC,KAAK,EAAE,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO;gBACrD,MAAM,EAAE,GAAG,CAAC,EAAE;gBACd,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS;gBACT,kBAAkB,EAAE,cAAc,CAAC,GAAG,EAAE,SAAS,CAAC;gBAClD,QAAQ,EAAE,YAAY,CAAC,EAAE;gBACzB,SAAS,EAAE,YAAY,CAAC,GAAG;gBAC3B,iBAAiB,EAAE,IAAI;aACxB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,iBAAiB,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,sBAAsB,CAC7B,QAA2B,EAC3B,WAAmB,EACnB,MAAc,EACd,GAAuC,EACvC,GAAS,EACT,UAAoC;IAEpC,MAAM,MAAM,GAAG,uBAAuB,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,yBAAyB,GAAG,IAAI,CAAC,CAAC,CAAC;IAEvG,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAC1B;;;;;;;aAOS,CACV,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,CAAyE,CAAC;IAE3G,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,IAAI,GAAG,CAAC,GAAG,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1D,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/health/stuck-task-recovery.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Automatic recovery actions for stuck tasks.
+ *
+ * This builds on detectStuckTasks() and applies a conservative set of actions:
+ * - orphaned_task: reset task to pending, release any task lock, increment failure_count, log incident
+ * - hanging_invocation: kill runner process, remove runner row, reset task (or skip after repeated failures)
+ * - zombie_runner/dead_runner: stop runner (best-effort) and reset its current_task_id (if any)
+ *
+ * Note: The repo intentionally approximates "invocation started/completed" using tasks.updated_at,
+ * task_invocations.created_at, and runners.heartbeat_at. Recovery follows those same signals.
+ */
+import type Database from 'better-sqlite3';
+import type { SteroidsConfig } from '../config/loader.js';
+import { type StuckTaskDetectionConfig, type StuckTaskDetectionReport } from './stuck-task-detector.js';
+export interface StuckTaskRecoveryConfig extends StuckTaskDetectionConfig {
+    autoRecover?: boolean;
+    maxRecoveryAttempts?: number;
+    maxIncidentsPerHour?: number;
+    killGraceMs?: number;
+}
+export type RecoveryResolution = 'auto_restart' | 'skipped' | 'escalated' | 'none';
+export interface RecoveryAction {
+    kind: 'task' | 'runner';
+    targetId: string;
+    failureMode: string;
+    resolution: RecoveryResolution;
+    reason: string;
+}
+export interface RecoverStuckTasksOptions {
+    projectPath: string;
+    projectDb: Database.Database;
+    globalDb: Database.Database;
+    config?: SteroidsConfig;
+    now?: Date;
+    dryRun?: boolean;
+    isPidAlive?: (pid: number) => boolean;
+    killPid?: (pid: number, graceMs: number) => Promise<boolean>;
+}
+export interface RecoverStuckTasksResult {
+    report: StuckTaskDetectionReport;
+    actions: RecoveryAction[];
+    skippedDueToSafetyLimit: boolean;
+}
+export declare function recoverStuckTasks(options: RecoverStuckTasksOptions): Promise<RecoverStuckTasksResult>;
+//# sourceMappingURL=stuck-task-recovery.d.ts.map

package/dist/health/stuck-task-recovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stuck-task-recovery.d.ts","sourceRoot":"","sources":["../../src/health/stuck-task-recovery.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAE3C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAG1D,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAK9B,MAAM,0BAA0B,CAAC;AAElC,MAAM,WAAW,uBAAwB,SAAQ,wBAAwB;IACvE,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAEnF,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC7B,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC5B,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IACtC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;CAC9D;AAED,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,wBAAwB,CAAC;IACjC,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,uBAAuB,EAAE,OAAO,CAAC;CAClC;AAuTD,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAqD3G"}