steroids-cli 0.18.1 → 0.18.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/commands/task-runtime-cleanup.d.ts.map +1 -1
- package/dist/commands/task-runtime-cleanup.js +6 -2
- package/dist/commands/task-runtime-cleanup.js.map +1 -1
- package/dist/commands/tasks-reset.d.ts.map +1 -1
- package/dist/commands/tasks-reset.js +2 -1
- package/dist/commands/tasks-reset.js.map +1 -1
- package/dist/health/stuck-task-recovery.d.ts.map +1 -1
- package/dist/health/stuck-task-recovery.js +16 -0
- package/dist/health/stuck-task-recovery.js.map +1 -1
- package/dist/monitor/scanner.d.ts.map +1 -1
- package/dist/monitor/scanner.js +32 -0
- package/dist/monitor/scanner.js.map +1 -1
- package/dist/runners/abandoned-runners.d.ts +21 -0
- package/dist/runners/abandoned-runners.d.ts.map +1 -0
- package/dist/runners/abandoned-runners.js +100 -0
- package/dist/runners/abandoned-runners.js.map +1 -0
- package/dist/runners/wakeup-global-cleanup.d.ts.map +1 -1
- package/dist/runners/wakeup-global-cleanup.js +2 -79
- package/dist/runners/wakeup-global-cleanup.js.map +1 -1
- package/dist/runners/wakeup-sanitise-recovery.d.ts +6 -2
- package/dist/runners/wakeup-sanitise-recovery.d.ts.map +1 -1
- package/dist/runners/wakeup-sanitise-recovery.js.map +1 -1
- package/dist/runners/wakeup-sanitise-runtime.d.ts +17 -0
- package/dist/runners/wakeup-sanitise-runtime.d.ts.map +1 -0
- package/dist/runners/wakeup-sanitise-runtime.js +108 -0
- package/dist/runners/wakeup-sanitise-runtime.js.map +1 -0
- package/dist/runners/wakeup-sanitise.d.ts +2 -1
- package/dist/runners/wakeup-sanitise.d.ts.map +1 -1
- package/dist/runners/wakeup-sanitise.js +13 -95
- package/dist/runners/wakeup-sanitise.js.map +1 -1
- package/dist/self-heal/reload-sweep.d.ts +24 -0
- package/dist/self-heal/reload-sweep.d.ts.map +1 -0
- package/dist/self-heal/reload-sweep.js +119 -0
- package/dist/self-heal/reload-sweep.js.map +1 -0
- package/dist/tasks/resettable-statuses.d.ts +4 -0
- package/dist/tasks/resettable-statuses.d.ts.map +1 -0
- package/dist/tasks/resettable-statuses.js +14 -0
- package/dist/tasks/resettable-statuses.js.map +1 -0
- package/package.json +1 -1
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import type { openDatabase } from '../database/connection.js';
|
|
6
6
|
import type { openGlobalDatabase } from './global-db.js';
|
|
7
|
-
import type { SanitiseSummary } from './wakeup-sanitise.js';
|
|
8
7
|
export interface StaleInvocationRow {
|
|
9
8
|
id: number;
|
|
10
9
|
task_id: string;
|
|
@@ -13,9 +12,14 @@ export interface StaleInvocationRow {
|
|
|
13
12
|
runner_id: string | null;
|
|
14
13
|
task_status: string | null;
|
|
15
14
|
}
|
|
15
|
+
export interface InvocationRecoverySummary {
|
|
16
|
+
recoveredApprovals: number;
|
|
17
|
+
recoveredRejects: number;
|
|
18
|
+
closedStaleInvocations: number;
|
|
19
|
+
}
|
|
16
20
|
export declare function parseReviewerDecisionFromInvocationLogContent(raw: string): 'approve' | 'reject' | null;
|
|
17
21
|
export declare function parseReviewerDecisionFromLog(projectPath: string, invocationId: number): 'approve' | 'reject' | null;
|
|
18
22
|
export declare function shouldSkipInvocation(row: StaleInvocationRow, activeTaskIds: Set<string>, hasActiveMergeLock: boolean, hasActiveParallelRunner: boolean, globalDb: ReturnType<typeof openGlobalDatabase>['db']): boolean;
|
|
19
23
|
export declare function isRunnerProcessDead(runnerId: string | null, globalDb: ReturnType<typeof openGlobalDatabase>['db']): boolean;
|
|
20
|
-
export declare function recoverOrphanedInvocation(projectDb: ReturnType<typeof openDatabase>['db'], projectPath: string, row: StaleInvocationRow, dryRun: boolean, summary:
|
|
24
|
+
export declare function recoverOrphanedInvocation(projectDb: ReturnType<typeof openDatabase>['db'], projectPath: string, row: StaleInvocationRow, dryRun: boolean, summary: InvocationRecoverySummary, source: string): Promise<void>;
|
|
21
25
|
//# sourceMappingURL=wakeup-sanitise-recovery.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wakeup-sanitise-recovery.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise-recovery.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"wakeup-sanitise-recovery.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise-recovery.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AAG9D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAYzD,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED,MAAM,WAAW,yBAAyB;IACxC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,wBAAgB,6CAA6C,CAC3D,GAAG,EAAE,MAAM,GACV,SAAS,GAAG,QAAQ,GAAG,IAAI,CA6B7B;AAED,wBAAgB,4BAA4B,CAC1C,WAAW,EAAE,MAAM,EACnB,YAAY,EAAE,MAAM,GACnB,SAAS,GAAG,QAAQ,GAAG,IAAI,CAU7B;AAED,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,kBAAkB,EACvB,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,EAC1B,kBAAkB,EAAE,OAAO,EAC3B,uBAAuB,EAAE,OAAO,EAChC,QAAQ,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,IAAI,CAAC,GACpD,OAAO,CAqBT;AAED,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,MAAM,GAAG,IAAI,EACvB,QAAQ,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,IAAI,CAAC,GACpD,OAAO,CAeT;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,IAAI,CAAC,EAChD,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,kBAAkB,EACvB,MAAM,EAAE,OAAO,EACf,OAAO,EAAE,yBAAyB,EAClC,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,IAAI,CAAC,CA2If"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wakeup-sanitise-recovery.js","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise-recovery.ts"],"names":[],"mappings":";AAAA;;;GAGG;;
|
|
1
|
+
{"version":3,"file":"wakeup-sanitise-recovery.js","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise-recovery.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AAmCH,sGA+BC;AAED,oEAaC;AAED,oDA2BC;AAED,kDAkBC;AAED,8DAkJC;AApRD,qCAAqC;AACrC,yCAAiC;AACjC,qCAAuC;AAEvC,mDAAiD;AAGjD,6FAA0F;AAC1F,+FAGsD;AACtD,iFAI+C;AAiB/C,SAAgB,6CAA6C,CAC3D,GAAW;IAEX,MAAM,cAAc,GAAa,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAIhC,CAAC;YACF,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;gBAC7F,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,kDAAkD;QACpD,CAAC;IACH,CAAC;IAED,MAAM,eAAe,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAE7F,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;QACxC,MAAM,QAAQ,GAAG,IAAA,yDAA2B,EAAC,SAAS,CAAC,CAAC,QAAQ,CAAC;QACjE,IAAI,QAAQ,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QAC7C,IAAI,QAAQ,KAAK,QAAQ;YAAE,OAAO,QAAQ,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAgB,4BAA4B,CAC1C,WAAmB,EACnB,YAAoB;IAEpB,MAAM,OAAO,GAAG,IAAA,gBAAI,EAAC,WAAW,EAAE,WAAW,EAAE,aAAa,EAAE,GAAG,YAAY,MAAM,CAAC,CAAC;IACrF,IAAI,CAAC,IAAA,oBAAU,EAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAA,sBAAY,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3C,OAAO,6CAA6C,CAAC,GAAG,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAgB,oBAAoB,CAClC,GAAuB,EACvB,aAA0B,EAC1B,kBAA2B,EAC3B,uBAAgC,EAChC,QAAqD;IAErD,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IAChD,+EAA+E;IAC/E,IAAI,kBAAkB,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB;QAAE,OAAO,IAAI,CAAC;IACrG,IAAI,uBAAuB,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;QAC7C,MAAM,SAAS,GAAG,QAAQ;aACvB,OAAO,CAAC,sCAAsC,CAAC;aAC/C,GAAG,CAAC,GAAG,CAAC,SAAS,CAAuC,CAAC;QAC5D,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,SAAS,CAAC,GAAG,KAAK,IAAI,EAAE,CAAC;gBAC3B,IAAI,CAAC;oBACH,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;oBAC/B,OAAO,IAAI,CAAC;gBACd,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,KAAK,CAAC;gBACf,CAAC;YACH,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAgB,mBAAmB,CACjC,QAAuB,EACvB,QAAqD;IAErD,mGAAmG;IACnG,oFAAoF;IACpF,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5B,MAAM,SAAS,GAAG,QAAQ;SACvB,OAAO,CAAC,sCAAsC,CAAC;SAC/C,GAAG,CAAC,QAAQ,CAAuC,CAAC;IACvD,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,SAAS,CAAC,GAAG,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IACzC,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC/B,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,yBAAyB,CAC7C,SAAgD,EAChD,WAAmB,EACnB,GAAuB,EACvB,MAAe,EACf,OAAkC,EAClC,MAAc;IAEd,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;IAClE,MAAM,gBAAgB,GACpB,GAAG,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,4BAA4B,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAErF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,IAAI,gBAAgB,KAAK,SAAS,IAAI,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACnE,SAAS;iBACN,OAAO,CACN;;;;+CAIqC,CACtC;iBACA,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,0BAA0B,MAAM,iCAAiC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAE7G,MAAM,IAAI,GAAG,SAAS;iBACnB,OAAO,CAAC,2EAA2E,CAAC;iBACpF,GAAG,CAAC,GAAG,CAAC,OAAO,CAAqF,CAAC;YAExG,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,iBAAiB,GAAG,IAAA,6CAAqB,EAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBACrF,MAAM,cAAc,GAAG,IAAA,6CAAqB,EAAC,WAAW,EAAE,iBAAiB,CAAC,CAAC;gBAE7E,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,CAAC;oBACvB,IAAA,sDAA8B,EAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC;gBAClE,CAAC;qBAAM,CAAC;oBACN,MAAM,OAAO,GAAG,IAAA,oDAAqB,EAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC;oBACzE,MAAM,IAAA,mDAAoB,EAAC,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE;wBACnD,KAAK,EAAE,cAAc;wBACrB,KAAK,EACH,OAAO,CAAC,IAAI,KAAK,UAAU;4BACzB,CAAC,CAAC,0BAA0B,MAAM,0CAA0C;4BAC5E,CAAC,CAAC,0BAA0B,MAAM,wCAAwC;wBAC9E,MAAM,EAAE,IAAA,sBAAU,EAAC,WAAW,CAAC;wBAC/B,WAAW;wBACX,iBAAiB,EAAE,WAAW;qBAC/B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;aAAM,IAAI,gBAAgB,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;YACzE,SAAS;iBACN,OAAO,CACN;;;;+CAIqC,CACtC;iBACA,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,0BAA0B,MAAM,gCAAgC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAE5G,MAAM,YAAY,GAAG,SAAS;iBAC3B,OAAO,CACN;;;8CAGoC,CACrC;iBACA,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAEpB,IAAI,YAAY,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC7B,SAAS;qBACN,OAAO,CACN;;wFAE4E,CAC7E;qBACA,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,0BAA0B,MAAM,uBAAuB,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;gBAC1F,SAAS;qBACN,OAAO,CAAC,0CAA0C,CAAC;qBACnD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,SAAS;iBACN,OAAO,CACN;;;;+CAIqC,CACtC;iBACA,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,aAAa,MAAM,uCAAuC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAEtG,MAAM,WAAW,GAAG,SAAS;iBAC1B,OAAO,CACN;mDACyC,CAC1C;iBACA,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpB,IAAI,WAAW,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC5B,SAAS;qBACN,OAAO,CAAC,0CAA0C,CAAC;qBACnD,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtB,CAAC;YAED,kFAAkF;YAClF,IAAI,GAAG,CAAC,WAAW,KAAK,eAAe,EAAE,CAAC;gBACxC,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;oBACxD,oEAAoE;oBACpE,SAAS;yBACN,OAAO,CACN;yDAC2C,CAC5C;yBACA,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;gBACtB,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;oBAC1C,uDAAuD;oBACvD,MAAM,cAAc,GAAG,4BAA4B,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;oBACzE,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;wBACjC,kDAAkD;wBAClD,SAAS;6BACN,OAAO,CACN;2DAC2C,CAC5C;6BACA,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtB,CAAC;yBAAM,CAAC;wBACN,qEAAqE;wBACrE,SAAS;6BACN,OAAO,CACN;2DAC2C,CAC5C;6BACA,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;oBACtB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,KAAK,SAAS,IAAI,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACnE,OAAO,CAAC,kBAAkB,IAAI,CAAC,CAAC;IAClC,CAAC;SAAM,IAAI,gBAAgB,KAAK,QAAQ,IAAI,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACzE,OAAO,CAAC,gBAAgB,IAAI,CAAC,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,sBAAsB,IAAI,CAAC,CAAC;IACtC,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
export interface RuntimeSanitiseSummary {
|
|
3
|
+
recoveredApprovals: number;
|
|
4
|
+
recoveredRejects: number;
|
|
5
|
+
closedStaleInvocations: number;
|
|
6
|
+
releasedTaskLocks: number;
|
|
7
|
+
}
|
|
8
|
+
export interface RuntimeSanitiseOptions {
|
|
9
|
+
globalDb: Database.Database;
|
|
10
|
+
projectDb: Database.Database;
|
|
11
|
+
projectPath: string;
|
|
12
|
+
dryRun: boolean;
|
|
13
|
+
staleInvocationTimeoutSec: number;
|
|
14
|
+
}
|
|
15
|
+
export declare function reconcileInvocationRuntimeState(options: RuntimeSanitiseOptions): Promise<RuntimeSanitiseSummary>;
|
|
16
|
+
export declare function runtimeSanitiseActionCount(summary: RuntimeSanitiseSummary): number;
|
|
17
|
+
//# sourceMappingURL=wakeup-sanitise-runtime.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wakeup-sanitise-runtime.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise-runtime.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAS3C,MAAM,WAAW,sBAAsB;IACrC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,sBAAsB;IACrC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC5B,SAAS,EAAE,QAAQ,CAAC,QAAQ,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC;IAChB,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,wBAAsB,+BAA+B,CACnD,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,CAAC,CA0HjC;AAED,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,sBAAsB,GAAG,MAAM,CAOlF"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.reconcileInvocationRuntimeState = reconcileInvocationRuntimeState;
|
|
4
|
+
exports.runtimeSanitiseActionCount = runtimeSanitiseActionCount;
|
|
5
|
+
const clone_js_1 = require("../parallel/clone.js");
|
|
6
|
+
const wakeup_sanitise_recovery_js_1 = require("./wakeup-sanitise-recovery.js");
|
|
7
|
+
async function reconcileInvocationRuntimeState(options) {
|
|
8
|
+
const { globalDb, projectDb, projectPath, dryRun, staleInvocationTimeoutSec } = options;
|
|
9
|
+
const summary = {
|
|
10
|
+
recoveredApprovals: 0,
|
|
11
|
+
recoveredRejects: 0,
|
|
12
|
+
closedStaleInvocations: 0,
|
|
13
|
+
releasedTaskLocks: 0,
|
|
14
|
+
};
|
|
15
|
+
const staleCutoffMs = Date.now() - staleInvocationTimeoutSec * 1000;
|
|
16
|
+
const activeRunnerTaskRows = globalDb
|
|
17
|
+
.prepare(`SELECT r.current_task_id, r.parallel_session_id
|
|
18
|
+
FROM runners r
|
|
19
|
+
LEFT JOIN parallel_sessions ps ON ps.id = r.parallel_session_id
|
|
20
|
+
WHERE (r.project_path = ? OR ps.project_path = ?)
|
|
21
|
+
AND r.status = 'running'
|
|
22
|
+
AND r.heartbeat_at > datetime('now', '-5 minutes')
|
|
23
|
+
AND (r.current_task_id IS NOT NULL OR r.parallel_session_id IS NOT NULL)`)
|
|
24
|
+
.all(projectPath, projectPath);
|
|
25
|
+
const activeTaskIds = new Set(activeRunnerTaskRows
|
|
26
|
+
.map((row) => row.current_task_id)
|
|
27
|
+
.filter((taskId) => typeof taskId === 'string' && taskId.length > 0));
|
|
28
|
+
const hasActiveParallelRunner = activeRunnerTaskRows.some((row) => typeof row.parallel_session_id === 'string' && row.parallel_session_id.length > 0);
|
|
29
|
+
let hasActiveMergeLock = false;
|
|
30
|
+
try {
|
|
31
|
+
const projectId = (0, clone_js_1.getProjectHash)(projectPath);
|
|
32
|
+
const nowMs = Date.now();
|
|
33
|
+
const mergeLockRows = globalDb
|
|
34
|
+
.prepare('SELECT heartbeat_at FROM workspace_merge_locks WHERE project_id = ?')
|
|
35
|
+
.all(projectId);
|
|
36
|
+
hasActiveMergeLock = mergeLockRows.some((row) => {
|
|
37
|
+
const heartbeatMs = Date.parse(row.heartbeat_at);
|
|
38
|
+
return Number.isFinite(heartbeatMs) && nowMs - heartbeatMs < 90_000;
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
hasActiveMergeLock = false;
|
|
43
|
+
}
|
|
44
|
+
const staleInvocations = projectDb
|
|
45
|
+
.prepare(`SELECT i.id, i.task_id, i.role, i.started_at_ms, i.runner_id, t.status AS task_status
|
|
46
|
+
FROM task_invocations i
|
|
47
|
+
LEFT JOIN tasks t ON t.id = i.task_id
|
|
48
|
+
WHERE i.status = 'running'
|
|
49
|
+
AND i.started_at_ms IS NOT NULL
|
|
50
|
+
AND i.started_at_ms <= ?
|
|
51
|
+
ORDER BY i.started_at_ms ASC`)
|
|
52
|
+
.all(staleCutoffMs);
|
|
53
|
+
for (const row of staleInvocations) {
|
|
54
|
+
if ((0, wakeup_sanitise_recovery_js_1.shouldSkipInvocation)(row, activeTaskIds, hasActiveMergeLock, hasActiveParallelRunner, globalDb)) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
await (0, wakeup_sanitise_recovery_js_1.recoverOrphanedInvocation)(projectDb, projectPath, row, dryRun, summary, 'stale timeout');
|
|
58
|
+
}
|
|
59
|
+
const deadRunnerCutoffMs = Date.now() - 120_000;
|
|
60
|
+
const recentRunningInvocations = projectDb
|
|
61
|
+
.prepare(`SELECT i.id, i.task_id, i.role, i.started_at_ms, i.runner_id, t.status AS task_status
|
|
62
|
+
FROM task_invocations i
|
|
63
|
+
LEFT JOIN tasks t ON t.id = i.task_id
|
|
64
|
+
WHERE i.status = 'running'
|
|
65
|
+
AND i.started_at_ms IS NOT NULL
|
|
66
|
+
AND i.started_at_ms > ?
|
|
67
|
+
AND i.started_at_ms <= ?
|
|
68
|
+
ORDER BY i.started_at_ms ASC`)
|
|
69
|
+
.all(staleCutoffMs, deadRunnerCutoffMs);
|
|
70
|
+
for (const row of recentRunningInvocations) {
|
|
71
|
+
const isRebaseRole = row.role === 'rebase_coder' || row.role === 'rebase_reviewer';
|
|
72
|
+
if (activeTaskIds.has(row.task_id) || (hasActiveMergeLock && !isRebaseRole)) {
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (!(0, wakeup_sanitise_recovery_js_1.isRunnerProcessDead)(row.runner_id, globalDb)) {
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
await (0, wakeup_sanitise_recovery_js_1.recoverOrphanedInvocation)(projectDb, projectPath, row, dryRun, summary, 'dead runner');
|
|
79
|
+
}
|
|
80
|
+
if (dryRun) {
|
|
81
|
+
return summary;
|
|
82
|
+
}
|
|
83
|
+
const orphanedLocks = projectDb
|
|
84
|
+
.prepare(`SELECT tl.task_id FROM task_locks tl
|
|
85
|
+
WHERE NOT EXISTS (
|
|
86
|
+
SELECT 1 FROM task_invocations i
|
|
87
|
+
WHERE i.task_id = tl.task_id AND i.status = 'running'
|
|
88
|
+
)
|
|
89
|
+
AND tl.task_id NOT IN (${activeTaskIds.size > 0 ? [...activeTaskIds].map(() => '?').join(',') : "'__none__'"})`)
|
|
90
|
+
.all(...(activeTaskIds.size > 0 ? [...activeTaskIds] : []));
|
|
91
|
+
if (orphanedLocks.length === 0) {
|
|
92
|
+
return summary;
|
|
93
|
+
}
|
|
94
|
+
const taskIds = orphanedLocks.map((row) => row.task_id);
|
|
95
|
+
const placeholders = taskIds.map(() => '?').join(',');
|
|
96
|
+
const released = projectDb
|
|
97
|
+
.prepare(`DELETE FROM task_locks WHERE task_id IN (${placeholders})`)
|
|
98
|
+
.run(...taskIds);
|
|
99
|
+
summary.releasedTaskLocks += released.changes;
|
|
100
|
+
return summary;
|
|
101
|
+
}
|
|
102
|
+
function runtimeSanitiseActionCount(summary) {
|
|
103
|
+
return (summary.recoveredApprovals +
|
|
104
|
+
summary.recoveredRejects +
|
|
105
|
+
summary.closedStaleInvocations +
|
|
106
|
+
summary.releasedTaskLocks);
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=wakeup-sanitise-runtime.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wakeup-sanitise-runtime.js","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise-runtime.ts"],"names":[],"mappings":";;AAwBA,0EA4HC;AAED,gEAOC;AA5JD,mDAAsD;AACtD,+EAKuC;AAiBhC,KAAK,UAAU,+BAA+B,CACnD,OAA+B;IAE/B,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,EAAE,yBAAyB,EAAE,GAAG,OAAO,CAAC;IACxF,MAAM,OAAO,GAA2B;QACtC,kBAAkB,EAAE,CAAC;QACrB,gBAAgB,EAAE,CAAC;QACnB,sBAAsB,EAAE,CAAC;QACzB,iBAAiB,EAAE,CAAC;KACrB,CAAC;IAEF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,yBAAyB,GAAG,IAAI,CAAC;IACpE,MAAM,oBAAoB,GAAG,QAAQ;SAClC,OAAO,CACN;;;;;;kFAM4E,CAC7E;SACA,GAAG,CAAC,WAAW,EAAE,WAAW,CAG3B,CAAC;IAEL,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,oBAAoB;SACjB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,eAAe,CAAC;SACjC,MAAM,CAAC,CAAC,MAAM,EAAoB,EAAE,CAAC,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CACzF,CAAC;IACF,MAAM,uBAAuB,GAAG,oBAAoB,CAAC,IAAI,CACvD,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,GAAG,CAAC,mBAAmB,KAAK,QAAQ,IAAI,GAAG,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAC3F,CAAC;IAEF,IAAI,kBAAkB,GAAG,KAAK,CAAC;IAC/B,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,IAAA,yBAAc,EAAC,WAAW,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,aAAa,GAAG,QAAQ;aAC3B,OAAO,CAAC,qEAAqE,CAAC;aAC9E,GAAG,CAAC,SAAS,CAAoC,CAAC;QACrD,kBAAkB,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACjD,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,KAAK,GAAG,WAAW,GAAG,MAAM,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,kBAAkB,GAAG,KAAK,CAAC;IAC7B,CAAC;IAED,MAAM,gBAAgB,GAAG,SAAS;SAC/B,OAAO,CACN;;;;;;oCAM8B,CAC/B;SACA,GAAG,CAAC,aAAa,CAA8B,CAAC;IAEnD,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACnC,IAAI,IAAA,kDAAoB,EAAC,GAAG,EAAE,aAAa,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,QAAQ,CAAC,EAAE,CAAC;YACpG,SAAS;QACX,CAAC;QACD,MAAM,IAAA,uDAAyB,EAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;IACjG,CAAC;IAED,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,wBAAwB,GAAG,SAAS;SACvC,OAAO,CACN;;;;;;;oCAO8B,CAC/B;SACA,GAAG,CAAC,aAAa,EAAE,kBAAkB,CAA8B,CAAC;IAEvE,KAAK,MAAM,GAAG,IAAI,wBAAwB,EAAE,CAAC;QAC3C,MAAM,YAAY,GAAG,GAAG,CAAC,IAAI,KAAK,cAAc,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,CAAC;QACnF,IAAI,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YAC5E,SAAS;QACX,CAAC;QACD,IAAI,CAAC,IAAA,iDAAmB,EAAC,GAAG,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC;YAClD,SAAS;QACX,CAAC;QACD,MAAM,IAAA,uDAAyB,EAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC;IAC/F,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,aAAa,GAAG,SAAS;SAC5B,OAAO,CACN;;;;;gCAMG,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,YACzE,GAAG,CACL;SACA,GAAG,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAA+B,CAAC;IAE5F,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,SAAS;SACvB,OAAO,CAAC,4CAA4C,YAAY,GAAG,CAAC;SACpE,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IACnB,OAAO,CAAC,iBAAiB,IAAI,QAAQ,CAAC,OAAO,CAAC;IAE9C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAgB,0BAA0B,CAAC,OAA+B;IACxE,OAAO,CACL,OAAO,CAAC,kBAAkB;QAC1B,OAAO,CAAC,gBAAgB;QACxB,OAAO,CAAC,sBAAsB;QAC9B,OAAO,CAAC,iBAAiB,CAC1B,CAAC;AACJ,CAAC"}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
import { openDatabase } from '../database/connection.js';
|
|
5
5
|
import { openGlobalDatabase } from './global-db.js';
|
|
6
|
+
import { parseReviewerDecisionFromInvocationLogContent } from './wakeup-sanitise-recovery.js';
|
|
6
7
|
export interface SanitiseSettings {
|
|
7
8
|
enabled: boolean;
|
|
8
9
|
intervalMinutes: number;
|
|
@@ -21,7 +22,7 @@ export interface SanitiseSummary {
|
|
|
21
22
|
}
|
|
22
23
|
export declare function getSanitiseSettings(projectPath: string): SanitiseSettings;
|
|
23
24
|
export declare function shouldRunPeriodicSanitise(db: ReturnType<typeof openGlobalDatabase>['db'], projectPath: string, intervalMinutes: number): boolean;
|
|
24
|
-
export { parseReviewerDecisionFromInvocationLogContent } from './wakeup-sanitise-recovery.js';
|
|
25
25
|
export declare function runPeriodicSanitiseForProject(globalDb: ReturnType<typeof openGlobalDatabase>['db'], projectDb: ReturnType<typeof openDatabase>['db'], projectPath: string, dryRun: boolean): Promise<SanitiseSummary>;
|
|
26
|
+
export { parseReviewerDecisionFromInvocationLogContent };
|
|
26
27
|
export declare function sanitisedActionCount(summary: SanitiseSummary): number;
|
|
27
28
|
//# sourceMappingURL=wakeup-sanitise.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wakeup-sanitise.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"wakeup-sanitise.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAEpD,OAAO,EACL,6CAA6C,EAC9C,MAAM,+BAA+B,CAAC;AAGvC,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,yBAAyB,EAAE,MAAM,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,OAAO,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,oBAAoB,EAAE,MAAM,CAAC;CAC9B;AAKD,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,MAAM,GAAG,gBAAgB,CAezE;AAMD,wBAAgB,yBAAyB,CACvC,EAAE,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,IAAI,CAAC,EAC/C,WAAW,EAAE,MAAM,EACnB,eAAe,EAAE,MAAM,GACtB,OAAO,CAkBT;AA2PD,wBAAsB,6BAA6B,CACjD,QAAQ,EAAE,UAAU,CAAC,OAAO,kBAAkB,CAAC,CAAC,IAAI,CAAC,EACrD,SAAS,EAAE,UAAU,CAAC,OAAO,YAAY,CAAC,CAAC,IAAI,CAAC,EAChD,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,eAAe,CAAC,CA2C1B;AAED,OAAO,EAAE,6CAA6C,EAAE,CAAC;AAEzD,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,eAAe,GAAG,MAAM,CAUrE"}
|
|
@@ -10,7 +10,8 @@ exports.runPeriodicSanitiseForProject = runPeriodicSanitiseForProject;
|
|
|
10
10
|
exports.sanitisedActionCount = sanitisedActionCount;
|
|
11
11
|
const loader_js_1 = require("../config/loader.js");
|
|
12
12
|
const wakeup_sanitise_recovery_js_1 = require("./wakeup-sanitise-recovery.js");
|
|
13
|
-
|
|
13
|
+
Object.defineProperty(exports, "parseReviewerDecisionFromInvocationLogContent", { enumerable: true, get: function () { return wakeup_sanitise_recovery_js_1.parseReviewerDecisionFromInvocationLogContent; } });
|
|
14
|
+
const wakeup_sanitise_runtime_js_1 = require("./wakeup-sanitise-runtime.js");
|
|
14
15
|
const DEFAULT_SANITISE_INTERVAL_MINUTES = 5;
|
|
15
16
|
const DEFAULT_SANITISE_INVOCATION_TIMEOUT_SEC = 1800;
|
|
16
17
|
function getSanitiseSettings(projectPath) {
|
|
@@ -42,8 +43,6 @@ function markPeriodicSanitiseRun(db, projectPath) {
|
|
|
42
43
|
db.prepare(`INSERT INTO _global_schema (key, value) VALUES (?, datetime('now'))
|
|
43
44
|
ON CONFLICT(key) DO UPDATE SET value = excluded.value`).run(getSanitiseSchemaKey(projectPath));
|
|
44
45
|
}
|
|
45
|
-
var wakeup_sanitise_recovery_js_2 = require("./wakeup-sanitise-recovery.js");
|
|
46
|
-
Object.defineProperty(exports, "parseReviewerDecisionFromInvocationLogContent", { enumerable: true, get: function () { return wakeup_sanitise_recovery_js_2.parseReviewerDecisionFromInvocationLogContent; } });
|
|
47
46
|
async function sanitiseProjectState(globalDb, projectDb, projectPath, dryRun, staleInvocationTimeoutSec) {
|
|
48
47
|
const summary = {
|
|
49
48
|
ran: true,
|
|
@@ -56,98 +55,17 @@ async function sanitiseProjectState(globalDb, projectDb, projectPath, dryRun, st
|
|
|
56
55
|
recoveredDisputedTasks: 0,
|
|
57
56
|
recoveredFailedTasks: 0,
|
|
58
57
|
};
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
const activeTaskIds = new Set(activeRunnerTaskRows
|
|
71
|
-
.map((row) => row.current_task_id)
|
|
72
|
-
.filter((taskId) => typeof taskId === 'string' && taskId.length > 0));
|
|
73
|
-
const hasActiveParallelRunner = activeRunnerTaskRows.some((row) => typeof row.parallel_session_id === 'string' && row.parallel_session_id.length > 0);
|
|
74
|
-
let hasActiveMergeLock = false;
|
|
75
|
-
try {
|
|
76
|
-
const STALE_LOCK_TTL_MS = 90_000;
|
|
77
|
-
const projectId = (0, clone_js_1.getProjectHash)(projectPath);
|
|
78
|
-
const mergeLockRows = globalDb
|
|
79
|
-
.prepare('SELECT heartbeat_at FROM workspace_merge_locks WHERE project_id = ?')
|
|
80
|
-
.all(projectId);
|
|
81
|
-
const nowMs = Date.now();
|
|
82
|
-
hasActiveMergeLock = mergeLockRows.some((row) => {
|
|
83
|
-
const hbMs = Date.parse(row.heartbeat_at);
|
|
84
|
-
return Number.isFinite(hbMs) && (nowMs - hbMs) < STALE_LOCK_TTL_MS;
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
catch {
|
|
88
|
-
hasActiveMergeLock = false;
|
|
89
|
-
}
|
|
90
|
-
// ── Pass 1: Stale invocations (exceeded timeout, any runner state) ──
|
|
91
|
-
const staleInvocations = projectDb
|
|
92
|
-
.prepare(`SELECT i.id, i.task_id, i.role, i.started_at_ms, i.runner_id, t.status AS task_status
|
|
93
|
-
FROM task_invocations i
|
|
94
|
-
LEFT JOIN tasks t ON t.id = i.task_id
|
|
95
|
-
WHERE i.status = 'running'
|
|
96
|
-
AND i.started_at_ms IS NOT NULL
|
|
97
|
-
AND i.started_at_ms <= ?
|
|
98
|
-
ORDER BY i.started_at_ms ASC`)
|
|
99
|
-
.all(staleCutoffMs);
|
|
100
|
-
for (const row of staleInvocations) {
|
|
101
|
-
if ((0, wakeup_sanitise_recovery_js_1.shouldSkipInvocation)(row, activeTaskIds, hasActiveMergeLock, hasActiveParallelRunner, globalDb)) {
|
|
102
|
-
continue;
|
|
103
|
-
}
|
|
104
|
-
await (0, wakeup_sanitise_recovery_js_1.recoverOrphanedInvocation)(projectDb, projectPath, row, dryRun, summary, 'stale timeout');
|
|
105
|
-
}
|
|
106
|
-
// ── Pass 2: Dead-runner orphans (runner process is dead, invocation >2 min old) ──
|
|
107
|
-
// Catches invocations left 'running' by crashed/killed runners without waiting 30 min.
|
|
108
|
-
const deadRunnerCutoffMs = Date.now() - 120_000;
|
|
109
|
-
const recentRunningInvocations = projectDb
|
|
110
|
-
.prepare(`SELECT i.id, i.task_id, i.role, i.started_at_ms, i.runner_id, t.status AS task_status
|
|
111
|
-
FROM task_invocations i
|
|
112
|
-
LEFT JOIN tasks t ON t.id = i.task_id
|
|
113
|
-
WHERE i.status = 'running'
|
|
114
|
-
AND i.started_at_ms IS NOT NULL
|
|
115
|
-
AND i.started_at_ms > ?
|
|
116
|
-
AND i.started_at_ms <= ?
|
|
117
|
-
ORDER BY i.started_at_ms ASC`)
|
|
118
|
-
.all(staleCutoffMs, deadRunnerCutoffMs);
|
|
119
|
-
for (const row of recentRunningInvocations) {
|
|
120
|
-
const isRebaseRole = row.role === 'rebase_coder' || row.role === 'rebase_reviewer';
|
|
121
|
-
if (activeTaskIds.has(row.task_id) || (hasActiveMergeLock && !isRebaseRole)) {
|
|
122
|
-
continue;
|
|
123
|
-
}
|
|
124
|
-
// Only recover if we can confirm the runner process is dead
|
|
125
|
-
if (!(0, wakeup_sanitise_recovery_js_1.isRunnerProcessDead)(row.runner_id, globalDb)) {
|
|
126
|
-
continue;
|
|
127
|
-
}
|
|
128
|
-
await (0, wakeup_sanitise_recovery_js_1.recoverOrphanedInvocation)(projectDb, projectPath, row, dryRun, summary, 'dead runner');
|
|
129
|
-
}
|
|
130
|
-
// ── Pass 3: Orphaned task locks (task has no running invocations and no active runner) ──
|
|
131
|
-
if (!dryRun) {
|
|
132
|
-
const orphanedLocks = projectDb
|
|
133
|
-
.prepare(`SELECT tl.task_id FROM task_locks tl
|
|
134
|
-
WHERE NOT EXISTS (
|
|
135
|
-
SELECT 1 FROM task_invocations i
|
|
136
|
-
WHERE i.task_id = tl.task_id AND i.status = 'running'
|
|
137
|
-
)
|
|
138
|
-
AND tl.task_id NOT IN (${activeTaskIds.size > 0
|
|
139
|
-
? [...activeTaskIds].map(() => '?').join(',')
|
|
140
|
-
: "'__none__'"})`)
|
|
141
|
-
.all(...(activeTaskIds.size > 0 ? [...activeTaskIds] : []));
|
|
142
|
-
if (orphanedLocks.length > 0) {
|
|
143
|
-
const taskIds = orphanedLocks.map((r) => r.task_id);
|
|
144
|
-
const placeholders = taskIds.map(() => '?').join(',');
|
|
145
|
-
const released = projectDb
|
|
146
|
-
.prepare(`DELETE FROM task_locks WHERE task_id IN (${placeholders})`)
|
|
147
|
-
.run(...taskIds);
|
|
148
|
-
summary.releasedTaskLocks += released.changes;
|
|
149
|
-
}
|
|
150
|
-
}
|
|
58
|
+
const runtimeSummary = await (0, wakeup_sanitise_runtime_js_1.reconcileInvocationRuntimeState)({
|
|
59
|
+
globalDb,
|
|
60
|
+
projectDb,
|
|
61
|
+
projectPath,
|
|
62
|
+
dryRun,
|
|
63
|
+
staleInvocationTimeoutSec,
|
|
64
|
+
});
|
|
65
|
+
summary.recoveredApprovals += runtimeSummary.recoveredApprovals;
|
|
66
|
+
summary.recoveredRejects += runtimeSummary.recoveredRejects;
|
|
67
|
+
summary.closedStaleInvocations += runtimeSummary.closedStaleInvocations;
|
|
68
|
+
summary.releasedTaskLocks += runtimeSummary.releasedTaskLocks;
|
|
151
69
|
if (!dryRun) {
|
|
152
70
|
const releasedTaskLocks = projectDb
|
|
153
71
|
.prepare(`DELETE FROM task_locks WHERE expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ', 'now')`)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wakeup-sanitise.js","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise.ts"],"names":[],"mappings":";AAAA;;GAEG;;;
|
|
1
|
+
{"version":3,"file":"wakeup-sanitise.js","sourceRoot":"","sources":["../../src/runners/wakeup-sanitise.ts"],"names":[],"mappings":";AAAA;;GAEG;;;AA+BH,kDAeC;AAMD,8DAsBC;AA2PD,sEAgDC;AAID,oDAUC;AA/XD,mDAAiD;AACjD,+EAEuC;AAgX9B,8HAjXP,2EAA6C,OAiXO;AA/WtD,6EAA+E;AAoB/E,MAAM,iCAAiC,GAAG,CAAC,CAAC;AAC5C,MAAM,uCAAuC,GAAG,IAAI,CAAC;AAErD,SAAgB,mBAAmB,CAAC,WAAmB;IACrD,MAAM,MAAM,GAAG,IAAA,sBAAU,EAAC,WAAW,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAEnC,MAAM,OAAO,GAAG,MAAM,CAAC,eAAe,IAAI,IAAI,CAAC;IAC/C,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,CAC9B,CAAC,EACD,MAAM,CAAC,MAAM,CAAC,uBAAuB,IAAI,iCAAiC,CAAC,CAC5E,CAAC;IACF,MAAM,yBAAyB,GAAG,IAAI,CAAC,GAAG,CACxC,EAAE,EACF,MAAM,CAAC,MAAM,CAAC,4BAA4B,IAAI,uCAAuC,CAAC,CACvF,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,yBAAyB,EAAE,CAAC;AACjE,CAAC;AAED,SAAS,oBAAoB,CAAC,WAAmB;IAC/C,OAAO,6BAA6B,WAAW,EAAE,CAAC;AACpD,CAAC;AAED,SAAgB,yBAAyB,CACvC,EAA+C,EAC/C,WAAmB,EACnB,eAAuB;IAEvB,MAAM,GAAG,GAAG,oBAAoB,CAAC,WAAW,CAAC,CAAC;IAC9C,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CAAC,gDAAgD,CAAC;SACzD,GAAG,CAAC,GAAG,CAAkC,CAAC;IAE7C,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CACpB;;;gBAGY,CACb,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,eAAe,UAAU,CAAgC,CAAC;IAE/E,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,SAAS,uBAAuB,CAC9B,EAA+C,EAC/C,WAAmB;IAEnB,EAAE,CAAC,OAAO,CACR;2DACuD,CACxD,CAAC,GAAG,CAAC,oBAAoB,CAAC,WAAW,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,QAAqD,EACrD,SAAgD,EAChD,WAAmB,EACnB,MAAe,EACf,yBAAiC;IAEjC,MAAM,OAAO,GAAoB;QAC/B,GAAG,EAAE,IAAI;QACT,MAAM,EAAE,IAAI;QACZ,kBAAkB,EAAE,CAAC;QACrB,gBAAgB,EAAE,CAAC;QACnB,sBAAsB,EAAE,CAAC;QACzB,iBAAiB,EAAE,CAAC;QACpB,oBAAoB,EAAE,CAAC;QACvB,sBAAsB,EAAE,CAAC;QACzB,oBAAoB,EAAE,CAAC;KACxB,CAAC;IAEF,MAAM,cAAc,GAAG,MAAM,IAAA,4DAA+B,EAAC;QAC3D,QAAQ;QACR,SAAS;QACT,WAAW;QACX,MAAM;QACN,yBAAyB;KAC1B,CAAC,CAAC;IACH,OAAO,CAAC,kBAAkB,IAAI,cAAc,CAAC,kBAAkB,CAAC;IAChE,OAAO,CAAC,gBAAgB,IAAI,cAAc,CAAC,gBAAgB,CAAC;IAC5D,OAAO,CAAC,sBAAsB,IAAI,cAAc,CAAC,sBAAsB,CAAC;IACxE,OAAO,CAAC,iBAAiB,IAAI,cAAc,CAAC,iBAAiB,CAAC;IAE9D,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,iBAAiB,GAAG,SAAS;aAChC,OAAO,CAAC,kFAAkF,CAAC;aAC3F,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,iBAAiB,IAAI,iBAAiB,CAAC,OAAO,CAAC;QAEvD,MAAM,oBAAoB,GAAG,SAAS;aACnC,OAAO,CAAC,qFAAqF,CAAC;aAC9F,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,oBAAoB,GAAG,oBAAoB,CAAC,OAAO,CAAC;QAE5D,mFAAmF;QACnF,oFAAoF;QACpF,kFAAkF;QAClF,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,SAAS;iBAC3B,OAAO,CACN;;;;;;eAMK,CACN;iBACA,GAAG,EAA2B,CAAC;YAElC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC1C,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,SAAS;qBACN,OAAO,CACN;4BACgB,YAAY,GAAG,CAChC;qBACA,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;gBAEf,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,CACjC;;;oCAG0B,CAC3B,CAAC;gBACF,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;oBACrB,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACpB,CAAC;gBACD,2CAA2C;gBAC3C,SAAS;qBACN,OAAO,CAAC,4CAA4C,YAAY,GAAG,CAAC;qBACpE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;YACjB,CAAC;YACD,OAAO,CAAC,sBAAsB,GAAG,YAAY,CAAC,MAAM,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,sFAAsF;QACxF,CAAC;QAED,mDAAmD;QACnD,4EAA4E;QAC5E,oEAAoE;QACpE,8EAA8E;QAC9E,wEAAwE;QACxE,uEAAuE;QACvE,0EAA0E;QAC1E,sEAAsE;QACtE,IAAI,CAAC;YACH,MAAM,iBAAiB,GAAG,CAAC,CAAC;YAC5B,MAAM,UAAU,GAAG,SAAS;iBACzB,OAAO,CACN;;;;;;;;;;;;;;mBAcS,CACV;iBACA,GAAG,CAAC,iBAAiB,CAA0C,CAAC;YAEnE,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACxC,MAAM,YAAY,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClD,SAAS;qBACN,OAAO,CACN;;;;;;;;;;;;4BAYgB,YAAY,GAAG,CAChC;qBACA,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;gBAEf,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,CACjC;;;oCAG0B,CAC3B,CAAC;gBACF,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;oBAC7B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;gBACpC,CAAC;gBACD,SAAS;qBACN,OAAO,CAAC,4CAA4C,YAAY,GAAG,CAAC;qBACpE,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;gBAEf,0CAA0C;gBAC1C,+EAA+E;gBAC/E,4EAA4E;gBAC5E,gDAAgD;gBAChD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CACjC,SAAS;qBACN,OAAO,CACN,sDAAsD,YAAY,GAAG,CACtE;qBACA,GAAG,CAAC,GAAG,GAAG,CAAC;qBACX,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,UAAoB,CAAC;qBACvC,MAAM,CAAC,OAAO,CAAC,CACnB,CAAC;gBAEF,IAAI,mBAAmB,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACjC,yEAAyE;oBACzE,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;oBACrC,MAAM,KAAK,GAAG,CAAC,GAAG,mBAAmB,CAAC,CAAC;oBACvC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,EAAG,CAAC;wBAC/B,MAAM,UAAU,GAAG,SAAS;6BACzB,OAAO,CACN,6EAA6E,CAC9E;6BACA,GAAG,CAAC,SAAS,CAAkC,CAAC;wBACnD,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;4BAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC;gCAClC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;gCAC7B,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;4BAC3B,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;wBACxB,MAAM,OAAO,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;wBAChC,MAAM,cAAc,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACxD,MAAM,MAAM,GAAG,SAAS;6BACrB,OAAO,CACN;wCACwB,cAAc;4EACsB,CAC7D;6BACA,GAAG,CAAC,GAAG,OAAO,CAA0C,CAAC;wBAE5D,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BACtB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;4BACxC,MAAM,gBAAgB,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;4BAC1D,SAAS;iCACN,OAAO,CACN;;;;;;kCAMgB,gBAAgB,GAAG,CACpC;iCACA,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;4BAEnB,MAAM,SAAS,GAAG,SAAS,CAAC,OAAO,CACjC;;;0CAG0B,CAC3B,CAAC;4BACF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gCACvB,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;4BAChC,CAAC;4BACD,SAAS;iCACN,OAAO,CAAC,4CAA4C,gBAAgB,GAAG,CAAC;iCACxE,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;4BAEnB,OAAO,CAAC,oBAAoB,IAAI,MAAM,CAAC,MAAM,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,8CAA8C;YAChD,CAAC;YACD,OAAO,CAAC,oBAAoB,IAAI,UAAU,CAAC,MAAM,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,kCAAkC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAEM,KAAK,UAAU,6BAA6B,CACjD,QAAqD,EACrD,SAAgD,EAChD,WAAmB,EACnB,MAAe;IAEf,MAAM,QAAQ,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAClD,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO;YACL,GAAG,EAAE,KAAK;YACV,MAAM,EAAE,UAAU;YAClB,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,sBAAsB,EAAE,CAAC;YACzB,iBAAiB,EAAE,CAAC;YACpB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,CAAC;YACzB,oBAAoB,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,CAAC,eAAe,CAAC,EAAE,CAAC;QAChF,OAAO;YACL,GAAG,EAAE,KAAK;YACV,MAAM,EAAE,kBAAkB;YAC1B,kBAAkB,EAAE,CAAC;YACrB,gBAAgB,EAAE,CAAC;YACnB,sBAAsB,EAAE,CAAC;YACzB,iBAAiB,EAAE,CAAC;YACpB,oBAAoB,EAAE,CAAC;YACvB,sBAAsB,EAAE,CAAC;YACzB,oBAAoB,EAAE,CAAC;SACxB,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,oBAAoB,CACxC,QAAQ,EACR,SAAS,EACT,WAAW,EACX,MAAM,EACN,QAAQ,CAAC,yBAAyB,CACnC,CAAC;IAEF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,uBAAuB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAID,SAAgB,oBAAoB,CAAC,OAAwB;IAC3D,OAAO,CACL,OAAO,CAAC,kBAAkB;QAC1B,OAAO,CAAC,gBAAgB;QACxB,OAAO,CAAC,sBAAsB;QAC9B,OAAO,CAAC,iBAAiB;QACzB,OAAO,CAAC,oBAAoB;QAC5B,OAAO,CAAC,sBAAsB;QAC9B,OAAO,CAAC,oBAAoB,CAC7B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export type ReloadSelfHealSource = 'runners_page' | 'task_page' | 'project_tasks_page';
|
|
2
|
+
export interface ReloadSelfHealOptions {
|
|
3
|
+
source: ReloadSelfHealSource;
|
|
4
|
+
projectPath?: string;
|
|
5
|
+
}
|
|
6
|
+
export interface ReloadSelfHealProjectResult {
|
|
7
|
+
projectPath: string;
|
|
8
|
+
recoveredActions: number;
|
|
9
|
+
skippedRecoveryDueToSafetyLimit: boolean;
|
|
10
|
+
error?: string;
|
|
11
|
+
}
|
|
12
|
+
export interface ReloadSelfHealResult {
|
|
13
|
+
cleanedRunnerCount: number;
|
|
14
|
+
projects: ReloadSelfHealProjectResult[];
|
|
15
|
+
}
|
|
16
|
+
export interface ReloadSelfHealScheduleResult {
|
|
17
|
+
scheduled: boolean;
|
|
18
|
+
reason: 'scheduled' | 'already_running' | 'cooldown';
|
|
19
|
+
}
|
|
20
|
+
export declare const RELOAD_SELF_HEAL_COOLDOWN_MS = 5000;
|
|
21
|
+
export declare function runReloadSelfHealNow(options: ReloadSelfHealOptions): Promise<ReloadSelfHealResult>;
|
|
22
|
+
export declare function scheduleReloadSelfHeal(options: ReloadSelfHealOptions): ReloadSelfHealScheduleResult;
|
|
23
|
+
export declare function resetReloadSelfHealStateForTests(): void;
|
|
24
|
+
//# sourceMappingURL=reload-sweep.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reload-sweep.d.ts","sourceRoot":"","sources":["../../src/self-heal/reload-sweep.ts"],"names":[],"mappings":"AAcA,MAAM,MAAM,oBAAoB,GAAG,cAAc,GAAG,WAAW,GAAG,oBAAoB,CAAC;AAEvF,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,oBAAoB,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,2BAA2B;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,+BAA+B,EAAE,OAAO,CAAC;IACzC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,oBAAoB;IACnC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,2BAA2B,EAAE,CAAC;CACzC;AAED,MAAM,WAAW,4BAA4B;IAC3C,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,EAAE,WAAW,GAAG,iBAAiB,GAAG,UAAU,CAAC;CACtD;AAED,eAAO,MAAM,4BAA4B,OAAQ,CAAC;AAgElD,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC,oBAAoB,CAAC,CAwB/B;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,qBAAqB,GAC7B,4BAA4B,CAoB9B;AAED,wBAAgB,gCAAgC,IAAI,IAAI,CAGvD"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.RELOAD_SELF_HEAL_COOLDOWN_MS = void 0;
|
|
4
|
+
exports.runReloadSelfHealNow = runReloadSelfHealNow;
|
|
5
|
+
exports.scheduleReloadSelfHeal = scheduleReloadSelfHeal;
|
|
6
|
+
exports.resetReloadSelfHealStateForTests = resetReloadSelfHealStateForTests;
|
|
7
|
+
const node_fs_1 = require("node:fs");
|
|
8
|
+
const node_path_1 = require("node:path");
|
|
9
|
+
const loader_js_1 = require("../config/loader.js");
|
|
10
|
+
const connection_js_1 = require("../database/connection.js");
|
|
11
|
+
const stuck_task_recovery_js_1 = require("../health/stuck-task-recovery.js");
|
|
12
|
+
const abandoned_runners_js_1 = require("../runners/abandoned-runners.js");
|
|
13
|
+
const global_db_connection_js_1 = require("../runners/global-db-connection.js");
|
|
14
|
+
const projects_js_1 = require("../runners/projects.js");
|
|
15
|
+
const wakeup_sanitise_js_1 = require("../runners/wakeup-sanitise.js");
|
|
16
|
+
const wakeup_sanitise_runtime_js_1 = require("../runners/wakeup-sanitise-runtime.js");
|
|
17
|
+
exports.RELOAD_SELF_HEAL_COOLDOWN_MS = 5_000;
|
|
18
|
+
let activeSweep = null;
|
|
19
|
+
let lastCompletedAt = 0;
|
|
20
|
+
function projectDbExists(projectPath) {
|
|
21
|
+
return (0, node_fs_1.existsSync)((0, node_path_1.join)(projectPath, '.steroids', 'steroids.db'));
|
|
22
|
+
}
|
|
23
|
+
function resolveProjectPaths(projectPath) {
|
|
24
|
+
if (!projectPath) {
|
|
25
|
+
return (0, projects_js_1.getRegisteredProjects)(false).map((project) => project.path);
|
|
26
|
+
}
|
|
27
|
+
const registeredProjects = (0, projects_js_1.getRegisteredProjects)(false);
|
|
28
|
+
const isEnabled = registeredProjects.some((project) => project.path === projectPath);
|
|
29
|
+
return isEnabled ? [projectPath] : [];
|
|
30
|
+
}
|
|
31
|
+
async function recoverProject(globalDb, projectPath) {
|
|
32
|
+
if (!(0, node_fs_1.existsSync)(projectPath) || !projectDbExists(projectPath)) {
|
|
33
|
+
return {
|
|
34
|
+
projectPath,
|
|
35
|
+
recoveredActions: 0,
|
|
36
|
+
skippedRecoveryDueToSafetyLimit: false,
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
const config = (0, loader_js_1.loadConfig)(projectPath);
|
|
41
|
+
const { db: projectDb, close } = (0, connection_js_1.openDatabase)(projectPath);
|
|
42
|
+
try {
|
|
43
|
+
const recovery = await (0, stuck_task_recovery_js_1.recoverStuckTasks)({
|
|
44
|
+
projectPath,
|
|
45
|
+
projectDb,
|
|
46
|
+
globalDb,
|
|
47
|
+
config,
|
|
48
|
+
dryRun: false,
|
|
49
|
+
});
|
|
50
|
+
const runtimeSummary = await (0, wakeup_sanitise_runtime_js_1.reconcileInvocationRuntimeState)({
|
|
51
|
+
globalDb,
|
|
52
|
+
projectDb,
|
|
53
|
+
projectPath,
|
|
54
|
+
dryRun: false,
|
|
55
|
+
staleInvocationTimeoutSec: (0, wakeup_sanitise_js_1.getSanitiseSettings)(projectPath).staleInvocationTimeoutSec,
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
projectPath,
|
|
59
|
+
recoveredActions: recovery.actions.length + (0, wakeup_sanitise_runtime_js_1.runtimeSanitiseActionCount)(runtimeSummary),
|
|
60
|
+
skippedRecoveryDueToSafetyLimit: recovery.skippedDueToSafetyLimit,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
close();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
return {
|
|
69
|
+
projectPath,
|
|
70
|
+
recoveredActions: 0,
|
|
71
|
+
skippedRecoveryDueToSafetyLimit: false,
|
|
72
|
+
error: error instanceof Error ? error.message : String(error),
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
async function runReloadSelfHealNow(options) {
|
|
77
|
+
const { db: globalDb, close } = (0, global_db_connection_js_1.openGlobalDatabase)();
|
|
78
|
+
try {
|
|
79
|
+
const cleanupResults = (0, abandoned_runners_js_1.cleanupAbandonedRunners)(globalDb, {
|
|
80
|
+
dryRun: false,
|
|
81
|
+
log: () => { },
|
|
82
|
+
});
|
|
83
|
+
const cleanedRunnerCount = cleanupResults.reduce((count, result) => count + (result.staleRunners ?? 0), 0);
|
|
84
|
+
const projects = [];
|
|
85
|
+
for (const projectPath of resolveProjectPaths(options.projectPath)) {
|
|
86
|
+
projects.push(await recoverProject(globalDb, projectPath));
|
|
87
|
+
}
|
|
88
|
+
return {
|
|
89
|
+
cleanedRunnerCount,
|
|
90
|
+
projects,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
close();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
function scheduleReloadSelfHeal(options) {
|
|
98
|
+
if (activeSweep) {
|
|
99
|
+
return { scheduled: false, reason: 'already_running' };
|
|
100
|
+
}
|
|
101
|
+
if (Date.now() - lastCompletedAt < exports.RELOAD_SELF_HEAL_COOLDOWN_MS) {
|
|
102
|
+
return { scheduled: false, reason: 'cooldown' };
|
|
103
|
+
}
|
|
104
|
+
activeSweep = runReloadSelfHealNow(options)
|
|
105
|
+
.catch(() => ({
|
|
106
|
+
cleanedRunnerCount: 0,
|
|
107
|
+
projects: [],
|
|
108
|
+
}))
|
|
109
|
+
.finally(() => {
|
|
110
|
+
lastCompletedAt = Date.now();
|
|
111
|
+
activeSweep = null;
|
|
112
|
+
});
|
|
113
|
+
return { scheduled: true, reason: 'scheduled' };
|
|
114
|
+
}
|
|
115
|
+
function resetReloadSelfHealStateForTests() {
|
|
116
|
+
activeSweep = null;
|
|
117
|
+
lastCompletedAt = 0;
|
|
118
|
+
}
|
|
119
|
+
//# sourceMappingURL=reload-sweep.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reload-sweep.js","sourceRoot":"","sources":["../../src/self-heal/reload-sweep.ts"],"names":[],"mappings":";;;AAsGA,oDA0BC;AAED,wDAsBC;AAED,4EAGC;AA7JD,qCAAqC;AACrC,yCAAiC;AACjC,mDAAiD;AACjD,6DAAyD;AACzD,6EAAqE;AACrE,0EAA0E;AAC1E,gFAAwE;AACxE,wDAA+D;AAC/D,sEAAoE;AACpE,sFAG+C;AA0BlC,QAAA,4BAA4B,GAAG,KAAK,CAAC;AAElD,IAAI,WAAW,GAAyC,IAAI,CAAC;AAC7D,IAAI,eAAe,GAAG,CAAC,CAAC;AAExB,SAAS,eAAe,CAAC,WAAmB;IAC1C,OAAO,IAAA,oBAAU,EAAC,IAAA,gBAAI,EAAC,WAAW,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAoB;IAC/C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO,IAAA,mCAAqB,EAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACrE,CAAC;IAED,MAAM,kBAAkB,GAAG,IAAA,mCAAqB,EAAC,KAAK,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;IACrF,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AACxC,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,QAAa,EAAE,WAAmB;IAC9D,IAAI,CAAC,IAAA,oBAAU,EAAC,WAAW,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,EAAE,CAAC;QAC9D,OAAO;YACL,WAAW;YACX,gBAAgB,EAAE,CAAC;YACnB,+BAA+B,EAAE,KAAK;SACvC,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAA,sBAAU,EAAC,WAAW,CAAC,CAAC;QACvC,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,IAAA,4BAAY,EAAC,WAAW,CAAC,CAAC;QAC3D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAA,0CAAiB,EAAC;gBACvC,WAAW;gBACX,SAAS;gBACT,QAAQ;gBACR,MAAM;gBACN,MAAM,EAAE,KAAK;aACd,CAAC,CAAC;YACH,MAAM,cAAc,GAAG,MAAM,IAAA,4DAA+B,EAAC;gBAC3D,QAAQ;gBACR,SAAS;gBACT,WAAW;gBACX,MAAM,EAAE,KAAK;gBACb,yBAAyB,EAAE,IAAA,wCAAmB,EAAC,WAAW,CAAC,CAAC,yBAAyB;aACtF,CAAC,CAAC;YACH,OAAO;gBACL,WAAW;gBACX,gBAAgB,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,GAAG,IAAA,uDAA0B,EAAC,cAAc,CAAC;gBACtF,+BAA+B,EAAE,QAAQ,CAAC,uBAAuB;aAClE,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,WAAW;YACX,gBAAgB,EAAE,CAAC;YACnB,+BAA+B,EAAE,KAAK;YACtC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,oBAAoB,CACxC,OAA8B;IAE9B,MAAM,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,IAAA,4CAAkB,GAAE,CAAC;IACrD,IAAI,CAAC;QACH,MAAM,cAAc,GAAG,IAAA,8CAAuB,EAAC,QAAQ,EAAE;YACvD,MAAM,EAAE,KAAK;YACb,GAAG,EAAE,GAAG,EAAE,GAAE,CAAC;SACd,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAG,cAAc,CAAC,MAAM,CAC9C,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC,EACrD,CAAC,CACF,CAAC;QAEF,MAAM,QAAQ,GAAkC,EAAE,CAAC;QACnD,KAAK,MAAM,WAAW,IAAI,mBAAmB,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,CAAC;YACnE,QAAQ,CAAC,IAAI,CAAC,MAAM,cAAc,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,OAAO;YACL,kBAAkB;YAClB,QAAQ;SACT,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,KAAK,EAAE,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAgB,sBAAsB,CACpC,OAA8B;IAE9B,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;IACzD,CAAC;IAED,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,GAAG,oCAA4B,EAAE,CAAC;QAChE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAClD,CAAC;IAED,WAAW,GAAG,oBAAoB,CAAC,OAAO,CAAC;SACxC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC;QACZ,kBAAkB,EAAE,CAAC;QACrB,QAAQ,EAAE,EAAE;KACb,CAAC,CAAC;SACF,OAAO,CAAC,GAAG,EAAE;QACZ,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC,CAAC,CAAC;IAEL,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;AAClD,CAAC;AAED,SAAgB,gCAAgC;IAC9C,WAAW,GAAG,IAAI,CAAC;IACnB,eAAe,GAAG,CAAC,CAAC;AACtB,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const PROJECT_RESETTABLE_STATUSES: readonly ["failed", "disputed", "blocked_error", "blocked_conflict"];
|
|
2
|
+
export type ProjectResettableStatus = (typeof PROJECT_RESETTABLE_STATUSES)[number];
|
|
3
|
+
export declare function isProjectResettableStatus(status: string): status is ProjectResettableStatus;
|
|
4
|
+
//# sourceMappingURL=resettable-statuses.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resettable-statuses.d.ts","sourceRoot":"","sources":["../../src/tasks/resettable-statuses.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,2BAA2B,sEAKE,CAAC;AAE3C,MAAM,MAAM,uBAAuB,GAAG,CAAC,OAAO,2BAA2B,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnF,wBAAgB,yBAAyB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,IAAI,uBAAuB,CAE3F"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PROJECT_RESETTABLE_STATUSES = void 0;
|
|
4
|
+
exports.isProjectResettableStatus = isProjectResettableStatus;
|
|
5
|
+
exports.PROJECT_RESETTABLE_STATUSES = [
|
|
6
|
+
'failed',
|
|
7
|
+
'disputed',
|
|
8
|
+
'blocked_error',
|
|
9
|
+
'blocked_conflict',
|
|
10
|
+
];
|
|
11
|
+
function isProjectResettableStatus(status) {
|
|
12
|
+
return exports.PROJECT_RESETTABLE_STATUSES.includes(status);
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=resettable-statuses.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"resettable-statuses.js","sourceRoot":"","sources":["../../src/tasks/resettable-statuses.ts"],"names":[],"mappings":";;;AAWA,8DAEC;AAXY,QAAA,2BAA2B,GAAG;IACzC,QAAQ;IACR,UAAU;IACV,eAAe;IACf,kBAAkB;CACsB,CAAC;AAI3C,SAAgB,yBAAyB,CAAC,MAAc;IACtD,OAAQ,mCAAiD,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAC7E,CAAC"}
|