steroids-cli 0.6.4 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/completion.js +5 -1
- package/dist/commands/completion.js.map +1 -1
- package/dist/commands/health-stuck.d.ts +5 -0
- package/dist/commands/health-stuck.d.ts.map +1 -0
- package/dist/commands/health-stuck.js +312 -0
- package/dist/commands/health-stuck.js.map +1 -0
- package/dist/commands/health.d.ts.map +1 -1
- package/dist/commands/health.js +14 -1
- package/dist/commands/health.js.map +1 -1
- package/dist/config/loader.d.ts +8 -0
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +14 -0
- package/dist/config/loader.js.map +1 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +40 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/database/queries.d.ts +2 -0
- package/dist/database/queries.d.ts.map +1 -1
- package/dist/database/queries.js.map +1 -1
- package/dist/database/schema.d.ts +2 -2
- package/dist/database/schema.d.ts.map +1 -1
- package/dist/database/schema.js +21 -0
- package/dist/database/schema.js.map +1 -1
- package/dist/health/stuck-task-detector.d.ts +131 -0
- package/dist/health/stuck-task-detector.d.ts.map +1 -0
- package/dist/health/stuck-task-detector.js +238 -0
- package/dist/health/stuck-task-detector.js.map +1 -0
- package/dist/health/stuck-task-recovery.d.ts +45 -0
- package/dist/health/stuck-task-recovery.d.ts.map +1 -0
- package/dist/health/stuck-task-recovery.js +312 -0
- package/dist/health/stuck-task-recovery.js.map +1 -0
- package/dist/migrations/manifest.d.ts.map +1 -1
- package/dist/migrations/manifest.js +2 -1
- package/dist/migrations/manifest.js.map +1 -1
- package/dist/runners/wakeup.d.ts +2 -0
- package/dist/runners/wakeup.d.ts.map +1 -1
- package/dist/runners/wakeup.js +144 -93
- package/dist/runners/wakeup.js.map +1 -1
- package/migrations/009_add_incidents_and_failure_tracking.sql +35 -0
- package/migrations/manifest.json +9 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/migrations/manifest.ts"],"names":[],"mappings":";AAAA;;;GAGG;;
|
|
1
|
+
{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/migrations/manifest.ts"],"names":[],"mappings":";AAAA;;;GAGG;;AA6HH,wDAGC;AAKD,oDAEC;AAKD,sDAEC;AAKD,kDASC;AAKD,gDAkBC;AAKD,sCAcC;AAKD,sCAIC;AAKD,4CAmCC;AAgCD,8CAIC;AAKD,wCAQC;AAKD,sDAMC;AAKD,wDAMC;AAKD,4CAEC;AAKD,0DASC;AAjVD,qCAA6E;AAC7E,yCAA0C;AAC1C,6CAAyC;AACzC,qCAAkC;AAElC,iEAAiE;AACjE,IAAI,YAAY,GAAkB,IAAI,CAAC;AAEvC;;;GAGG;AACH,SAAS,cAAc;IACrB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IAEtC,mDAAmD;IACnD,MAAM,UAAU,GAAa,EAAE,CAAC;IAEhC,uEAAuE;IACvE,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QACpB,IAAI,GAAG,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,GAAG,GAAG,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,0DAA0D;IAC1D,IAAI,OAAO,OAAO,KAAK,WAAW,IAAI,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC7D,IAAI,GAAG,GAAG,IAAA,mBAAO,EAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,GAAG,GAAG,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,iEAAiE;IACjE,mDAAmD;IACnD,IAAI,OAAO,SAAS,KAAK,WAAW,EAAE,CAAC;QACrC,IAAI,GAAG,GAAG,SAAS,CAAC;QACpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrB,GAAG,GAAG,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IAED,+EAA+E;IAC/E,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,IAAA,gBAAI,EAAC,GAAG,EAAE,cAAc,CAAC,CAAC;QAC1C,IAAI,IAAA,oBAAU,EAAC,OAAO,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAA,sBAAY,EAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBACvD,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;oBAChC,YAAY,GAAG,GAAG,CAAC;oBACnB,OAAO,GAAG,CAAC;gBACb,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,MAAM,MAAM,GAAG,IAAA,gBAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,cAAc,CAAC,CAAC;IACnD,IAAI,IAAA,oBAAU,EAAC,MAAM,CAAC,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAA,sBAAY,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;YACtD,IAAI,GAAG,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBAChC,YAAY,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;gBAC7B,OAAO,OAAO,CAAC,GAAG,EAAE,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IAC1B,CAAC;IAED,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;AAChE,CAAC;AAyCD,MAAM,SAAS,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,WAAW;AAClD,MAAM,mBAAmB,GAAG,IAAA,gBAAI,EAAC,IAAA,iBAAO,GAAE,EAAE,WAAW,CAAC,CAAC;AACzD,MAAM,SAAS,GAAG,IAAA,gBAAI,EAAC,mBAAmB,EAAE,YAAY,CAAC,CAAC;AAC1D,MAAM,UAAU,GAAG,IAAA,gBAAI,EAAC,SAAS,EAAE,qBAAqB,CAAC,CAAC;AAE1D;;GAEG;AACH,SAAgB,sBAAsB;IACpC,gDAAgD;IAChD,OAAO,IAAA,gBAAI,EAAC,cAAc,EAAE,EAAE,YAAY,EAAE,eAAe,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,SAAgB,oBAAoB,CAAC,QAAgB;IACnD,OAAO,IAAA,gBAAI,EAAC,cAAc,EAAE,EAAE,YAAY,EAAE,QAAQ,CAAC,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAgB,qBAAqB;IACnC,OAAO,IAAA,gBAAI,EAAC,SAAS,EAAE,eAAe,CAAC,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAgB,mBAAmB;IACjC,MAAM,YAAY,GAAG,sBAAsB,EAAE,CAAC;IAE9C,IAAI,CAAC,IAAA,oBAAU,EAAC,YAAY,CAAC,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,kCAAkC,YAAY,EAAE,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,OAAO,GAAG,IAAA,sBAAY,EAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACpD,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED;;GAEG;AACH,SAAgB,kBAAkB;IAChC,IAAI,CAAC,IAAA,oBAAU,EAAC,UAAU,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,IAAA,sBAAY,EAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvD,MAAM,KAAK,GAAkB,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAEtD,gCAAgC;QAChC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,SAAS,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,QAAQ,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,aAAa,CAAC,QAA2B;IACvD,IAAI,CAAC,IAAA,oBAAU,EAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,IAAA,mBAAS,EAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,MAAM,KAAK,GAAkB;QAC3B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;QACrB,QAAQ;KACT,CAAC;IAEF,IAAA,uBAAa,EAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1D,mDAAmD;IACnD,IAAA,uBAAa,EAAC,qBAAqB,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,SAAgB,aAAa,CAAC,OAAe;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAsB,CAAC;IAC1D,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB,CAAC,QAA2B;IAC1D,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,OAAO,QAAQ,CAAC,eAAe,KAAK,QAAQ,IAAI,QAAQ,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvD,CAAC;IAED,gCAAgC;IAChC,KAAK,MAAM,SAAS,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC5C,sBAAsB,CAAC,SAAS,CAAC,CAAC;IACpC,CAAC;IAED,mCAAmC;IACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,4BAA4B,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClF,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QAC/B,IAAI,KAAK,KAAK,QAAQ,CAAC,eAAe,EAAE,CAAC;YACvC,MAAM,IAAI,KAAK,CACb,oBAAoB,QAAQ,CAAC,eAAe,0CAA0C,KAAK,GAAG,CAC/F,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,KAAqB;IACnD,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,IAAI,KAAK,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,EAAE,eAAe,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,EAAE,eAAe,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,EAAE,sBAAsB,CAAC,CAAC;IAC/D,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,UAAU,IAAI,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,aAAa,KAAK,CAAC,EAAE,qBAAqB,CAAC,CAAC;IAC9D,CAAC;IAED,2CAA2C;AAC7C,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB,CAAC,OAAe;IAC/C,MAAM,IAAI,GAAG,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC;IAClC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrB,OAAO,UAAU,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,SAAgB,cAAc,CAAC,OAAe,EAAE,gBAAwB;IACtE,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,wBAAwB;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,MAAM,MAAM,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC1C,OAAO,MAAM,KAAK,gBAAgB,CAAC;AACrC,CAAC;AAED;;GAEG;AACH,SAAgB,qBAAqB,CACnC,QAA2B,EAC3B,iBAAqC;IAErC,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7D,OAAO,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAChE,CAAC;AAED;;GAEG;AACH,SAAgB,sBAAsB,CACpC,QAA2B,EAC3B,iBAAqC;IAErC,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChE,OAAO,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED;;GAEG;AACH,SAAgB,gBAAgB,CAAC,QAA2B,EAAE,EAAU;IACtE,OAAO,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,CAAC;AAC5D,CAAC;AAED;;GAEG;AACH,SAAgB,uBAAuB,CAAC,QAA2B;IACjE,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IAC5B,CAAC;IAED,OAAO;QACL,GAAG,EAAE,CAAC,EAAE,gCAAgC;QACxC,GAAG,EAAE,QAAQ,CAAC,eAAe;KAC9B,CAAC;AACJ,CAAC"}
|
package/dist/runners/wakeup.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wakeup.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup.ts"],"names":[],"mappings":"AAAA;;GAEG;
|
|
1
|
+
{"version":3,"file":"wakeup.d.ts","sourceRoot":"","sources":["../../src/runners/wakeup.ts"],"names":[],"mappings":"AAAA;;GAEG;AAaH,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,aAAa,CAAC;IACrE,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,+BAA+B,CAAC,EAAE,OAAO,CAAC;CAC3C;AA8BD;;;GAGG;AACH,wBAAgB,yBAAyB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAgBtE;AA2DD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,GAAG,IAAI,CAUjD;AAED,wBAAsB,MAAM,CAAC,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAsLjF;AAED;;GAEG;AACH,wBAAsB,iBAAiB,IAAI,OAAO,CAAC;IACjD,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC,CAyCD"}
|
package/dist/runners/wakeup.js
CHANGED
|
@@ -14,6 +14,9 @@ const lock_js_1 = require("./lock.js");
|
|
|
14
14
|
const global_db_js_1 = require("./global-db.js");
|
|
15
15
|
const heartbeat_js_1 = require("./heartbeat.js");
|
|
16
16
|
const projects_js_1 = require("./projects.js");
|
|
17
|
+
const connection_js_1 = require("../database/connection.js");
|
|
18
|
+
const loader_js_1 = require("../config/loader.js");
|
|
19
|
+
const stuck_task_recovery_js_1 = require("../health/stuck-task-recovery.js");
|
|
17
20
|
/**
|
|
18
21
|
* Check if a project has pending work
|
|
19
22
|
*/
|
|
@@ -136,117 +139,165 @@ async function wakeup(options = {}) {
|
|
|
136
139
|
recordWakeupTime();
|
|
137
140
|
}
|
|
138
141
|
// Step 1: Clean up stale runners first
|
|
139
|
-
const
|
|
142
|
+
const global = (0, global_db_js_1.openGlobalDatabase)();
|
|
140
143
|
try {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
144
|
+
try {
|
|
145
|
+
const staleRunners = (0, heartbeat_js_1.findStaleRunners)(global.db);
|
|
146
|
+
if (staleRunners.length > 0) {
|
|
147
|
+
log(`Found ${staleRunners.length} stale runner(s), cleaning up...`);
|
|
148
|
+
if (!dryRun) {
|
|
149
|
+
for (const runner of staleRunners) {
|
|
150
|
+
if (runner.pid) {
|
|
151
|
+
killProcess(runner.pid);
|
|
152
|
+
}
|
|
153
|
+
global.db.prepare('DELETE FROM runners WHERE id = ?').run(runner.id);
|
|
148
154
|
}
|
|
149
|
-
db.prepare('DELETE FROM runners WHERE id = ?').run(runner.id);
|
|
150
155
|
}
|
|
156
|
+
results.push({
|
|
157
|
+
action: 'cleaned',
|
|
158
|
+
reason: `Cleaned ${staleRunners.length} stale runner(s)`,
|
|
159
|
+
staleRunners: staleRunners.length,
|
|
160
|
+
});
|
|
151
161
|
}
|
|
152
|
-
results.push({
|
|
153
|
-
action: 'cleaned',
|
|
154
|
-
reason: `Cleaned ${staleRunners.length} stale runner(s)`,
|
|
155
|
-
staleRunners: staleRunners.length,
|
|
156
|
-
});
|
|
157
162
|
}
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
close();
|
|
161
|
-
}
|
|
162
|
-
// Step 2: Clean zombie lock if present
|
|
163
|
-
const lockStatus = (0, lock_js_1.checkLockStatus)();
|
|
164
|
-
if (lockStatus.isZombie && lockStatus.pid) {
|
|
165
|
-
log(`Found zombie lock (PID: ${lockStatus.pid}), cleaning...`);
|
|
166
|
-
if (!dryRun) {
|
|
167
|
-
(0, lock_js_1.removeLock)();
|
|
163
|
+
catch {
|
|
164
|
+
// ignore global DB issues; wakeup will still attempt per-project checks
|
|
168
165
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
action: 'none',
|
|
177
|
-
reason: 'No registered projects',
|
|
178
|
-
pendingTasks: 0,
|
|
179
|
-
});
|
|
180
|
-
return results;
|
|
181
|
-
}
|
|
182
|
-
log(`Checking ${registeredProjects.length} registered project(s)...`);
|
|
183
|
-
// Step 4: Check each project and start runners as needed
|
|
184
|
-
for (const project of registeredProjects) {
|
|
185
|
-
// Skip if project directory doesn't exist
|
|
186
|
-
if (!(0, node_fs_1.existsSync)(project.path)) {
|
|
187
|
-
log(`Skipping ${project.path}: directory not found`);
|
|
188
|
-
results.push({
|
|
189
|
-
action: 'none',
|
|
190
|
-
reason: 'Directory not found',
|
|
191
|
-
projectPath: project.path,
|
|
192
|
-
});
|
|
193
|
-
continue;
|
|
194
|
-
}
|
|
195
|
-
// Skip if project already has an active runner
|
|
196
|
-
if (hasActiveRunnerForProject(project.path)) {
|
|
197
|
-
log(`Skipping ${project.path}: runner already active`);
|
|
198
|
-
results.push({
|
|
199
|
-
action: 'none',
|
|
200
|
-
reason: 'Runner already active',
|
|
201
|
-
projectPath: project.path,
|
|
202
|
-
});
|
|
203
|
-
continue;
|
|
166
|
+
// Step 2: Clean zombie lock if present
|
|
167
|
+
const lockStatus = (0, lock_js_1.checkLockStatus)();
|
|
168
|
+
if (lockStatus.isZombie && lockStatus.pid) {
|
|
169
|
+
log(`Found zombie lock (PID: ${lockStatus.pid}), cleaning...`);
|
|
170
|
+
if (!dryRun) {
|
|
171
|
+
(0, lock_js_1.removeLock)();
|
|
172
|
+
}
|
|
204
173
|
}
|
|
205
|
-
//
|
|
206
|
-
|
|
207
|
-
|
|
174
|
+
// Step 3: Get all registered projects from global registry
|
|
175
|
+
const registeredProjects = (0, projects_js_1.getRegisteredProjects)(false); // enabled only
|
|
176
|
+
if (registeredProjects.length === 0) {
|
|
177
|
+
log('No registered projects found');
|
|
178
|
+
log('Run "steroids projects add <path>" to register a project');
|
|
208
179
|
results.push({
|
|
209
180
|
action: 'none',
|
|
210
|
-
reason: 'No
|
|
211
|
-
|
|
212
|
-
});
|
|
213
|
-
continue;
|
|
214
|
-
}
|
|
215
|
-
// Start runner for this project
|
|
216
|
-
log(`Starting runner for: ${project.path}`);
|
|
217
|
-
if (dryRun) {
|
|
218
|
-
results.push({
|
|
219
|
-
action: 'would_start',
|
|
220
|
-
reason: `Would start runner (dry-run)`,
|
|
221
|
-
projectPath: project.path,
|
|
181
|
+
reason: 'No registered projects',
|
|
182
|
+
pendingTasks: 0,
|
|
222
183
|
});
|
|
223
|
-
|
|
184
|
+
return results;
|
|
224
185
|
}
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
186
|
+
log(`Checking ${registeredProjects.length} registered project(s)...`);
|
|
187
|
+
// Step 4: Check each project and start runners as needed
|
|
188
|
+
for (const project of registeredProjects) {
|
|
189
|
+
// Skip if project directory doesn't exist
|
|
190
|
+
if (!(0, node_fs_1.existsSync)(project.path)) {
|
|
191
|
+
log(`Skipping ${project.path}: directory not found`);
|
|
192
|
+
results.push({
|
|
193
|
+
action: 'none',
|
|
194
|
+
reason: 'Directory not found',
|
|
195
|
+
projectPath: project.path,
|
|
196
|
+
});
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
// Skip if project already has an active runner
|
|
200
|
+
// Check for pending work
|
|
201
|
+
const hasWork = await projectHasPendingWork(project.path);
|
|
202
|
+
if (!hasWork) {
|
|
203
|
+
log(`Skipping ${project.path}: no pending tasks`);
|
|
204
|
+
results.push({
|
|
205
|
+
action: 'none',
|
|
206
|
+
reason: 'No pending tasks',
|
|
207
|
+
projectPath: project.path,
|
|
208
|
+
});
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
// Step 4a: Recover stuck tasks (best-effort) before deciding whether to (re)start a runner.
|
|
212
|
+
// This is what unblocks orphaned/infinite-hang scenarios without manual intervention.
|
|
213
|
+
let recoveredActions = 0;
|
|
214
|
+
let skippedRecoveryDueToSafetyLimit = false;
|
|
215
|
+
try {
|
|
216
|
+
const { db: projectDb, close: closeProjectDb } = (0, connection_js_1.openDatabase)(project.path);
|
|
217
|
+
try {
|
|
218
|
+
const config = (0, loader_js_1.loadConfig)(project.path);
|
|
219
|
+
const recovery = await (0, stuck_task_recovery_js_1.recoverStuckTasks)({
|
|
220
|
+
projectPath: project.path,
|
|
221
|
+
projectDb,
|
|
222
|
+
globalDb: global.db,
|
|
223
|
+
config,
|
|
224
|
+
dryRun,
|
|
225
|
+
});
|
|
226
|
+
recoveredActions = recovery.actions.length;
|
|
227
|
+
skippedRecoveryDueToSafetyLimit = recovery.skippedDueToSafetyLimit;
|
|
228
|
+
if (recoveredActions > 0 && !quiet) {
|
|
229
|
+
log(`Recovered ${recoveredActions} stuck item(s) in ${project.path}`);
|
|
230
|
+
}
|
|
231
|
+
if (skippedRecoveryDueToSafetyLimit && !quiet) {
|
|
232
|
+
log(`Skipping auto-recovery in ${project.path}: safety limit hit (maxIncidentsPerHour)`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
finally {
|
|
236
|
+
closeProjectDb();
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
catch {
|
|
240
|
+
// If recovery can't run (DB missing/corrupt), we still proceed with runner checks.
|
|
241
|
+
}
|
|
242
|
+
// Skip if project already has an active runner (after recovery, which may have killed/removed it).
|
|
243
|
+
if (hasActiveRunnerForProject(project.path)) {
|
|
244
|
+
log(`Skipping ${project.path}: runner already active`);
|
|
245
|
+
results.push({
|
|
246
|
+
action: 'none',
|
|
247
|
+
reason: recoveredActions > 0
|
|
248
|
+
? `Runner already active (recovered ${recoveredActions} stuck item(s))`
|
|
249
|
+
: 'Runner already active',
|
|
250
|
+
projectPath: project.path,
|
|
251
|
+
recoveredActions,
|
|
252
|
+
skippedRecoveryDueToSafetyLimit,
|
|
253
|
+
});
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
// Start runner for this project
|
|
257
|
+
log(`Starting runner for: ${project.path}`);
|
|
258
|
+
if (dryRun) {
|
|
259
|
+
results.push({
|
|
260
|
+
action: 'would_start',
|
|
261
|
+
reason: recoveredActions > 0 ? `Recovered ${recoveredActions} stuck item(s); would start runner (dry-run)` : `Would start runner (dry-run)`,
|
|
262
|
+
projectPath: project.path,
|
|
263
|
+
recoveredActions,
|
|
264
|
+
skippedRecoveryDueToSafetyLimit,
|
|
265
|
+
});
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const startResult = startRunner(project.path);
|
|
269
|
+
if (startResult) {
|
|
270
|
+
results.push({
|
|
271
|
+
action: 'started',
|
|
272
|
+
reason: recoveredActions > 0 ? `Recovered ${recoveredActions} stuck item(s); started runner` : `Started runner`,
|
|
273
|
+
pid: startResult.pid,
|
|
274
|
+
projectPath: project.path,
|
|
275
|
+
recoveredActions,
|
|
276
|
+
skippedRecoveryDueToSafetyLimit,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
results.push({
|
|
281
|
+
action: 'none',
|
|
282
|
+
reason: recoveredActions > 0 ? `Recovered ${recoveredActions} stuck item(s); failed to start runner` : 'Failed to start runner',
|
|
283
|
+
projectPath: project.path,
|
|
284
|
+
recoveredActions,
|
|
285
|
+
skippedRecoveryDueToSafetyLimit,
|
|
286
|
+
});
|
|
287
|
+
}
|
|
233
288
|
}
|
|
234
|
-
|
|
289
|
+
// If no specific results, add a summary
|
|
290
|
+
if (results.length === 0) {
|
|
235
291
|
results.push({
|
|
236
292
|
action: 'none',
|
|
237
|
-
reason: '
|
|
238
|
-
projectPath: project.path,
|
|
293
|
+
reason: 'No action needed',
|
|
239
294
|
});
|
|
240
295
|
}
|
|
296
|
+
return results;
|
|
241
297
|
}
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
results.push({
|
|
245
|
-
action: 'none',
|
|
246
|
-
reason: 'No action needed',
|
|
247
|
-
});
|
|
298
|
+
finally {
|
|
299
|
+
global.close();
|
|
248
300
|
}
|
|
249
|
-
return results;
|
|
250
301
|
}
|
|
251
302
|
/**
|
|
252
303
|
* Check if wake-up is needed without taking action
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wakeup.js","sourceRoot":"","sources":["../../src/runners/wakeup.ts"],"names":[],"mappings":";AAAA;;GAEG;;
|
|
1
|
+
{"version":3,"file":"wakeup.js","sourceRoot":"","sources":["../../src/runners/wakeup.ts"],"names":[],"mappings":";AAAA;;GAEG;;AA8DH,8DAgBC;AA8DD,8CAUC;AAED,wBAsLC;AAKD,8CA4CC;AA7XD,qCAAqC;AACrC,yCAAiC;AACjC,2DAA2C;AAC3C,uCAAwD;AACxD,iDAAoD;AACpD,iDAAkD;AAClD,+CAAsD;AACtD,6DAAyD;AACzD,mDAAiD;AACjD,6EAAqE;AAmBrE;;GAEG;AACH,KAAK,UAAU,qBAAqB,CAAC,WAAmB;IACtD,MAAM,MAAM,GAAG,IAAA,gBAAI,EAAC,WAAW,EAAE,WAAW,EAAE,aAAa,CAAC,CAAC;IAC7D,IAAI,CAAC,IAAA,oBAAU,EAAC,MAAM,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,CAAC;QACH,2CAA2C;QAC3C,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAEpD,MAAM,MAAM,GAAG,EAAE;aACd,OAAO,CACN;8DACsD,CACvD;aACA,GAAG,EAAuB,CAAC;QAE9B,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAgB,yBAAyB,CAAC,WAAmB;IAC3D,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAA,iCAAkB,GAAE,CAAC;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE;aACX,OAAO,CACN;;;0DAGkD,CACnD;aACA,GAAG,CAAC,WAAW,CAA8B,CAAC;QAEjD,OAAO,GAAG,KAAK,SAAS,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,KAAK,EAAE,CAAC;IACV,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC9B,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,WAAmB;IACtC,IAAI,CAAC;QACH,6EAA6E;QAC7E,0EAA0E;QAC1E,+DAA+D;QAC/D,MAAM,IAAI,GAAG,CAAC,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAExE,MAAM,KAAK,GAAG,IAAA,0BAAK,EAAC,UAAU,EAAE,IAAI,EAAE;YACpC,QAAQ,EAAE,IAAI;YACd,KAAK,EAAE,QAAQ;SAChB,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH;;GAEG;AACH,SAAS,gBAAgB;IACvB,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAA,iCAAkB,GAAE,CAAC;IAC3C,IAAI,CAAC;QACH,EAAE,CAAC,OAAO,CACR;8DACwD,CACzD,CAAC,GAAG,EAAE,CAAC;IACV,CAAC;YAAS,CAAC;QACT,KAAK,EAAE,CAAC;IACV,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAgB,iBAAiB;IAC/B,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAA,iCAAkB,GAAE,CAAC;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE;aACX,OAAO,CAAC,+DAA+D,CAAC;aACxE,GAAG,EAAmC,CAAC;QAC1C,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC5B,CAAC;YAAS,CAAC;QACT,KAAK,EAAE,CAAC;IACV,CAAC;AACH,CAAC;AAEM,KAAK,UAAU,MAAM,CAAC,UAAyB,EAAE;IACtD,MAAM,EAAE,KAAK,GAAG,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,OAAO,GAAmB,EAAE,CAAC;IAEnC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAQ,EAAE;QAChC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC,CAAC;IAEF,oDAAoD;IACpD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,gBAAgB,EAAE,CAAC;IACrB,CAAC;IAED,uCAAuC;IACvC,MAAM,MAAM,GAAG,IAAA,iCAAkB,GAAE,CAAC;IACpC,IAAI,CAAC;QACH,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAA,+BAAgB,EAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACjD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,GAAG,CAAC,SAAS,YAAY,CAAC,MAAM,kCAAkC,CAAC,CAAC;gBACpE,IAAI,CAAC,MAAM,EAAE,CAAC;oBACZ,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;wBAClC,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;4BACf,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;wBAC1B,CAAC;wBACD,MAAM,CAAC,EAAE,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;oBACvE,CAAC;gBACH,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,WAAW,YAAY,CAAC,MAAM,kBAAkB;oBACxD,YAAY,EAAE,YAAY,CAAC,MAAM;iBAClC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wEAAwE;QAC1E,CAAC;QAEH,uCAAuC;QACvC,MAAM,UAAU,GAAG,IAAA,yBAAe,GAAE,CAAC;QACrC,IAAI,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC;YAC1C,GAAG,CAAC,2BAA2B,UAAU,CAAC,GAAG,gBAAgB,CAAC,CAAC;YAC/D,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,IAAA,oBAAU,GAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,2DAA2D;QAC3D,MAAM,kBAAkB,GAAG,IAAA,mCAAqB,EAAC,KAAK,CAAC,CAAC,CAAC,eAAe;QAExE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,8BAA8B,CAAC,CAAC;YACpC,GAAG,CAAC,0DAA0D,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,wBAAwB;gBAChC,YAAY,EAAE,CAAC;aAChB,CAAC,CAAC;YACH,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,GAAG,CAAC,YAAY,kBAAkB,CAAC,MAAM,2BAA2B,CAAC,CAAC;QAEtE,yDAAyD;QACzD,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACzC,0CAA0C;YAC1C,IAAI,CAAC,IAAA,oBAAU,EAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC9B,GAAG,CAAC,YAAY,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC;gBACrD,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,qBAAqB;oBAC7B,WAAW,EAAE,OAAO,CAAC,IAAI;iBAC1B,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,+CAA+C;YAC/C,yBAAyB;YACzB,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC1D,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,GAAG,CAAC,YAAY,OAAO,CAAC,IAAI,oBAAoB,CAAC,CAAC;gBAClD,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,kBAAkB;oBAC1B,WAAW,EAAE,OAAO,CAAC,IAAI;iBAC1B,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,4FAA4F;YAC5F,sFAAsF;YACtF,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,+BAA+B,GAAG,KAAK,CAAC;YAC5C,IAAI,CAAC;gBACH,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,IAAA,4BAAY,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBAC5E,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,IAAA,sBAAU,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;oBACxC,MAAM,QAAQ,GAAG,MAAM,IAAA,0CAAiB,EAAC;wBACvC,WAAW,EAAE,OAAO,CAAC,IAAI;wBACzB,SAAS;wBACT,QAAQ,EAAE,MAAM,CAAC,EAAE;wBACnB,MAAM;wBACN,MAAM;qBACP,CAAC,CAAC;oBACH,gBAAgB,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC;oBAC3C,+BAA+B,GAAG,QAAQ,CAAC,uBAAuB,CAAC;oBACnE,IAAI,gBAAgB,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;wBACnC,GAAG,CAAC,aAAa,gBAAgB,qBAAqB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;oBACxE,CAAC;oBACD,IAAI,+BAA+B,IAAI,CAAC,KAAK,EAAE,CAAC;wBAC9C,GAAG,CAAC,6BAA6B,OAAO,CAAC,IAAI,0CAA0C,CAAC,CAAC;oBAC3F,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,cAAc,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,mFAAmF;YACrF,CAAC;YAED,mGAAmG;YACnG,IAAI,yBAAyB,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,GAAG,CAAC,YAAY,OAAO,CAAC,IAAI,yBAAyB,CAAC,CAAC;gBACvD,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,MAAM;oBACd,MAAM,EACJ,gBAAgB,GAAG,CAAC;wBAClB,CAAC,CAAC,oCAAoC,gBAAgB,iBAAiB;wBACvE,CAAC,CAAC,uBAAuB;oBAC7B,WAAW,EAAE,OAAO,CAAC,IAAI;oBACzB,gBAAgB;oBAChB,+BAA+B;iBAChC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,gCAAgC;YAChC,GAAG,CAAC,wBAAwB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;YAE5C,IAAI,MAAM,EAAE,CAAC;gBACX,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,aAAa;oBACrB,MAAM,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,gBAAgB,8CAA8C,CAAC,CAAC,CAAC,8BAA8B;oBAC3I,WAAW,EAAE,OAAO,CAAC,IAAI;oBACzB,gBAAgB;oBAChB,+BAA+B;iBAChC,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC9C,IAAI,WAAW,EAAE,CAAC;gBAChB,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,SAAS;oBACjB,MAAM,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,gBAAgB,gCAAgC,CAAC,CAAC,CAAC,gBAAgB;oBAC/G,GAAG,EAAE,WAAW,CAAC,GAAG;oBACpB,WAAW,EAAE,OAAO,CAAC,IAAI;oBACzB,gBAAgB;oBAChB,+BAA+B;iBAChC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,MAAM;oBACd,MAAM,EAAE,gBAAgB,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,gBAAgB,wCAAwC,CAAC,CAAC,CAAC,wBAAwB;oBAC/H,WAAW,EAAE,OAAO,CAAC,IAAI;oBACzB,gBAAgB;oBAChB,+BAA+B;iBAChC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,IAAI,CAAC;gBACX,MAAM,EAAE,MAAM;gBACd,MAAM,EAAE,kBAAkB;aAC3B,CAAC,CAAC;QACL,CAAC;QAEC,OAAO,OAAO,CAAC;IACjB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,KAAK,EAAE,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;GAEG;AACI,KAAK,UAAU,iBAAiB;IAIrC,MAAM,UAAU,GAAG,IAAA,yBAAe,GAAE,CAAC;IAErC,IAAI,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,GAAG,EAAE,CAAC;QACxC,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,IAAA,iCAAkB,GAAE,CAAC;QAC3C,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAA,+BAAgB,EAAC,EAAE,CAAC,CAAC;YAC1C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,OAAO;oBACL,MAAM,EAAE,IAAI;oBACZ,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,+BAA+B;iBAC9D,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,mBAAmB,EAAE,CAAC;QACxD,CAAC;gBAAS,CAAC;YACT,KAAK,EAAE,CAAC;QACV,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,QAAQ,EAAE,CAAC;QACxB,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,2BAA2B,EAAE,CAAC;IAC/D,CAAC;IAED,4BAA4B;IAC5B,MAAM,kBAAkB,GAAG,IAAA,mCAAqB,EAAC,KAAK,CAAC,CAAC;IACxD,IAAI,gBAAgB,GAAG,CAAC,CAAC;IAEzB,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;QACzC,IAAI,IAAA,oBAAU,EAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,qBAAqB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YAC5E,gBAAgB,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO;YACL,MAAM,EAAE,IAAI;YACZ,MAAM,EAAE,GAAG,gBAAgB,gCAAgC;SAC5D,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE,CAAC;AACvD,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
-- Migration: Add incidents table and failure tracking for stuck-task recovery
|
|
2
|
+
-- Supports automatic recovery actions (reset orphaned tasks, escalate after repeats)
|
|
3
|
+
|
|
4
|
+
-- UP
|
|
5
|
+
CREATE TABLE IF NOT EXISTS incidents (
|
|
6
|
+
id TEXT PRIMARY KEY,
|
|
7
|
+
task_id TEXT REFERENCES tasks(id),
|
|
8
|
+
runner_id TEXT,
|
|
9
|
+
failure_mode TEXT NOT NULL, -- orphaned_task | hanging_invocation | zombie_runner | dead_runner
|
|
10
|
+
detected_at TEXT NOT NULL,
|
|
11
|
+
resolved_at TEXT,
|
|
12
|
+
resolution TEXT, -- auto_restart | skipped | escalated | none
|
|
13
|
+
details TEXT, -- JSON string with relevant diagnostics
|
|
14
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
15
|
+
);
|
|
16
|
+
|
|
17
|
+
CREATE INDEX IF NOT EXISTS idx_incidents_task ON incidents(task_id);
|
|
18
|
+
CREATE INDEX IF NOT EXISTS idx_incidents_detected ON incidents(detected_at);
|
|
19
|
+
CREATE INDEX IF NOT EXISTS idx_incidents_unresolved ON incidents(resolved_at) WHERE resolved_at IS NULL;
|
|
20
|
+
|
|
21
|
+
-- Add failure tracking to tasks (forward-only; SQLite can't drop columns)
|
|
22
|
+
ALTER TABLE tasks ADD COLUMN failure_count INTEGER NOT NULL DEFAULT 0;
|
|
23
|
+
ALTER TABLE tasks ADD COLUMN last_failure_at TEXT;
|
|
24
|
+
|
|
25
|
+
CREATE INDEX IF NOT EXISTS idx_tasks_failures ON tasks(failure_count) WHERE failure_count > 0;
|
|
26
|
+
|
|
27
|
+
-- DOWN
|
|
28
|
+
-- Note: SQLite doesn't support DROP COLUMN directly
|
|
29
|
+
-- Forward-only migration for tasks table changes
|
|
30
|
+
DROP INDEX IF EXISTS idx_tasks_failures;
|
|
31
|
+
DROP INDEX IF EXISTS idx_incidents_unresolved;
|
|
32
|
+
DROP INDEX IF EXISTS idx_incidents_detected;
|
|
33
|
+
DROP INDEX IF EXISTS idx_incidents_task;
|
|
34
|
+
DROP TABLE IF EXISTS incidents;
|
|
35
|
+
|
package/migrations/manifest.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": "0.2.1",
|
|
3
|
-
"latestDbVersion":
|
|
3
|
+
"latestDbVersion": 9,
|
|
4
4
|
"migrations": [
|
|
5
5
|
{
|
|
6
6
|
"id": 1,
|
|
@@ -65,6 +65,14 @@
|
|
|
65
65
|
"description": "Add skipped column to sections table for excluding sections from runner",
|
|
66
66
|
"checksum": "",
|
|
67
67
|
"cliVersion": "0.4.51"
|
|
68
|
+
},
|
|
69
|
+
{
|
|
70
|
+
"id": 9,
|
|
71
|
+
"name": "009_add_incidents_and_failure_tracking",
|
|
72
|
+
"file": "009_add_incidents_and_failure_tracking.sql",
|
|
73
|
+
"description": "Add incidents table and task failure counters for stuck-task recovery",
|
|
74
|
+
"checksum": "",
|
|
75
|
+
"cliVersion": "0.6.5"
|
|
68
76
|
}
|
|
69
77
|
]
|
|
70
78
|
}
|