pi-para 0.7.1 → 0.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/CONFIGURATION.md +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +47 -23
- package/dist/index.js.map +1 -1
- package/dist/scheduler/controls.d.ts.map +1 -1
- package/dist/scheduler/controls.js +2 -1
- package/dist/scheduler/controls.js.map +1 -1
- package/dist/scheduler/index.d.ts +10 -0
- package/dist/scheduler/index.d.ts.map +1 -1
- package/dist/scheduler/index.js +62 -2
- package/dist/scheduler/index.js.map +1 -1
- package/dist/scheduler/session-capture.d.ts +8 -0
- package/dist/scheduler/session-capture.d.ts.map +1 -1
- package/dist/scheduler/session-capture.js +58 -4
- package/dist/scheduler/session-capture.js.map +1 -1
- package/dist/scheduler/state.d.ts +3 -0
- package/dist/scheduler/state.d.ts.map +1 -1
- package/dist/scheduler/state.js +18 -0
- package/dist/scheduler/state.js.map +1 -1
- package/dist/state.d.ts +6 -1
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +55 -11
- package/dist/state.js.map +1 -1
- package/package.json +1 -1
- package/src/index.ts +45 -22
- package/src/scheduler/controls.ts +2 -1
- package/src/scheduler/index.ts +60 -2
- package/src/scheduler/session-capture.ts +68 -6
- package/src/scheduler/state.ts +21 -0
- package/src/state.ts +54 -10
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.js","sourceRoot":"","sources":["../../src/scheduler/state.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAmChD,MAAM,OAAO,gBAAgB;IAClB,EAAE,CAAoB;IAE/B,YAAY,MAAM,GAAG,YAAY,EAAE,CAAC,eAAe;QACjD,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED,OAAO;QACL,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC
|
|
1
|
+
{"version":3,"file":"state.js","sourceRoot":"","sources":["../../src/scheduler/state.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAmChD,MAAM,OAAO,gBAAgB;IAClB,EAAE,CAAoB;IAE/B,YAAY,MAAM,GAAG,YAAY,EAAE,CAAC,eAAe;QACjD,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAED,OAAO;QACL,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KA8CZ,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,CAAC,GAAW;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,GAAG,CAAkC,CAAC;QACtH,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC5B,CAAC;IAED,QAAQ,CAAC,GAAW,EAAE,KAAa;QACjC,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,+EAA+E,CAAC;aACxF,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,WAAW,CAAC,GAAW;QACrB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wCAAwC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrE,CAAC;IAED,OAAO,CAAC,QAAgB,EAAE,OAAgB,EAAE,OAAuB,EAAE;QACnE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAChF,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI9B,CAAC,CAAC,GAAG,CACJ,QAAQ,EACR,WAAW,EACX,IAAI,CAAC,QAAQ,IAAI,CAAC,EAClB,WAAW,CAAC,WAAW,EAAE,EACzB,IAAI,CAAC,SAAS,IAAI,IAAI,EACtB,GAAG,CAAC,WAAW,EAAE,EACjB,GAAG,CAAC,WAAW,EAAE,CAClB,CAAC;QACF,OAAO,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACxC,CAAC;IAED,SAAS,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE;QACxB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;;OAK3B,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAyB,CAAC;YAElD,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACvB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIf,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;YAClC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACvB,OAAO,SAAS,CAAC,EAAE,GAAG,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAC7G,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC;gBAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAA,CAAC;YAC1C,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,QAAQ,CAAC,EAAU,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QACnC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yEAAyE,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;IACxH,CAAC;IAED,IAAI,CAAC,EAAU,EAAE,KAAa,EAAE,OAA6D,EAAE;QAC7F,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG;YAAE,OAAO;QACjB,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;QAC1C,IAAI,GAAG,CAAC,QAAQ,GAAG,WAAW,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC/C,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIf,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,2EAA2E,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;QAC1H,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IAC9E,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,4CAA4C,CAAC,CAAC,GAAG,CAAC,EAAE,CAAyB,CAAC;QAC1G,OAAO,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;IAED,IAAI,CAAC,MAAoB;QACvB,MAAM,IAAI,GAAG,MAAM;YACjB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,gEAAgE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAe;YAC7G,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,+CAA+C,CAAC,CAAC,GAAG,EAAgB,CAAC;QACzF,OAAO,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC7B,CAAC;IAED,cAAc,CAAC,GAAG,GAAG,IAAI,IAAI,EAAE;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI9B,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7C,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,WAAW,CAAC,QAAiB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QAC7C,MAAM,MAAM,GAAG,QAAQ;YACrB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIjB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,QAAQ,CAAC;YACtD,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAIjB,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,WAAW,CAAC,OAA8C,EAAE;QAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ;YACxB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8EAA8E,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAiB;YAC3I,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,0DAA0D,CAAC,CAAC,GAAG,CAAC,KAAK,CAAiB,CAAC;QAC3G,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IACpC,CAAC;IAED,cAAc,CAAC,EAAU;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,EAAE,CAA2B,CAAC;QAC9G,OAAO,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5C,CAAC;IAED,aAAa,CACX,QAAgB,EAChB,OAAgB,EAChB,MAAc,EACd,SAAe,EACf,UAAuB,EACvB,UAAyB,EACzB,KAAoB;QAEpB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAIf,CAAC,CAAC,GAAG,CACJ,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC,EAC7B,MAAM,EACN,SAAS,CAAC,WAAW,EAAE,EACvB,UAAU,EAAE,WAAW,EAAE,IAAI,IAAI,EACjC,UAAU,EACV,KAAK,CACN,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,SAAiB;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI3B,CAAC,CAAC,GAAG,CAAC,SAAS,CAAyB,CAAC;QAC1C,OAAO,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACrC,CAAC;CACF;AA0BD,SAAS,SAAS,CAAC,GAAa;IAC9B,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAY;QAChD,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,SAAS,EAAE,GAAG,CAAC,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAe;IACvC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,OAAO,EAAE,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAY,CAAC,CAAC,CAAC,IAAI;QAC1E,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,UAAU,EAAE,GAAG,CAAC,WAAW;QAC3B,UAAU,EAAE,GAAG,CAAC,WAAW;QAC3B,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC"}
|
package/dist/state.d.ts
CHANGED
|
@@ -24,7 +24,12 @@ export declare class StateDB {
|
|
|
24
24
|
private db;
|
|
25
25
|
constructor(wikiDir: string);
|
|
26
26
|
private migrate;
|
|
27
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* Check if a session file has already been processed with the same content.
|
|
29
|
+
* Uses a stat (size+mtime) fast path before falling back to a full content
|
|
30
|
+
* hash, so registry-wide scans stay cheap even with multi-hundred-MB
|
|
31
|
+
* session files. A hash match under a changed stat refreshes the cache.
|
|
32
|
+
*/
|
|
28
33
|
isProcessed(sessionPath: string): boolean;
|
|
29
34
|
/** Get the stored hash for a session (or null if not processed). */
|
|
30
35
|
getHash(sessionPath: string): string | null;
|
package/dist/state.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAOH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,EAAE,CAAoB;gBAElB,OAAO,EAAE,MAAM;IAO3B,OAAO,CAAC,OAAO;
|
|
1
|
+
{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAOH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,qBAAa,OAAO;IAClB,OAAO,CAAC,EAAE,CAAoB;gBAElB,OAAO,EAAE,MAAM;IAO3B,OAAO,CAAC,OAAO;IAmCf;;;;;OAKG;IACH,WAAW,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IA2BzC,oEAAoE;IACpE,OAAO,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAO3C,+CAA+C;IAC/C,aAAa,CACX,WAAW,EAAE,MAAM,EACnB,KAAK,EAAE,MAAM,EACb,YAAY,EAAE,MAAM,EAAE,EACtB,YAAY,EAAE,MAAM,EAAE,GACrB,IAAI;IAqBP,0CAA0C;IAC1C,aAAa,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAatE,yCAAyC;IACzC,SAAS,IAAI,gBAAgB,EAAE;IAe/B,4DAA4D;IAC5D,UAAU,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,gBAAgB,EAAE;IAiBlE,mCAAmC;IACnC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI;IAOpC,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAM1C,0CAA0C;IAC1C,iBAAiB,CACf,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,EAAE,MAAM,EAAE,EACd,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,GAChB,IAAI;IAiBP,4DAA4D;IAC5D,gBAAgB,CAAC,KAAK,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,OAAO,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,WAAW,EAAE;IAoC/F,4CAA4C;IAC5C,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAMrC,KAAK,IAAI,IAAI;CAGd"}
|
package/dist/state.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import Database from "better-sqlite3";
|
|
7
7
|
import { createHash } from "node:crypto";
|
|
8
|
-
import { readFileSync } from "node:fs";
|
|
8
|
+
import { readFileSync, statSync } from "node:fs";
|
|
9
9
|
import { join } from "node:path";
|
|
10
10
|
export class StateDB {
|
|
11
11
|
db;
|
|
@@ -39,16 +39,50 @@ export class StateDB {
|
|
|
39
39
|
updated_at TEXT NOT NULL
|
|
40
40
|
);
|
|
41
41
|
`);
|
|
42
|
-
|
|
43
|
-
|
|
42
|
+
// Stat fast path for isProcessed(): when size+mtime are unchanged since
|
|
43
|
+
// the recorded hash, skip re-hashing the whole session file. Critical for
|
|
44
|
+
// startup catch-up scans over registries with very large session JSONLs.
|
|
45
|
+
try {
|
|
46
|
+
this.db.exec("ALTER TABLE processed_sessions ADD COLUMN file_size INTEGER");
|
|
47
|
+
}
|
|
48
|
+
catch { /* column exists */ }
|
|
49
|
+
try {
|
|
50
|
+
this.db.exec("ALTER TABLE processed_sessions ADD COLUMN file_mtime_ms INTEGER");
|
|
51
|
+
}
|
|
52
|
+
catch { /* column exists */ }
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Check if a session file has already been processed with the same content.
|
|
56
|
+
* Uses a stat (size+mtime) fast path before falling back to a full content
|
|
57
|
+
* hash, so registry-wide scans stay cheap even with multi-hundred-MB
|
|
58
|
+
* session files. A hash match under a changed stat refreshes the cache.
|
|
59
|
+
*/
|
|
44
60
|
isProcessed(sessionPath) {
|
|
45
61
|
const row = this.db
|
|
46
|
-
.prepare("SELECT content_hash FROM processed_sessions WHERE session_path = ?")
|
|
62
|
+
.prepare("SELECT content_hash, file_size, file_mtime_ms FROM processed_sessions WHERE session_path = ?")
|
|
47
63
|
.get(sessionPath);
|
|
48
64
|
if (!row)
|
|
49
65
|
return false;
|
|
66
|
+
let stat;
|
|
67
|
+
try {
|
|
68
|
+
stat = statSync(sessionPath);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
// mtimeMs is a float and loses sub-millisecond precision across
|
|
74
|
+
// utimes/copy round-trips; compare with a small tolerance. Size is exact.
|
|
75
|
+
if (row.file_size !== null && row.file_mtime_ms !== null &&
|
|
76
|
+
row.file_size === stat.size && Math.abs(row.file_mtime_ms - stat.mtimeMs) <= 2) {
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
50
79
|
const currentHash = hashFile(sessionPath);
|
|
51
|
-
|
|
80
|
+
if (row.content_hash !== currentHash)
|
|
81
|
+
return false;
|
|
82
|
+
this.db
|
|
83
|
+
.prepare("UPDATE processed_sessions SET file_size = ?, file_mtime_ms = ? WHERE session_path = ?")
|
|
84
|
+
.run(stat.size, Math.floor(stat.mtimeMs), sessionPath);
|
|
85
|
+
return true;
|
|
52
86
|
}
|
|
53
87
|
/** Get the stored hash for a session (or null if not processed). */
|
|
54
88
|
getHash(sessionPath) {
|
|
@@ -60,24 +94,26 @@ export class StateDB {
|
|
|
60
94
|
/** Record a successfully processed session. */
|
|
61
95
|
recordSuccess(sessionPath, scope, pagesCreated, pagesUpdated) {
|
|
62
96
|
const hash = hashFile(sessionPath);
|
|
97
|
+
const stat = statFile(sessionPath);
|
|
63
98
|
this.db
|
|
64
99
|
.prepare(`
|
|
65
100
|
INSERT OR REPLACE INTO processed_sessions
|
|
66
|
-
(session_path, content_hash, processed_at, scope, pages_created, pages_updated, error)
|
|
67
|
-
VALUES (?, ?, ?, ?, ?, ?, NULL)
|
|
101
|
+
(session_path, content_hash, processed_at, scope, pages_created, pages_updated, error, file_size, file_mtime_ms)
|
|
102
|
+
VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?)
|
|
68
103
|
`)
|
|
69
|
-
.run(sessionPath, hash, new Date().toISOString(), scope, JSON.stringify(pagesCreated), JSON.stringify(pagesUpdated));
|
|
104
|
+
.run(sessionPath, hash, new Date().toISOString(), scope, JSON.stringify(pagesCreated), JSON.stringify(pagesUpdated), stat?.size ?? null, stat ? Math.floor(stat.mtimeMs) : null);
|
|
70
105
|
}
|
|
71
106
|
/** Record a failed processing attempt. */
|
|
72
107
|
recordFailure(sessionPath, scope, error) {
|
|
73
108
|
const hash = hashFile(sessionPath);
|
|
109
|
+
const stat = statFile(sessionPath);
|
|
74
110
|
this.db
|
|
75
111
|
.prepare(`
|
|
76
112
|
INSERT OR REPLACE INTO processed_sessions
|
|
77
|
-
(session_path, content_hash, processed_at, scope, pages_created, pages_updated, error)
|
|
78
|
-
VALUES (?, ?, ?, ?, '[]', '[]', ?)
|
|
113
|
+
(session_path, content_hash, processed_at, scope, pages_created, pages_updated, error, file_size, file_mtime_ms)
|
|
114
|
+
VALUES (?, ?, ?, ?, '[]', '[]', ?, ?, ?)
|
|
79
115
|
`)
|
|
80
|
-
.run(sessionPath, hash, new Date().toISOString(), scope, error);
|
|
116
|
+
.run(sessionPath, hash, new Date().toISOString(), scope, error, stat?.size ?? null, stat ? Math.floor(stat.mtimeMs) : null);
|
|
81
117
|
}
|
|
82
118
|
/** Get all failed sessions for retry. */
|
|
83
119
|
getFailed() {
|
|
@@ -153,6 +189,14 @@ export class StateDB {
|
|
|
153
189
|
this.db.close();
|
|
154
190
|
}
|
|
155
191
|
}
|
|
192
|
+
function statFile(filePath) {
|
|
193
|
+
try {
|
|
194
|
+
return statSync(filePath);
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
return null;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
156
200
|
function hashFile(filePath) {
|
|
157
201
|
try {
|
|
158
202
|
const content = readFileSync(filePath);
|
package/dist/state.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state.js","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"state.js","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAqBjC,MAAM,OAAO,OAAO;IACV,EAAE,CAAoB;IAE9B,YAAY,OAAe;QACzB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;QAC/C,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO,EAAE,CAAC;IACjB,CAAC;IAEO,OAAO;QACb,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;KAsBZ,CAAC,CAAC;QACH,wEAAwE;QACxE,0EAA0E;QAC1E,yEAAyE;QACzE,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,6DAA6D,CAAC,CAAC;QAC9E,CAAC;QAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC;QAC/B,IAAI,CAAC;YACH,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,iEAAiE,CAAC,CAAC;QAClF,CAAC;QAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC;IACjC,CAAC;IAED;;;;;OAKG;IACH,WAAW,CAAC,WAAmB;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,8FAA8F,CAAC;aACvG,GAAG,CAAC,WAAW,CAAiG,CAAC;QACpH,IAAI,CAAC,GAAG;YAAE,OAAO,KAAK,CAAC;QAEvB,IAAI,IAAuC,CAAC;QAC5C,IAAI,CAAC;YACH,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;QACD,gEAAgE;QAChE,0EAA0E;QAC1E,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI;YACtD,GAAG,CAAC,SAAS,KAAK,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACjF,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,WAAW,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;QAC1C,IAAI,GAAG,CAAC,YAAY,KAAK,WAAW;YAAE,OAAO,KAAK,CAAC;QACnD,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,uFAAuF,CAAC;aAChG,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,WAAW,CAAC,CAAC;QACzD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oEAAoE;IACpE,OAAO,CAAC,WAAmB;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,oEAAoE,CAAC;aAC7E,GAAG,CAAC,WAAW,CAAyC,CAAC;QAC5D,OAAO,GAAG,EAAE,YAAY,IAAI,IAAI,CAAC;IACnC,CAAC;IAED,+CAA+C;IAC/C,aAAa,CACX,WAAmB,EACnB,KAAa,EACb,YAAsB,EACtB,YAAsB;QAEtB,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC;;;;OAIR,CAAC;aACD,GAAG,CACF,WAAW,EACX,IAAI,EACJ,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACxB,KAAK,EACL,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAC5B,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,EAC5B,IAAI,EAAE,IAAI,IAAI,IAAI,EAClB,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CACvC,CAAC;IACN,CAAC;IAED,0CAA0C;IAC1C,aAAa,CAAC,WAAmB,EAAE,KAAa,EAAE,KAAa;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;QACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC;;;;OAIR,CAAC;aACD,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,EAC5D,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClE,CAAC;IAED,yCAAyC;IACzC,SAAS;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,0DAA0D,CAAC;aACnE,GAAG,EAQF,CAAC;QACL,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,4DAA4D;IAC5D,UAAU,CAAC,KAAc,EAAE,QAAgB,EAAE;QAC3C,MAAM,GAAG,GAAG,KAAK;YACf,CAAC,CAAC,qFAAqF;YACvF,CAAC,CAAC,qEAAqE,CAAC;QAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAQ7C,CAAC;QACH,OAAO,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;IAChC,CAAC;IAED,mCAAmC;IACnC,QAAQ,CAAC,GAAW;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,8CAA8C,CAAC;aACvD,GAAG,CAAC,GAAG,CAAkC,CAAC;QAC7C,OAAO,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC;IAC5B,CAAC;IAED,QAAQ,CAAC,GAAW,EAAE,KAAa;QACjC,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,gEAAgE,CAAC;aACzE,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACrB,CAAC;IAED,0CAA0C;IAC1C,iBAAiB,CACf,IAAY,EACZ,QAAgB,EAChB,KAAe,EACf,IAAc,EACd,cAAsB,EACtB,SAAiB;QAEjB,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC;;;;OAIR,CAAC;aACD,GAAG,CACF,IAAI,EACJ,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EACrB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EACpB,cAAc,EACd,SAAS,CACV,CAAC;IACN,CAAC;IAED,4DAA4D;IAC5D,gBAAgB,CAAC,KAA8D;QAC7E,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,uDAAuD,CAAC;aAChE,GAAG,EAOF,CAAC;QAEL,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnC,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,IAAI,IAAI,CAAa;YACrD,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAa;YACnD,cAAc,EAAE,GAAG,CAAC,eAAe;YACnC,SAAS,EAAE,GAAG,CAAC,UAAU;SAC1B,CAAC,CAAC,CAAC;QAEJ,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAE7B,+DAA+D;QAC/D,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YAC5B,IACE,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC;gBACxB,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAClD,CAAC;gBACD,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO,IAAI,CAAC;YAC5C,OAAO,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5D,CAAC,CAAC,CAAC;IACL,CAAC;IAED,4CAA4C;IAC5C,iBAAiB,CAAC,IAAY;QAC5B,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,2CAA2C,CAAC;aACpD,GAAG,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,QAAgB;IAChC,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB;IAChC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,GAQrB;IACC,OAAO;QACL,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE;QACtB,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC;QACnD,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC;QACnD,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -128,6 +128,7 @@ export default async function piPara(pi: ExtensionAPI): Promise<void> {
|
|
|
128
128
|
|
|
129
129
|
let store: QMDStore | null = null;
|
|
130
130
|
let currentScope: ProjectScope | null = null;
|
|
131
|
+
let startupMaintenanceTimer: ReturnType<typeof setTimeout> | null = null;
|
|
131
132
|
let storeDisabled = false;
|
|
132
133
|
|
|
133
134
|
|
|
@@ -301,28 +302,44 @@ export default async function piPara(pi: ExtensionAPI): Promise<void> {
|
|
|
301
302
|
})
|
|
302
303
|
: null;
|
|
303
304
|
|
|
304
|
-
//
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
305
|
+
// Startup maintenance (embed backlog drain + capture catch-up) is
|
|
306
|
+
// deliberately deferred and detached: it must never run inside this
|
|
307
|
+
// awaited session_start handler, where registry scans over large session
|
|
308
|
+
// files previously froze fresh sessions. It is also skipped entirely in
|
|
309
|
+
// non-interactive children (squad workers, RPC probes) — they only append
|
|
310
|
+
// their own completed sessions for the next interactive session to pick up.
|
|
311
|
+
const interactive = ctx.hasUI === true;
|
|
312
|
+
if (scheduler && interactive && ((embedEnabled && store) || startupCatchupEnabled)) {
|
|
313
|
+
const envDelay = Number(process.env.PI_PARA_STARTUP_DELAY_MS);
|
|
314
|
+
const startupDelayMs = Number.isFinite(envDelay) && envDelay >= 0 ? envDelay : 30_000;
|
|
315
|
+
startupMaintenanceTimer = setTimeout(() => {
|
|
316
|
+
startupMaintenanceTimer = null;
|
|
317
|
+
void (async () => {
|
|
318
|
+
if (embedEnabled && store) {
|
|
319
|
+
try {
|
|
320
|
+
scheduler.embedEnabled = true;
|
|
321
|
+
scheduler.enqueue("qmd-embed", {}, { dedupeKey: "qmd-embed", priority: 5 });
|
|
322
|
+
} catch {
|
|
323
|
+
// Background embedding is best-effort; BM25 search works regardless.
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
if (startupCatchupEnabled) {
|
|
327
|
+
try {
|
|
328
|
+
const stateDb = new StateDB(wikiDir);
|
|
329
|
+
await enqueueCompletedSessionsFromRegistry(scheduler, wikiDir, { stateDb });
|
|
330
|
+
stateDb.close();
|
|
331
|
+
} catch {
|
|
332
|
+
// Startup capture catch-up is best-effort.
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
try {
|
|
336
|
+
await scheduler.tick();
|
|
337
|
+
} catch {
|
|
338
|
+
// Tick failures are recorded in scheduler history.
|
|
339
|
+
}
|
|
340
|
+
})();
|
|
341
|
+
}, startupDelayMs);
|
|
342
|
+
startupMaintenanceTimer.unref?.();
|
|
326
343
|
}
|
|
327
344
|
|
|
328
345
|
if (loadedConfig.userConfig?.webWiki.enabled && !printMode) {
|
|
@@ -364,6 +381,12 @@ export default async function piPara(pi: ExtensionAPI): Promise<void> {
|
|
|
364
381
|
// -- session_shutdown: register session for scheduler, close store ---------
|
|
365
382
|
|
|
366
383
|
pi.on("session_shutdown", async (_event, ctx) => {
|
|
384
|
+
// 0. Cancel any not-yet-fired deferred startup maintenance.
|
|
385
|
+
if (startupMaintenanceTimer) {
|
|
386
|
+
clearTimeout(startupMaintenanceTimer);
|
|
387
|
+
startupMaintenanceTimer = null;
|
|
388
|
+
}
|
|
389
|
+
|
|
367
390
|
// 1. Register session as completed (scheduler catch-up processes later)
|
|
368
391
|
const sessionFile = ctx.sessionManager.getSessionFile();
|
|
369
392
|
if (sessionFile) {
|
|
@@ -31,7 +31,8 @@ export async function queueCaptureRecent(options: QueueCaptureRecentOptions): Pr
|
|
|
31
31
|
const scheduler = new WikiScheduler({ wikiDir: options.wikiDir, dbPath, enabled: true });
|
|
32
32
|
const stateDb = new StateDB(options.wikiDir);
|
|
33
33
|
try {
|
|
34
|
-
|
|
34
|
+
// Explicit user request: capture even sessions that are still fresh.
|
|
35
|
+
const queuedIds = enqueueCompletedSessions(scheduler, eligible, { stateDb, quietMs: 0 });
|
|
35
36
|
return { scanned: entries.length, eligible: eligible.length, queuedIds };
|
|
36
37
|
} finally {
|
|
37
38
|
stateDb.close();
|
package/src/scheduler/index.ts
CHANGED
|
@@ -135,6 +135,12 @@ export class WikiScheduler {
|
|
|
135
135
|
if (store) await reindex(store);
|
|
136
136
|
this.markDirty();
|
|
137
137
|
await gitCommit(this.wikiDir, "wiki: rebuild index and refresh search");
|
|
138
|
+
// Orphaned-vector GC: every page edit creates a new content generation
|
|
139
|
+
// whose old vectors are never deleted by qmd's write path. Left alone,
|
|
140
|
+
// the store grows unboundedly (observed: 1,070MB for 1,011 docs, 98.5%
|
|
141
|
+
// garbage) and every embed/search pass slows down until the interactive
|
|
142
|
+
// session visibly stalls. Bounded daily GC keeps it flat.
|
|
143
|
+
if (store) this.maybeCollectOrphanedVectors(store);
|
|
138
144
|
} finally {
|
|
139
145
|
releaseLease(this.state.db, leaseKey, this.holderId);
|
|
140
146
|
}
|
|
@@ -146,6 +152,29 @@ export class WikiScheduler {
|
|
|
146
152
|
}
|
|
147
153
|
}
|
|
148
154
|
|
|
155
|
+
/**
|
|
156
|
+
* Run orphaned-vector GC at most once per day. DELETEs are fast (ms–s at
|
|
157
|
+
* healthy sizes); the pathological first run on a badly bloated store is a
|
|
158
|
+
* one-time cost that prevents the far worse alternative of unbounded growth.
|
|
159
|
+
*/
|
|
160
|
+
private maybeCollectOrphanedVectors(store: QMDStore): void {
|
|
161
|
+
try {
|
|
162
|
+
const last = this.state.getValue(VECTOR_GC_KEY);
|
|
163
|
+
if (last && Date.now() - Date.parse(last) < VECTOR_GC_INTERVAL_MS) return;
|
|
164
|
+
// Record the attempt time first so a crashing/slow GC cannot retry-loop.
|
|
165
|
+
this.state.setValue(VECTOR_GC_KEY, new Date().toISOString());
|
|
166
|
+
const internal = (store as { internal?: { cleanupOrphanedVectors?: () => number; cleanupOrphanedContent?: () => number } }).internal;
|
|
167
|
+
if (!internal?.cleanupOrphanedVectors) return;
|
|
168
|
+
const chunks = internal.cleanupOrphanedVectors();
|
|
169
|
+
const hashes = internal.cleanupOrphanedContent?.() ?? 0;
|
|
170
|
+
// Background code in the interactive process must stay silent (no
|
|
171
|
+
// console); record the outcome durably instead.
|
|
172
|
+
this.state.setValue(VECTOR_GC_RESULT_KEY, JSON.stringify({ at: new Date().toISOString(), chunks, hashes }));
|
|
173
|
+
} catch {
|
|
174
|
+
// GC is best-effort; search and embedding work regardless.
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
149
178
|
/**
|
|
150
179
|
* Generate vector embeddings for documents QMD reports as pending.
|
|
151
180
|
* Skips fast when the backlog is empty. Throws when the embedding pass
|
|
@@ -158,6 +187,11 @@ export class WikiScheduler {
|
|
|
158
187
|
// No real embedding provider configured — embed calls would hit the inert
|
|
159
188
|
// shim endpoint (slow TCP timeouts on Windows) and always fail. Skip.
|
|
160
189
|
if (!storeHasApiProviders(store)) return;
|
|
190
|
+
// A credential/billing failure (401/402/403) cannot heal by retrying every
|
|
191
|
+
// tick and every session start; it needs a human (top up credits, fix the
|
|
192
|
+
// key). Pause embedding durably and resume automatically after backoff.
|
|
193
|
+
const backoffUntil = this.state.getValue(EMBED_BACKOFF_KEY);
|
|
194
|
+
if (backoffUntil && Date.parse(backoffUntil) > Date.now()) return;
|
|
161
195
|
|
|
162
196
|
const status = await store.getStatus();
|
|
163
197
|
const pending = status.needsEmbedding ?? 0;
|
|
@@ -170,18 +204,42 @@ export class WikiScheduler {
|
|
|
170
204
|
const diagnostics = [...new Set(attempt.diagnostics)].join(" | ");
|
|
171
205
|
if (!attempt.ok) {
|
|
172
206
|
const reason = attempt.error instanceof Error ? attempt.error.message : String(attempt.error);
|
|
173
|
-
throw
|
|
207
|
+
throw this.embedFailure(diagnostics ? `${reason} — ${diagnostics}` : reason);
|
|
174
208
|
}
|
|
175
209
|
const result = attempt.value;
|
|
176
210
|
if (result.errors > 0) {
|
|
177
|
-
throw
|
|
211
|
+
throw this.embedFailure(
|
|
178
212
|
`qmd embed finished with ${result.errors} error(s) (${result.chunksEmbedded} chunk(s) embedded, ${pending} doc(s) were pending)${diagnostics ? ` — ${diagnostics}` : ""}`,
|
|
179
213
|
);
|
|
180
214
|
}
|
|
215
|
+
this.state.deleteValue(EMBED_BACKOFF_KEY);
|
|
181
216
|
} finally {
|
|
182
217
|
releaseLease(this.state.db, leaseKey, this.holderId);
|
|
183
218
|
}
|
|
184
219
|
}
|
|
220
|
+
|
|
221
|
+
/** Record durable backoff for credential/billing embed failures, then return the error. */
|
|
222
|
+
private embedFailure(message: string): Error {
|
|
223
|
+
if (isCredentialEmbedError(message)) {
|
|
224
|
+
const until = new Date(Date.now() + EMBED_CREDENTIAL_BACKOFF_MS).toISOString();
|
|
225
|
+
try {
|
|
226
|
+
this.state.setValue(EMBED_BACKOFF_KEY, until);
|
|
227
|
+
} catch { /* backoff bookkeeping is best-effort */ }
|
|
228
|
+
return new Error(`${message} — embedding paused until ${until} (credential/billing error; fix the provider and it resumes automatically, BM25 search keeps working)`);
|
|
229
|
+
}
|
|
230
|
+
return new Error(message);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const EMBED_BACKOFF_KEY = "qmd-embed-backoff-until";
|
|
235
|
+
const VECTOR_GC_KEY = "qmd-vector-gc-last-run";
|
|
236
|
+
const VECTOR_GC_RESULT_KEY = "qmd-vector-gc-last-result";
|
|
237
|
+
const VECTOR_GC_INTERVAL_MS = 24 * 60 * 60_000;
|
|
238
|
+
const EMBED_CREDENTIAL_BACKOFF_MS = 6 * 60 * 60_000;
|
|
239
|
+
|
|
240
|
+
/** Credential/billing embed failures (401/402/403) that retries cannot fix. */
|
|
241
|
+
export function isCredentialEmbedError(message: string): boolean {
|
|
242
|
+
return /\bAPI 40[123]\b/.test(message);
|
|
185
243
|
}
|
|
186
244
|
|
|
187
245
|
const schedulers = new Map<string, WikiScheduler>();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { appendFile, readFile } from "node:fs/promises";
|
|
2
|
-
import { existsSync, readFileSync } from "node:fs";
|
|
1
|
+
import { appendFile, readFile, rename, writeFile } from "node:fs/promises";
|
|
2
|
+
import { existsSync, readFileSync, statSync } from "node:fs";
|
|
3
3
|
import { basename, dirname, join } from "node:path";
|
|
4
4
|
import type { Model } from "@earendil-works/pi-ai";
|
|
5
5
|
import type { QMDStore } from "qmd-engine";
|
|
@@ -62,31 +62,93 @@ export async function appendCompletedSession(
|
|
|
62
62
|
return { timestamp, sessionPath };
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
/**
|
|
66
|
+
* A session file modified this recently is treated as still active: capturing
|
|
67
|
+
* it now would both re-read a growing file and produce a summary that is
|
|
68
|
+
* immediately stale, so it stays in the registry for a later pass.
|
|
69
|
+
*/
|
|
70
|
+
export const ACTIVE_SESSION_QUIET_MS = 10 * 60_000;
|
|
71
|
+
|
|
65
72
|
export async function enqueueCompletedSessionsFromRegistry(
|
|
66
73
|
scheduler: WikiScheduler,
|
|
67
74
|
wikiDir: string,
|
|
68
|
-
opts: { stateDb?: StateDB } = {},
|
|
75
|
+
opts: { stateDb?: StateDB; quietMs?: number } = {},
|
|
69
76
|
): Promise<number[]> {
|
|
70
77
|
const entries = await readCompletedSessionRegistry(wikiDir);
|
|
71
|
-
|
|
78
|
+
const { ids, survivors } = scanCompletedSessions(scheduler, entries, opts);
|
|
79
|
+
// Compact the registry: keep one line per still-relevant session so the
|
|
80
|
+
// file (and the startup scan) cannot grow without bound. Processed and
|
|
81
|
+
// deleted sessions are dropped; unprocessed and still-active ones survive.
|
|
82
|
+
if (survivors.length < entries.length) {
|
|
83
|
+
await compactCompletedSessionRegistry(wikiDir, entries, survivors);
|
|
84
|
+
}
|
|
85
|
+
return ids;
|
|
72
86
|
}
|
|
73
87
|
|
|
74
88
|
export function enqueueCompletedSessions(
|
|
75
89
|
scheduler: WikiScheduler,
|
|
76
90
|
entries: CompletedSessionEntry[],
|
|
77
|
-
opts: { stateDb?: StateDB } = {},
|
|
91
|
+
opts: { stateDb?: StateDB; quietMs?: number } = {},
|
|
78
92
|
): number[] {
|
|
93
|
+
return scanCompletedSessions(scheduler, entries, opts).ids;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function scanCompletedSessions(
|
|
97
|
+
scheduler: WikiScheduler,
|
|
98
|
+
entries: CompletedSessionEntry[],
|
|
99
|
+
opts: { stateDb?: StateDB; quietMs?: number } = {},
|
|
100
|
+
): { ids: number[]; survivors: CompletedSessionEntry[] } {
|
|
101
|
+
const quietMs = opts.quietMs ?? ACTIVE_SESSION_QUIET_MS;
|
|
79
102
|
const ids: number[] = [];
|
|
103
|
+
const survivors: CompletedSessionEntry[] = [];
|
|
104
|
+
const seen = new Set<string>();
|
|
80
105
|
for (const entry of entries) {
|
|
106
|
+
if (seen.has(entry.sessionPath)) continue;
|
|
107
|
+
seen.add(entry.sessionPath);
|
|
81
108
|
if (!existsSync(entry.sessionPath)) continue;
|
|
109
|
+
// Still-active session: skip this pass but keep it registered.
|
|
110
|
+
try {
|
|
111
|
+
if (Date.now() - statSync(entry.sessionPath).mtimeMs < quietMs) {
|
|
112
|
+
survivors.push(entry);
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
} catch {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
82
118
|
if (opts.stateDb?.isProcessed(entry.sessionPath)) continue;
|
|
119
|
+
survivors.push(entry);
|
|
83
120
|
ids.push(scheduler.enqueue(
|
|
84
121
|
"capture-session",
|
|
85
122
|
{ sessionPath: entry.sessionPath, scope: detectScopeFromSessionPath(entry.sessionPath) } satisfies CaptureSessionPayload,
|
|
86
123
|
{ dedupeKey: `capture-session:${entry.sessionPath}`, priority: 20 },
|
|
87
124
|
));
|
|
88
125
|
}
|
|
89
|
-
return ids;
|
|
126
|
+
return { ids, survivors };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Atomically rewrite the registry with the surviving entries, merging any
|
|
131
|
+
* lines appended concurrently since `snapshot` was read.
|
|
132
|
+
*/
|
|
133
|
+
async function compactCompletedSessionRegistry(
|
|
134
|
+
wikiDir: string,
|
|
135
|
+
snapshot: CompletedSessionEntry[],
|
|
136
|
+
survivors: CompletedSessionEntry[],
|
|
137
|
+
): Promise<void> {
|
|
138
|
+
const registryPath = join(wikiDir, ".completed-sessions");
|
|
139
|
+
try {
|
|
140
|
+
const snapshotPaths = new Set(snapshot.map((entry) => entry.sessionPath));
|
|
141
|
+
const current = await readCompletedSessionRegistry(wikiDir);
|
|
142
|
+
const appendedSinceSnapshot = current.filter((entry) => !snapshotPaths.has(entry.sessionPath));
|
|
143
|
+
const lines = [...survivors, ...appendedSinceSnapshot]
|
|
144
|
+
.map((entry) => `${entry.timestamp}|${entry.sessionPath}`)
|
|
145
|
+
.join("\n");
|
|
146
|
+
const tmpPath = `${registryPath}.tmp-${process.pid}`;
|
|
147
|
+
await writeFile(tmpPath, lines ? `${lines}\n` : "", "utf-8");
|
|
148
|
+
await rename(tmpPath, registryPath);
|
|
149
|
+
} catch {
|
|
150
|
+
// Compaction is an optimization; the registry stays valid without it.
|
|
151
|
+
}
|
|
90
152
|
}
|
|
91
153
|
|
|
92
154
|
export function createCaptureSessionHandler(deps: CaptureSessionDeps): SchedulerTaskHandler {
|
package/src/scheduler/state.ts
CHANGED
|
@@ -89,9 +89,30 @@ export class SchedulerStateDB {
|
|
|
89
89
|
duration_ms INTEGER,
|
|
90
90
|
error TEXT
|
|
91
91
|
);
|
|
92
|
+
|
|
93
|
+
CREATE TABLE IF NOT EXISTS scheduler_kv (
|
|
94
|
+
key TEXT PRIMARY KEY,
|
|
95
|
+
value TEXT NOT NULL,
|
|
96
|
+
updated_at TEXT NOT NULL
|
|
97
|
+
);
|
|
92
98
|
`);
|
|
93
99
|
}
|
|
94
100
|
|
|
101
|
+
getValue(key: string): string | null {
|
|
102
|
+
const row = this.db.prepare("SELECT value FROM scheduler_kv WHERE key = ?").get(key) as { value: string } | undefined;
|
|
103
|
+
return row?.value ?? null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
setValue(key: string, value: string): void {
|
|
107
|
+
this.db
|
|
108
|
+
.prepare("INSERT OR REPLACE INTO scheduler_kv (key, value, updated_at) VALUES (?, ?, ?)")
|
|
109
|
+
.run(key, value, new Date().toISOString());
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
deleteValue(key: string): void {
|
|
113
|
+
this.db.prepare("DELETE FROM scheduler_kv WHERE key = ?").run(key);
|
|
114
|
+
}
|
|
115
|
+
|
|
95
116
|
enqueue(taskName: string, payload: unknown, opts: EnqueueOptions = {}): number {
|
|
96
117
|
const now = opts.now ?? new Date();
|
|
97
118
|
const availableAt = opts.availableAt ?? now;
|