pi-para 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +0 -9
- package/CHANGELOG.md +22 -0
- package/CONFIGURATION.md +5 -23
- package/README.md +7 -31
- package/SETUP.md +5 -3
- package/STRUCTURE.md +0 -9
- package/dist/atomic-write.d.ts +2 -0
- package/dist/atomic-write.d.ts.map +1 -0
- package/dist/atomic-write.js +18 -0
- package/dist/atomic-write.js.map +1 -0
- package/dist/capture.d.ts +34 -0
- package/dist/capture.d.ts.map +1 -0
- package/dist/capture.js +461 -0
- package/dist/capture.js.map +1 -0
- package/dist/cli-legacy.d.ts +4 -0
- package/dist/cli-legacy.d.ts.map +1 -0
- package/dist/cli-legacy.js +41 -0
- package/dist/cli-legacy.js.map +1 -0
- package/dist/cli.d.ts +8 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +427 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands.d.ts +15 -0
- package/dist/commands.d.ts.map +1 -0
- package/dist/commands.js +791 -0
- package/dist/commands.js.map +1 -0
- package/dist/config.d.ts +94 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +228 -0
- package/dist/config.js.map +1 -0
- package/dist/context.d.ts +63 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +311 -0
- package/dist/context.js.map +1 -0
- package/dist/credentials.d.ts +27 -0
- package/dist/credentials.d.ts.map +1 -0
- package/dist/credentials.js +85 -0
- package/dist/credentials.js.map +1 -0
- package/dist/daemon.d.ts +49 -0
- package/dist/daemon.d.ts.map +1 -0
- package/dist/daemon.js +226 -0
- package/dist/daemon.js.map +1 -0
- package/dist/doctor.d.ts +20 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +197 -0
- package/dist/doctor.js.map +1 -0
- package/dist/frontmatter.d.ts +59 -0
- package/dist/frontmatter.d.ts.map +1 -0
- package/dist/frontmatter.js +170 -0
- package/dist/frontmatter.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +357 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest.d.ts +88 -0
- package/dist/ingest.d.ts.map +1 -0
- package/dist/ingest.js +105 -0
- package/dist/ingest.js.map +1 -0
- package/dist/link-discovery.d.ts +40 -0
- package/dist/link-discovery.d.ts.map +1 -0
- package/dist/link-discovery.js +122 -0
- package/dist/link-discovery.js.map +1 -0
- package/dist/link-utils.d.ts +30 -0
- package/dist/link-utils.d.ts.map +1 -0
- package/dist/link-utils.js +145 -0
- package/dist/link-utils.js.map +1 -0
- package/dist/lint.d.ts +39 -0
- package/dist/lint.d.ts.map +1 -0
- package/dist/lint.js +827 -0
- package/dist/lint.js.map +1 -0
- package/dist/maintainer.d.ts +24 -0
- package/dist/maintainer.d.ts.map +1 -0
- package/dist/maintainer.js +358 -0
- package/dist/maintainer.js.map +1 -0
- package/dist/model-resolver.d.ts +30 -0
- package/dist/model-resolver.d.ts.map +1 -0
- package/dist/model-resolver.js +71 -0
- package/dist/model-resolver.js.map +1 -0
- package/dist/paths.d.ts +20 -0
- package/dist/paths.d.ts.map +1 -0
- package/dist/paths.js +33 -0
- package/dist/paths.js.map +1 -0
- package/dist/processor.d.ts +14 -0
- package/dist/processor.d.ts.map +1 -0
- package/dist/processor.js +361 -0
- package/dist/processor.js.map +1 -0
- package/dist/qmd-providers.d.ts +12 -0
- package/dist/qmd-providers.d.ts.map +1 -0
- package/dist/qmd-providers.js +45 -0
- package/dist/qmd-providers.js.map +1 -0
- package/dist/query.d.ts +69 -0
- package/dist/query.d.ts.map +1 -0
- package/dist/query.js +120 -0
- package/dist/query.js.map +1 -0
- package/dist/raw.d.ts +57 -0
- package/dist/raw.d.ts.map +1 -0
- package/dist/raw.js +219 -0
- package/dist/raw.js.map +1 -0
- package/dist/redact.d.ts +21 -0
- package/dist/redact.d.ts.map +1 -0
- package/dist/redact.js +143 -0
- package/dist/redact.js.map +1 -0
- package/dist/repair.d.ts +11 -0
- package/dist/repair.d.ts.map +1 -0
- package/dist/repair.js +41 -0
- package/dist/repair.js.map +1 -0
- package/dist/scheduler/controls.d.ts +35 -0
- package/dist/scheduler/controls.d.ts.map +1 -0
- package/dist/scheduler/controls.js +98 -0
- package/dist/scheduler/controls.js.map +1 -0
- package/dist/scheduler/index.d.ts +70 -0
- package/dist/scheduler/index.d.ts.map +1 -0
- package/dist/scheduler/index.js +245 -0
- package/dist/scheduler/index.js.map +1 -0
- package/dist/scheduler/leases.d.ts +22 -0
- package/dist/scheduler/leases.d.ts.map +1 -0
- package/dist/scheduler/leases.js +74 -0
- package/dist/scheduler/leases.js.map +1 -0
- package/dist/scheduler/session-capture.d.ts +33 -0
- package/dist/scheduler/session-capture.d.ts.map +1 -0
- package/dist/scheduler/session-capture.js +108 -0
- package/dist/scheduler/session-capture.js.map +1 -0
- package/dist/scheduler/state.d.ts +56 -0
- package/dist/scheduler/state.d.ts.map +1 -0
- package/dist/scheduler/state.js +210 -0
- package/dist/scheduler/state.js.map +1 -0
- package/dist/scope.d.ts +54 -0
- package/dist/scope.d.ts.map +1 -0
- package/dist/scope.js +161 -0
- package/dist/scope.js.map +1 -0
- package/dist/scripts/bulk-link-fix.d.ts +9 -0
- package/dist/scripts/bulk-link-fix.d.ts.map +1 -0
- package/dist/scripts/bulk-link-fix.js +106 -0
- package/dist/scripts/bulk-link-fix.js.map +1 -0
- package/dist/session-tools.d.ts +33 -0
- package/dist/session-tools.d.ts.map +1 -0
- package/dist/session-tools.js +211 -0
- package/dist/session-tools.js.map +1 -0
- package/dist/settings.d.ts +37 -0
- package/dist/settings.d.ts.map +1 -0
- package/dist/settings.js +176 -0
- package/dist/settings.js.map +1 -0
- package/dist/setup.d.ts +29 -0
- package/dist/setup.d.ts.map +1 -0
- package/dist/setup.js +90 -0
- package/dist/setup.js.map +1 -0
- package/dist/state.d.ts +54 -0
- package/dist/state.d.ts.map +1 -0
- package/dist/state.js +176 -0
- package/dist/state.js.map +1 -0
- package/dist/status.d.ts +22 -0
- package/dist/status.d.ts.map +1 -0
- package/dist/status.js +93 -0
- package/dist/status.js.map +1 -0
- package/dist/store.d.ts +71 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +282 -0
- package/dist/store.js.map +1 -0
- package/dist/summarize.d.ts +60 -0
- package/dist/summarize.d.ts.map +1 -0
- package/dist/summarize.js +266 -0
- package/dist/summarize.js.map +1 -0
- package/dist/tag-registry.d.ts +56 -0
- package/dist/tag-registry.d.ts.map +1 -0
- package/dist/tag-registry.js +146 -0
- package/dist/tag-registry.js.map +1 -0
- package/dist/templates/prompts.d.ts +51 -0
- package/dist/templates/prompts.d.ts.map +1 -0
- package/dist/templates/prompts.js +203 -0
- package/dist/templates/prompts.js.map +1 -0
- package/dist/tools.d.ts +26 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +1198 -0
- package/dist/tools.js.map +1 -0
- package/dist/watcher.d.ts +25 -0
- package/dist/watcher.d.ts.map +1 -0
- package/dist/watcher.js +84 -0
- package/dist/watcher.js.map +1 -0
- package/dist/webui/client/dist/index.html +1885 -0
- package/dist/webui/server.d.ts +31 -0
- package/dist/webui/server.d.ts.map +1 -0
- package/dist/webui/server.js +566 -0
- package/dist/webui/server.js.map +1 -0
- package/dist/wiki-tool-guidance.d.ts +6 -0
- package/dist/wiki-tool-guidance.d.ts.map +1 -0
- package/dist/wiki-tool-guidance.js +70 -0
- package/dist/wiki-tool-guidance.js.map +1 -0
- package/dist/wiki.d.ts +108 -0
- package/dist/wiki.d.ts.map +1 -0
- package/dist/wiki.js +497 -0
- package/dist/wiki.js.map +1 -0
- package/package.json +2 -1
- package/scripts/build-assets.mjs +1 -13
- package/skills/setup/SKILL.md +1 -1
- package/src/cli.ts +0 -70
- package/src/commands.ts +78 -3
- package/src/config.ts +0 -25
- package/src/doctor.ts +21 -1
- package/src/index.ts +16 -38
- package/src/scheduler/index.ts +64 -0
- package/src/settings.ts +88 -0
- package/src/setup.ts +1 -1
- package/src/templates/prompts.ts +0 -56
- package/scripts/gepa/dataset.py +0 -192
- package/scripts/gepa/lm_providers.py +0 -472
- package/scripts/gepa/metric.py +0 -279
- package/scripts/gepa/optimize.py +0 -367
- package/scripts/gepa/program.py +0 -99
- package/scripts/gepa/pyproject.toml +0 -19
- package/scripts/gepa/run-all.sh +0 -53
- package/scripts/gepa/wiki_reader.py +0 -113
- package/src/gepa/index.ts +0 -287
- package/src/gepa/optimized/capture-prompt.txt +0 -65
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type QueueItem, type QueueStatus, type SchedulerHistoryItem } from "./state.js";
|
|
2
|
+
export interface QueueCaptureRecentOptions {
|
|
3
|
+
wikiDir: string;
|
|
4
|
+
dbPath?: string;
|
|
5
|
+
hours?: number;
|
|
6
|
+
now?: Date;
|
|
7
|
+
}
|
|
8
|
+
export interface QueueCaptureRecentResult {
|
|
9
|
+
scanned: number;
|
|
10
|
+
eligible: number;
|
|
11
|
+
queuedIds: number[];
|
|
12
|
+
}
|
|
13
|
+
export declare function queueCaptureRecent(options: QueueCaptureRecentOptions): Promise<QueueCaptureRecentResult>;
|
|
14
|
+
export declare function listSchedulerTasks(wikiDir: string, opts?: {
|
|
15
|
+
dbPath?: string;
|
|
16
|
+
status?: QueueStatus;
|
|
17
|
+
}): QueueItem[];
|
|
18
|
+
export declare function retryFailedSchedulerTasks(wikiDir: string, opts?: {
|
|
19
|
+
dbPath?: string;
|
|
20
|
+
taskName?: string;
|
|
21
|
+
now?: Date;
|
|
22
|
+
}): number;
|
|
23
|
+
export declare function getSchedulerTask(wikiDir: string, id: number, opts?: {
|
|
24
|
+
dbPath?: string;
|
|
25
|
+
}): QueueItem | null;
|
|
26
|
+
export declare function listSchedulerHistory(wikiDir: string, opts?: {
|
|
27
|
+
dbPath?: string;
|
|
28
|
+
taskName?: string;
|
|
29
|
+
limit?: number;
|
|
30
|
+
}): SchedulerHistoryItem[];
|
|
31
|
+
export declare function formatQueueItems(items: QueueItem[]): string;
|
|
32
|
+
export declare function formatQueueCaptureRecentResult(result: QueueCaptureRecentResult): string;
|
|
33
|
+
export declare function formatQueueItem(item: QueueItem | null): string;
|
|
34
|
+
export declare function formatSchedulerHistory(items: SchedulerHistoryItem[]): string;
|
|
35
|
+
//# sourceMappingURL=controls.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"controls.d.ts","sourceRoot":"","sources":["../../src/scheduler/controls.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoB,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,KAAK,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAG3G,MAAM,WAAW,yBAAyB;IACxC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ;AAED,MAAM,WAAW,wBAAwB;IACvC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED,wBAAsB,kBAAkB,CAAC,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAoB9G;AAED,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GACnD,SAAS,EAAE,CAOb;AAED,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,IAAI,CAAA;CAAO,GAC5D,MAAM,CAOR;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,EAAE,EAAE,MAAM,EACV,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,SAAS,GAAG,IAAI,CAOlB;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GAChE,oBAAoB,EAAE,CAOxB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,CAK3D;AAED,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,wBAAwB,GAAG,MAAM,CAMvF;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,GAAG,MAAM,CAU9D;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,oBAAoB,EAAE,GAAG,MAAM,CAO5E"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { getParaPaths } from "../paths.js";
|
|
2
|
+
import { StateDB } from "../state.js";
|
|
3
|
+
import { WikiScheduler } from "./index.js";
|
|
4
|
+
import { SchedulerStateDB } from "./state.js";
|
|
5
|
+
import { enqueueCompletedSessions, readCompletedSessionRegistry } from "./session-capture.js";
|
|
6
|
+
export async function queueCaptureRecent(options) {
|
|
7
|
+
const now = options.now ?? new Date();
|
|
8
|
+
const hours = options.hours ?? 24;
|
|
9
|
+
const cutoff = now.getTime() - hours * 60 * 60 * 1000;
|
|
10
|
+
const entries = await readCompletedSessionRegistry(options.wikiDir);
|
|
11
|
+
const eligible = entries.filter((entry) => {
|
|
12
|
+
const ts = Date.parse(entry.timestamp);
|
|
13
|
+
return Number.isFinite(ts) && ts >= cutoff;
|
|
14
|
+
});
|
|
15
|
+
const dbPath = options.dbPath ?? getParaPaths({ wikiDir: options.wikiDir }).schedulerDbPath;
|
|
16
|
+
const scheduler = new WikiScheduler({ wikiDir: options.wikiDir, dbPath, enabled: true });
|
|
17
|
+
const stateDb = new StateDB(options.wikiDir);
|
|
18
|
+
try {
|
|
19
|
+
const queuedIds = enqueueCompletedSessions(scheduler, eligible, { stateDb });
|
|
20
|
+
return { scanned: entries.length, eligible: eligible.length, queuedIds };
|
|
21
|
+
}
|
|
22
|
+
finally {
|
|
23
|
+
stateDb.close();
|
|
24
|
+
scheduler.stop();
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
export function listSchedulerTasks(wikiDir, opts = {}) {
|
|
28
|
+
const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
|
|
29
|
+
try {
|
|
30
|
+
return db.list(opts.status);
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
db.close();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export function retryFailedSchedulerTasks(wikiDir, opts = {}) {
|
|
37
|
+
const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
|
|
38
|
+
try {
|
|
39
|
+
return db.retryFailed(opts.taskName, opts.now);
|
|
40
|
+
}
|
|
41
|
+
finally {
|
|
42
|
+
db.close();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
export function getSchedulerTask(wikiDir, id, opts = {}) {
|
|
46
|
+
const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
|
|
47
|
+
try {
|
|
48
|
+
return db.get(id);
|
|
49
|
+
}
|
|
50
|
+
finally {
|
|
51
|
+
db.close();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export function listSchedulerHistory(wikiDir, opts = {}) {
|
|
55
|
+
const db = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath);
|
|
56
|
+
try {
|
|
57
|
+
return db.listHistory({ taskName: opts.taskName, limit: opts.limit });
|
|
58
|
+
}
|
|
59
|
+
finally {
|
|
60
|
+
db.close();
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export function formatQueueItems(items) {
|
|
64
|
+
if (items.length === 0)
|
|
65
|
+
return "No scheduler tasks queued.";
|
|
66
|
+
return items
|
|
67
|
+
.map((item) => `${item.id} ${item.status} ${item.taskName} attempts=${item.attempts} dedupe=${item.dedupeKey ?? "-"}`)
|
|
68
|
+
.join("\n");
|
|
69
|
+
}
|
|
70
|
+
export function formatQueueCaptureRecentResult(result) {
|
|
71
|
+
return [
|
|
72
|
+
`Scanned ${result.scanned} completed-session entr${result.scanned === 1 ? "y" : "ies"}.`,
|
|
73
|
+
`Eligible within window: ${result.eligible}.`,
|
|
74
|
+
`Queued ${result.queuedIds.length} capture task${result.queuedIds.length === 1 ? "" : "s"}.`,
|
|
75
|
+
].join("\n");
|
|
76
|
+
}
|
|
77
|
+
export function formatQueueItem(item) {
|
|
78
|
+
if (!item)
|
|
79
|
+
return "Scheduler task not found.";
|
|
80
|
+
return [
|
|
81
|
+
`Task ${item.id}: ${item.taskName}`,
|
|
82
|
+
`Status: ${item.status}`,
|
|
83
|
+
`Attempts: ${item.attempts}`,
|
|
84
|
+
`Available: ${item.availableAt}`,
|
|
85
|
+
`Dedupe: ${item.dedupeKey ?? "-"}`,
|
|
86
|
+
`Payload: ${JSON.stringify(item.payload, null, 2)}`,
|
|
87
|
+
].join("\n");
|
|
88
|
+
}
|
|
89
|
+
export function formatSchedulerHistory(items) {
|
|
90
|
+
if (items.length === 0)
|
|
91
|
+
return "No scheduler history.";
|
|
92
|
+
return items.map((item) => {
|
|
93
|
+
const duration = item.durationMs === null ? "-" : `${item.durationMs}ms`;
|
|
94
|
+
const error = item.error ? ` error=${item.error.slice(0, 80)}` : "";
|
|
95
|
+
return `${item.id} ${item.status} ${item.taskName} duration=${duration}${error}`;
|
|
96
|
+
}).join("\n");
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=controls.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"controls.js","sourceRoot":"","sources":["../../src/scheduler/controls.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAA+D,MAAM,YAAY,CAAC;AAC3G,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,sBAAsB,CAAC;AAe9F,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAAkC;IACzE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IACtD,MAAM,OAAO,GAAG,MAAM,4BAA4B,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE;QACxC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACvC,OAAO,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC;IAC5F,MAAM,SAAS,GAAG,IAAI,aAAa,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IACzF,MAAM,OAAO,GAAG,IAAI,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,wBAAwB,CAAC,SAAS,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC3E,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,KAAK,EAAE,CAAC;QAChB,SAAS,CAAC,IAAI,EAAE,CAAC;IACnB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAAe,EACf,OAAkD,EAAE;IAEpD,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;IAC1F,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,yBAAyB,CACvC,OAAe,EACf,OAA2D,EAAE;IAE7D,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;IAC1F,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,EAAU,EACV,OAA4B,EAAE;IAE9B,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;IAC1F,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,OAAe,EACf,OAA+D,EAAE;IAEjE,MAAM,EAAE,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;IAC1F,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACxE,CAAC;YAAS,CAAC;QACT,EAAE,CAAC,KAAK,EAAE,CAAC;IACb,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAkB;IACjD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,4BAA4B,CAAC;IAC5D,OAAO,KAAK;SACT,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,aAAa,IAAI,CAAC,QAAQ,WAAW,IAAI,CAAC,SAAS,IAAI,GAAG,EAAE,CAAC;SACrH,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,MAAgC;IAC7E,OAAO;QACL,WAAW,MAAM,CAAC,OAAO,0BAA0B,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;QACxF,2BAA2B,MAAM,CAAC,QAAQ,GAAG;QAC7C,UAAU,MAAM,CAAC,SAAS,CAAC,MAAM,gBAAgB,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG;KAC7F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAsB;IACpD,IAAI,CAAC,IAAI;QAAE,OAAO,2BAA2B,CAAC;IAC9C,OAAO;QACL,QAAQ,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ,EAAE;QACnC,WAAW,IAAI,CAAC,MAAM,EAAE;QACxB,aAAa,IAAI,CAAC,QAAQ,EAAE;QAC5B,cAAc,IAAI,CAAC,WAAW,EAAE;QAChC,WAAW,IAAI,CAAC,SAAS,IAAI,GAAG,EAAE;QAClC,YAAY,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;KACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA6B;IAClE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,uBAAuB,CAAC;IACvD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC;QACzE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,OAAO,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,aAAa,QAAQ,GAAG,KAAK,EAAE,CAAC;IACnF,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { QMDStore } from "qmd-engine";
|
|
2
|
+
import { SchedulerStateDB, type QueueItem } from "./state.js";
|
|
3
|
+
export type SchedulerTaskHandler = (item: QueueItem, scheduler: WikiScheduler) => Promise<void>;
|
|
4
|
+
export interface WikiSchedulerOptions {
|
|
5
|
+
wikiDir: string;
|
|
6
|
+
dbPath?: string;
|
|
7
|
+
intervalMs?: number;
|
|
8
|
+
enabled?: boolean;
|
|
9
|
+
storeProvider?: () => QMDStore | null;
|
|
10
|
+
markDirty?: () => void;
|
|
11
|
+
handlers?: Record<string, SchedulerTaskHandler>;
|
|
12
|
+
/**
|
|
13
|
+
* When true, wiki-maintenance chains a deduplicated `qmd-embed` task after
|
|
14
|
+
* reindexing so new/changed pages get vector embeddings in the background.
|
|
15
|
+
* Default false so standalone/test schedulers keep BM25-only behavior.
|
|
16
|
+
*/
|
|
17
|
+
embedEnabled?: boolean;
|
|
18
|
+
}
|
|
19
|
+
export declare class WikiScheduler {
|
|
20
|
+
readonly wikiDir: string;
|
|
21
|
+
readonly holderId: string;
|
|
22
|
+
readonly state: SchedulerStateDB;
|
|
23
|
+
private timer;
|
|
24
|
+
private running;
|
|
25
|
+
private stopped;
|
|
26
|
+
private closed;
|
|
27
|
+
private readonly intervalMs;
|
|
28
|
+
private readonly enabled;
|
|
29
|
+
private readonly storeProvider;
|
|
30
|
+
private readonly markDirty;
|
|
31
|
+
private handlers;
|
|
32
|
+
embedEnabled: boolean;
|
|
33
|
+
constructor(opts: WikiSchedulerOptions);
|
|
34
|
+
start(): void;
|
|
35
|
+
stop(): void;
|
|
36
|
+
private closeState;
|
|
37
|
+
enqueue(taskName: string, payload?: unknown, opts?: {
|
|
38
|
+
priority?: number;
|
|
39
|
+
dedupeKey?: string;
|
|
40
|
+
availableAt?: Date;
|
|
41
|
+
}): number;
|
|
42
|
+
registerHandlers(handlers: Record<string, SchedulerTaskHandler>): void;
|
|
43
|
+
tick(): Promise<number>;
|
|
44
|
+
private runItem;
|
|
45
|
+
private runWikiMaintenance;
|
|
46
|
+
/**
|
|
47
|
+
* Generate vector embeddings for documents QMD reports as pending.
|
|
48
|
+
* Skips fast when the backlog is empty. Throws when the embedding pass
|
|
49
|
+
* reports errors so the scheduler records the failure and retries with
|
|
50
|
+
* backoff — BM25 search keeps working regardless.
|
|
51
|
+
*/
|
|
52
|
+
private runQmdEmbed;
|
|
53
|
+
}
|
|
54
|
+
export declare function getWikiScheduler(opts: WikiSchedulerOptions): WikiScheduler;
|
|
55
|
+
export declare function startWikiScheduler(opts: WikiSchedulerOptions): WikiScheduler;
|
|
56
|
+
export declare function stopWikiScheduler(wikiDir: string, dbPath?: string): void;
|
|
57
|
+
export declare function enqueueWikiMaintenance(wikiDir: string, store: QMDStore, markDirty: () => void, opts?: {
|
|
58
|
+
delayMs?: number;
|
|
59
|
+
dbPath?: string;
|
|
60
|
+
}): number | null;
|
|
61
|
+
/**
|
|
62
|
+
* Enqueue a deduplicated background embedding pass (e.g. at session startup
|
|
63
|
+
* to drain a pre-existing needsEmbedding backlog). No-op scheduling cost when
|
|
64
|
+
* the backlog is empty — the task exits after one getStatus() call.
|
|
65
|
+
*/
|
|
66
|
+
export declare function enqueueQmdEmbed(wikiDir: string, store: QMDStore, opts?: {
|
|
67
|
+
dbPath?: string;
|
|
68
|
+
}): number;
|
|
69
|
+
export declare function resetSchedulersForTests(): void;
|
|
70
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scheduler/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAM3C,OAAO,EAAE,gBAAgB,EAAE,KAAK,SAAS,EAAE,MAAM,YAAY,CAAC;AAE9D,MAAM,MAAM,oBAAoB,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAEhG,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,aAAa,CAAC,EAAE,MAAM,QAAQ,GAAG,IAAI,CAAC;IACtC,SAAS,CAAC,EAAE,MAAM,IAAI,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IAChD;;;;OAIG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED,qBAAa,aAAa;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC;IACjC,OAAO,CAAC,KAAK,CAA+C;IAC5D,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAwB;IACtD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAa;IACvC,OAAO,CAAC,QAAQ,CAAuC;IACvD,YAAY,EAAE,OAAO,CAAC;gBAEV,IAAI,EAAE,oBAAoB;IAgBtC,KAAK,IAAI,IAAI;IASb,IAAI,IAAI,IAAI;IAOZ,OAAO,CAAC,UAAU;IAMlB,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,GAAE,OAAY,EAAE,IAAI,GAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,IAAI,CAAA;KAAO,GAAG,MAAM;IAKlI,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,GAAG,IAAI;IAIhE,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;YAmBf,OAAO;YAqBP,kBAAkB;IAoBhC;;;;;OAKG;YACW,WAAW;CAqB1B;AAKD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,oBAAoB,GAAG,aAAa,CAU1E;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,oBAAoB,GAAG,aAAa,CAI5E;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAWxE;AAED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,QAAQ,EACf,SAAS,EAAE,MAAM,IAAI,EACrB,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/C,MAAM,GAAG,IAAI,CA2Bf;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,QAAQ,EACf,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAC7B,MAAM,CAUR;AAED,wBAAgB,uBAAuB,IAAI,IAAI,CAO9C"}
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { gitCommit, rebuildIndex } from "../wiki.js";
|
|
3
|
+
import { reindex } from "../store.js";
|
|
4
|
+
import { getParaPaths } from "../paths.js";
|
|
5
|
+
import { acquireLease, releaseLease } from "./leases.js";
|
|
6
|
+
import { SchedulerStateDB } from "./state.js";
|
|
7
|
+
export class WikiScheduler {
|
|
8
|
+
wikiDir;
|
|
9
|
+
holderId;
|
|
10
|
+
state;
|
|
11
|
+
timer = null;
|
|
12
|
+
running = false;
|
|
13
|
+
stopped = false;
|
|
14
|
+
closed = false;
|
|
15
|
+
intervalMs;
|
|
16
|
+
enabled;
|
|
17
|
+
storeProvider;
|
|
18
|
+
markDirty;
|
|
19
|
+
handlers;
|
|
20
|
+
embedEnabled;
|
|
21
|
+
constructor(opts) {
|
|
22
|
+
this.wikiDir = opts.wikiDir;
|
|
23
|
+
this.holderId = `scheduler:${process.pid}:${randomUUID()}`;
|
|
24
|
+
this.state = new SchedulerStateDB(opts.dbPath ?? getParaPaths({ wikiDir: opts.wikiDir }).schedulerDbPath);
|
|
25
|
+
this.intervalMs = opts.intervalMs ?? 15 * 60_000;
|
|
26
|
+
this.enabled = opts.enabled ?? true;
|
|
27
|
+
this.storeProvider = opts.storeProvider ?? (() => null);
|
|
28
|
+
this.markDirty = opts.markDirty ?? (() => { });
|
|
29
|
+
this.embedEnabled = opts.embedEnabled ?? false;
|
|
30
|
+
this.handlers = {
|
|
31
|
+
"wiki-maintenance": async () => this.runWikiMaintenance(),
|
|
32
|
+
"qmd-embed": async () => this.runQmdEmbed(),
|
|
33
|
+
...(opts.handlers ?? {}),
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
start() {
|
|
37
|
+
if (!this.enabled || this.timer || this.closed)
|
|
38
|
+
return;
|
|
39
|
+
this.stopped = false;
|
|
40
|
+
this.state.requeueRunning();
|
|
41
|
+
void this.tick();
|
|
42
|
+
this.timer = setInterval(() => void this.tick(), this.intervalMs);
|
|
43
|
+
this.timer.unref?.();
|
|
44
|
+
}
|
|
45
|
+
stop() {
|
|
46
|
+
this.stopped = true;
|
|
47
|
+
if (this.timer)
|
|
48
|
+
clearInterval(this.timer);
|
|
49
|
+
this.timer = null;
|
|
50
|
+
if (!this.running)
|
|
51
|
+
this.closeState();
|
|
52
|
+
}
|
|
53
|
+
closeState() {
|
|
54
|
+
if (this.closed)
|
|
55
|
+
return;
|
|
56
|
+
this.closed = true;
|
|
57
|
+
this.state.close();
|
|
58
|
+
}
|
|
59
|
+
enqueue(taskName, payload = {}, opts = {}) {
|
|
60
|
+
if (this.closed)
|
|
61
|
+
throw new Error("Wiki scheduler is stopped.");
|
|
62
|
+
return this.state.enqueue(taskName, payload, opts);
|
|
63
|
+
}
|
|
64
|
+
registerHandlers(handlers) {
|
|
65
|
+
this.handlers = { ...this.handlers, ...handlers };
|
|
66
|
+
}
|
|
67
|
+
async tick() {
|
|
68
|
+
if (!this.enabled || this.running || this.stopped || this.closed)
|
|
69
|
+
return 0;
|
|
70
|
+
this.running = true;
|
|
71
|
+
let processed = 0;
|
|
72
|
+
try {
|
|
73
|
+
for (;;) {
|
|
74
|
+
if (this.stopped)
|
|
75
|
+
break;
|
|
76
|
+
const item = this.state.claimNext();
|
|
77
|
+
if (!item)
|
|
78
|
+
break;
|
|
79
|
+
await this.runItem(item);
|
|
80
|
+
processed++;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
finally {
|
|
84
|
+
this.running = false;
|
|
85
|
+
if (this.stopped)
|
|
86
|
+
this.closeState();
|
|
87
|
+
}
|
|
88
|
+
return processed;
|
|
89
|
+
}
|
|
90
|
+
async runItem(item) {
|
|
91
|
+
const handler = this.handlers[item.taskName];
|
|
92
|
+
const started = new Date();
|
|
93
|
+
if (!handler) {
|
|
94
|
+
this.state.fail(item.id, `Unknown scheduler task: ${item.taskName}`, { now: started, maxAttempts: 1 });
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
await handler(item, this);
|
|
99
|
+
if (this.stopped || this.closed)
|
|
100
|
+
return;
|
|
101
|
+
const finished = new Date();
|
|
102
|
+
this.state.complete(item.id, finished);
|
|
103
|
+
this.state.recordHistory(item.taskName, item.payload, "done", started, finished, finished.getTime() - started.getTime(), null);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
if (this.stopped || this.closed)
|
|
107
|
+
return;
|
|
108
|
+
const retryAt = new Date(Date.now() + Math.min(60_000, 1_000 * 2 ** item.attempts));
|
|
109
|
+
this.state.fail(item.id, err instanceof Error ? err.message : String(err), { retryAt, now: new Date() });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
async runWikiMaintenance() {
|
|
113
|
+
const leaseKey = `wiki-maintenance:${this.wikiDir}`;
|
|
114
|
+
if (!acquireLease(this.state.db, leaseKey, this.holderId, { ttlMs: 5 * 60_000 }))
|
|
115
|
+
return;
|
|
116
|
+
try {
|
|
117
|
+
await rebuildIndex(this.wikiDir);
|
|
118
|
+
const store = this.storeProvider();
|
|
119
|
+
if (store)
|
|
120
|
+
await reindex(store);
|
|
121
|
+
this.markDirty();
|
|
122
|
+
await gitCommit(this.wikiDir, "wiki: rebuild index and refresh search");
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
releaseLease(this.state.db, leaseKey, this.holderId);
|
|
126
|
+
}
|
|
127
|
+
// Chain background embedding so changed pages become vector-searchable.
|
|
128
|
+
// Runs as its own task (deduplicated) so reindex latency stays low and
|
|
129
|
+
// embedding failures are recorded/retried instead of breaking maintenance.
|
|
130
|
+
if (this.embedEnabled) {
|
|
131
|
+
this.enqueue("qmd-embed", {}, { dedupeKey: "qmd-embed", priority: 5 });
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Generate vector embeddings for documents QMD reports as pending.
|
|
136
|
+
* Skips fast when the backlog is empty. Throws when the embedding pass
|
|
137
|
+
* reports errors so the scheduler records the failure and retries with
|
|
138
|
+
* backoff — BM25 search keeps working regardless.
|
|
139
|
+
*/
|
|
140
|
+
async runQmdEmbed() {
|
|
141
|
+
const store = this.storeProvider();
|
|
142
|
+
if (!store)
|
|
143
|
+
return;
|
|
144
|
+
const status = await store.getStatus();
|
|
145
|
+
const pending = status.needsEmbedding ?? 0;
|
|
146
|
+
if (pending === 0)
|
|
147
|
+
return;
|
|
148
|
+
const leaseKey = `qmd-embed:${this.wikiDir}`;
|
|
149
|
+
if (!acquireLease(this.state.db, leaseKey, this.holderId, { ttlMs: 15 * 60_000 }))
|
|
150
|
+
return;
|
|
151
|
+
try {
|
|
152
|
+
const result = await store.embed();
|
|
153
|
+
if (result.errors > 0) {
|
|
154
|
+
throw new Error(`qmd embed finished with ${result.errors} error(s) (${result.chunksEmbedded} chunk(s) embedded, ${pending} doc(s) were pending)`);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
finally {
|
|
158
|
+
releaseLease(this.state.db, leaseKey, this.holderId);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const schedulers = new Map();
|
|
163
|
+
const debounceTimers = new Map();
|
|
164
|
+
export function getWikiScheduler(opts) {
|
|
165
|
+
const key = opts.dbPath ?? getParaPaths({ wikiDir: opts.wikiDir }).schedulerDbPath;
|
|
166
|
+
const existing = schedulers.get(key);
|
|
167
|
+
if (existing) {
|
|
168
|
+
if (opts.handlers)
|
|
169
|
+
existing.registerHandlers(opts.handlers);
|
|
170
|
+
return existing;
|
|
171
|
+
}
|
|
172
|
+
const scheduler = new WikiScheduler(opts);
|
|
173
|
+
schedulers.set(key, scheduler);
|
|
174
|
+
return scheduler;
|
|
175
|
+
}
|
|
176
|
+
export function startWikiScheduler(opts) {
|
|
177
|
+
const scheduler = getWikiScheduler(opts);
|
|
178
|
+
scheduler.start();
|
|
179
|
+
return scheduler;
|
|
180
|
+
}
|
|
181
|
+
export function stopWikiScheduler(wikiDir, dbPath) {
|
|
182
|
+
const key = dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath;
|
|
183
|
+
const timer = debounceTimers.get(key);
|
|
184
|
+
if (timer) {
|
|
185
|
+
clearTimeout(timer);
|
|
186
|
+
debounceTimers.delete(key);
|
|
187
|
+
}
|
|
188
|
+
const scheduler = schedulers.get(key);
|
|
189
|
+
if (!scheduler)
|
|
190
|
+
return;
|
|
191
|
+
scheduler.stop();
|
|
192
|
+
schedulers.delete(key);
|
|
193
|
+
}
|
|
194
|
+
export function enqueueWikiMaintenance(wikiDir, store, markDirty, opts = {}) {
|
|
195
|
+
const key = opts.dbPath ?? getParaPaths({ wikiDir }).schedulerDbPath;
|
|
196
|
+
const existingTimer = debounceTimers.get(key);
|
|
197
|
+
if (existingTimer)
|
|
198
|
+
clearTimeout(existingTimer);
|
|
199
|
+
const scheduler = getWikiScheduler({
|
|
200
|
+
wikiDir,
|
|
201
|
+
dbPath: opts.dbPath,
|
|
202
|
+
storeProvider: () => store,
|
|
203
|
+
markDirty,
|
|
204
|
+
});
|
|
205
|
+
const delayMs = opts.delayMs ?? 2_000;
|
|
206
|
+
if (delayMs <= 0) {
|
|
207
|
+
const id = scheduler.enqueue("wiki-maintenance", {}, { dedupeKey: "wiki-maintenance", priority: 10 });
|
|
208
|
+
void scheduler.tick();
|
|
209
|
+
return id;
|
|
210
|
+
}
|
|
211
|
+
const timer = setTimeout(() => {
|
|
212
|
+
debounceTimers.delete(key);
|
|
213
|
+
scheduler.enqueue("wiki-maintenance", {}, { dedupeKey: "wiki-maintenance", priority: 10 });
|
|
214
|
+
void scheduler.tick();
|
|
215
|
+
}, delayMs);
|
|
216
|
+
timer.unref?.();
|
|
217
|
+
debounceTimers.set(key, timer);
|
|
218
|
+
return null;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Enqueue a deduplicated background embedding pass (e.g. at session startup
|
|
222
|
+
* to drain a pre-existing needsEmbedding backlog). No-op scheduling cost when
|
|
223
|
+
* the backlog is empty — the task exits after one getStatus() call.
|
|
224
|
+
*/
|
|
225
|
+
export function enqueueQmdEmbed(wikiDir, store, opts = {}) {
|
|
226
|
+
const scheduler = getWikiScheduler({
|
|
227
|
+
wikiDir,
|
|
228
|
+
dbPath: opts.dbPath,
|
|
229
|
+
storeProvider: () => store,
|
|
230
|
+
});
|
|
231
|
+
scheduler.embedEnabled = true;
|
|
232
|
+
const id = scheduler.enqueue("qmd-embed", {}, { dedupeKey: "qmd-embed", priority: 5 });
|
|
233
|
+
void scheduler.tick();
|
|
234
|
+
return id;
|
|
235
|
+
}
|
|
236
|
+
export function resetSchedulersForTests() {
|
|
237
|
+
for (const timer of debounceTimers.values())
|
|
238
|
+
clearTimeout(timer);
|
|
239
|
+
debounceTimers.clear();
|
|
240
|
+
for (const [key, scheduler] of schedulers) {
|
|
241
|
+
scheduler.stop();
|
|
242
|
+
schedulers.delete(key);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/scheduler/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAkB,MAAM,YAAY,CAAC;AAoB9D,MAAM,OAAO,aAAa;IACf,OAAO,CAAS;IAChB,QAAQ,CAAS;IACjB,KAAK,CAAmB;IACzB,KAAK,GAA0C,IAAI,CAAC;IACpD,OAAO,GAAG,KAAK,CAAC;IAChB,OAAO,GAAG,KAAK,CAAC;IAChB,MAAM,GAAG,KAAK,CAAC;IACN,UAAU,CAAS;IACnB,OAAO,CAAU;IACjB,aAAa,CAAwB;IACrC,SAAS,CAAa;IAC/B,QAAQ,CAAuC;IACvD,YAAY,CAAU;IAEtB,YAAY,IAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,aAAa,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,EAAE,CAAC;QAC3D,IAAI,CAAC,KAAK,GAAG,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;QAC1G,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,GAAG,MAAM,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC;QACpC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC9C,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC;QAC/C,IAAI,CAAC,QAAQ,GAAG;YACd,kBAAkB,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,kBAAkB,EAAE;YACzD,WAAW,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE;YAC3C,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;SACzB,CAAC;IACJ,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACvD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACjE,IAAI,CAAC,KAAgC,CAAC,KAAK,EAAE,EAAE,CAAC;IACnD,CAAC;IAED,IAAI;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,IAAI,CAAC,UAAU,EAAE,CAAC;IACvC,CAAC;IAEO,UAAU;QAChB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,OAAO,CAAC,QAAgB,EAAE,UAAmB,EAAE,EAAE,OAAsE,EAAE;QACvH,IAAI,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACrD,CAAC;IAED,gBAAgB,CAAC,QAA8C;QAC7D,IAAI,CAAC,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,GAAG,QAAQ,EAAE,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC;YACH,SAAS,CAAC;gBACR,IAAI,IAAI,CAAC,OAAO;oBAAE,MAAM;gBACxB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBACpC,IAAI,CAAC,IAAI;oBAAE,MAAM;gBACjB,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;gBACzB,SAAS,EAAE,CAAC;YACd,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,IAAI,CAAC,OAAO;gBAAE,IAAI,CAAC,UAAU,EAAE,CAAC;QACtC,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,IAAe;QACnC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,2BAA2B,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;YACvG,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YAC1B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO;YACxC,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;YACvC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC;QACjI,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM;gBAAE,OAAO;YACxC,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE,EAAE,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,QAAQ,GAAG,oBAAoB,IAAI,CAAC,OAAO,EAAE,CAAC;QACpD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC;YAAE,OAAO;QACzF,IAAI,CAAC;YACH,MAAM,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACnC,IAAI,KAAK;gBAAE,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,wCAAwC,CAAC,CAAC;QAC1E,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvD,CAAC;QACD,wEAAwE;QACxE,uEAAuE;QACvE,2EAA2E;QAC3E,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACnC,IAAI,CAAC,KAAK;YAAE,OAAO;QAEnB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,SAAS,EAAE,CAAC;QACvC,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;QAC3C,IAAI,OAAO,KAAK,CAAC;YAAE,OAAO;QAE1B,MAAM,QAAQ,GAAG,aAAa,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7C,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,EAAE,GAAG,MAAM,EAAE,CAAC;YAAE,OAAO;QAC1F,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,2BAA2B,MAAM,CAAC,MAAM,cAAc,MAAM,CAAC,cAAc,uBAAuB,OAAO,uBAAuB,CACjI,CAAC;YACJ,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;AACpD,MAAM,cAAc,GAAG,IAAI,GAAG,EAAyC,CAAC;AAExE,MAAM,UAAU,gBAAgB,CAAC,IAA0B;IACzD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC;IACnF,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACrC,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,IAAI,CAAC,QAAQ;YAAE,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5D,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;IAC1C,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;IAC/B,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAA0B;IAC3D,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACzC,SAAS,CAAC,KAAK,EAAE,CAAC;IAClB,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAe,EAAE,MAAe;IAChE,MAAM,GAAG,GAAG,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC;IAChE,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,KAAK,EAAE,CAAC;QACV,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7B,CAAC;IACD,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,SAAS,CAAC,IAAI,EAAE,CAAC;IACjB,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,OAAe,EACf,KAAe,EACf,SAAqB,EACrB,OAA8C,EAAE;IAEhD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,eAAe,CAAC;IACrE,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,aAAa;QAAE,YAAY,CAAC,aAAa,CAAC,CAAC;IAE/C,MAAM,SAAS,GAAG,gBAAgB,CAAC;QACjC,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,aAAa,EAAE,GAAG,EAAE,CAAC,KAAK;QAC1B,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC;IACtC,IAAI,OAAO,IAAI,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACtG,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;QAC5B,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,SAAS,CAAC,OAAO,CAAC,kBAAkB,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3F,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC,EAAE,OAAO,CAAC,CAAC;IACX,KAAgC,CAAC,KAAK,EAAE,EAAE,CAAC;IAC5C,cAAc,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,KAAe,EACf,OAA4B,EAAE;IAE9B,MAAM,SAAS,GAAG,gBAAgB,CAAC;QACjC,OAAO;QACP,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,aAAa,EAAE,GAAG,EAAE,CAAC,KAAK;KAC3B,CAAC,CAAC;IACH,SAAS,CAAC,YAAY,GAAG,IAAI,CAAC;IAC9B,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC;IACvF,KAAK,SAAS,CAAC,IAAI,EAAE,CAAC;IACtB,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,UAAU,uBAAuB;IACrC,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,MAAM,EAAE;QAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjE,cAAc,CAAC,KAAK,EAAE,CAAC;IACvB,KAAK,MAAM,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,UAAU,EAAE,CAAC;QAC1C,SAAS,CAAC,IAAI,EAAE,CAAC;QACjB,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
export interface LeaseRecord {
|
|
3
|
+
leaseKey: string;
|
|
4
|
+
holderId: string;
|
|
5
|
+
expiresAt: string;
|
|
6
|
+
heartbeatAt: string;
|
|
7
|
+
metadataJson: string | null;
|
|
8
|
+
}
|
|
9
|
+
export declare function ensureLeaseSchema(db: Database.Database): void;
|
|
10
|
+
export declare function createLeaseHolder(prefix?: string): string;
|
|
11
|
+
export declare function acquireLease(db: Database.Database, leaseKey: string, holderId: string, opts?: {
|
|
12
|
+
ttlMs?: number;
|
|
13
|
+
now?: Date;
|
|
14
|
+
metadata?: unknown;
|
|
15
|
+
}): boolean;
|
|
16
|
+
export declare function renewLease(db: Database.Database, leaseKey: string, holderId: string, opts?: {
|
|
17
|
+
ttlMs?: number;
|
|
18
|
+
now?: Date;
|
|
19
|
+
}): boolean;
|
|
20
|
+
export declare function releaseLease(db: Database.Database, leaseKey: string, holderId: string): boolean;
|
|
21
|
+
export declare function getLease(db: Database.Database, leaseKey: string): LeaseRecord | null;
|
|
22
|
+
//# sourceMappingURL=leases.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leases.d.ts","sourceRoot":"","sources":["../../src/scheduler/leases.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B;AAED,wBAAgB,iBAAiB,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAU7D;AAED,wBAAgB,iBAAiB,CAAC,MAAM,SAAY,GAAG,MAAM,CAE5D;AAED,wBAAgB,YAAY,CAC1B,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;CAAO,GAC5D,OAAO,CAgCT;AAED,wBAAgB,UAAU,CACxB,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,GAAG,CAAC,EAAE,IAAI,CAAA;CAAO,GACxC,OAAO,CAUT;AAED,wBAAgB,YAAY,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAI/F;AAED,wBAAgB,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAQpF"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import Database from "better-sqlite3";
|
|
2
|
+
import { randomUUID } from "node:crypto";
|
|
3
|
+
export function ensureLeaseSchema(db) {
|
|
4
|
+
db.exec(`
|
|
5
|
+
CREATE TABLE IF NOT EXISTS scheduler_leases (
|
|
6
|
+
lease_key TEXT PRIMARY KEY,
|
|
7
|
+
holder_id TEXT NOT NULL,
|
|
8
|
+
expires_at TEXT NOT NULL,
|
|
9
|
+
heartbeat_at TEXT NOT NULL,
|
|
10
|
+
metadata_json TEXT
|
|
11
|
+
);
|
|
12
|
+
`);
|
|
13
|
+
}
|
|
14
|
+
export function createLeaseHolder(prefix = "pi-para") {
|
|
15
|
+
return `${prefix}:${process.pid}:${randomUUID()}`;
|
|
16
|
+
}
|
|
17
|
+
export function acquireLease(db, leaseKey, holderId, opts = {}) {
|
|
18
|
+
ensureLeaseSchema(db);
|
|
19
|
+
const now = opts.now ?? new Date();
|
|
20
|
+
const ttlMs = opts.ttlMs ?? 2 * 60_000;
|
|
21
|
+
const expiresAt = new Date(now.getTime() + ttlMs).toISOString();
|
|
22
|
+
const nowIso = now.toISOString();
|
|
23
|
+
const metadataJson = opts.metadata === undefined ? null : JSON.stringify(opts.metadata);
|
|
24
|
+
db.exec("BEGIN IMMEDIATE");
|
|
25
|
+
try {
|
|
26
|
+
const current = db.prepare("SELECT holder_id, expires_at FROM scheduler_leases WHERE lease_key = ?").get(leaseKey);
|
|
27
|
+
const expired = !current || Date.parse(current.expires_at) <= now.getTime();
|
|
28
|
+
const sameHolder = current?.holder_id === holderId;
|
|
29
|
+
if (!expired && !sameHolder) {
|
|
30
|
+
db.exec("ROLLBACK");
|
|
31
|
+
return false;
|
|
32
|
+
}
|
|
33
|
+
db.prepare(`
|
|
34
|
+
INSERT OR REPLACE INTO scheduler_leases
|
|
35
|
+
(lease_key, holder_id, expires_at, heartbeat_at, metadata_json)
|
|
36
|
+
VALUES (?, ?, ?, ?, ?)
|
|
37
|
+
`).run(leaseKey, holderId, expiresAt, nowIso, metadataJson);
|
|
38
|
+
db.exec("COMMIT");
|
|
39
|
+
return true;
|
|
40
|
+
}
|
|
41
|
+
catch (err) {
|
|
42
|
+
try {
|
|
43
|
+
db.exec("ROLLBACK");
|
|
44
|
+
}
|
|
45
|
+
catch { }
|
|
46
|
+
throw err;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
export function renewLease(db, leaseKey, holderId, opts = {}) {
|
|
50
|
+
ensureLeaseSchema(db);
|
|
51
|
+
const now = opts.now ?? new Date();
|
|
52
|
+
const expiresAt = new Date(now.getTime() + (opts.ttlMs ?? 2 * 60_000)).toISOString();
|
|
53
|
+
const result = db.prepare(`
|
|
54
|
+
UPDATE scheduler_leases
|
|
55
|
+
SET expires_at = ?, heartbeat_at = ?
|
|
56
|
+
WHERE lease_key = ? AND holder_id = ?
|
|
57
|
+
`).run(expiresAt, now.toISOString(), leaseKey, holderId);
|
|
58
|
+
return result.changes === 1;
|
|
59
|
+
}
|
|
60
|
+
export function releaseLease(db, leaseKey, holderId) {
|
|
61
|
+
ensureLeaseSchema(db);
|
|
62
|
+
const result = db.prepare("DELETE FROM scheduler_leases WHERE lease_key = ? AND holder_id = ?").run(leaseKey, holderId);
|
|
63
|
+
return result.changes === 1;
|
|
64
|
+
}
|
|
65
|
+
export function getLease(db, leaseKey) {
|
|
66
|
+
ensureLeaseSchema(db);
|
|
67
|
+
const row = db.prepare(`
|
|
68
|
+
SELECT lease_key as leaseKey, holder_id as holderId, expires_at as expiresAt,
|
|
69
|
+
heartbeat_at as heartbeatAt, metadata_json as metadataJson
|
|
70
|
+
FROM scheduler_leases WHERE lease_key = ?
|
|
71
|
+
`).get(leaseKey);
|
|
72
|
+
return row ?? null;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=leases.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"leases.js","sourceRoot":"","sources":["../../src/scheduler/leases.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAUzC,MAAM,UAAU,iBAAiB,CAAC,EAAqB;IACrD,EAAE,CAAC,IAAI,CAAC;;;;;;;;GAQP,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,MAAM,GAAG,SAAS;IAClD,OAAO,GAAG,MAAM,IAAI,OAAO,CAAC,GAAG,IAAI,UAAU,EAAE,EAAE,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,EAAqB,EACrB,QAAgB,EAChB,QAAgB,EAChB,OAA2D,EAAE;IAE7D,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IAChE,MAAM,MAAM,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAExF,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAC3B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,EAAE,CAAC,OAAO,CAAC,wEAAwE,CAAC,CAAC,GAAG,CAAC,QAAQ,CAEpG,CAAC;QAEd,MAAM,OAAO,GAAG,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;QAC5E,MAAM,UAAU,GAAG,OAAO,EAAE,SAAS,KAAK,QAAQ,CAAC;QACnD,IAAI,CAAC,OAAO,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5B,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACpB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,EAAE,CAAC,OAAO,CAAC;;;;KAIV,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAC5D,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QACrC,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,EAAqB,EACrB,QAAgB,EAChB,QAAgB,EAChB,OAAuC,EAAE;IAEzC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC;IACnC,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;IACrF,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC;;;;GAIzB,CAAC,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACzD,OAAO,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,EAAqB,EAAE,QAAgB,EAAE,QAAgB;IACpF,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtB,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,oEAAoE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACxH,OAAO,MAAM,CAAC,OAAO,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,EAAqB,EAAE,QAAgB;IAC9D,iBAAiB,CAAC,EAAE,CAAC,CAAC;IACtB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;;;GAItB,CAAC,CAAC,GAAG,CAAC,QAAQ,CAA4B,CAAC;IAC5C,OAAO,GAAG,IAAI,IAAI,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Model } from "@earendil-works/pi-ai";
|
|
2
|
+
import type { QMDStore } from "qmd-engine";
|
|
3
|
+
import { StateDB } from "../state.js";
|
|
4
|
+
import { type ProcessResult } from "../processor.js";
|
|
5
|
+
import type { WikiScheduler, SchedulerTaskHandler } from "./index.js";
|
|
6
|
+
export interface CompletedSessionEntry {
|
|
7
|
+
timestamp: string;
|
|
8
|
+
sessionPath: string;
|
|
9
|
+
}
|
|
10
|
+
export interface CaptureSessionPayload {
|
|
11
|
+
sessionPath: string;
|
|
12
|
+
scope?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface CaptureSessionDeps {
|
|
15
|
+
wikiDir: string;
|
|
16
|
+
storeProvider: () => QMDStore | null;
|
|
17
|
+
model?: Model<any>;
|
|
18
|
+
getApiKey?: (provider: string) => Promise<string | undefined>;
|
|
19
|
+
stateDb?: StateDB;
|
|
20
|
+
processor?: (sessionPath: string, wikiDir: string, store: QMDStore, scope: string, model: Model<any>, getApiKey: (provider: string) => Promise<string | undefined>) => Promise<ProcessResult>;
|
|
21
|
+
}
|
|
22
|
+
export declare function parseCompletedSessionLine(line: string): CompletedSessionEntry | null;
|
|
23
|
+
export declare function readCompletedSessionRegistry(wikiDir: string): Promise<CompletedSessionEntry[]>;
|
|
24
|
+
export declare function appendCompletedSession(wikiDir: string, sessionPath: string, timestamp?: string): Promise<CompletedSessionEntry>;
|
|
25
|
+
export declare function enqueueCompletedSessionsFromRegistry(scheduler: WikiScheduler, wikiDir: string, opts?: {
|
|
26
|
+
stateDb?: StateDB;
|
|
27
|
+
}): Promise<number[]>;
|
|
28
|
+
export declare function enqueueCompletedSessions(scheduler: WikiScheduler, entries: CompletedSessionEntry[], opts?: {
|
|
29
|
+
stateDb?: StateDB;
|
|
30
|
+
}): number[];
|
|
31
|
+
export declare function createCaptureSessionHandler(deps: CaptureSessionDeps): SchedulerTaskHandler;
|
|
32
|
+
export declare function detectScopeFromSessionPath(sessionPath: string): string;
|
|
33
|
+
//# sourceMappingURL=session-capture.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"session-capture.d.ts","sourceRoot":"","sources":["../../src/scheduler/session-capture.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAkB,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAErE,OAAO,KAAK,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEtE,MAAM,WAAW,qBAAqB;IACpC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,qBAAqB;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,QAAQ,GAAG,IAAI,CAAC;IACrC,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IAC9D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,CACV,WAAW,EAAE,MAAM,EACnB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,QAAQ,EACf,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,EACjB,SAAS,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,KACzD,OAAO,CAAC,aAAa,CAAC,CAAC;CAC7B;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,GAAG,IAAI,CAMpF;AAED,wBAAsB,4BAA4B,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAQpG;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,SAAS,SAA2B,GACnC,OAAO,CAAC,qBAAqB,CAAC,CAIhC;AAED,wBAAsB,oCAAoC,CACxD,SAAS,EAAE,aAAa,EACxB,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC/B,OAAO,CAAC,MAAM,EAAE,CAAC,CAGnB;AAED,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,aAAa,EACxB,OAAO,EAAE,qBAAqB,EAAE,EAChC,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC/B,MAAM,EAAE,CAYV;AAED,wBAAgB,2BAA2B,CAAC,IAAI,EAAE,kBAAkB,GAAG,oBAAoB,CA+B1F;AAED,wBAAgB,0BAA0B,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAgBtE"}
|