pi-vault-mind 0.16.34 → 0.16.37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/README.md +35 -20
  3. package/dist/runtime/headless-sync-worker.d.ts +84 -0
  4. package/dist/runtime/headless-sync-worker.js +319 -0
  5. package/dist/src/agent-docs.d.ts +6 -0
  6. package/dist/src/agent-docs.js +60 -0
  7. package/dist/src/agent-queue.d.ts +1 -1
  8. package/dist/src/agent-queue.js +3 -2
  9. package/dist/src/agents/BroadcasterAgent.js +3 -2
  10. package/dist/src/agents/HeavyLifterAgent.js +3 -2
  11. package/dist/src/agents/ManagerAgent.js +3 -2
  12. package/dist/src/agents/MinerAgent.js +3 -2
  13. package/dist/src/capabilities.d.ts +31 -0
  14. package/dist/src/capabilities.js +264 -0
  15. package/dist/src/collection-index.d.ts +10 -0
  16. package/dist/src/collection-index.js +15 -0
  17. package/dist/src/commands.js +657 -78
  18. package/dist/src/config-keys.d.ts +1 -1
  19. package/dist/src/config-keys.js +5 -0
  20. package/dist/src/dispatch-writeback.d.ts +5 -0
  21. package/dist/src/dispatch-writeback.js +20 -0
  22. package/dist/src/docling.d.ts +3 -0
  23. package/dist/src/docling.js +43 -0
  24. package/dist/src/embedding-providers.d.ts +9 -8
  25. package/dist/src/embedding-providers.js +22 -10
  26. package/dist/src/engine.js +4 -0
  27. package/dist/src/graph.d.ts +20 -0
  28. package/dist/src/graph.js +199 -21
  29. package/dist/src/headless-rebuild.d.ts +83 -0
  30. package/dist/src/headless-rebuild.js +654 -0
  31. package/dist/src/lance.js +16 -4
  32. package/dist/src/markdown-import.d.ts +18 -0
  33. package/dist/src/markdown-import.js +193 -0
  34. package/dist/src/modal-client.d.ts +104 -0
  35. package/dist/src/modal-client.js +73 -1
  36. package/dist/src/modal-config.d.ts +6 -4
  37. package/dist/src/modal-config.js +15 -6
  38. package/dist/src/personalize.js +8 -2
  39. package/dist/src/server.js +3 -2
  40. package/dist/src/status.d.ts +2 -0
  41. package/dist/src/status.js +6 -0
  42. package/dist/src/sync.d.ts +25 -1
  43. package/dist/src/sync.js +320 -0
  44. package/dist/src/tool-catalog.d.ts +2 -2
  45. package/dist/src/tool-catalog.js +42 -2
  46. package/dist/src/tools.js +114 -33
  47. package/dist/src/transcription/engine.d.ts +144 -0
  48. package/dist/src/transcription/engine.js +548 -0
  49. package/dist/src/transcription/local-server.d.ts +30 -0
  50. package/dist/src/transcription/local-server.js +92 -0
  51. package/dist/src/transcription/note-template.d.ts +32 -0
  52. package/dist/src/transcription/note-template.js +231 -0
  53. package/dist/src/transcription/notebooklm.d.ts +33 -0
  54. package/dist/src/transcription/notebooklm.js +145 -0
  55. package/dist/src/transcription/source-resolution.d.ts +51 -0
  56. package/dist/src/transcription/source-resolution.js +84 -0
  57. package/dist/src/transcription/types.d.ts +133 -0
  58. package/dist/src/transcription/types.js +11 -0
  59. package/dist/src/types.d.ts +69 -1
  60. package/dist/src/types.js +25 -0
  61. package/dist/src/utils.js +4 -0
  62. package/dist/src/vault-link-graph.d.ts +29 -0
  63. package/dist/src/vault-link-graph.js +119 -0
  64. package/dist/src/vault-tools.js +23 -2
  65. package/dist/src/vm-handlers.js +16 -23
  66. package/dist/src/vm-script.d.ts +39 -0
  67. package/dist/src/vm-script.js +197 -0
  68. package/dist/src/watcher.d.ts +5 -1
  69. package/dist/src/watcher.js +158 -58
  70. package/dist/test/agent-docs.test.js +28 -0
  71. package/dist/test/agent-queue.test.js +9 -8
  72. package/dist/test/agents/Agents.test.js +6 -5
  73. package/dist/test/capabilities.test.js +87 -0
  74. package/dist/test/check-deps-script.test.js +8 -1
  75. package/dist/test/commands.test.js +253 -5
  76. package/dist/test/config-merge.test.js +14 -0
  77. package/dist/test/embedding-providers.test.js +27 -0
  78. package/dist/test/headless-rebuild.test.js +394 -0
  79. package/dist/test/headless-sync-worker.test.js +152 -0
  80. package/dist/test/index.test.js +24 -0
  81. package/dist/test/markdown-import.test.js +66 -0
  82. package/dist/test/modal-client.test.js +51 -0
  83. package/dist/test/modal-config.test.js +22 -1
  84. package/dist/test/personalize.test.js +63 -0
  85. package/dist/test/pvm-setup.test.js +61 -1
  86. package/dist/test/tool-catalog.test.js +22 -4
  87. package/dist/test/transcription-engine.test.js +382 -0
  88. package/dist/test/transcription-local-server.test.js +65 -0
  89. package/dist/test/transcription-note-template.test.js +256 -0
  90. package/dist/test/transcription-source-resolution.test.js +194 -0
  91. package/dist/test/utils.test.js +7 -0
  92. package/dist/test/vault-pi-script.test.js +49 -0
  93. package/dist/test/vm-script-tool.test.js +21 -0
  94. package/dist/test/vm-transcribe-tool.test.js +134 -0
  95. package/package.json +5 -2
  96. package/pi-vault-mind.config.example.json +8 -0
  97. package/scripts/check-deps.sh +5 -5
  98. package/scripts/pvm-setup.mjs +53 -10
  99. package/scripts/vault-pi.sh +36 -11
  100. package/skills/vault-mind/SKILL.md +61 -3
  101. package/skills/vault-mind-miner/SKILL.md +20 -6
  102. package/skills/vault-mind-setup/references/obsidian-vault-structure.md +1 -1
  103. package/skills/vault-mind-setup/references/pi-extension-wiring.md +12 -12
package/CHANGELOG.md CHANGED
@@ -1,5 +1,26 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.16.36 — 2026-08-30
4
+
5
+ ### Changed
6
+
7
+ - **Node 24 runtime floor.** The supported Node.js floor is now `24.19.0+`, matching the current local/runtime toolchain and simplifying support expectations.
8
+ - **GitHub workflows disabled.** The repository no longer runs the generic `ci.yml` or `release-obsidian.yml` automation. Verification and publishing are currently a local operator workflow; the former workflow files are preserved under `.github/workflows-disabled/` for future reuse.
9
+
10
+ ## 0.16.35 — 2026-08-30
11
+
12
+ ### Added
13
+
14
+ - **Watcher request-block preservation.** The vault watcher no longer destructively replaces `@agent` markers with subagent output. The original request block is preserved in the source note; only the running-status line is updated with a brief completion summary. Requests are reviewable and the vault note remains a faithful record of what was asked.
15
+ - **Reserved `@agent:main` identity.** The `@agent:main` marker is recognized by the watcher but never auto-dispatched to a background subagent. It is reserved for the interactive main session. Use `/vm main sweep [file-or-folder]` to collect pending `@agent:main` markers and address them in the current session.
16
+ - **Stock `system.md` and `AGENTS.md` baseline provisioning.** `pvm setup` provisions vault-local `system.md` (agent system prompt) and vault-root `AGENTS.md` (agent conventions) from curated stock baselines when those files do not already exist, giving every fresh vault a working agent identity without manual authoring. Baselines are demarcated with `<!-- vm:baseline:start/end -->` tags so `/vm personalize` can update only the personalization section without touching the baseline.
17
+ - **`vm_script` tool and run-sdk scaffold.** A sandboxed JavaScript/TypeScript execution tool (`vm_script`) is scaffolded using the `run` QuickJS package. Scripts execute with no ambient filesystem, network, or timer access beyond declared host functions (`vm.*`, `vault.read`, `log`). Enabled by default for manager and main agents. Follow-up: HITL interruption path, `vault.write`, and `pvm run <script>` CLI subcommand.
18
+
19
+ ### Changed
20
+
21
+ - **docling replaces any2md.** The Miner's document-conversion path (`vm_ingest`) now uses `docling` (via `uvx` or a globally installed `pip install docling`) for PDF, DOCX, HTML, and URL normalization. The `any2md` npm package is no longer a dependency or runtime requirement.
22
+
23
+
3
24
  ## 0.16.34 — 2026-08-29
4
25
 
5
26
  ### Added
package/README.md CHANGED
@@ -12,13 +12,13 @@ Passive Obsidian vault extension for the [pi](https://github.com/mariozechner/pi
12
12
 
13
13
  ## Features
14
14
 
15
- - **Passive File Watcher** — drop `@agent-Miner`, `@agent-Broadcaster`, etc. in any Obsidian note. Save the file. The watcher detects the marker, groups by role, and dispatches isolated subagent forks (`vault-mind-{role}` agents). No chat pollution, no manual triggers.
15
+ - **Passive File Watcher** — drop `@agent:<role>` (or `@agent-<role>`) markers in any Obsidian note. Save. The watcher detects the marker, groups by role, and dispatches isolated subagent forks. The original request block is preserved in the note; only the running-status line is updated with the result summary. `@agent:main` is reserved for the current interactive session and is never auto-dispatched — sweep it manually with `/vm main sweep`. No chat pollution, no manual triggers.
16
16
  - **Multi-Agent Architecture** — five specialist agents under the `vault-mind-{role}` skill naming: **Manager** (interactive orchestrator), **Miner** (research + entity extraction), **Broadcaster** (NotebookLM artifacts), **Heavy-Lifter** (external delegation), **Watcher** (passive file observer).
17
17
  - **LanceDB Vector + FTS + Graph** — hybrid semantic + keyword search with automatic entity extraction and BFS graph traversal. All local, no external binaries.
18
18
  - **JSONL Source-of-Truth + LanceDB Index** — every fact lives in a durable, human-readable, version-control-friendly `collections/*.jsonl` file. The LanceDB index is derived and rebuildable via `/vm reindex --all --reembed`.
19
19
  - **Bidirectional Obsidian Sync** — substantial entries (`>200 chars` or tagged `decision`/`insight`/`requirement`) auto-write to `Vault/Agent/Inbox/`. Graph entities render as Obsidian Canvas files.
20
20
  - **Vault-Scoped Configuration Surface** — Vault Mind setup/config now lives under `<vault>/.vault-mind/`, with the Obsidian setup wizard and `/vm setup` both writing the vault-local surface instead of a shared global config.
21
- - **Interactive Setup Wizard** — the Obsidian setup wizard handles first-run runtime/install/provider/folder/preferences/review flow, and `/vm setup` remains available for CLI/repair use.
21
+ - **Audio Transcription Lane** — `vm_transcribe` handles audio/video separately from `vm_ingest`, preferring a vault-configured Modal HTTP endpoint for high-quality jobs when `vaultMind.transcription.remoteUrl` is set, while keeping a safe local `faster-whisper` fallback and `/vm transcribe status|config|use|local` ops surface for switching lanes and managing the local server.
22
22
 
23
23
  ## Architecture
24
24
 
@@ -171,19 +171,13 @@ The Broadcaster agent can generate podcasts, study guides, and slide decks from
171
171
 
172
172
  ### Prerequisites
173
173
 
174
- - **Node.js** 22.19.0+ (matches `engines` in `package.json`)
174
+ - **Node.js** 24.19.0+ (matches `engines` in `package.json`)
175
175
  - **Embedding Provider** — choose one:
176
176
  - `@xenova/transformers` — built-in, no external deps (uses all-MiniLM-L6-v2, offline-capable)
177
177
  - `ollama` — requires Ollama running locally with `embeddinggemma` (higher quality)
178
178
  - `modal` — optional, **bring-your-own deploy**: offload embedding + bulk re-index to a cloud GPU service and sync vectors down for offline search (see [docs/integrations/MODAL_EMBEDDING.md](docs/integrations/MODAL_EMBEDDING.md)). Default behavior is unchanged until you opt in.
179
179
 
180
- > **On the Modal provider:** the local providers above work for everyone with
181
- > zero infrastructure — that's the default. `modal` is an **optional, self-hosted
182
- > tier**: you deploy your own copy of the service (`uvx modal deploy modal/app.py`
183
- > from a clone of this repo) to your own Modal account and point the extension at
184
- > *your* URL + token. There is no shared/hosted endpoint — a deployment's URL and
185
- > bearer token are private to whoever owns it. (A standalone PyPI package for the
186
- > server may come later.)
180
+ > **On the Modal transcription path:** this is **bring-your-own deploy**. You deploy your own Modal HTTP endpoint from the `audio-transcription` repo, store its URL under `vaultMind.transcription.remoteUrl`, and Vault Mind can prefer that remote high-quality path by default. If no remote transcription URL is configured, `vm_transcribe` falls back to the local `faster-whisper` lane.
187
181
 
188
182
  ### 1. Install and set up the vault-local runtime
189
183
 
@@ -200,8 +194,10 @@ context, model-routing, task, and subagent extensions inside
200
194
  `<vault>/.vault-mind/.pi/agent/`; installs the curated `obsidian-cli`,
201
195
  `obsidian-markdown`, `obsidian-bases`, and `json-canvas` skills; and
202
196
  merge-safely scaffolds Vault Mind, model-router, and main-agent configuration.
203
- Existing user configuration, collections, model choices, and `system.md` are
204
- preserved.
197
+ On first run it also creates stock `.pi/agent/system.md` and vault-root
198
+ `AGENTS.md` baseline regions. All of these are preserved on repeat runs —
199
+ existing user configuration, collections, model choices, and personalizations
200
+ are never overwritten.
205
201
 
206
202
  Refresh only extensions, tools, and skills without touching vault
207
203
  configuration:
@@ -360,16 +356,33 @@ applies no boundary while the persona is `main`.
360
356
  ### Use specialist agents without changing the main session
361
357
 
362
358
  Normally, leave the interactive session on `main` and dispatch specialist work
363
- through the passive watcher:
359
+ through the passive watcher. Both marker forms are equivalent:
364
360
 
365
361
  ```text
366
- @agent-Miner
362
+ @agent:miner
367
363
  Research and extract the durable facts from this note.
368
364
  ```
369
365
 
370
- The watcher runs that work as an isolated specialist job. It does not turn the
371
- interactive terminal into the Miner persona. The same distinction applies to
372
- `@agent-Broadcaster`, `@agent-Manager`, and `@agent-Heavy-Lifter`.
366
+ or
367
+
368
+ ```text
369
+ @agent-miner
370
+ Research and extract the durable facts from this note.
371
+ ```
372
+
373
+ The watcher runs that work as an isolated specialist job, preserving the
374
+ request block in the source note and replacing only the running-status line
375
+ with a result summary. It does not turn the interactive terminal into the
376
+ Miner persona. The same applies to `@agent:broadcaster`, `@agent:manager`,
377
+ and `@agent:heavy-lifter`.
378
+
379
+ `@agent:main` is reserved for the current interactive session and is never
380
+ auto-dispatched. To sweep reserved markers into the active session:
381
+
382
+ ```text
383
+ /vm main sweep
384
+ /vm main sweep path/to/note.md
385
+ ```
373
386
 
374
387
  ### Explicitly change the interactive persona
375
388
 
@@ -535,8 +548,9 @@ belongs in `vaultMind.identities` or an explicit environment override.
535
548
  | `vm_describe` | Introspect schema, count, and sample entries |
536
549
  | `vm_stats` | Dashboard: counts, sizes, LanceDB status |
537
550
  | `vm_export` | Export to JSON, CSV, or Markdown |
538
- | `vm_promote` | Promote entries between collections via pending queue |
539
-
551
+ | `vm_ingest` | Convert URLs and documents to markdown via docling |
552
+ | `vm_transcribe` | Transcribe audio/video, generate transcript notes + raw artifacts |
553
+ | `vm_script` | Execute sandboxed JS/TS with vm.*, vault.read, log.* |
540
554
  ## Commands
541
555
 
542
556
  | Command | Purpose |
@@ -555,7 +569,8 @@ belongs in `vaultMind.identities` or an explicit environment override.
555
569
  | `/vm embedding status \| use \| model \| models \| pull` | Manage embedding provider |
556
570
  | `/vm identity status \| set <role> \| on \| off` | Inspect or explicitly control the interactive agent persona boundary |
557
571
  | `/vm remote status \| config \| sync \| jobs \| migrate` | Manage remote embedding + vector sync |
558
- | `/vm watcher start \| stop \| status` | Manage the passive file watcher |
572
+ | `/vm watcher start \| stop \| status` | Manage the passive file watcher |
573
+ | `/vm main sweep [file]` | Sweep `@agent:main` markers into the current interactive session |
559
574
  | `/vm server status` | Show HTTP server status, port, and uptime |
560
575
 
561
576
  ## Documentation
@@ -0,0 +1,84 @@
1
+ import { type HeadlessRebuildAfterReindexDependencies, type HeadlessRebuildDependencies } from "../src/headless-rebuild.js";
2
+ import type { UniversalConfig } from "../src/types.js";
3
+ type WorkerPhase = "credentials" | "auth" | "sync" | "embed" | "graph" | "config";
4
+ export declare class HeadlessSyncWorkerError extends Error {
5
+ readonly phase: WorkerPhase;
6
+ constructor(phase: WorkerPhase);
7
+ }
8
+ export interface HeadlessSyncWorkerRequest {
9
+ operationId: string;
10
+ graphId: string;
11
+ collections: string[];
12
+ }
13
+ interface SecretClient {
14
+ secrets: {
15
+ resolve(reference: string): Promise<string>;
16
+ };
17
+ }
18
+ interface VectorRevision {
19
+ collection: string;
20
+ model: string;
21
+ dim: number;
22
+ watermark: number;
23
+ revision: string;
24
+ }
25
+ interface GraphRevisionRow {
26
+ graph_id: string;
27
+ revision: string;
28
+ source_fingerprint: string;
29
+ updated_at: string;
30
+ }
31
+ interface RevisionManifest {
32
+ vectors: Array<VectorRevision & {
33
+ updated_at: string;
34
+ }>;
35
+ graphs: GraphRevisionRow[];
36
+ }
37
+ export interface HeadlessSyncWorkerDependencies {
38
+ env: Record<string, string | undefined>;
39
+ workspaceRoot: string;
40
+ createClient(options: {
41
+ auth: string;
42
+ integrationName: string;
43
+ integrationVersion: string;
44
+ }): Promise<SecretClient>;
45
+ execFile(command: string, args: string[], options: {
46
+ cwd: string;
47
+ env: Record<string, string | undefined>;
48
+ }): Promise<unknown>;
49
+ runRebuild(request: {
50
+ graphId: string;
51
+ clonePath: string;
52
+ wait: true;
53
+ }, deps: Partial<HeadlessRebuildDependencies> & Pick<HeadlessRebuildAfterReindexDependencies, "reindex">): Promise<{
54
+ graphId: string;
55
+ sourceFingerprint: string;
56
+ jobId?: string;
57
+ jobStatus?: string;
58
+ }>;
59
+ reindexRemote(cfg: UniversalConfig, collections: string[], vaultPath: string, options: {
60
+ syncDown: false;
61
+ }): Promise<Array<{
62
+ collection: string;
63
+ job?: {
64
+ status: string;
65
+ };
66
+ error?: string;
67
+ model?: string;
68
+ dim?: number | null;
69
+ remote_collection?: string;
70
+ }>>;
71
+ readRevisionsManifest(baseUrl: string, token: string): Promise<{
72
+ vectors: VectorRevision[];
73
+ graphs: GraphRevisionRow[];
74
+ }>;
75
+ fetchNamespaceWatermark(baseUrl: string, token: string, collection: string, model: string, dim: number): Promise<number>;
76
+ publishRevisions(baseUrl: string, token: string, manifest: RevisionManifest): Promise<void>;
77
+ now(): string;
78
+ }
79
+ export declare function runHeadlessSyncWorker(request: HeadlessSyncWorkerRequest, deps: HeadlessSyncWorkerDependencies): Promise<{
80
+ graphId: string;
81
+ graphJobId?: string;
82
+ sourceFingerprint: string;
83
+ }>;
84
+ export {};
@@ -0,0 +1,319 @@
1
+ import { execFile as nodeExecFile } from "node:child_process";
2
+ import * as fs from "node:fs";
3
+ import * as path from "node:path";
4
+ import { promisify } from "node:util";
5
+ import { runHeadlessRebuildAfterReindex, } from "../src/headless-rebuild.js";
6
+ import { reindexRemote } from "../src/sync.js";
7
+ const execFileAsync = promisify(nodeExecFile);
8
+ const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/;
9
+ export class HeadlessSyncWorkerError extends Error {
10
+ phase;
11
+ constructor(phase) {
12
+ super(`${phase} failed`);
13
+ this.name = "HeadlessSyncWorkerError";
14
+ this.phase = phase;
15
+ }
16
+ }
17
+ function required(env, key, phase) {
18
+ const value = env[key];
19
+ if (!value)
20
+ throw new HeadlessSyncWorkerError(phase);
21
+ return value;
22
+ }
23
+ function assertSafe(value, phase) {
24
+ if (!IDENTIFIER.test(value))
25
+ throw new HeadlessSyncWorkerError(phase);
26
+ return value;
27
+ }
28
+ function reference(env, key) {
29
+ const value = required(env, key, "config");
30
+ if (!value.startsWith("op://") || value.length > 512)
31
+ throw new HeadlessSyncWorkerError("config");
32
+ return value;
33
+ }
34
+ function optionalReference(env, key) {
35
+ const value = env[key];
36
+ if (!value)
37
+ return undefined;
38
+ return reference(env, key);
39
+ }
40
+ function selectConfig(raw, collections, token) {
41
+ let parsed;
42
+ try {
43
+ parsed = JSON.parse(raw);
44
+ }
45
+ catch {
46
+ throw new HeadlessSyncWorkerError("config");
47
+ }
48
+ if (!parsed || typeof parsed !== "object")
49
+ throw new HeadlessSyncWorkerError("config");
50
+ const cfg = parsed;
51
+ const embeddingUrl = cfg.vaultMind?.embedding?.remoteUrl;
52
+ if (!cfg.collections ||
53
+ typeof cfg.collections !== "object" ||
54
+ !cfg.vaultMind ||
55
+ typeof embeddingUrl !== "string" ||
56
+ embeddingUrl.length === 0) {
57
+ throw new HeadlessSyncWorkerError("config");
58
+ }
59
+ const selected = {};
60
+ for (const collection of collections) {
61
+ const definition = cfg.collections[collection];
62
+ if (!definition)
63
+ throw new HeadlessSyncWorkerError("config");
64
+ selected[collection] = definition;
65
+ }
66
+ cfg.collections = selected;
67
+ cfg.vaultMind.embedding = {
68
+ ...cfg.vaultMind.embedding,
69
+ remoteApiKey: token,
70
+ };
71
+ return { config: cfg, embeddingUrl };
72
+ }
73
+ function sanitizeError(error, phase) {
74
+ if (error instanceof HeadlessSyncWorkerError)
75
+ throw error;
76
+ throw new HeadlessSyncWorkerError(phase);
77
+ }
78
+ export async function runHeadlessSyncWorker(request, deps) {
79
+ assertSafe(request.operationId, "config");
80
+ assertSafe(request.graphId, "config");
81
+ if (request.collections.length === 0 ||
82
+ new Set(request.collections).size !== request.collections.length) {
83
+ throw new HeadlessSyncWorkerError("config");
84
+ }
85
+ for (const collection of request.collections)
86
+ assertSafe(collection, "config");
87
+ const workspace = path.resolve(deps.workspaceRoot, request.operationId);
88
+ const home = path.join(workspace, ".home");
89
+ const env = { ...deps.env, HOME: home };
90
+ fs.mkdirSync(home, { recursive: true, mode: 0o700 });
91
+ required(deps.env, "OP_SERVICE_ACCOUNT_TOKEN", "credentials");
92
+ const embeddingToken = required(deps.env, "PVM_REMOTE_EMBEDDING_API_KEY", "credentials");
93
+ let email;
94
+ let password;
95
+ let mfa;
96
+ let e2ePassword;
97
+ let vault;
98
+ try {
99
+ const resolve = async (ref) => {
100
+ const result = (await deps.execFile("op", ["read", "--no-newline", ref], {
101
+ cwd: workspace,
102
+ env,
103
+ }));
104
+ if (typeof result.stdout !== "string" || result.stdout.length === 0) {
105
+ throw new HeadlessSyncWorkerError("credentials");
106
+ }
107
+ return result.stdout;
108
+ };
109
+ const mfaRef = optionalReference(deps.env, "PVM_HEADLESS_OP_MFA_REF");
110
+ const e2eRef = optionalReference(deps.env, "PVM_HEADLESS_OP_E2E_PASSWORD_REF");
111
+ [email, password, mfa, e2ePassword, vault] = await Promise.all([
112
+ resolve(reference(deps.env, "PVM_HEADLESS_OP_EMAIL_REF")),
113
+ resolve(reference(deps.env, "PVM_HEADLESS_OP_PASSWORD_REF")),
114
+ mfaRef ? resolve(mfaRef) : Promise.resolve(undefined),
115
+ e2eRef ? resolve(e2eRef) : Promise.resolve(undefined),
116
+ resolve(reference(deps.env, "PVM_HEADLESS_OP_VAULT_REF")),
117
+ ]);
118
+ let parsedVault;
119
+ try {
120
+ parsedVault = JSON.parse(vault);
121
+ }
122
+ catch {
123
+ throw new HeadlessSyncWorkerError("credentials");
124
+ }
125
+ if (!parsedVault ||
126
+ typeof parsedVault !== "object" ||
127
+ !("vault_id" in parsedVault) ||
128
+ typeof parsedVault.vault_id !== "string" ||
129
+ !IDENTIFIER.test(parsedVault.vault_id)) {
130
+ throw new HeadlessSyncWorkerError("credentials");
131
+ }
132
+ vault = parsedVault.vault_id;
133
+ }
134
+ catch (error) {
135
+ return sanitizeError(error, "credentials");
136
+ }
137
+ try {
138
+ const loginArgs = ["login", "--email", email, "--password", password];
139
+ if (mfa)
140
+ loginArgs.push("--mfa", mfa);
141
+ await deps.execFile("ob", loginArgs, { cwd: workspace, env });
142
+ const setupArgs = [
143
+ "sync-setup",
144
+ "--vault",
145
+ vault,
146
+ "--path",
147
+ workspace,
148
+ "--password",
149
+ e2ePassword,
150
+ "--device-name",
151
+ `pvm-headless-${request.operationId}`,
152
+ ];
153
+ if (!e2ePassword)
154
+ setupArgs.splice(setupArgs.indexOf("--password"), 2);
155
+ await deps.execFile("ob", setupArgs, { cwd: workspace, env });
156
+ await deps.execFile("ob", ["sync", "--path", workspace], { cwd: workspace, env });
157
+ }
158
+ catch (error) {
159
+ return sanitizeError(error, "auth");
160
+ }
161
+ let cfg;
162
+ let embeddingUrl;
163
+ try {
164
+ const configPath = path.join(workspace, ".vault-mind", "vault-mind.config.json");
165
+ const rawConfig = fs.readFileSync(configPath, "utf8");
166
+ ({ config: cfg, embeddingUrl } = selectConfig(rawConfig, request.collections, embeddingToken));
167
+ }
168
+ catch (error) {
169
+ return sanitizeError(error, "config");
170
+ }
171
+ let checkpoint;
172
+ let reindexRevisions;
173
+ try {
174
+ checkpoint = await deps.runRebuild({ graphId: request.graphId, clonePath: workspace, wait: true }, {
175
+ runObSync: async (clonePath) => {
176
+ await deps.execFile("ob", ["sync", "--path", clonePath], { cwd: clonePath, env });
177
+ },
178
+ reindex: async (resolvedConfig, clonePath) => {
179
+ const selectedConfig = {
180
+ ...cfg,
181
+ ...resolvedConfig,
182
+ collections: cfg.collections,
183
+ };
184
+ const results = await deps.reindexRemote(selectedConfig, request.collections, clonePath, {
185
+ syncDown: false,
186
+ });
187
+ if (results.some((result) => result.error || result.job?.status !== "done")) {
188
+ throw new HeadlessSyncWorkerError("embed");
189
+ }
190
+ reindexRevisions = results
191
+ .filter((result) => result.job?.status === "done" &&
192
+ !result.error &&
193
+ !!result.model &&
194
+ !!result.remote_collection)
195
+ .map((result) => ({
196
+ collection: result.remote_collection,
197
+ model: result.model,
198
+ dim: (result.dim ?? 0),
199
+ revision: result.job.status,
200
+ watermark: 0,
201
+ }));
202
+ },
203
+ });
204
+ }
205
+ catch (error) {
206
+ return sanitizeError(error, error instanceof HeadlessSyncWorkerError ? error.phase : "graph");
207
+ }
208
+ if (checkpoint.jobStatus !== "done" || !checkpoint.jobId)
209
+ throw new HeadlessSyncWorkerError("graph");
210
+ let vectors;
211
+ try {
212
+ vectors = reindexRevisions;
213
+ const remoteCollections = new Set(vectors.map((vector) => vector.collection));
214
+ if (vectors.length !== request.collections.length ||
215
+ vectors.some((vector) => !remoteCollections.has(vector.collection) ||
216
+ !(typeof vector.dim === "number" && vector.dim > 0) ||
217
+ !Number.isSafeInteger(vector.watermark) ||
218
+ vector.watermark < 0)) {
219
+ throw new HeadlessSyncWorkerError("embed");
220
+ }
221
+ const watermarks = await Promise.all(vectors.map(async (vector) => {
222
+ try {
223
+ return await deps.fetchNamespaceWatermark(embeddingUrl, embeddingToken, vector.collection, vector.model, vector.dim);
224
+ }
225
+ catch (error) {
226
+ throw new HeadlessSyncWorkerError("embed");
227
+ }
228
+ }));
229
+ vectors = vectors.map((vector, index) => ({
230
+ ...vector,
231
+ watermark: watermarks[index],
232
+ }));
233
+ }
234
+ catch (error) {
235
+ return sanitizeError(error, "embed");
236
+ }
237
+ try {
238
+ const existing = await deps.readRevisionsManifest(embeddingUrl, embeddingToken);
239
+ const selectedRemote = new Set(vectors.map((vector) => vector.collection));
240
+ const survivingVectors = existing.vectors
241
+ .filter((vector) => !selectedRemote.has(vector.collection))
242
+ .map((vector) => ({ ...vector, updated_at: deps.now() }));
243
+ await deps.publishRevisions(embeddingUrl, embeddingToken, {
244
+ vectors: [
245
+ ...survivingVectors,
246
+ ...vectors.map((vector) => ({
247
+ ...vector,
248
+ updated_at: deps.now(),
249
+ })),
250
+ ],
251
+ graphs: [
252
+ ...existing.graphs,
253
+ {
254
+ graph_id: checkpoint.graphId,
255
+ revision: checkpoint.jobId,
256
+ source_fingerprint: checkpoint.sourceFingerprint,
257
+ updated_at: deps.now(),
258
+ },
259
+ ],
260
+ });
261
+ }
262
+ catch (error) {
263
+ return sanitizeError(error, "graph");
264
+ }
265
+ return {
266
+ graphId: checkpoint.graphId,
267
+ graphJobId: checkpoint.jobId,
268
+ sourceFingerprint: checkpoint.sourceFingerprint,
269
+ };
270
+ }
271
+ const defaultDependencies = {
272
+ env: process.env,
273
+ workspaceRoot: process.env.PVM_HEADLESS_WORKSPACE_ROOT || "/pvm-vault/workspaces",
274
+ execFile: async (command, args, options) => execFileAsync(command, args, options),
275
+ runRebuild: runHeadlessRebuildAfterReindex,
276
+ reindexRemote: reindexRemote,
277
+ readRevisionsManifest: async (baseUrl, token) => {
278
+ const target = baseUrl.replace(/\/$/, "");
279
+ const response = await fetch(`${target}/sync/revisions`, {
280
+ method: "GET",
281
+ headers: { Authorization: `Bearer ${token}` },
282
+ });
283
+ if (!response.ok)
284
+ throw new Error("revision read failed");
285
+ return (await response.json());
286
+ },
287
+ fetchNamespaceWatermark: async (baseUrl, token, collection, model, dim) => {
288
+ const target = baseUrl.replace(/\/$/, "");
289
+ const params = new URLSearchParams({ collection, model, dim: String(dim) });
290
+ const response = await fetch(`${target}/sync/namespace-watermark?${params.toString()}`, {
291
+ method: "GET",
292
+ headers: { Authorization: `Bearer ${token}` },
293
+ });
294
+ if (!response.ok)
295
+ throw new Error("namespace watermark read failed");
296
+ const body = (await response.json());
297
+ return body.watermark;
298
+ },
299
+ publishRevisions: async (baseUrl, token, manifest) => {
300
+ const response = await fetch(`${baseUrl.replace(/\/$/, "")}/sync/revisions`, {
301
+ method: "POST",
302
+ headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
303
+ body: JSON.stringify(manifest),
304
+ });
305
+ if (!response.ok)
306
+ throw new Error("revision publish failed");
307
+ },
308
+ now: () => new Date().toISOString(),
309
+ };
310
+ if (import.meta.url === `file://${process.argv[1]}`) {
311
+ const [operationId, graphId, collectionsJson] = process.argv.slice(2);
312
+ void (async () => {
313
+ const collections = JSON.parse(collectionsJson || "[]");
314
+ const result = await runHeadlessSyncWorker({ operationId: operationId || "", graphId: graphId || "", collections }, defaultDependencies);
315
+ process.stdout.write(JSON.stringify(result));
316
+ })().catch(() => {
317
+ process.exitCode = 1;
318
+ });
319
+ }
@@ -0,0 +1,6 @@
1
+ export declare const BASELINE_START = "<!-- vm:baseline:start -->";
2
+ export declare const BASELINE_END = "<!-- vm:baseline:end -->";
3
+ export declare const PERSONALIZATION_START = "<!-- vm:personalization:start -->";
4
+ export declare const PERSONALIZATION_END = "<!-- vm:personalization:end -->";
5
+ export declare const buildDefaultSystemMd: (vaultPath: string) => string;
6
+ export declare const buildDefaultAgentsMd: () => string;
@@ -0,0 +1,60 @@
1
+ import * as path from "node:path";
2
+ export const BASELINE_START = "<!-- vm:baseline:start -->";
3
+ export const BASELINE_END = "<!-- vm:baseline:end -->";
4
+ export const PERSONALIZATION_START = "<!-- vm:personalization:start -->";
5
+ export const PERSONALIZATION_END = "<!-- vm:personalization:end -->";
6
+ export const buildDefaultSystemMd = (vaultPath) => {
7
+ const vaultName = path.basename(vaultPath);
8
+ return `# Pi main agent: ${vaultName}
9
+
10
+ ${BASELINE_START}
11
+ You are the unrestricted main interactive agent for this vault. Bundled specialist skills are available resources; their presence does not change your active identity. Adopt a specialist identity only when explicitly launched as that subagent.
12
+
13
+ ## Operating baseline
14
+ - Read \`00-system/SYSTEM.md\` first when it exists. If it does not exist, fall back to a vault-root \`SYSTEM.md\`. Then read the vault-root \`AGENTS.md\`.
15
+ - Prefer the vault's own operating manual and schemas over generic assumptions.
16
+ - Use the \`vm_*\` tools, collections, and queue as the durable system of record for agent work.
17
+ - Prefer \`uv\` for Python tools and CLIs. Use \`pnpm\`/\`npm\` for JavaScript and TypeScript packages such as \`run\`.
18
+ - Use \`docling\` for document conversion in ingest flows.
19
+ - Use \`vm_script\` for reusable sandboxed JS/TS workflows when the task fits the approved host-function surface.
20
+ - Use \`vm_transcribe\` for audio/video transcription (voice notes, podcasts, NotebookLM audio overviews). Defaults come from \`vaultMind.transcription\`: prefer the remote Modal HTTP endpoint when \`remoteUrl\` is configured, otherwise fall back to the local engine. Keep this lane separate from \`vm_ingest\`/docling.
21
+ - Use the \`nlm\` CLI for NotebookLM source inventory lookups. Do not use MCP as the primary path for NotebookLM notebook/source queries.
22
+ - Never auto-publish, never fabricate facts, and keep human approval at external side effects.
23
+ - The watcher recognizes \`@agent:<role>\` and \`@agent-<role>\` anywhere in watchable files. \`@agent:main\` is reserved for human-authored review and follow-up notes; it is never auto-dispatched. Automated transcript preparation, source linking, and note population use \`@agent:audio-overviews\`.
24
+ ${BASELINE_END}
25
+
26
+ ${PERSONALIZATION_START}
27
+ Add vault-specific priorities, domain language, and working preferences here. /vm personalize may update only this section.
28
+ ${PERSONALIZATION_END}
29
+ `;
30
+ };
31
+ export const buildDefaultAgentsMd = () => `# AGENTS: pi-vault-mind operating baseline
32
+
33
+ ${BASELINE_START}
34
+ ## Agent roles
35
+ - \`main\` — the interactive agent you work with directly.
36
+ - \`manager\` — synthesis, orchestration, and roadmap work.
37
+ - \`miner\` — extraction, ingestion, and structured knowledge capture.
38
+ - \`broadcaster\` — human-facing artifact generation.
39
+ - \`heavy-lifter\` — large-context implementation and deeper delegated work.
40
+ - \`personalization\` — resolves annotated feedback on generated system/agent docs.
41
+ - \`audio-overviews\` — automated worker for NotebookLM audio overview transcription, source resolution, and note population. Uses deterministic scripts/functions rather than freeform improvisation.
42
+
43
+ ## Watcher + inline requests
44
+ - Use \`@agent:<role>\` or \`@agent-<role>\` in watchable files to request agent help.
45
+ - \`@agent:main\` is recognized but never auto-dispatched; it is for human-authored review notes and follow-up only — not automated transcript generation.
46
+ - Auto-dispatched watcher jobs preserve the request block in the source note and replace only the running-status line with the result summary.
47
+ - Watcher dispatches should be logged to collections so they can be queried later.
48
+
49
+ ## Tooling baseline
50
+ - Prefer \`uv\` for Python tools.
51
+ - Prefer reusable \`vm_script\` workflows over ad-hoc one-off scripts when the workflow should be repeatable.
52
+ - Use \`vm_transcribe\` for audio/video transcription; keep this lane separate from \`vm_ingest\`/docling.
53
+ - Use \`nlm\` for NotebookLM source inventory. Do not use MCP as the primary path for NotebookLM lookups.
54
+ - Keep reusable content, test, and update workflows in skills/tools rather than scattered shell snippets.
55
+ ${BASELINE_END}
56
+
57
+ ${PERSONALIZATION_START}
58
+ Add vault-specific agent roster details, domain-specific prompts, and delegation rules here. /vm personalize may update only this section.
59
+ ${PERSONALIZATION_END}
60
+ `;
@@ -75,7 +75,7 @@ export declare const markJobFailed: (vaultPath: string, id: string, error: strin
75
75
  export declare const markJobRunning: (vaultPath: string, id: string) => AgentJob | undefined;
76
76
  /** Mark a pending job as cancelled. */
77
77
  export declare const cancelJob: (vaultPath: string, id: string) => AgentJob | undefined;
78
- /** True if the file still contains the dispatch marker for this job. */
78
+ /** True if the file still contains the pending dispatch status placeholder. */
79
79
  export declare const markerStillPresent: (filePath: string, dispatchId: string) => boolean;
80
80
  /** Scan a vault's queue and mark any running job whose marker is gone as done. */
81
81
  export declare const sweepCompletedJobs: (vaultPath: string) => AgentJob[];
@@ -12,6 +12,7 @@
12
12
  */
13
13
  import * as fs from "node:fs";
14
14
  import * as path from "node:path";
15
+ import { dispatchStatusLine } from "./dispatch-writeback.js";
15
16
  import { loadConfig, resolveVaultFile } from "./utils.js";
16
17
  const queueSubscribers = new Set();
17
18
  /** Subscribe to queue events. Returns an unsubscribe function. */
@@ -138,11 +139,11 @@ export const cancelJob = (vaultPath, id) => {
138
139
  return undefined;
139
140
  return updateJob(vaultPath, id, "cancelled");
140
141
  };
141
- /** True if the file still contains the dispatch marker for this job. */
142
+ /** True if the file still contains the pending dispatch status placeholder. */
142
143
  export const markerStillPresent = (filePath, dispatchId) => {
143
144
  try {
144
145
  const content = fs.readFileSync(filePath, "utf-8");
145
- return content.includes(`<!-- pi-dispatch:${dispatchId} -->`);
146
+ return content.includes(dispatchStatusLine(dispatchId));
146
147
  }
147
148
  catch {
148
149
  return false;
@@ -6,6 +6,7 @@
6
6
  * Writes output only under Agent/Presentations/.
7
7
  */
8
8
  import { Agent } from "../agent-bus/Agent.js";
9
+ import { buildDispatchResultLine, dispatchStatusLine } from "../dispatch-writeback.js";
9
10
  export class BroadcasterAgent extends Agent {
10
11
  taskQueue = null;
11
12
  constructor(config, bus, llm) {
@@ -58,8 +59,8 @@ export class BroadcasterAgent extends Agent {
58
59
  path: job.filePath,
59
60
  edits: [
60
61
  {
61
- oldText: `<!-- pi-dispatch:${job.dispatchId} -->`,
62
- newText: `> Broadcaster completed. See dispatch ${job.dispatchId}.`,
62
+ oldText: dispatchStatusLine(job.dispatchId),
63
+ newText: buildDispatchResultLine(job.dispatchId, `Broadcaster completed. See dispatch ${job.dispatchId}.`),
63
64
  },
64
65
  ],
65
66
  });