pi-vault-mind 0.14.2 → 0.16.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/CHANGELOG.md +293 -0
  2. package/README.md +5 -5
  3. package/agents/broadcaster.agent.md +2 -2
  4. package/agents/heavy-lifter.agent.md +1 -1
  5. package/agents/main.agent.md +29 -0
  6. package/agents/manager.agent.md +1 -1
  7. package/agents/miner.agent.md +1 -1
  8. package/dist/packages/obsidian/src/client.js +331 -0
  9. package/dist/packages/obsidian/src/types.js +1 -0
  10. package/dist/src/activity.d.ts +49 -0
  11. package/dist/src/activity.js +175 -0
  12. package/dist/src/agent-bus/Agent.d.ts +9 -1
  13. package/dist/src/agent-bus/Agent.js +18 -2
  14. package/dist/src/agent-bus/index.d.ts +5 -5
  15. package/dist/src/agent-bus/index.js +1 -1
  16. package/dist/src/agent-queue.js +12 -3
  17. package/dist/src/agents/BroadcasterAgent.d.ts +1 -1
  18. package/dist/src/agents/BroadcasterAgent.js +2 -2
  19. package/dist/src/agents/HeavyLifterAgent.d.ts +1 -1
  20. package/dist/src/agents/HeavyLifterAgent.js +2 -2
  21. package/dist/src/agents/ManagerAgent.js +1 -1
  22. package/dist/src/agents/MinerAgent.d.ts +1 -1
  23. package/dist/src/agents/MinerAgent.js +2 -2
  24. package/dist/src/agents/index.d.ts +2 -2
  25. package/dist/src/agents/index.js +2 -2
  26. package/dist/src/auth.d.ts +17 -15
  27. package/dist/src/auth.js +26 -37
  28. package/dist/src/autosync.js +2 -2
  29. package/dist/src/commands.d.ts +2 -19
  30. package/dist/src/commands.js +301 -231
  31. package/dist/src/config-keys.d.ts +16 -0
  32. package/dist/src/config-keys.js +45 -0
  33. package/dist/src/discover-schema.js +4 -4
  34. package/dist/src/edits.d.ts +21 -0
  35. package/dist/src/edits.js +75 -0
  36. package/dist/src/embedding-probe.d.ts +34 -0
  37. package/dist/src/embedding-probe.js +142 -0
  38. package/dist/src/embedding-providers.d.ts +133 -0
  39. package/dist/src/embedding-providers.js +173 -0
  40. package/dist/src/embedding-secrets.d.ts +38 -0
  41. package/dist/src/embedding-secrets.js +291 -0
  42. package/dist/src/engine.d.ts +6 -0
  43. package/dist/src/engine.js +78 -6
  44. package/dist/src/events.js +7 -10
  45. package/dist/src/extension-packages.d.ts +9 -0
  46. package/dist/src/extension-packages.js +14 -0
  47. package/dist/src/git.d.ts +19 -0
  48. package/dist/src/git.js +78 -0
  49. package/dist/src/graph.d.ts +2 -1
  50. package/dist/src/graph.js +47 -13
  51. package/dist/src/identities-config.d.ts +6 -0
  52. package/dist/src/identities-config.js +39 -0
  53. package/dist/src/identity-injector.js +1 -1
  54. package/dist/src/index.js +9 -26
  55. package/dist/src/lance.d.ts +9 -0
  56. package/dist/src/lance.js +143 -203
  57. package/dist/src/modal-client.d.ts +16 -0
  58. package/dist/src/modal-client.js +16 -0
  59. package/dist/src/modal-config.d.ts +37 -14
  60. package/dist/src/modal-config.js +59 -32
  61. package/dist/src/model-router.d.ts +27 -0
  62. package/dist/src/model-router.js +67 -0
  63. package/dist/src/models.d.ts +42 -0
  64. package/dist/src/models.js +78 -0
  65. package/dist/src/personalize.js +17 -3
  66. package/dist/src/scaffold.d.ts +48 -0
  67. package/dist/src/scaffold.js +246 -0
  68. package/dist/src/server.d.ts +47 -4
  69. package/dist/src/server.js +988 -221
  70. package/dist/src/session-search.d.ts +63 -0
  71. package/dist/src/session-search.js +379 -0
  72. package/dist/src/settings-ui.d.ts +2 -2
  73. package/dist/src/settings-ui.js +76 -135
  74. package/dist/src/sync.d.ts +3 -3
  75. package/dist/src/sync.js +19 -15
  76. package/dist/src/tool-catalog.d.ts +18 -0
  77. package/dist/src/tool-catalog.js +232 -0
  78. package/dist/src/tools/marksman.js +1 -1
  79. package/dist/src/tools.js +111 -65
  80. package/dist/src/types.d.ts +127 -71
  81. package/dist/src/utils.d.ts +57 -6
  82. package/dist/src/utils.js +114 -28
  83. package/dist/src/vault-tools.d.ts +34 -0
  84. package/dist/src/vault-tools.js +99 -0
  85. package/dist/src/vault-writer.js +4 -1
  86. package/dist/src/vm-handlers.js +140 -43
  87. package/dist/src/watcher.js +9 -9
  88. package/dist/test/activity-tool.test.js +54 -0
  89. package/dist/test/agent-bus/Agent.test.js +174 -0
  90. package/dist/test/agent-bus/AgentLoader.test.js +180 -0
  91. package/dist/test/agent-bus/AgentRegistry.test.js +86 -0
  92. package/dist/test/agent-bus/LLMProvider.test.js +162 -0
  93. package/dist/test/agent-bus/TaskQueue.test.js +256 -0
  94. package/dist/test/agent-dispatch-context.test.js +73 -0
  95. package/dist/test/agent-queue.test.js +1 -1
  96. package/dist/test/agents/Agents.test.js +282 -0
  97. package/dist/test/auth.test.js +57 -56
  98. package/dist/test/bridge-identity.test.js +34 -3
  99. package/dist/test/classify-search-mode.test.js +51 -0
  100. package/dist/test/commands.test.js +410 -0
  101. package/dist/test/config-merge.test.js +98 -0
  102. package/dist/test/context-capture.test.js +56 -0
  103. package/dist/test/discover-schema.test.js +8 -5
  104. package/dist/test/dispatch.test.js +45 -91
  105. package/dist/test/edits.test.js +144 -0
  106. package/dist/test/embedding-probe.test.js +424 -0
  107. package/dist/test/embedding-providers.test.js +157 -0
  108. package/dist/test/embedding-secrets.test.js +210 -0
  109. package/dist/test/engine-identity.test.js +1 -1
  110. package/dist/test/events.test.js +675 -0
  111. package/dist/test/git-bridge.test.js +160 -0
  112. package/dist/test/graph.test.js +464 -0
  113. package/dist/test/helpers/server.js +81 -0
  114. package/dist/test/identity-injector.test.js +197 -0
  115. package/dist/test/index.test.js +5 -7
  116. package/dist/test/lance-modal.test.js +67 -2
  117. package/dist/test/lance.test.js +429 -0
  118. package/dist/test/modal-client.test.js +1 -1
  119. package/dist/test/modal-config.test.js +92 -79
  120. package/dist/test/model-readiness.test.js +122 -0
  121. package/dist/test/model-router.test.js +99 -0
  122. package/dist/test/obsidian-client.test.js +42 -0
  123. package/dist/test/personalize.test.js +1 -1
  124. package/dist/test/rest-models.test.js +153 -0
  125. package/dist/test/rest-queue.test.js +135 -87
  126. package/dist/test/rest-setup.test.js +625 -115
  127. package/dist/test/rest-vm.test.js +355 -94
  128. package/dist/test/server-cors.test.js +48 -0
  129. package/dist/test/server-ws.test.js +61 -34
  130. package/dist/test/session-crud.test.js +323 -0
  131. package/dist/test/session-search.test.js +139 -0
  132. package/dist/test/settings-ui.test.js +113 -4
  133. package/dist/test/sync.test.js +12 -12
  134. package/dist/test/tombstone.test.js +8 -10
  135. package/dist/test/tool-catalog.test.js +86 -0
  136. package/dist/test/utils.test.js +588 -0
  137. package/dist/test/vault-tools.test.js +103 -12
  138. package/dist/test/vault-writer.test.js +371 -0
  139. package/dist/test/vm-handlers.test.js +620 -0
  140. package/extension-packages.json +9 -0
  141. package/package.json +107 -90
  142. package/pi-vault-mind.config.example.json +24 -2
  143. package/scripts/check-deps.sh +35 -32
  144. package/scripts/cli-e2e.mjs +215 -0
  145. package/scripts/configuration-e2e.mjs +783 -0
  146. package/scripts/e2e-commands.mjs +391 -0
  147. package/scripts/e2e-ws.mjs +3 -2
  148. package/scripts/fetch-modal-token.sh +4 -3
  149. package/scripts/generate-config-keys-json.mjs +45 -0
  150. package/scripts/generate-extension-packages-json.mjs +55 -0
  151. package/scripts/pi-session-query +0 -0
  152. package/scripts/publish-obsidian-plugin.sh +0 -0
  153. package/scripts/reset-test-vault.sh +2 -2
  154. package/scripts/setup-test-config.sh +0 -0
  155. package/scripts/setup-vault-pi.sh +48 -9
  156. package/scripts/test-slash-commands.mjs +2 -2
  157. package/scripts/vault-pi.sh +12 -3
  158. package/skills/github-release-notes-file/SKILL.md +42 -0
  159. package/skills/obsidian-guided-test-session/SKILL.md +110 -0
  160. package/skills/obsidian-interactive-test/SKILL.md +88 -0
  161. package/skills/obsidian-plugin-publish/SKILL.md +58 -0
  162. package/skills/pi-vault-mind-release/SKILL.md +141 -0
  163. package/skills/pre-test-release/SKILL.md +116 -0
  164. package/dist/test/auto-indexer.test.js +0 -2
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Canonical flat key paths for vaultMind config access.
3
+ *
4
+ * This is the SINGLE SOURCE OF TRUTH for config key string literals used
5
+ * by the setup wizard, CLI, HTTP server, and Obsidian plugin. When a field
6
+ * is renamed in src/types.ts, update ONLY this file and regenerate the JSON
7
+ * (scripts/generate-config-keys-json.mjs).
8
+ *
9
+ * Two shapes are provided:
10
+ * - EMBEDDING_FLAT_KEYS: bare key names on vaultMind.embedding (for dynamic access)
11
+ * - VAULT_MIND_CONFIG_KEYS: fully-qualified dotted paths (for static access)
12
+ */
13
+ /** Bare key names on vaultMind.embedding — used by setup wizard, CLI, HTTP, and plugin. */
14
+ export declare const EMBEDDING_FLAT_KEYS: readonly ["remoteUrl", "localUrl", "model", "apiKey", "dim", "useTransformers", "workspace", "remoteApiKey", "remoteReadApiKey", "remoteWriteApiKey", "fallback", "sync", "collectionModels", "coalesce"];
15
+ /** Fully-qualified dotted key paths that the plugin's reconfigure form reads/writes. */
16
+ export declare const VAULT_MIND_CONFIG_KEYS: readonly ["vaultMind.embedding.remoteUrl", "vaultMind.embedding.localUrl", "vaultMind.embedding.model", "vaultMind.embedding.workspace", "vaultMind.embedding.dim", "vaultMind.embedding.useTransformers", "vaultMind.embedding.remoteApiKey", "vaultMind.embedding.fallback", "vaultMind.embedding.sync", "vaultMind.ftsEnabled", "vaultMind.graph", "vaultMind.vaults.default.autoStart", "extensionCompatibility.pi-context.enabled"];
@@ -0,0 +1,45 @@
1
+ /**
2
+ * Canonical flat key paths for vaultMind config access.
3
+ *
4
+ * This is the SINGLE SOURCE OF TRUTH for config key string literals used
5
+ * by the setup wizard, CLI, HTTP server, and Obsidian plugin. When a field
6
+ * is renamed in src/types.ts, update ONLY this file and regenerate the JSON
7
+ * (scripts/generate-config-keys-json.mjs).
8
+ *
9
+ * Two shapes are provided:
10
+ * - EMBEDDING_FLAT_KEYS: bare key names on vaultMind.embedding (for dynamic access)
11
+ * - VAULT_MIND_CONFIG_KEYS: fully-qualified dotted paths (for static access)
12
+ */
13
+ /** Bare key names on vaultMind.embedding — used by setup wizard, CLI, HTTP, and plugin. */
14
+ export const EMBEDDING_FLAT_KEYS = [
15
+ "remoteUrl",
16
+ "localUrl",
17
+ "model",
18
+ "apiKey",
19
+ "dim",
20
+ "useTransformers",
21
+ "workspace",
22
+ "remoteApiKey",
23
+ "remoteReadApiKey",
24
+ "remoteWriteApiKey",
25
+ "fallback",
26
+ "sync",
27
+ "collectionModels",
28
+ "coalesce",
29
+ ];
30
+ /** Fully-qualified dotted key paths that the plugin's reconfigure form reads/writes. */
31
+ export const VAULT_MIND_CONFIG_KEYS = [
32
+ "vaultMind.embedding.remoteUrl",
33
+ "vaultMind.embedding.localUrl",
34
+ "vaultMind.embedding.model",
35
+ "vaultMind.embedding.workspace",
36
+ "vaultMind.embedding.dim",
37
+ "vaultMind.embedding.useTransformers",
38
+ "vaultMind.embedding.remoteApiKey",
39
+ "vaultMind.embedding.fallback",
40
+ "vaultMind.embedding.sync",
41
+ "vaultMind.ftsEnabled",
42
+ "vaultMind.graph",
43
+ "vaultMind.vaults.default.autoStart",
44
+ "extensionCompatibility.pi-context.enabled",
45
+ ];
@@ -1,7 +1,7 @@
1
1
  import * as fs from "node:fs";
2
2
  import * as path from "node:path";
3
3
  import { Type } from "typebox";
4
- import { ensureDir, findConfig } from "./utils.js";
4
+ import { ensureDir, getConfigPath } from "./utils.js";
5
5
  /** Infer a single JSON value's type. */
6
6
  export const inferType = (value) => {
7
7
  if (value === null || value === undefined)
@@ -88,7 +88,7 @@ export const discoverSchemaFromFile = (filePath, sample = 100) => {
88
88
  };
89
89
  /** Read the mutable project config object, or an empty object if none exists. */
90
90
  const readProjectConfig = (cwd) => {
91
- const cfgPath = findConfig(cwd).project || path.join(cwd, "pi-vault-mind.config.json");
91
+ const cfgPath = getConfigPath(cwd);
92
92
  if (!fs.existsSync(cfgPath))
93
93
  return {};
94
94
  try {
@@ -100,7 +100,7 @@ const readProjectConfig = (cwd) => {
100
100
  };
101
101
  /** Persist the project config with standard formatting. */
102
102
  const writeProjectConfig = (cwd, obj) => {
103
- const cfgPath = findConfig(cwd).project || path.join(cwd, "pi-vault-mind.config.json");
103
+ const cfgPath = getConfigPath(cwd);
104
104
  ensureDir(cfgPath);
105
105
  fs.writeFileSync(cfgPath, `${JSON.stringify(obj, null, 2)}\n`, "utf-8");
106
106
  return cfgPath;
@@ -151,7 +151,7 @@ export const registerDiscoverSchemaTool = (pi) => {
151
151
  description: "Infer the schema of an unconfigured .jsonl file by sampling its first N rows and register it as a collection.",
152
152
  promptSnippet: 'discover_schema(file="data/events.jsonl", name="events", sample=100)',
153
153
  promptGuidelines: [
154
- "Use when you have a .jsonl file that is not yet mapped in pi-vault-mind.config.json.",
154
+ "Use when you have a .jsonl file that is not yet mapped in .vault-mind/vault-mind.config.json.",
155
155
  "The tool reads the first N rows, infers field names and JSON types, and writes a CollectionDef to the project config.",
156
156
  ],
157
157
  parameters: Type.Object({
@@ -0,0 +1,21 @@
1
+ export interface PendingEdit {
2
+ id: string;
3
+ path: string;
4
+ agentId: string;
5
+ role: string;
6
+ dispatchId: string;
7
+ oldText: string;
8
+ newText: string;
9
+ status: "pending" | "applied" | "rejected";
10
+ createdAt: number;
11
+ }
12
+ export declare function listEdits(cwd: string, filePath?: string): PendingEdit[];
13
+ export declare function getEdit(cwd: string, id: string): PendingEdit | undefined;
14
+ export declare function createEdit(cwd: string, edit: Omit<PendingEdit, "id" | "status" | "createdAt">): PendingEdit;
15
+ export declare function applyEdit(cwd: string, id: string): {
16
+ ok: true;
17
+ } | {
18
+ ok: false;
19
+ error: string;
20
+ };
21
+ export declare function rejectEdit(cwd: string, id: string): boolean;
@@ -0,0 +1,75 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import * as fs from "node:fs";
3
+ import * as path from "node:path";
4
+ import { getConfigPath } from "./utils.js";
5
+ function editsPath(cwd) {
6
+ return path.join(path.dirname(getConfigPath(cwd)), "edits.jsonl");
7
+ }
8
+ export function listEdits(cwd, filePath) {
9
+ const p = editsPath(cwd);
10
+ if (!fs.existsSync(p))
11
+ return [];
12
+ const edits = fs
13
+ .readFileSync(p, "utf-8")
14
+ .split("\n")
15
+ .filter(Boolean)
16
+ .map((line) => {
17
+ try {
18
+ return JSON.parse(line);
19
+ }
20
+ catch {
21
+ return null;
22
+ }
23
+ })
24
+ .filter((e) => e !== null);
25
+ if (filePath)
26
+ return edits.filter((e) => e.path === filePath);
27
+ return edits;
28
+ }
29
+ export function getEdit(cwd, id) {
30
+ return listEdits(cwd).find((e) => e.id === id);
31
+ }
32
+ export function createEdit(cwd, edit) {
33
+ const full = {
34
+ ...edit,
35
+ id: randomUUID(),
36
+ status: "pending",
37
+ createdAt: Date.now(),
38
+ };
39
+ fs.appendFileSync(editsPath(cwd), `${JSON.stringify(full)}\n`, "utf-8");
40
+ return full;
41
+ }
42
+ export function applyEdit(cwd, id) {
43
+ const edit = getEdit(cwd, id);
44
+ if (edit?.status !== "pending")
45
+ return { ok: false, error: "Edit not found or already resolved" };
46
+ const filePath = path.join(cwd, edit.path);
47
+ if (!fs.existsSync(filePath))
48
+ return { ok: false, error: `File not found: ${edit.path}` };
49
+ const content = fs.readFileSync(filePath, "utf-8");
50
+ // Count occurrences — only replace if oldText appears exactly once
51
+ const occurrences = content.split(edit.oldText).length - 1;
52
+ if (occurrences === 0)
53
+ return { ok: false, error: "oldText not found in file" };
54
+ if (occurrences > 1)
55
+ return {
56
+ ok: false,
57
+ error: `oldText appears ${occurrences} times — ambiguous match, cannot auto-apply`,
58
+ };
59
+ const newContent = content.replace(edit.oldText, edit.newText);
60
+ fs.writeFileSync(filePath, newContent, "utf-8");
61
+ updateEditStatus(cwd, id, "applied");
62
+ return { ok: true };
63
+ }
64
+ export function rejectEdit(cwd, id) {
65
+ const edit = getEdit(cwd, id);
66
+ if (edit?.status !== "pending")
67
+ return false;
68
+ updateEditStatus(cwd, id, "rejected");
69
+ return true;
70
+ }
71
+ function updateEditStatus(cwd, id, status) {
72
+ const edits = listEdits(cwd);
73
+ const updated = edits.map((e) => (e.id === id ? { ...e, status } : e));
74
+ fs.writeFileSync(editsPath(cwd), `${updated.map((e) => JSON.stringify(e)).join("\n")}\n`, "utf-8");
75
+ }
@@ -0,0 +1,34 @@
1
+ import type { VaultMindConfig } from "./types.js";
2
+ export interface EmbeddingProbeRequest {
3
+ target: "local" | "remote";
4
+ url: string;
5
+ workspace?: string;
6
+ model?: string;
7
+ transientSecrets?: {
8
+ apiKey?: string;
9
+ readApiKey?: string;
10
+ writeApiKey?: string;
11
+ };
12
+ }
13
+ export interface EmbeddingModelOption {
14
+ id: string;
15
+ name: string;
16
+ dim: number | null;
17
+ }
18
+ export interface EmbeddingProbeResponse {
19
+ ok: boolean;
20
+ target: "local" | "remote";
21
+ endpoint: string;
22
+ models: readonly EmbeddingModelOption[];
23
+ selectedModelAvailable: boolean | null;
24
+ latencyMs: number | null;
25
+ error: {
26
+ code: "invalid_request" | "unauthorized" | "unreachable" | "provider_error";
27
+ message: string;
28
+ } | null;
29
+ }
30
+ export declare function probeEmbeddingEndpoint(request: EmbeddingProbeRequest, options?: {
31
+ fetch?: typeof fetch;
32
+ now?: () => number;
33
+ config?: VaultMindConfig;
34
+ }): Promise<EmbeddingProbeResponse>;
@@ -0,0 +1,142 @@
1
+ import { resolveEmbeddingSecret } from "./embedding-secrets.js";
2
+ const normalizeEndpointUrl = (url) => {
3
+ const suffixes = ["/v1/embeddings", "/v1", "/"];
4
+ let base = url;
5
+ let changed = true;
6
+ while (changed) {
7
+ changed = false;
8
+ for (const suffix of suffixes) {
9
+ if (base.endsWith(suffix)) {
10
+ base = base.slice(0, -suffix.length);
11
+ changed = true;
12
+ }
13
+ }
14
+ }
15
+ return base;
16
+ };
17
+ const isValidHttpUrl = (url) => {
18
+ try {
19
+ const parsed = new URL(url);
20
+ return parsed.protocol === "http:" || parsed.protocol === "https:";
21
+ }
22
+ catch {
23
+ return false;
24
+ }
25
+ };
26
+ const isString = (value) => typeof value === "string";
27
+ const isNumber = (value) => typeof value === "number" && Number.isFinite(value);
28
+ const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
29
+ const hasStringField = (obj, key) => key in obj && isString(obj[key]);
30
+ const hasNumberField = (obj, key) => key in obj && isNumber(obj[key]);
31
+ const isOllamaCatalog = (body) => {
32
+ if (!isObject(body) || !("models" in body) || !Array.isArray(body.models))
33
+ return false;
34
+ return body.models.every((m) => isObject(m) && hasStringField(m, "name"));
35
+ };
36
+ const isModalCatalog = (body) => {
37
+ if (!isObject(body) || !("models" in body) || !Array.isArray(body.models))
38
+ return false;
39
+ return body.models.every((m) => isObject(m) && hasStringField(m, "key") && hasNumberField(m, "native_dim"));
40
+ };
41
+ const isOpenAiCatalog = (body) => {
42
+ if (!isObject(body) || !("data" in body) || !Array.isArray(body.data))
43
+ return false;
44
+ return body.data.every((d) => isObject(d) && hasStringField(d, "id"));
45
+ };
46
+ const normalizeModels = (body) => {
47
+ if (isOllamaCatalog(body)) {
48
+ return body.models.map((m) => ({ id: m.name, name: m.name, dim: null }));
49
+ }
50
+ if (isModalCatalog(body)) {
51
+ return body.models.map((m) => ({ id: m.key, name: m.key, dim: m.native_dim }));
52
+ }
53
+ if (isOpenAiCatalog(body)) {
54
+ return body.data.map((d) => ({ id: d.id, name: d.id, dim: null }));
55
+ }
56
+ return null;
57
+ };
58
+ const sortModels = (models) => [...models].sort((a, b) => a.id.localeCompare(b.id));
59
+ const buildErrorResponse = (target, endpoint, code, message) => ({
60
+ ok: false,
61
+ target,
62
+ endpoint,
63
+ models: [],
64
+ selectedModelAvailable: null,
65
+ latencyMs: null,
66
+ error: { code, message },
67
+ });
68
+ const resolveAuthorizationHeader = (request, cfg) => {
69
+ const cwd = request.workspace;
70
+ if (request.target === "local") {
71
+ if (request.transientSecrets?.apiKey)
72
+ return request.transientSecrets.apiKey;
73
+ if (cfg)
74
+ return resolveEmbeddingSecret(cfg, "localApiKey", { cwd });
75
+ return undefined;
76
+ }
77
+ // Remote read path: never select write-scoped credentials.
78
+ if (request.transientSecrets?.readApiKey)
79
+ return request.transientSecrets.readApiKey;
80
+ if (request.transientSecrets?.apiKey)
81
+ return request.transientSecrets.apiKey;
82
+ if (cfg) {
83
+ const readToken = resolveEmbeddingSecret(cfg, "remoteReadApiKey", { cwd });
84
+ if (readToken)
85
+ return readToken;
86
+ return resolveEmbeddingSecret(cfg, "remoteApiKey", { cwd });
87
+ }
88
+ return undefined;
89
+ };
90
+ export async function probeEmbeddingEndpoint(request, options) {
91
+ const fetchImpl = options?.fetch ?? fetch;
92
+ const nowImpl = options?.now ?? Date.now;
93
+ const cfg = options?.config;
94
+ const base = normalizeEndpointUrl(request.url);
95
+ if (request.target !== "local" && request.target !== "remote") {
96
+ return buildErrorResponse("local", base, "invalid_request", "Invalid target");
97
+ }
98
+ if (!isValidHttpUrl(base)) {
99
+ return buildErrorResponse(request.target, base, "invalid_request", "Invalid URL");
100
+ }
101
+ const target = request.target;
102
+ const discoveryUrls = target === "local"
103
+ ? [`${base}/api/tags`, `${base}/v1/models`]
104
+ : [`${base}/models`, `${base}/v1/models`];
105
+ const headers = {};
106
+ const auth = resolveAuthorizationHeader(request, cfg);
107
+ if (auth)
108
+ headers.Authorization = `Bearer ${auth}`;
109
+ const start = nowImpl();
110
+ try {
111
+ for (const url of discoveryUrls) {
112
+ const res = await fetchImpl(url, { method: "GET", headers });
113
+ if (res.status === 401) {
114
+ return buildErrorResponse(target, base, "unauthorized", "Authorization failed");
115
+ }
116
+ if (!res.ok) {
117
+ if (res.status === 404)
118
+ continue;
119
+ return buildErrorResponse(target, base, "provider_error", "Provider returned an error");
120
+ }
121
+ const body = (await res.json());
122
+ const models = normalizeModels(body);
123
+ if (models === null)
124
+ continue;
125
+ const sorted = sortModels(models);
126
+ const selectedModelAvailable = request.model == null ? null : sorted.some((m) => m.id === request.model);
127
+ return {
128
+ ok: true,
129
+ target,
130
+ endpoint: base,
131
+ models: sorted,
132
+ selectedModelAvailable,
133
+ latencyMs: nowImpl() - start,
134
+ error: null,
135
+ };
136
+ }
137
+ return buildErrorResponse(target, base, "provider_error", "Provider returned an error");
138
+ }
139
+ catch {
140
+ return buildErrorResponse(target, base, "unreachable", "Endpoint is unreachable");
141
+ }
142
+ }
@@ -0,0 +1,133 @@
1
+ /**
2
+ * Embedding provider interface + concrete implementations.
3
+ *
4
+ * The vector-store layer (currently `src/lance.ts`) is decoupled from the
5
+ * vector-generator layer (this file). A provider only knows how to turn
6
+ * text into `number[][]`; the store adapter wraps the provider into the
7
+ * store-specific class (e.g. LanceDB's `TextEmbeddingFunction`) for its
8
+ * schema/auto-embed needs. Adding a new vector store means writing a new
9
+ * adapter; adding a new provider means implementing this interface.
10
+ *
11
+ * ── Provider contract ─────────────────────────────────────────────────────
12
+ * `embed(texts, task)` — same-length batch, one vector per text, in order.
13
+ * `dim()` — output dimensionality (the store needs this to
14
+ * build the schema; `undefined` is allowed when the
15
+ * dim is learned lazily from the first response).
16
+ * `init()` — optional one-time setup (network probe, model
17
+ * download). Idempotent.
18
+ *
19
+ * The coalescer in `src/embed-queue.ts` already consumes the same
20
+ * `EmbedFn = (texts, task) => Promise<number[][]>` shape — the coalescer
21
+ * path doesn't even need the provider class, just the function. We keep
22
+ * `EmbeddingProvider` as a class for the dim/init convenience, and the
23
+ * factory at the bottom of this file returns both the provider and its
24
+ * matching `EmbedFn` closure.
25
+ */
26
+ import type { EmbedTask } from "./embed-queue.js";
27
+ import type { VaultMindConfig } from "./types.js";
28
+ /**
29
+ * The store-agnostic provider contract. A store adapter (e.g. the Lance
30
+ * adapter in `lance.ts`) wraps an instance of this into the store's own
31
+ * embedding-function class. The coalescer in `embed-queue.ts` consumes
32
+ * just the `embed()` method via the `EmbedFn` closure.
33
+ */
34
+ export interface EmbeddingProvider {
35
+ /** Identifier for logging + the coalescer cache key. */
36
+ readonly id: string;
37
+ /**
38
+ * Output dimensionality. May be `undefined` when the dim is unknown
39
+ * until the first embed call completes (e.g. Modal's `/embed`
40
+ * response carries the dim; the store schema is updated lazily).
41
+ */
42
+ dim(): number | undefined;
43
+ /**
44
+ * Embed a batch of texts. Same length out as in, one vector per text,
45
+ * in order. `task` selects query vs document so providers can pick a
46
+ * task-specific prompt (Ollama / Modal both use this).
47
+ */
48
+ embed(texts: string[], task: EmbedTask): Promise<number[][]>;
49
+ /**
50
+ * One-time setup. Idempotent — calling twice is a no-op. Network
51
+ * probes, model downloads, etc. should happen here. Implementations
52
+ * should never throw on a best-effort connectivity check; the caller
53
+ * will fall back if the actual `embed` call fails later.
54
+ */
55
+ init(): Promise<void>;
56
+ }
57
+ /** Default bearer for local /v1/embeddings endpoints (Ollama, LM Studio, …). */
58
+ export declare const DEFAULT_OLLAMA_API_KEY = "ollama";
59
+ export interface OpenAICompatibleProviderOptions {
60
+ url: string;
61
+ model: string;
62
+ /**
63
+ * Bearer token. Defaults to `"ollama"` for local endpoints; pass
64
+ * `null` explicitly to omit the header (e.g. unauthenticated remote
65
+ * servers that don't validate the header). Most production endpoints
66
+ * should pass an actual token.
67
+ */
68
+ apiKey?: string | null;
69
+ /** Output dim, if known up front. */
70
+ dims?: number;
71
+ }
72
+ /**
73
+ * OpenAI-compatible embedding provider. Works with any `/v1/embeddings`
74
+ * endpoint — Ollama, LM Studio, vLLM, Modal, OpenAI, etc. The endpoint
75
+ * contract is the OpenAI `/v1/embeddings` request/response shape.
76
+ */
77
+ export declare class OpenAICompatibleProvider implements EmbeddingProvider {
78
+ readonly id: string;
79
+ private readonly url;
80
+ private readonly model;
81
+ private readonly apiKey;
82
+ private readonly dimsValue;
83
+ constructor(options: OpenAICompatibleProviderOptions);
84
+ dim(): number | undefined;
85
+ init(): Promise<void>;
86
+ embed(texts: string[], _task: EmbedTask): Promise<number[][]>;
87
+ }
88
+ /**
89
+ * Offline transformers.js provider. Works on any Node.js with no network
90
+ * after the model is downloaded. Fixed dim 384, fixed model
91
+ * `Xenova/all-MiniLM-L6-v2`. The dim is small enough to fit in the
92
+ * `useTransformers: true` escape hatch (a fallback when no URL is
93
+ * configured) but it does NOT share a vector space with `embeddinggemma`,
94
+ * so it's only useful for local dev / testing.
95
+ */
96
+ export declare class TransformersProvider implements EmbeddingProvider {
97
+ readonly id = "transformers:Xenova/all-MiniLM-L6-v2";
98
+ private extractor;
99
+ private initPromise;
100
+ dim(): number | undefined;
101
+ init(): Promise<void>;
102
+ embed(texts: string[]): Promise<number[][]>;
103
+ }
104
+ /**
105
+ * Modal cloud embedding provider. Hits the deployed ASGI app's `/embed`
106
+ * endpoint (see `src/modal-client.ts`). The dim is learned lazily from
107
+ * the first response and cached by the caller.
108
+ */
109
+ export declare class ModalProvider implements EmbeddingProvider {
110
+ readonly id: string;
111
+ private readonly cfg;
112
+ private readonly collection;
113
+ private readonly dimsValue;
114
+ constructor(cfg: VaultMindConfig, collection?: string, dimsValue?: number);
115
+ dim(): number | undefined;
116
+ init(): Promise<void>;
117
+ embed(texts: string[], task: EmbedTask): Promise<number[][]>;
118
+ }
119
+ /**
120
+ * Pick the right provider for a vault config. Resolution order matches the
121
+ * pre-refactor behavior in `lance.ts:getEmbeddingFunction`:
122
+ *
123
+ * 1. `embedding.remoteUrl` → Modal cloud
124
+ * 2. `embedding.localUrl` → OpenAI-compatible (Ollama, LM Studio, etc.)
125
+ * 3. `embedding.useTransformers !== false` → transformers.js offline
126
+ *
127
+ * To add a new provider, add a class above and a branch here. No
128
+ * downstream code (lance, coalescer, plugin) needs to change.
129
+ */
130
+ export declare const createProvider: (cfg: VaultMindConfig, opts?: {
131
+ collection?: string;
132
+ dims?: number;
133
+ }) => Promise<EmbeddingProvider>;
@@ -0,0 +1,173 @@
1
+ import { resolveEmbeddingSecret } from "./embedding-secrets.js";
2
+ import { createModalClient, resolveDim, resolveModel } from "./modal-config.js";
3
+ // ── OpenAI-compatible provider (any /v1/embeddings endpoint) ──────────────
4
+ /** Default bearer for local /v1/embeddings endpoints (Ollama, LM Studio, …). */
5
+ export const DEFAULT_OLLAMA_API_KEY = "ollama";
6
+ /**
7
+ * OpenAI-compatible embedding provider. Works with any `/v1/embeddings`
8
+ * endpoint — Ollama, LM Studio, vLLM, Modal, OpenAI, etc. The endpoint
9
+ * contract is the OpenAI `/v1/embeddings` request/response shape.
10
+ */
11
+ export class OpenAICompatibleProvider {
12
+ id;
13
+ url;
14
+ model;
15
+ apiKey;
16
+ dimsValue;
17
+ constructor(options) {
18
+ this.url = options.url;
19
+ this.model = options.model;
20
+ // 0.16.0+ default: "ollama" for local endpoints. Matches
21
+ // AgentModelProvider's behavior. Callers passing `null` explicitly
22
+ // get no Authorization header (some local servers reject "Bearer
23
+ // ollama" as invalid auth).
24
+ this.apiKey = options.apiKey === null ? null : (options.apiKey ?? DEFAULT_OLLAMA_API_KEY);
25
+ this.dimsValue = options.dims;
26
+ this.id = `openai-compatible:${options.url}:${options.model}`;
27
+ }
28
+ dim() {
29
+ return this.dimsValue;
30
+ }
31
+ async init() {
32
+ // Best-effort connectivity check; failures are non-fatal.
33
+ try {
34
+ const baseUrl = this.url.replace(/\/v1\/embeddings\/?$/, "").replace(/\/+$/, "");
35
+ const headers = {};
36
+ if (this.apiKey)
37
+ headers.Authorization = `Bearer ${this.apiKey}`;
38
+ await fetch(`${baseUrl}/v1/models`, { headers });
39
+ }
40
+ catch (err) {
41
+ console.warn(`[pi-vault-mind] Cannot reach embedding endpoint at ${this.url}: ${err}`);
42
+ }
43
+ }
44
+ async embed(texts, _task) {
45
+ const endpoint = this.url.endsWith("/v1/embeddings")
46
+ ? this.url
47
+ : `${this.url.replace(/\/+$/, "")}/v1/embeddings`;
48
+ const headers = { "Content-Type": "application/json" };
49
+ if (this.apiKey)
50
+ headers.Authorization = `Bearer ${this.apiKey}`;
51
+ const resp = await fetch(endpoint, {
52
+ method: "POST",
53
+ headers,
54
+ body: JSON.stringify({ model: this.model, input: texts }),
55
+ });
56
+ if (!resp.ok) {
57
+ const body = await resp.text().catch(() => "");
58
+ throw new Error(`Embed failed: ${resp.status} ${body}`);
59
+ }
60
+ const data = (await resp.json());
61
+ if (!data.data || !Array.isArray(data.data)) {
62
+ throw new Error("Invalid embedding response");
63
+ }
64
+ return data.data.map((d) => d.embedding);
65
+ }
66
+ }
67
+ // ── Transformers.js provider (fully offline) ───────────────────────────────
68
+ /**
69
+ * Offline transformers.js provider. Works on any Node.js with no network
70
+ * after the model is downloaded. Fixed dim 384, fixed model
71
+ * `Xenova/all-MiniLM-L6-v2`. The dim is small enough to fit in the
72
+ * `useTransformers: true` escape hatch (a fallback when no URL is
73
+ * configured) but it does NOT share a vector space with `embeddinggemma`,
74
+ * so it's only useful for local dev / testing.
75
+ */
76
+ export class TransformersProvider {
77
+ id = "transformers:Xenova/all-MiniLM-L6-v2";
78
+ extractor = null;
79
+ initPromise = null;
80
+ dim() {
81
+ return 384;
82
+ }
83
+ async init() {
84
+ if (!this.initPromise) {
85
+ this.initPromise = (async () => {
86
+ const { pipeline } = await import("@xenova/transformers");
87
+ this.extractor = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
88
+ })();
89
+ }
90
+ return this.initPromise;
91
+ }
92
+ async embed(texts) {
93
+ await this.init();
94
+ const embeddings = [];
95
+ for (const text of texts) {
96
+ const result = await this.extractor(text, { pooling: "mean", normalize: true });
97
+ embeddings.push(Array.from(result.data));
98
+ }
99
+ return embeddings;
100
+ }
101
+ }
102
+ // ── Modal provider (cloud, /embed endpoint) ────────────────────────────────
103
+ /**
104
+ * Modal cloud embedding provider. Hits the deployed ASGI app's `/embed`
105
+ * endpoint (see `src/modal-client.ts`). The dim is learned lazily from
106
+ * the first response and cached by the caller.
107
+ */
108
+ export class ModalProvider {
109
+ id;
110
+ cfg;
111
+ collection;
112
+ dimsValue;
113
+ constructor(cfg, collection, dimsValue) {
114
+ this.cfg = cfg;
115
+ this.collection = collection;
116
+ this.dimsValue = dimsValue;
117
+ const model = resolveModel(cfg, collection);
118
+ this.id = `modal:${model}:${dimsValue ?? "native"}`;
119
+ }
120
+ dim() {
121
+ return this.dimsValue;
122
+ }
123
+ async init() {
124
+ // No-op: Modal client is built per-embed; no persistent connection.
125
+ }
126
+ async embed(texts, task) {
127
+ const client = createModalClient(this.cfg);
128
+ if (!client) {
129
+ throw new Error("Modal provider not configured (missing baseUrl or token).");
130
+ }
131
+ const res = await client.embed(texts, {
132
+ model: resolveModel(this.cfg, this.collection),
133
+ dim: resolveDim(this.cfg, this.collection),
134
+ task,
135
+ });
136
+ return res.vectors;
137
+ }
138
+ }
139
+ // ── Factory ────────────────────────────────────────────────────────────────
140
+ /**
141
+ * Pick the right provider for a vault config. Resolution order matches the
142
+ * pre-refactor behavior in `lance.ts:getEmbeddingFunction`:
143
+ *
144
+ * 1. `embedding.remoteUrl` → Modal cloud
145
+ * 2. `embedding.localUrl` → OpenAI-compatible (Ollama, LM Studio, etc.)
146
+ * 3. `embedding.useTransformers !== false` → transformers.js offline
147
+ *
148
+ * To add a new provider, add a class above and a branch here. No
149
+ * downstream code (lance, coalescer, plugin) needs to change.
150
+ */
151
+ export const createProvider = async (cfg, opts = {}) => {
152
+ if (cfg.embedding.remoteUrl) {
153
+ const provider = new ModalProvider(cfg, opts.collection, opts.dims);
154
+ await provider.init();
155
+ return provider;
156
+ }
157
+ if (cfg.embedding.localUrl) {
158
+ const provider = new OpenAICompatibleProvider({
159
+ url: cfg.embedding.localUrl,
160
+ model: cfg.embedding.model || "embeddinggemma",
161
+ apiKey: resolveEmbeddingSecret(cfg, "localApiKey") ?? DEFAULT_OLLAMA_API_KEY,
162
+ dims: opts.dims,
163
+ });
164
+ await provider.init();
165
+ return provider;
166
+ }
167
+ if (cfg.embedding.useTransformers !== false) {
168
+ const provider = new TransformersProvider();
169
+ await provider.init();
170
+ return provider;
171
+ }
172
+ throw new Error("No embedding provider configured. Set remoteUrl, localUrl, or useTransformers: true.");
173
+ };