pi-harness-runtime 1.1.410 → 1.1.412
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/harness/notification-settings.ts +130 -0
- package/harness/skill-remote-source.ts +157 -0
- package/harness/skill-sync.ts +74 -22
- package/harness/skills-startup.ts +202 -0
- package/harness/startup-report.ts +98 -56
- package/harness-startup-status.ts +31 -2
- package/index.ts +125 -31
- package/package.json +1 -1
- package/packages/a2a-adapter/package.json +1 -1
- package/packages/agent-policy/package.json +1 -1
- package/packages/architecture-generator/package.json +1 -1
- package/packages/auth/package.json +1 -1
- package/packages/autonomous-refactor/package.json +1 -1
- package/packages/autonomous-runtime/package.json +1 -1
- package/packages/autonomous-runtime/test/types.test.ts +59 -32
- package/packages/cache-strategy/package.json +1 -1
- package/packages/capability-registry/package.json +1 -1
- package/packages/checkpoint/package.json +1 -1
- package/packages/cli-plugin-sdk/package.json +1 -1
- package/packages/clipboard/package.json +1 -1
- package/packages/clipboard-plugin/package.json +1 -1
- package/packages/code-generation/package.json +1 -1
- package/packages/code-review/package.json +1 -1
- package/packages/codex-adapter/package.json +1 -1
- package/packages/codex-watcher/package.json +1 -1
- package/packages/config-capture/package.json +1 -1
- package/packages/connection-runtime/package.json +1 -1
- package/packages/context-compiler/package.json +1 -1
- package/packages/context-discovery/package.json +1 -1
- package/packages/context-manager/package.json +1 -1
- package/packages/contracts/package.json +1 -1
- package/packages/cookie-sanitizer/package.json +1 -1
- package/packages/cookie-sanitizer/test/sync.test.ts +7 -1
- package/packages/cost-optimizer/package.json +1 -1
- package/packages/dependency-analyzer/dist/src/imports.js +3 -3
- package/packages/dependency-analyzer/dist/src/imports.js.map +1 -1
- package/packages/dependency-analyzer/package.json +1 -1
- package/packages/dependency-analyzer/src/imports.ts +3 -3
- package/packages/dependency-analyzer/test/analyzer.test.ts +1 -0
- package/packages/django-plugin/package.json +1 -1
- package/packages/doc-generator/package.json +1 -1
- package/packages/durable-review/package.json +1 -1
- package/packages/durable-review/test/acceptance.test.ts +24 -2
- package/packages/durable-review/test/smoke.test.ts +85 -47
- package/packages/e2e-tester/package.json +1 -1
- package/packages/evaluation-engine/package.json +1 -1
- package/packages/evaluation-runner/package.json +1 -1
- package/packages/event-bus/package.json +1 -1
- package/packages/event-bus/test/bus.test.ts +16 -4
- package/packages/event-sink/package.json +1 -1
- package/packages/event-store/package.json +1 -1
- package/packages/experience-replay/package.json +1 -1
- package/packages/feedback-collector/package.json +1 -1
- package/packages/file-copy-helper/package.json +1 -1
- package/packages/framework-detector/package.json +1 -1
- package/packages/framework-plugin-sdk/package.json +1 -1
- package/packages/frappe-plugin/package.json +1 -1
- package/packages/generic-web-plugin/package.json +1 -1
- package/packages/health-monitor/package.json +1 -1
- package/packages/intent-analyzer/package.json +1 -1
- package/packages/jev-judge/package.json +1 -1
- package/packages/knowledge-graph/package.json +1 -1
- package/packages/knowledge-retrieval/package.json +1 -1
- package/packages/laravel-plugin/package.json +1 -1
- package/packages/learning-engine/package.json +1 -1
- package/packages/logging/package.json +1 -1
- package/packages/mcp-adapter/package.json +1 -1
- package/packages/memory-engine/package.json +1 -1
- package/packages/milestone-manager/package.json +1 -1
- package/packages/model-registry/package.json +1 -1
- package/packages/nextjs-plugin/package.json +1 -1
- package/packages/notification/package.json +1 -1
- package/packages/observability/package.json +1 -1
- package/packages/okf-indexer/package.json +1 -1
- package/packages/paperclip-integration/package.json +1 -1
- package/packages/performance-optimizer/package.json +1 -1
- package/packages/permission-mode/package.json +1 -1
- package/packages/privilege-broker/package.json +1 -1
- package/packages/project-analyzer/package.json +1 -1
- package/packages/project-bootstrap/package.json +1 -1
- package/packages/projection-engine/package.json +1 -1
- package/packages/prompt-compiler/package.json +1 -1
- package/packages/prompt-versioning/package.json +1 -1
- package/packages/provider-adapter-sdk/package.json +1 -1
- package/packages/provider-router/package.json +1 -1
- package/packages/provider-selector/package.json +1 -1
- package/packages/providers/package.json +1 -1
- package/packages/qdrant-skills/dist/client.d.ts.map +1 -1
- package/packages/qdrant-skills/dist/client.js +21 -8
- package/packages/qdrant-skills/dist/client.js.map +1 -1
- package/packages/qdrant-skills/dist/embedder.d.ts +37 -0
- package/packages/qdrant-skills/dist/embedder.d.ts.map +1 -1
- package/packages/qdrant-skills/dist/embedder.js +65 -1
- package/packages/qdrant-skills/dist/embedder.js.map +1 -1
- package/packages/qdrant-skills/dist/index.d.ts +1 -1
- package/packages/qdrant-skills/dist/index.d.ts.map +1 -1
- package/packages/qdrant-skills/dist/index.js +1 -1
- package/packages/qdrant-skills/dist/index.js.map +1 -1
- package/packages/qdrant-skills/package.json +1 -1
- package/packages/qdrant-skills/src/client.ts +19 -8
- package/packages/qdrant-skills/src/embedder.ts +103 -1
- package/packages/qdrant-skills/src/index.ts +5 -0
- package/packages/qdrant-skills/test/collection-info.test.ts +129 -0
- package/packages/qdrant-skills/test/embedder-batch.test.ts +184 -0
- package/packages/quota-manager/package.json +1 -1
- package/packages/rate-limiter/package.json +1 -1
- package/packages/react-vite-plugin/package.json +1 -1
- package/packages/relay/package.json +1 -1
- package/packages/release-manager/package.json +1 -1
- package/packages/requirement-compiler/package.json +1 -1
- package/packages/runtime/package.json +1 -1
- package/packages/scheduler/package.json +1 -1
- package/packages/scheduler-adapter/package.json +1 -1
- package/packages/session/package.json +1 -1
- package/packages/session-api/package.json +1 -1
- package/packages/session-export/package.json +1 -1
- package/packages/shared-context/package.json +1 -1
- package/packages/skill-mcp-client/package.json +1 -1
- package/packages/skill-registry/package.json +1 -1
- package/packages/skills/dist/index.d.ts +19 -1
- package/packages/skills/dist/index.d.ts.map +1 -1
- package/packages/skills/dist/index.js +7 -2
- package/packages/skills/dist/index.js.map +1 -1
- package/packages/skills/dist/loader.d.ts +273 -3
- package/packages/skills/dist/loader.d.ts.map +1 -1
- package/packages/skills/dist/loader.js +525 -36
- package/packages/skills/dist/loader.js.map +1 -1
- package/packages/skills/dist/remote-source.d.ts +107 -0
- package/packages/skills/dist/remote-source.d.ts.map +1 -0
- package/packages/skills/dist/remote-source.js +79 -0
- package/packages/skills/dist/remote-source.js.map +1 -0
- package/packages/skills/dist/vector-schema.d.ts +58 -0
- package/packages/skills/dist/vector-schema.d.ts.map +1 -0
- package/packages/skills/dist/vector-schema.js +55 -0
- package/packages/skills/dist/vector-schema.js.map +1 -0
- package/packages/skills/package.json +1 -1
- package/packages/skills/src/index.ts +28 -5
- package/packages/skills/src/loader.ts +688 -53
- package/packages/skills/src/remote-source.ts +148 -0
- package/packages/skills/src/vector-schema.ts +72 -0
- package/packages/skills/test/init-skills.test.ts +107 -111
- package/packages/skills/test/remote-skills.test.ts +242 -0
- package/packages/skills/test/remote-source.test.ts +248 -0
- package/packages/skills/test/vector-indexing.test.ts +205 -0
- package/packages/sprint-planner/package.json +1 -1
- package/packages/subscription-engine/package.json +1 -1
- package/packages/task-compiler/package.json +1 -1
- package/packages/telegram/package.json +1 -1
- package/packages/tencentdb-memory/package.json +1 -1
- package/packages/tencentdb-sync/package.json +1 -1
- package/packages/test-data-generator/package.json +1 -1
- package/packages/test-generator/package.json +1 -1
- package/packages/todo-bd-sync/package.json +1 -1
- package/packages/token-estimation/package.json +1 -1
- package/packages/token-optimizer/package.json +1 -1
- package/packages/tooling-preference/package.json +1 -1
- package/packages/trajectory/package.json +1 -1
- package/packages/tui/package.json +1 -1
- package/packages/two-phase-review/package.json +1 -1
- package/packages/types/package.json +1 -1
- package/packages/voice-api/package.json +1 -1
- package/packages/webmcp-tester/package.json +1 -1
- package/packages/wiki/package.json +1 -1
- package/packages/workflow-events/package.json +1 -1
- package/packages/workspace-scanner/package.json +1 -1
- package/packages/workspace-scanner/test/scanner.test.ts +4 -3
- package/packages/worktree/package.json +1 -1
- package/packages/write-review/package.json +1 -1
- package/scripts/sync-to-qdrant.ts +79 -14
- package/packages/autonomous-runtime/test/autonomous-runtime.test.ts +0 -194
- package/packages/autonomous-runtime/test/inbox.test.ts +0 -80
|
@@ -7,6 +7,22 @@
|
|
|
7
7
|
import { existsSync, readFileSync } from "node:fs";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { parseSkillFile } from "./parser.js";
|
|
10
|
+
import { LEGACY_SKILL_COLLECTION, LEGACY_SKILL_COLLECTION_DIMENSIONS, LEGACY_SKILL_COLLECTION_MODEL, SKILL_COLLECTION, describeSkillSchemaMismatch, } from "./vector-schema.js";
|
|
11
|
+
import { describeRemoteFailure, isStale, skillFromRemote, } from "./remote-source.js";
|
|
12
|
+
/**
|
|
13
|
+
* The vector size of the runtime, used when the installed Qdrant package
|
|
14
|
+
* does not report the size of its model.
|
|
15
|
+
*
|
|
16
|
+
* The embedder owns the real value; this is the fallback for an older
|
|
17
|
+
* package build.
|
|
18
|
+
*/
|
|
19
|
+
const RUNTIME_VECTOR_DIMENSIONS = 1536;
|
|
20
|
+
/** The distance of the runtime skill collection. */
|
|
21
|
+
const RUNTIME_VECTOR_DISTANCE = "Cosine";
|
|
22
|
+
/**
|
|
23
|
+
* The default number of skills one indexing request carries.
|
|
24
|
+
*/
|
|
25
|
+
const DEFAULT_INDEX_BATCH_SIZE = 32;
|
|
10
26
|
/** Resolve the Qdrant skill module path (monorepo sibling or npm package) */
|
|
11
27
|
async function resolveQdrantModules() {
|
|
12
28
|
// Try the monorepo sibling first (the compiled package, so the module
|
|
@@ -15,7 +31,21 @@ async function resolveQdrantModules() {
|
|
|
15
31
|
try {
|
|
16
32
|
// A dynamic import: this module is ESM, where `require` does not exist.
|
|
17
33
|
const mod = (await import(/* @vite-ignore */ base + "/index.js"));
|
|
18
|
-
return
|
|
34
|
+
return {
|
|
35
|
+
createQdrantClient: mod.createQdrantClient,
|
|
36
|
+
createCollection: mod.createCollection,
|
|
37
|
+
createEmbedding: mod.createEmbedding,
|
|
38
|
+
// Absent in an older package build: the registry falls back to one
|
|
39
|
+
// request per text, still with a bounded concurrency.
|
|
40
|
+
createEmbeddingsBatch: typeof mod.createEmbeddingsBatch === "function"
|
|
41
|
+
? mod.createEmbeddingsBatch
|
|
42
|
+
: undefined,
|
|
43
|
+
getCollectionInfo: typeof mod.getCollectionInfo === "function"
|
|
44
|
+
? mod.getCollectionInfo
|
|
45
|
+
: undefined,
|
|
46
|
+
embeddingModel: typeof mod.EMBEDDING_MODEL === "string" ? mod.EMBEDDING_MODEL : undefined,
|
|
47
|
+
embeddingDimensions: typeof mod.EMBEDDING_DIMENSIONS === "number" ? mod.EMBEDDING_DIMENSIONS : undefined,
|
|
48
|
+
};
|
|
19
49
|
}
|
|
20
50
|
catch {
|
|
21
51
|
// Try next path
|
|
@@ -28,9 +58,17 @@ async function resolveQdrantModules() {
|
|
|
28
58
|
const specifier = "@pi-harness/qdrant-skills";
|
|
29
59
|
const mod = (await import(/* @vite-ignore */ specifier));
|
|
30
60
|
return {
|
|
31
|
-
createQdrantClient:
|
|
61
|
+
createQdrantClient: mod.createQdrantClient,
|
|
32
62
|
createCollection: mod.createCollection,
|
|
33
63
|
createEmbedding: mod.createEmbedding,
|
|
64
|
+
createEmbeddingsBatch: typeof mod.createEmbeddingsBatch === "function"
|
|
65
|
+
? mod.createEmbeddingsBatch
|
|
66
|
+
: undefined,
|
|
67
|
+
getCollectionInfo: typeof mod.getCollectionInfo === "function"
|
|
68
|
+
? mod.getCollectionInfo
|
|
69
|
+
: undefined,
|
|
70
|
+
embeddingModel: typeof mod.EMBEDDING_MODEL === "string" ? mod.EMBEDDING_MODEL : undefined,
|
|
71
|
+
embeddingDimensions: typeof mod.EMBEDDING_DIMENSIONS === "number" ? mod.EMBEDDING_DIMENSIONS : undefined,
|
|
34
72
|
};
|
|
35
73
|
}
|
|
36
74
|
catch {
|
|
@@ -65,7 +103,10 @@ export function getQdrantConfig() {
|
|
|
65
103
|
return {
|
|
66
104
|
url,
|
|
67
105
|
apiKey,
|
|
68
|
-
collection:
|
|
106
|
+
// The collection carries the model in its name: a collection built with
|
|
107
|
+
// a different embedding model holds vectors of a different size and
|
|
108
|
+
// cannot be searched with these vectors.
|
|
109
|
+
collection: process.env.QDRANT_COLLECTION || SKILL_COLLECTION,
|
|
69
110
|
};
|
|
70
111
|
}
|
|
71
112
|
/**
|
|
@@ -73,18 +114,62 @@ export function getQdrantConfig() {
|
|
|
73
114
|
*/
|
|
74
115
|
export class SkillRegistry {
|
|
75
116
|
skills = new Map();
|
|
117
|
+
/** The options of the registry. */
|
|
118
|
+
_options;
|
|
119
|
+
/**
|
|
120
|
+
* The skills that only exist in the vector store.
|
|
121
|
+
*
|
|
122
|
+
* They are kept apart from the local skills on purpose: `list()` must
|
|
123
|
+
* report the LOCAL skills, so a client can say how many skills it has on
|
|
124
|
+
* disk and how many it can reach through the vector store.
|
|
125
|
+
*/
|
|
126
|
+
remoteSkills = new Map();
|
|
127
|
+
/**
|
|
128
|
+
* The remote skills whose content is COMPLETE.
|
|
129
|
+
*
|
|
130
|
+
* A skill that only carries the truncated body of a vector store payload
|
|
131
|
+
* is not in this set - it is completed on the first load.
|
|
132
|
+
*/
|
|
133
|
+
_completeRemote = new Set();
|
|
134
|
+
/**
|
|
135
|
+
* The service that holds the server-side skills.
|
|
136
|
+
*
|
|
137
|
+
* `undefined` means "no service is configured".
|
|
138
|
+
*/
|
|
139
|
+
_remoteSource;
|
|
76
140
|
index = { entries: [], loadedAt: 0 };
|
|
77
141
|
loading = new Map();
|
|
142
|
+
// The vector index. `undefined` means "use Qdrant", `null` means "no index".
|
|
143
|
+
_indexer;
|
|
144
|
+
_batchSize;
|
|
145
|
+
// The skills that still have to be indexed, in registration order.
|
|
146
|
+
_indexQueue = [];
|
|
147
|
+
_indexFlushRunning = false;
|
|
148
|
+
_indexFlushScheduled = false;
|
|
149
|
+
// The content hash of the version of a skill that was queued or indexed.
|
|
150
|
+
_indexedHashes = new Map();
|
|
78
151
|
// Qdrant state (lazily initialized on first register if configured)
|
|
79
152
|
_qdrant = null;
|
|
80
153
|
_qdrantInitPromise = null;
|
|
81
154
|
_qdrantInitFailed = false;
|
|
155
|
+
_qdrantIndexer = null;
|
|
156
|
+
constructor(options = {}) {
|
|
157
|
+
this._options = options;
|
|
158
|
+
this._indexer = options.indexer;
|
|
159
|
+
this._batchSize = options.batchSize ?? DEFAULT_INDEX_BATCH_SIZE;
|
|
160
|
+
}
|
|
82
161
|
/**
|
|
83
162
|
* Check if Qdrant initialization has failed (useful for diagnostics).
|
|
84
163
|
*/
|
|
85
164
|
isQdrantFailed() {
|
|
86
165
|
return this._qdrantInitFailed;
|
|
87
166
|
}
|
|
167
|
+
/**
|
|
168
|
+
* Whether skills are still queued for the vector index.
|
|
169
|
+
*/
|
|
170
|
+
isIndexing() {
|
|
171
|
+
return this._indexFlushRunning || this._indexQueue.length > 0;
|
|
172
|
+
}
|
|
88
173
|
/**
|
|
89
174
|
* Initialize Qdrant client if configured.
|
|
90
175
|
* Uses a shared init promise to prevent race conditions when multiple
|
|
@@ -99,6 +184,20 @@ export class SkillRegistry {
|
|
|
99
184
|
return this._qdrantInitPromise;
|
|
100
185
|
}
|
|
101
186
|
async _doQdrantInit() {
|
|
187
|
+
// An injected client replaces the whole connection: a test exercises the
|
|
188
|
+
// search path without a network and without credentials.
|
|
189
|
+
const injected = this._options.qdrantClient;
|
|
190
|
+
if (injected) {
|
|
191
|
+
this._qdrant = {
|
|
192
|
+
client: injected,
|
|
193
|
+
embedder: this._options.embedder ?? {
|
|
194
|
+
createEmbedding: async () => ({ embedding: [], tokens: 0 }),
|
|
195
|
+
},
|
|
196
|
+
collection: this._options.collection ?? "pi-harness-skills-test",
|
|
197
|
+
ready: true,
|
|
198
|
+
};
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
102
201
|
const config = getQdrantConfig();
|
|
103
202
|
if (!config) {
|
|
104
203
|
this._qdrantInitFailed = true;
|
|
@@ -115,10 +214,17 @@ export class SkillRegistry {
|
|
|
115
214
|
url: config.url,
|
|
116
215
|
apiKey: config.apiKey,
|
|
117
216
|
});
|
|
118
|
-
|
|
217
|
+
// The collection must hold vectors of the size the embedder produces.
|
|
218
|
+
// An existing collection with a different size is NOT recreated here:
|
|
219
|
+
// that would destroy the skills of another model. The mismatch is
|
|
220
|
+
// reported instead (see getQdrantStats).
|
|
221
|
+
await mods.createCollection(client, config.collection, this._expectedVectorSchema(mods).dimensions);
|
|
119
222
|
this._qdrant = {
|
|
120
223
|
client,
|
|
121
|
-
embedder: {
|
|
224
|
+
embedder: {
|
|
225
|
+
createEmbedding: mods.createEmbedding,
|
|
226
|
+
createEmbeddingsBatch: mods.createEmbeddingsBatch,
|
|
227
|
+
},
|
|
122
228
|
collection: config.collection,
|
|
123
229
|
ready: true,
|
|
124
230
|
};
|
|
@@ -129,27 +235,238 @@ export class SkillRegistry {
|
|
|
129
235
|
}
|
|
130
236
|
}
|
|
131
237
|
/**
|
|
132
|
-
*
|
|
238
|
+
* The schema the runtime expects of the skill collection.
|
|
239
|
+
*
|
|
240
|
+
* The installed Qdrant package knows the size of its own model; an older
|
|
241
|
+
* build does not, and the fallback is used then.
|
|
133
242
|
*/
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
243
|
+
_expectedVectorSchema(mods) {
|
|
244
|
+
return {
|
|
245
|
+
dimensions: mods.embeddingDimensions ?? RUNTIME_VECTOR_DIMENSIONS,
|
|
246
|
+
distance: RUNTIME_VECTOR_DISTANCE,
|
|
247
|
+
model: mods.embeddingModel,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* The state of the skill vector store.
|
|
252
|
+
*
|
|
253
|
+
* The numbers are what the startup summary shows next to the local skill
|
|
254
|
+
* count, and the state says WHY there is no number: no credentials, an
|
|
255
|
+
* unreachable cluster, a collection that does not exist yet.
|
|
256
|
+
*
|
|
257
|
+
* The collection schema is checked here as well: a collection built with a
|
|
258
|
+
* different embedding model holds vectors the runtime cannot search with,
|
|
259
|
+
* and the reason has to be named - otherwise the vector search simply
|
|
260
|
+
* finds nothing.
|
|
261
|
+
*/
|
|
262
|
+
async getQdrantStats() {
|
|
263
|
+
const injected = this._options.qdrantClient;
|
|
264
|
+
const config = injected
|
|
265
|
+
? {
|
|
266
|
+
url: "",
|
|
267
|
+
apiKey: "",
|
|
268
|
+
collection: this._options.collection ?? "pi-harness-skills-test",
|
|
269
|
+
}
|
|
270
|
+
: getQdrantConfig();
|
|
271
|
+
if (!config)
|
|
272
|
+
return { state: "not-configured", count: 0 };
|
|
273
|
+
const mods = await resolveQdrantModules();
|
|
274
|
+
if (!mods) {
|
|
275
|
+
return {
|
|
276
|
+
state: "unavailable",
|
|
277
|
+
count: 0,
|
|
278
|
+
collection: config.collection,
|
|
279
|
+
error: "the Qdrant package is not available",
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
if (!mods.getCollectionInfo) {
|
|
283
|
+
return {
|
|
284
|
+
state: "unavailable",
|
|
285
|
+
count: 0,
|
|
286
|
+
collection: config.collection,
|
|
287
|
+
error: "the installed Qdrant package cannot read a collection",
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
try {
|
|
291
|
+
const client = injected ?? mods.createQdrantClient({ url: config.url, apiKey: config.apiKey });
|
|
292
|
+
const info = await mods.getCollectionInfo(client, config.collection);
|
|
293
|
+
if (!info) {
|
|
294
|
+
return {
|
|
295
|
+
state: "unavailable",
|
|
296
|
+
count: 0,
|
|
297
|
+
collection: config.collection,
|
|
298
|
+
error: "the collection could not be read",
|
|
299
|
+
};
|
|
300
|
+
}
|
|
301
|
+
if (!info.exists) {
|
|
302
|
+
return { state: "missing", count: 0, collection: config.collection };
|
|
303
|
+
}
|
|
304
|
+
const count = info.pointsCount ?? 0;
|
|
305
|
+
const stats = {
|
|
306
|
+
state: count > 0 ? "populated" : "empty",
|
|
307
|
+
count,
|
|
308
|
+
collection: config.collection,
|
|
309
|
+
dimensions: info.vectorsSize,
|
|
310
|
+
distance: info.vectorsDistance,
|
|
311
|
+
};
|
|
312
|
+
const mismatch = describeSkillSchemaMismatch({ dimensions: info.vectorsSize, distance: info.vectorsDistance }, this._expectedVectorSchema(mods));
|
|
313
|
+
if (mismatch) {
|
|
314
|
+
stats.schemaMismatch = mismatch;
|
|
315
|
+
// The legacy collection is a known cause - name it, so the reader
|
|
316
|
+
// does not have to know the history of the collection names.
|
|
317
|
+
if (config.collection === LEGACY_SKILL_COLLECTION) {
|
|
318
|
+
stats.schemaMismatch +=
|
|
319
|
+
` - the collection ${LEGACY_SKILL_COLLECTION} was created by scripts/sync-to-qdrant.ts` +
|
|
320
|
+
` with ${LEGACY_SKILL_COLLECTION_MODEL} (${LEGACY_SKILL_COLLECTION_DIMENSIONS} dimensions)`;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
return stats;
|
|
324
|
+
}
|
|
325
|
+
catch (err) {
|
|
326
|
+
return {
|
|
327
|
+
state: "unavailable",
|
|
328
|
+
count: 0,
|
|
329
|
+
collection: config.collection,
|
|
330
|
+
error: err instanceof Error ? err.message : String(err),
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/** The point of a skill in the vector index. */
|
|
335
|
+
_toQdrantPoint(skill) {
|
|
336
|
+
return {
|
|
138
337
|
id: this._hashId(skill.id),
|
|
139
338
|
name: skill.id,
|
|
140
339
|
description: skill.frontmatter.description,
|
|
141
340
|
body: skill.body.slice(0, 2000),
|
|
142
341
|
text: [skill.id, skill.frontmatter.name, skill.frontmatter.description, skill.body.slice(0, 1500)].join(" "),
|
|
143
342
|
};
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* The indexer of the registry: the injected one, or the Qdrant cluster.
|
|
346
|
+
*/
|
|
347
|
+
_getIndexer() {
|
|
348
|
+
if (this._indexer !== undefined)
|
|
349
|
+
return this._indexer;
|
|
350
|
+
if (!this._qdrantIndexer) {
|
|
351
|
+
this._qdrantIndexer = this._createQdrantIndexer();
|
|
150
352
|
}
|
|
151
|
-
|
|
152
|
-
|
|
353
|
+
return this._qdrantIndexer;
|
|
354
|
+
}
|
|
355
|
+
/**
|
|
356
|
+
* The Qdrant cluster as an indexer.
|
|
357
|
+
*
|
|
358
|
+
* The batch call of the embeddings endpoint carries many texts in one
|
|
359
|
+
* request. One request per skill exhausted the sockets of the process when
|
|
360
|
+
* a registry held hundreds of skills, and every other request of the
|
|
361
|
+
* program then starved.
|
|
362
|
+
*/
|
|
363
|
+
_createQdrantIndexer() {
|
|
364
|
+
const qdrant = this._qdrant;
|
|
365
|
+
if (!qdrant?.ready)
|
|
366
|
+
return null;
|
|
367
|
+
return {
|
|
368
|
+
embed: async (texts) => {
|
|
369
|
+
if (texts.length === 0)
|
|
370
|
+
return [];
|
|
371
|
+
const batch = qdrant.embedder.createEmbeddingsBatch;
|
|
372
|
+
if (batch) {
|
|
373
|
+
const result = await batch(texts);
|
|
374
|
+
return result.embeddings;
|
|
375
|
+
}
|
|
376
|
+
// An older build without the batch call: one request per text, but
|
|
377
|
+
// the caller keeps the concurrency bounded all the same.
|
|
378
|
+
const embeddings = [];
|
|
379
|
+
for (const text of texts) {
|
|
380
|
+
const { embedding } = await qdrant.embedder.createEmbedding(text);
|
|
381
|
+
embeddings.push(embedding);
|
|
382
|
+
}
|
|
383
|
+
return embeddings;
|
|
384
|
+
},
|
|
385
|
+
upsert: async (points) => {
|
|
386
|
+
await qdrant.client.upsert(qdrant.collection, { wait: false, points });
|
|
387
|
+
},
|
|
388
|
+
};
|
|
389
|
+
}
|
|
390
|
+
/**
|
|
391
|
+
* Queue a skill for the vector index.
|
|
392
|
+
*
|
|
393
|
+
* A skill whose text did not change is not queued again: the registry is
|
|
394
|
+
* filled on every scan, and re-embedding the same skill would pay for the
|
|
395
|
+
* same vector over and over.
|
|
396
|
+
*/
|
|
397
|
+
_scheduleIndex(skill) {
|
|
398
|
+
const point = this._toQdrantPoint(skill);
|
|
399
|
+
const hash = this._hashId(point.text);
|
|
400
|
+
if (this._indexedHashes.get(skill.id) === hash)
|
|
401
|
+
return;
|
|
402
|
+
this._indexedHashes.set(skill.id, hash);
|
|
403
|
+
this._indexQueue.push({ skill, point, hash });
|
|
404
|
+
this._scheduleIndexFlush();
|
|
405
|
+
}
|
|
406
|
+
/**
|
|
407
|
+
* Start the index flush, a microtask from now.
|
|
408
|
+
*
|
|
409
|
+
* A scan registers every skill in one go. Waiting for the microtask lets
|
|
410
|
+
* that burst share one request instead of starting one request per skill.
|
|
411
|
+
*/
|
|
412
|
+
_scheduleIndexFlush() {
|
|
413
|
+
if (this._indexFlushScheduled)
|
|
414
|
+
return;
|
|
415
|
+
this._indexFlushScheduled = true;
|
|
416
|
+
queueMicrotask(() => {
|
|
417
|
+
this._indexFlushScheduled = false;
|
|
418
|
+
this._runIndexFlush();
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Index the queued skills, one batch at a time.
|
|
423
|
+
*
|
|
424
|
+
* Only one batch is in flight, so the number of open connections stays
|
|
425
|
+
* bound no matter how many skills the registry holds.
|
|
426
|
+
*/
|
|
427
|
+
_runIndexFlush() {
|
|
428
|
+
if (this._indexFlushRunning)
|
|
429
|
+
return;
|
|
430
|
+
this._indexFlushRunning = true;
|
|
431
|
+
void this._flushIndexQueue().finally(() => {
|
|
432
|
+
this._indexFlushRunning = false;
|
|
433
|
+
// Skills may have arrived while the flush was running.
|
|
434
|
+
if (this._indexQueue.length > 0)
|
|
435
|
+
this._scheduleIndexFlush();
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
async _flushIndexQueue() {
|
|
439
|
+
if (this._indexer === undefined)
|
|
440
|
+
await this._ensureQdrant();
|
|
441
|
+
const indexer = this._getIndexer();
|
|
442
|
+
if (!indexer) {
|
|
443
|
+
// No vector index: the skills stay usable without vectors.
|
|
444
|
+
this._indexQueue = [];
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
while (this._indexQueue.length > 0) {
|
|
448
|
+
const batch = this._indexQueue.splice(0, this._batchSize);
|
|
449
|
+
try {
|
|
450
|
+
const vectors = await indexer.embed(batch.map((item) => item.point.text));
|
|
451
|
+
const points = [];
|
|
452
|
+
batch.forEach((item, position) => {
|
|
453
|
+
const vector = vectors[position];
|
|
454
|
+
if (!vector || vector.length === 0)
|
|
455
|
+
return;
|
|
456
|
+
points.push({ id: item.point.id, vector, payload: item.point });
|
|
457
|
+
});
|
|
458
|
+
if (points.length > 0)
|
|
459
|
+
await indexer.upsert(points);
|
|
460
|
+
}
|
|
461
|
+
catch {
|
|
462
|
+
// The vector index is optional. A failed batch may be retried by a
|
|
463
|
+
// later registration of the same skills.
|
|
464
|
+
for (const item of batch) {
|
|
465
|
+
if (this._indexedHashes.get(item.skill.id) === item.hash) {
|
|
466
|
+
this._indexedHashes.delete(item.skill.id);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
153
470
|
}
|
|
154
471
|
}
|
|
155
472
|
/** Simple deterministic ID for Qdrant (no bigints needed). */
|
|
@@ -166,8 +483,8 @@ export class SkillRegistry {
|
|
|
166
483
|
register(skill) {
|
|
167
484
|
this.skills.set(skill.id, skill);
|
|
168
485
|
this.updateIndex(skill);
|
|
169
|
-
//
|
|
170
|
-
this.
|
|
486
|
+
// Queue the skill for the vector index without blocking registration.
|
|
487
|
+
this._scheduleIndex(skill);
|
|
171
488
|
}
|
|
172
489
|
/**
|
|
173
490
|
* Unregister a skill by ID
|
|
@@ -185,7 +502,90 @@ export class SkillRegistry {
|
|
|
185
502
|
* Get a skill by ID
|
|
186
503
|
*/
|
|
187
504
|
get(skillId) {
|
|
188
|
-
|
|
505
|
+
// A server skill is only known after a vector search found it, so it is
|
|
506
|
+
// looked up second - the local skill always wins.
|
|
507
|
+
return this.skills.get(skillId) ?? this.remoteSkills.get(skillId);
|
|
508
|
+
}
|
|
509
|
+
/**
|
|
510
|
+
* The skills that only exist in the vector store.
|
|
511
|
+
*
|
|
512
|
+
* They are not part of `list()`: the local count must stay local.
|
|
513
|
+
*/
|
|
514
|
+
listRemote() {
|
|
515
|
+
return Array.from(this.remoteSkills.values());
|
|
516
|
+
}
|
|
517
|
+
/**
|
|
518
|
+
* The service that holds the server-side skills.
|
|
519
|
+
*
|
|
520
|
+
* `undefined` means "no service is configured". Setting a source clears
|
|
521
|
+
* the cached remote skills: a new service is a new authority, so the old
|
|
522
|
+
* content must not survive it.
|
|
523
|
+
*/
|
|
524
|
+
setRemoteSource(source) {
|
|
525
|
+
this._remoteSource = source;
|
|
526
|
+
this.remoteSkills.clear();
|
|
527
|
+
this._completeRemote.clear();
|
|
528
|
+
}
|
|
529
|
+
/**
|
|
530
|
+
* Ask the service for the skills it holds.
|
|
531
|
+
*
|
|
532
|
+
* A skill whose version changed is replaced by the new one. A record
|
|
533
|
+
* without a version never replaces a cached skill - an unknown version is
|
|
534
|
+
* not evidence of a change.
|
|
535
|
+
*/
|
|
536
|
+
async refreshRemoteSkills(options) {
|
|
537
|
+
if (!this._remoteSource) {
|
|
538
|
+
return { state: "not-configured", count: 0 };
|
|
539
|
+
}
|
|
540
|
+
let records;
|
|
541
|
+
try {
|
|
542
|
+
records = await this._remoteSource.list(options);
|
|
543
|
+
}
|
|
544
|
+
catch (error) {
|
|
545
|
+
return {
|
|
546
|
+
state: this._remoteFailureState(error),
|
|
547
|
+
count: 0,
|
|
548
|
+
error: describeRemoteFailure(error),
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
for (const record of records) {
|
|
552
|
+
const cached = this.remoteSkills.get(record.id);
|
|
553
|
+
if (!isStale(cached, record))
|
|
554
|
+
continue;
|
|
555
|
+
const skill = skillFromRemote(record);
|
|
556
|
+
this.remoteSkills.set(skill.id, skill);
|
|
557
|
+
if (record.contentComplete)
|
|
558
|
+
this._completeRemote.add(skill.id);
|
|
559
|
+
}
|
|
560
|
+
return {
|
|
561
|
+
state: records.length > 0 ? "populated" : "empty",
|
|
562
|
+
count: records.length,
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* The state of the service that holds the server-side skills.
|
|
567
|
+
*
|
|
568
|
+
* The states stay apart: a refused call is not an unreachable service and
|
|
569
|
+
* not an empty service.
|
|
570
|
+
*/
|
|
571
|
+
async getRemoteStats() {
|
|
572
|
+
if (!this._remoteSource) {
|
|
573
|
+
return { state: "not-configured", count: 0 };
|
|
574
|
+
}
|
|
575
|
+
return this.refreshRemoteSkills();
|
|
576
|
+
}
|
|
577
|
+
/**
|
|
578
|
+
* The state a failed call stands for.
|
|
579
|
+
*
|
|
580
|
+
* A refusal of the service is "denied" - the service answered, it just
|
|
581
|
+
* did not allow this call. Everything else is a service that could not be
|
|
582
|
+
* reached.
|
|
583
|
+
*/
|
|
584
|
+
_remoteFailureState(error) {
|
|
585
|
+
const code = error?.code;
|
|
586
|
+
if (code === "denied" || code === -32001 || code === -32002)
|
|
587
|
+
return "denied";
|
|
588
|
+
return "unavailable";
|
|
189
589
|
}
|
|
190
590
|
/**
|
|
191
591
|
* List all registered skills
|
|
@@ -283,10 +683,38 @@ export class SkillRegistry {
|
|
|
283
683
|
return results[0] ?? null;
|
|
284
684
|
}
|
|
285
685
|
// --- Qdrant vector search --------------------------------------------------
|
|
686
|
+
/**
|
|
687
|
+
* A skill that only exists in the vector store.
|
|
688
|
+
*
|
|
689
|
+
* The payload carries the name, the description and the body - enough to
|
|
690
|
+
* USE the skill on a machine that has no local copy of it.
|
|
691
|
+
*/
|
|
692
|
+
static skillFromPayload(payload, name) {
|
|
693
|
+
const description = typeof payload.description === "string" ? payload.description : "";
|
|
694
|
+
const body = typeof payload.body === "string" ? payload.body : "";
|
|
695
|
+
if (!description && !body)
|
|
696
|
+
return null;
|
|
697
|
+
return {
|
|
698
|
+
id: name,
|
|
699
|
+
// A server skill has no directory on this machine.
|
|
700
|
+
path: "",
|
|
701
|
+
frontmatter: { name, description },
|
|
702
|
+
body,
|
|
703
|
+
references: [],
|
|
704
|
+
examples: [],
|
|
705
|
+
metadata: { name, description },
|
|
706
|
+
loaded: true,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
286
709
|
/**
|
|
287
710
|
* Search skills by semantic similarity using Qdrant.
|
|
288
711
|
* Falls back to an empty result if Qdrant is not available.
|
|
289
712
|
*
|
|
713
|
+
* A hit that belongs to no local skill is a SERVER skill: it is returned
|
|
714
|
+
* with its payload instead of being dropped. Dropping it would make a
|
|
715
|
+
* shared skill invisible on every machine that has no local copy of it -
|
|
716
|
+
* which is exactly the case the vector store exists for.
|
|
717
|
+
*
|
|
290
718
|
* @param query Natural-language query
|
|
291
719
|
* @param limit Max results (default 5)
|
|
292
720
|
* @param threshold Minimum cosine score (default 0.5)
|
|
@@ -307,10 +735,20 @@ export class SkillRegistry {
|
|
|
307
735
|
for (const pt of raw.points ?? []) {
|
|
308
736
|
if (pt.score < threshold)
|
|
309
737
|
continue;
|
|
310
|
-
const
|
|
311
|
-
const
|
|
312
|
-
if (
|
|
313
|
-
|
|
738
|
+
const payload = pt.payload ?? {};
|
|
739
|
+
const name = typeof payload.name === "string" ? payload.name : undefined;
|
|
740
|
+
if (!name)
|
|
741
|
+
continue;
|
|
742
|
+
const local = this.skills.get(name);
|
|
743
|
+
if (local) {
|
|
744
|
+
results.push({ skill: local, score: pt.score, source: "local" });
|
|
745
|
+
continue;
|
|
746
|
+
}
|
|
747
|
+
const remote = SkillRegistry.skillFromPayload(payload, name);
|
|
748
|
+
if (!remote)
|
|
749
|
+
continue;
|
|
750
|
+
this.remoteSkills.set(remote.id, remote);
|
|
751
|
+
results.push({ skill: remote, score: pt.score, source: "remote" });
|
|
314
752
|
}
|
|
315
753
|
return results;
|
|
316
754
|
}
|
|
@@ -324,13 +762,34 @@ export class SkillRegistry {
|
|
|
324
762
|
* Useful after Qdrant becomes available or collection is rebuilt.
|
|
325
763
|
*/
|
|
326
764
|
async syncAllToQdrant(onProgress) {
|
|
327
|
-
|
|
328
|
-
|
|
765
|
+
if (this._indexer === undefined)
|
|
766
|
+
await this._ensureQdrant();
|
|
767
|
+
const indexer = this._getIndexer();
|
|
768
|
+
if (!indexer)
|
|
329
769
|
return;
|
|
330
770
|
const skills = this.list();
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
771
|
+
const total = skills.length;
|
|
772
|
+
let done = 0;
|
|
773
|
+
for (let i = 0; i < total; i += this._batchSize) {
|
|
774
|
+
const batch = skills.slice(i, i + this._batchSize);
|
|
775
|
+
const points = batch.map((skill) => this._toQdrantPoint(skill));
|
|
776
|
+
try {
|
|
777
|
+
const vectors = await indexer.embed(points.map((point) => point.text));
|
|
778
|
+
const payload = [];
|
|
779
|
+
points.forEach((point, position) => {
|
|
780
|
+
const vector = vectors[position];
|
|
781
|
+
if (!vector || vector.length === 0)
|
|
782
|
+
return;
|
|
783
|
+
payload.push({ id: point.id, vector, payload: point });
|
|
784
|
+
});
|
|
785
|
+
if (payload.length > 0)
|
|
786
|
+
await indexer.upsert(payload);
|
|
787
|
+
}
|
|
788
|
+
catch {
|
|
789
|
+
// The vector index is optional — a failed batch does not stop the sync.
|
|
790
|
+
}
|
|
791
|
+
done += batch.length;
|
|
792
|
+
onProgress?.(done, total);
|
|
334
793
|
}
|
|
335
794
|
}
|
|
336
795
|
/** Whether Qdrant is available and ready. */
|
|
@@ -341,7 +800,34 @@ export class SkillRegistry {
|
|
|
341
800
|
* Load a skill's full content (progressive disclosure)
|
|
342
801
|
*/
|
|
343
802
|
async load(skillId, options) {
|
|
344
|
-
|
|
803
|
+
let skill = this.skills.get(skillId) ?? this.remoteSkills.get(skillId);
|
|
804
|
+
// A skill that is not known yet may still live on the server. The
|
|
805
|
+
// service decides whether this user may see it - a refusal is not an
|
|
806
|
+
// error, it is an answer.
|
|
807
|
+
if (!skill && this._remoteSource) {
|
|
808
|
+
const record = await this._remoteSource.get(skillId);
|
|
809
|
+
if (!record)
|
|
810
|
+
return null;
|
|
811
|
+
skill = skillFromRemote(record);
|
|
812
|
+
this.remoteSkills.set(skill.id, skill);
|
|
813
|
+
if (record.contentComplete)
|
|
814
|
+
this._completeRemote.add(skill.id);
|
|
815
|
+
}
|
|
816
|
+
// A cached remote skill may carry only a truncated body. The service
|
|
817
|
+
// holds the complete content, so it is fetched once.
|
|
818
|
+
if (skill &&
|
|
819
|
+
!this.skills.has(skillId) &&
|
|
820
|
+
this._remoteSource &&
|
|
821
|
+
!this._completeRemote.has(skillId)) {
|
|
822
|
+
const record = await this._remoteSource.get(skillId);
|
|
823
|
+
if (record) {
|
|
824
|
+
const complete = skillFromRemote(record);
|
|
825
|
+
this.remoteSkills.set(complete.id, complete);
|
|
826
|
+
if (record.contentComplete)
|
|
827
|
+
this._completeRemote.add(complete.id);
|
|
828
|
+
skill = complete;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
345
831
|
if (!skill) {
|
|
346
832
|
return null;
|
|
347
833
|
}
|
|
@@ -371,9 +857,11 @@ export class SkillRegistry {
|
|
|
371
857
|
*/
|
|
372
858
|
async loadSkill(skill, options) {
|
|
373
859
|
const loadedSkill = { ...skill };
|
|
374
|
-
|
|
860
|
+
// A server skill has no directory on this machine - its body comes from
|
|
861
|
+
// the vector store, so nothing is read from the disk.
|
|
862
|
+
const skillMdPath = skill.path ? join(skill.path, "SKILL.md") : "";
|
|
375
863
|
// Re-parse to get fresh content
|
|
376
|
-
if (existsSync(skillMdPath)) {
|
|
864
|
+
if (skillMdPath && existsSync(skillMdPath)) {
|
|
377
865
|
const parsed = parseSkillFile(skillMdPath);
|
|
378
866
|
loadedSkill.body = parsed.body;
|
|
379
867
|
loadedSkill.frontmatter = parsed.frontmatter;
|
|
@@ -445,6 +933,7 @@ export class SkillRegistry {
|
|
|
445
933
|
*/
|
|
446
934
|
clear() {
|
|
447
935
|
this.skills.clear();
|
|
936
|
+
this.remoteSkills.clear();
|
|
448
937
|
this.index = { entries: [], loadedAt: 0 };
|
|
449
938
|
this.loading.clear();
|
|
450
939
|
}
|
|
@@ -458,14 +947,14 @@ export class SkillRegistry {
|
|
|
458
947
|
* Check if skill exists
|
|
459
948
|
*/
|
|
460
949
|
has(skillId) {
|
|
461
|
-
return this.skills.has(skillId);
|
|
950
|
+
return this.skills.has(skillId) || this.remoteSkills.has(skillId);
|
|
462
951
|
}
|
|
463
952
|
}
|
|
464
953
|
/**
|
|
465
954
|
* Create a new skill registry
|
|
466
955
|
*/
|
|
467
|
-
export function createSkillRegistry() {
|
|
468
|
-
return new SkillRegistry();
|
|
956
|
+
export function createSkillRegistry(options = {}) {
|
|
957
|
+
return new SkillRegistry(options);
|
|
469
958
|
}
|
|
470
959
|
/**
|
|
471
960
|
* Global skill registry instance (singleton)
|