pi-harness-runtime 1.1.416 → 1.1.418

package/packages/skills/src/skill-commands.ts

@@ -10,6 +10,7 @@ import { resolve } from "node:path";
 import type { ExtensionAPI, ExtensionContext, ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
 import { getGlobalSkillRegistry } from "./loader.js";
 import { scanForSkillsSync } from "./scanner.js";
+import { SKILL_MIN_SCORE } from "./vector-schema.js";
 
 /**
  * Skill command configuration
@@ -244,7 +245,7 @@ async function handleSkillSearch(
 
   // Try vector search if requested and available
   if (useVector && registry.isQdrantReady) {
-    const vectorResults = await registry.findVector(query, 5, 0.5);
+    const vectorResults = await registry.findVector(query, 5, SKILL_MIN_SCORE);
     if (vectorResults.length > 0) {
       backend = "vector";
       results = vectorResults.map(r => ({ skill: r.skill, score: r.score }));

package/packages/skills/src/vector-schema.ts

@@ -42,6 +42,36 @@ export const LEGACY_SKILL_COLLECTION_MODEL = "sentence-transformers/all-MiniLM-L
 /** The vector size of the legacy collection. */
 export const LEGACY_SKILL_COLLECTION_DIMENSIONS = 384;
 
+/**
+ * The score below which a hit of the runtime collection is not a hit.
+ *
+ * The score of a vector search is a cosine between two vectors, and its
+ * scale belongs to the model that produced them. Measured against the
+ * runtime collection (123 skills, `text-embedding-3-small`):
+ *
+ *   - a query that names a skill scores 0.31 to 0.56,
+ *   - a query about something else scores 0.10 to 0.23, and the wrong
+ *     skill that it matched tops out at 0.2305.
+ *
+ * The threshold sits between the two (0.27). The old value of 0.5 belonged
+ * to the MiniLM collection, whose vectors are not comparable, and it
+ * threw away even a clear match (0.47) - so a search of the runtime
+ * collection returned nothing at all.
+ *
+ * It lives here, next to the model and the collection, so that the two
+ * cannot drift apart.
+ */
+export const SKILL_MIN_SCORE = 0.27;
+
+/**
+ * The score below which a hit of the legacy collection is not a hit.
+ *
+ * The legacy collection holds MiniLM vectors. Their scale was never
+ * measured against a query, so the old value is kept. It is named here
+ * so it is not mistaken for the threshold of the runtime.
+ */
+export const LEGACY_SKILL_MIN_SCORE = 0.5;
+
 /** The schema a skill collection is expected to have. */
 export interface SkillVectorSchema {
 	/** The vector size. */

package/packages/skills/test/vector-threshold.test.ts

@@ -0,0 +1,108 @@
+/**
+ * The score that decides whether a hit of the vector store is a hit.
+ *
+ * The score of a vector search is a cosine between two vectors, and its
+ * scale belongs to the model that produced them. The runtime embeds with
+ * `text-embedding-3-small`, whose cosines for a query that names a skill
+ * land between 0.31 and 0.56 - while a query about something else reaches
+ * 0.23 at most.
+ *
+ * The threshold used to be 0.5, a value of the MiniLM collection, whose
+ * vectors are not comparable. Against the runtime collection it threw away
+ * every clear match (0.47) - a search of a filled collection returned
+ * nothing at all, and the skill was invisible.
+ *
+ * The threshold therefore lives next to the model, and the tests below
+ * hold it there.
+ */
+
+import { describe, expect, it } from "bun:test";
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+import {
+	createSkillRegistry,
+	type QdrantClientLike,
+	type QdrantEmbedderLike,
+} from "../src/loader.js";
+import { SKILL_MIN_SCORE } from "../src/vector-schema.js";
+
+/** A client that answers a query with the given points. */
+function fakeClient(points: Array<{ id: number; score: number }>): QdrantClientLike {
+	return {
+		async upsert(): Promise<void> {},
+		async query(): Promise<unknown> {
+			return { points };
+		},
+		async getCollection(): Promise<unknown> {
+			return { config: { params: { vectors: { size: 1536, distance: "Cosine" } } } };
+		},
+	};
+}
+
+/** An embedder that returns a fixed vector. */
+const fakeEmbedder: QdrantEmbedderLike = {
+	async createEmbedding() {
+		return { embedding: [0.1, 0.2, 0.3], tokens: 3 };
+	},
+};
+
+const registryWith = (scores: number[]) =>
+	createSkillRegistry({
+		qdrantClient: fakeClient(
+			scores.map((score, index) => ({
+				id: index + 1,
+				score,
+				payload: { name: `skill-${index}`, description: "d", body: "b" },
+			})),
+		),
+		embedder: fakeEmbedder,
+		collection: "pi-harness-skills-test",
+	});
+
+describe("the score that decides whether a hit is a hit", () => {
+	it("belongs to the model of the collection", () => {
+		// The measured values: a query that names a skill scores 0.31 to
+		// 0.56, a query about something else 0.10 to 0.23.
+		expect(SKILL_MIN_SCORE).toBeGreaterThan(0.23);
+		expect(SKILL_MIN_SCORE).toBeLessThan(0.31);
+	});
+
+	it("keeps a clear match that the old threshold of 0.5 threw away", async () => {
+		// 0.47 is a clear match of the runtime model - and below 0.5.
+		const registry = registryWith([0.47]);
+
+		const results = await registry.findVector("approve a purchase order");
+
+		expect(results).toHaveLength(1);
+		expect(results[0].score).toBe(0.47);
+	});
+
+	it("drops a hit that is noise", async () => {
+		const registry = registryWith([0.23]);
+
+		const results = await registry.findVector("cook a risotto");
+
+		expect(results).toHaveLength(0);
+	});
+
+	it("keeps the hits above it and drops the hits below it", async () => {
+		const registry = registryWith([0.55, 0.12, 0.4, 0.2]);
+
+		const results = await registry.findVector("a query");
+
+		expect(results.map((r) => r.score)).toEqual([0.55, 0.4]);
+	});
+
+	it("is the threshold of the skill command too", () => {
+		// The command passed 0.5 itself, so the fix of the default would
+		// not have reached the search of the user.
+		const source = readFileSync(
+			fileURLToPath(new URL("../src/skill-commands.ts", import.meta.url)),
+			"utf8",
+		);
+
+		expect(source).toContain("SKILL_MIN_SCORE");
+		expect(source).not.toMatch(/findVector\([^)]*,\s*5,\s*0\.5/);
+	});
+});

package/packages/sprint-planner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/sprint-planner",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "type": "module",
   "description": "Sprint planning engine for pi-harness",
   "main": "dist/index.js",

package/packages/subscription-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/subscription-engine",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/task-compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/task-compiler",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Task Compiler for pi-harness-runtime — produces deterministic task DAG from CompiledRequirement",
   "type": "module",
   "main": "./dist/index.js",

package/packages/telegram/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/telegram",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "pi-harness-runtime — Telegram inbound/outbound",
   "type": "module",
   "main": "./src/index.js",

package/packages/tencentdb-memory/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/tencentdb-memory",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "TencentDB Agent Memory client - sync skills to centralized knowledge base",
   "type": "module",
   "main": "src/index.ts",

package/packages/tencentdb-sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/tencentdb-sync",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Sync SKILL.md skills to TencentDB-Agent-Memory server (RFC-0105/0106)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/test-data-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/test-data-generator",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Test data generation for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/test-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/test-generator",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Auto-Test Generator — generate unit and E2E tests from source code (RFC-0013)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/todo-bd-sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/todo-bd-sync",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Two-way sync between rpiv-todo overlay and bd issue tracker",
   "type": "module",
   "main": "src/index.ts",

package/packages/token-estimation/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/token-estimation",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "pi-harness-runtime — token-estimation",
   "type": "module",
   "main": "./dist/index.js",

package/packages/token-optimizer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/token-optimizer",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Token Budget Optimizer — maximize prompt value within token limits (RFC-0011)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/tooling-preference/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/tooling-preference",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "pi-harness-runtime — JS/TS tooling preference (Biome vs ESLint/Prettier) proposal",
   "type": "module",
   "main": "./dist/index.js",

package/packages/trajectory/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/trajectory",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Trajectory capture for the write-review loop — persists cycle records for learning",
   "type": "module",
   "main": "./dist/src/index.js",

package/packages/tui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/tui",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "pi-harness-runtime — tui",
   "type": "module",
   "main": "./dist/index.js",

package/packages/two-phase-review/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/two-phase-review",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "pi-harness-runtime — two-phase implementation and review workflow (queue review signals, gate the phase, scope the diff)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/types",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Shared TypeScript types for pi-harness-runtime packages",
   "type": "module",
   "main": "./dist/index.js",

package/packages/voice-api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/voice-api",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "type": "module",
   "description": "Voice-to-text API using Whisper for Telegram and LINE integration",
   "main": "./dist/index.js",

package/packages/webmcp-tester/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/webmcp-tester",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "WebMCP-powered E2E testing for pi-harness - natural language test scenarios",
   "type": "module",
   "main": "./dist/index.js",

package/packages/wiki/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/wiki",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "LLM Wiki integration for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/workflow-events/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/workflow-events",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/workspace-scanner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/workspace-scanner",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/worktree/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/worktree",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "pi-harness-runtime — worktree",
   "type": "module",
   "main": "./dist/index.js",

package/packages/write-review/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/write-review",
-  "version": "1.1.416",
+  "version": "1.1.418",
   "description": "Two-agent write-review loop with blackboard coordination",
   "type": "module",
   "main": "src/index.ts",