pi-harness-runtime 1.1.412 → 1.1.414

package/packages/skills/src/loader.ts

@@ -22,6 +22,10 @@ import {
   LEGACY_SKILL_COLLECTION_MODEL,
   SKILL_COLLECTION,
   describeSkillSchemaMismatch,
+  skillPointPayload,
+  skillQdrantPoint,
+  stableHash,
+  type SkillPointPayload,
   type SkillVectorSchema,
 } from "./vector-schema.js";
 import {
@@ -39,11 +43,15 @@ import {
 // At runtime, packages/ is resolved from the monorepo root or node_modules.
 
 export interface QdrantSkillPoint {
+  /**
+   * The id of the point.
+   *
+   * It is the hash of the name in the payload, so the point id and the
+   * payload name always belong together.
+   */
   id: number;
-  name: string;
-  description: string;
-  body: string;
-  text: string;
+  /** The payload of the point. */
+  payload: SkillPointPayload;
 }
 
 export interface QdrantClientLike {
@@ -98,6 +106,29 @@ export interface QdrantSkillStats {
    * nothing does not have to guess why.
    */
   schemaMismatch?: string;
+  /**
+   * The collection of an earlier installation that still holds the skills.
+   *
+   * The runtime collection is empty (or does not exist yet) while the
+   * legacy collection holds the skills of the previous version. They are
+   * not lost - they live in a collection with another name, and one with
+   * another vector size.
+   */
+  legacy?: LegacyCollectionInfo;
+}
+
+/**
+ * The skills of the legacy collection.
+ */
+export interface LegacyCollectionInfo {
+  /** The name of the collection. */
+  collection: string;
+  /** The number of skill vectors it holds. */
+  count: number;
+  /** The vector size it was created with. */
+  dimensions?: number;
+  /** The embedding model it was created with. */
+  model?: string;
 }
 
 /** The information the Qdrant package can report about a collection. */
@@ -126,7 +157,7 @@ const RUNTIME_VECTOR_DISTANCE = "Cosine";
 export interface SkillVectorPoint {
   id: number;
   vector: number[];
-  payload: QdrantSkillPoint;
+  payload: SkillPointPayload;
 }
 
 /**
@@ -433,6 +464,43 @@ export class SkillRegistry {
     }
   }
 
+  /**
+   * The skills of the legacy collection.
+   *
+   * The legacy collection holds 384-dimensional vectors, so it cannot be
+   * searched with the vectors of the runtime. Its skills are not lost -
+   * they live in a collection with another name.
+   */
+  private async _readLegacyCollection(
+    client: QdrantClientLike,
+    mods: QdrantModules,
+  ): Promise<LegacyCollectionInfo | undefined> {
+    // The legacy collection IS the runtime collection when the user
+    // configured it explicitly - then there is no second collection.
+    if (this._options.collection === LEGACY_SKILL_COLLECTION) return undefined;
+    if (!mods.getCollectionInfo) return undefined;
+
+    try {
+      const info = await mods.getCollectionInfo(client, LEGACY_SKILL_COLLECTION);
+      if (!info?.exists) return undefined;
+      const count = info.pointsCount ?? 0;
+      if (count === 0) return undefined;
+      return {
+        collection: LEGACY_SKILL_COLLECTION,
+        count,
+        dimensions: info.vectorsSize,
+        model:
+          info.vectorsSize === LEGACY_SKILL_COLLECTION_DIMENSIONS
+            ? LEGACY_SKILL_COLLECTION_MODEL
+            : undefined,
+      };
+    } catch {
+      // The legacy collection is a bonus - a failure to read it must not
+      // hide the state of the runtime collection.
+      return undefined;
+    }
+  }
+
   /**
    * The schema the runtime expects of the skill collection.
    *
@@ -500,7 +568,15 @@ export class SkillRegistry {
         };
       }
       if (!info.exists) {
-        return { state: "missing", count: 0, collection: config.collection };
+        // The runtime collection does not exist (yet). An earlier
+        // installation may have left the skills in the legacy collection -
+        // naming them is more useful than an empty count.
+        return {
+          state: "missing",
+          count: 0,
+          collection: config.collection,
+          legacy: await this._readLegacyCollection(client, mods),
+        };
       }
 
       const count = info.pointsCount ?? 0;
@@ -512,6 +588,12 @@ export class SkillRegistry {
         distance: info.vectorsDistance,
       };
 
+      // An empty runtime collection can still have its skills in the
+      // legacy one - the same information, one collection further down.
+      if (count === 0) {
+        stats.legacy = await this._readLegacyCollection(client, mods);
+      }
+
       const mismatch = describeSkillSchemaMismatch(
         { dimensions: info.vectorsSize, distance: info.vectorsDistance },
         this._expectedVectorSchema(mods),
@@ -539,13 +621,17 @@ export class SkillRegistry {
 
   /** The point of a skill in the vector index. */
   private _toQdrantPoint(skill: Skill): QdrantSkillPoint {
-    return {
-      id: this._hashId(skill.id),
+    // The payload and the point id come from the shared schema, so the
+    // runtime and `scripts/sync-to-qdrant.ts` store the same point for the
+    // same skill. A writer that used its own id would leave two points
+    // behind - or overwrite an unrelated one.
+    const payload = skillPointPayload({
       name: skill.id,
+      title: skill.frontmatter.name,
       description: skill.frontmatter.description,
-      body: skill.body.slice(0, 2000),
-      text: [skill.id, skill.frontmatter.name, skill.frontmatter.description, skill.body.slice(0, 1500)].join(" "),
-    };
+      body: skill.body,
+    });
+    return skillQdrantPoint(payload);
   }
 
   /**
@@ -603,7 +689,7 @@ export class SkillRegistry {
    */
   private _scheduleIndex(skill: Skill): void {
     const point = this._toQdrantPoint(skill);
-    const hash = this._hashId(point.text);
+    const hash = stableHash(point.payload.text);
     if (this._indexedHashes.get(skill.id) === hash) return;
 
     this._indexedHashes.set(skill.id, hash);
@@ -655,12 +741,12 @@ export class SkillRegistry {
     while (this._indexQueue.length > 0) {
       const batch = this._indexQueue.splice(0, this._batchSize);
       try {
-        const vectors = await indexer.embed(batch.map((item) => item.point.text));
+        const vectors = await indexer.embed(batch.map((item) => item.point.payload.text));
         const points: SkillVectorPoint[] = [];
         batch.forEach((item, position) => {
           const vector = vectors[position];
           if (!vector || vector.length === 0) return;
-          points.push({ id: item.point.id, vector, payload: item.point });
+          points.push({ id: item.point.id, vector, payload: item.point.payload });
         });
         if (points.length > 0) await indexer.upsert(points);
       } catch {
@@ -675,15 +761,6 @@ export class SkillRegistry {
     }
   }
 
-  /** Simple deterministic ID for Qdrant (no bigints needed). */
-  private _hashId(id: string): number {
-    let h = 0;
-    for (let i = 0; i < id.length; i++) {
-      h = (Math.imul(31, h) + id.charCodeAt(i)) | 0;
-    }
-    return Math.abs(h);
-  }
-
   /**
    * Register a skill (auto-indexes to Qdrant if configured).
    */
@@ -1010,12 +1087,12 @@ export class SkillRegistry {
       const batch = skills.slice(i, i + this._batchSize);
       const points = batch.map((skill) => this._toQdrantPoint(skill));
       try {
-        const vectors = await indexer.embed(points.map((point) => point.text));
+        const vectors = await indexer.embed(points.map((point) => point.payload.text));
         const payload: SkillVectorPoint[] = [];
         points.forEach((point, position) => {
           const vector = vectors[position];
           if (!vector || vector.length === 0) return;
-          payload.push({ id: point.id, vector, payload: point });
+          payload.push({ id: point.id, vector, payload: point.payload });
         });
         if (payload.length > 0) await indexer.upsert(payload);
       } catch {

package/packages/skills/src/vector-schema.ts

@@ -51,12 +51,125 @@ export interface SkillVectorSchema {
 }
 
 /**
- * Why the collection does not match the expected schema - or null when it
- * does.
+ * The payload of a skill point.
  *
- * The message names the numbers, so the reader can tell WHICH schema the
- * collection was built with.
+ * Both writers of the collection - the runtime registry and
+ * `scripts/sync-to-qdrant.ts` - store the SAME payload under the SAME
+ * point id. A point is addressed by its id, so a writer that used a
+ * different id for the same skill would leave two points behind, and a
+ * writer whose id happened to collide with a foreign one would overwrite
+ * an unrelated skill.
  */
+export type SkillPointPayload = {
+  /** The name of the skill. The point id is derived from it. */
+  name: string;
+  /** What the skill does. */
+  description: string;
+  /** The body of the skill. */
+  body: string;
+  /** The text the vector was built from. */
+  text: string;
+};
+
+/**
+ * The body of a payload is truncated.
+ *
+ * A payload is an index, not a file store - the complete skill comes from
+ * the skill service.
+ */
+export const SKILL_PAYLOAD_BODY_LIMIT = 2000;
+
+/** The text of a payload is truncated as well. */
+export const SKILL_PAYLOAD_TEXT_LIMIT = 1500;
+
+/**
+ * A stable hash of a string.
+ *
+ * The same string always produces the same number, in every process and
+ * on every machine - a point id has to survive a restart.
+ *
+ * Two uses:
+ * - the id of a point is the hash of the name its payload carries, so the
+ *   point id and the payload name always belong together. Two writers
+ *   that store the same name address the same point.
+ * - the hash of the text of a skill says whether its content changed.
+ */
+export function stableHash(value: string): number {
+  let hash = 0;
+  for (let i = 0; i < value.length; i++) {
+    hash = (Math.imul(31, hash) + value.charCodeAt(i)) | 0;
+  }
+  return Math.abs(hash);
+}
+
+/**
+ * The text a skill vector is built from.
+ *
+ * The title is added when it differs from the name: a skill whose
+ * directory is named differently from its title carries both.
+ */
+export function skillPointText(input: {
+  /** The name the payload carries - the identity of the skill. */
+  name: string;
+  /** The title of the skill, when it differs from the name. */
+  title?: string;
+  /** What the skill does. */
+  description: string;
+  /** The body of the skill. */
+  body: string;
+}): string {
+  const parts = [input.name];
+  if (input.title && input.title !== input.name) parts.push(input.title);
+  parts.push(input.description);
+  parts.push(input.body.slice(0, SKILL_PAYLOAD_TEXT_LIMIT));
+  return parts.join(" ");
+}
+
+/**
+ * A point of the skill collection.
+ *
+ * The id is the hash of the name the payload carries, so the point id and
+ * the payload name always belong together: two writers that store the
+ * same name address the same point.
+ */
+export interface SkillQdrantPoint {
+  id: number;
+  payload: SkillPointPayload;
+}
+
+/**
+ * The point of a skill.
+ *
+ * The runtime registry and `scripts/sync-to-qdrant.ts` both build their
+ * points here, so they cannot drift apart. A writer that used its own id
+ * would leave a second point for every skill the other one indexes, and
+ * an id that happens to collide with a foreign one would overwrite an
+ * unrelated skill.
+ */
+export function skillQdrantPoint(payload: SkillPointPayload): SkillQdrantPoint {
+  return { id: stableHash(payload.name), payload };
+}
+
+/**
+ * The payload of a skill point.
+ */
+export function skillPointPayload(input: {
+  /** The name the payload carries - the identity of the skill. */
+  name: string;
+  /** The title of the skill, when it differs from the name. */
+  title?: string;
+  /** What the skill does. */
+  description: string;
+  /** The body of the skill. */
+  body: string;
+}): SkillPointPayload {
+  return {
+    name: input.name,
+    description: input.description,
+    body: input.body.slice(0, SKILL_PAYLOAD_BODY_LIMIT),
+    text: skillPointText(input),
+  };
+}
 export function describeSkillSchemaMismatch(
 	actual: { dimensions?: number; distance?: string },
 	expected: SkillVectorSchema,

package/packages/skills/test/remote-skills.test.ts

@@ -239,4 +239,54 @@ describe("the state of the vector store", () => {
 		expect(stats.state).toBe("missing");
 		expect(stats.count).toBe(0);
 	});
+
+	it("names the skills of the legacy collection", async () => {
+		// The runtime collection does not exist yet, but the collection of
+		// the previous version still holds the skills. They are not lost -
+		// the reader has to know where they are.
+		const client = {
+			async upsert(): Promise<void> {},
+			async query(): Promise<unknown> {
+				return { points: [] };
+			},
+			async getCollection(name: string): Promise<unknown> {
+				if (name === "pi-harness-skills") {
+					return {
+						points_count: 79,
+						config: { params: { vectors: { size: 384, distance: "Cosine" } } },
+					};
+				}
+				throw new Error("Not Found");
+			},
+		};
+		const registry = createSkillRegistry({
+			qdrantClient: client as never,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("missing");
+		expect(stats.legacy?.collection).toBe("pi-harness-skills");
+		expect(stats.legacy?.count).toBe(79);
+		expect(stats.legacy?.dimensions).toBe(384);
+		expect(stats.legacy?.model).toBe("sentence-transformers/all-MiniLM-L6-v2");
+	});
+
+	it("does not look for a second collection when the legacy one is the runtime one", async () => {
+		const client = fakeClient([], {
+			points_count: 0,
+			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.legacy).toBeUndefined();
+	});
 });

package/packages/skills/test/vector-schema.test.ts

@@ -0,0 +1,167 @@
+/**
+ * Regression tests for the point identity of the skill collection.
+ *
+ * Two writers fill the same collection: the runtime registry and
+ * `scripts/sync-to-qdrant.ts`. They used to disagree about the identity of
+ * a point - the runtime hashed the name of the skill, the script used a
+ * running number. The same skill then existed twice, and a running number
+ * could overwrite a point of an unrelated skill.
+ *
+ * The identity now comes from one shared place, and these tests hold it
+ * there.
+ */
+
+import { describe, expect, test } from "bun:test";
+
+import {
+	createSkillRegistry,
+	type SkillVectorIndexer,
+	type SkillVectorPoint,
+} from "../src/loader.js";
+import {
+	SKILL_PAYLOAD_BODY_LIMIT,
+	SKILL_PAYLOAD_TEXT_LIMIT,
+	skillPointPayload,
+	skillQdrantPoint,
+	stableHash,
+} from "../src/vector-schema.js";
+import type { Skill } from "../src/types.js";
+
+/** Wait until the registry has no index work left. */
+async function settle(registry: { isIndexing(): boolean }): Promise<void> {
+	for (let attempt = 0; attempt < 200 && registry.isIndexing(); attempt++) {
+		await new Promise((resolve) => setTimeout(resolve, 5));
+	}
+}
+
+/** A skill with the given id and body. */
+function makeSkill(id: string, body = `body of ${id}`): Skill {
+	return {
+		id,
+		path: `/tmp/skills/${id}`,
+		frontmatter: {
+			name: id,
+			description: `the ${id} skill`,
+		},
+		body,
+		metadata: {} as Skill["metadata"],
+	} as Skill;
+}
+
+/** An indexer that keeps the points it receives. */
+class RecordingIndexer implements SkillVectorIndexer {
+	readonly upsertCalls: SkillVectorPoint[][] = [];
+
+	async embed(texts: string[]): Promise<number[][]> {
+		return texts.map((text) => [text.length, 1, 2]);
+	}
+
+	async upsert(points: SkillVectorPoint[]): Promise<void> {
+		this.upsertCalls.push(points);
+	}
+}
+
+describe("the identity of a skill point", () => {
+	test("the id of the point is the hash of the name in the payload", () => {
+		const payload = skillPointPayload({
+			name: "a-skill",
+			description: "does something",
+			body: "the body",
+		});
+		const point = skillQdrantPoint(payload);
+
+		expect(point.id).toBe(stableHash(payload.name));
+	});
+
+	test("the same name always addresses the same point", () => {
+		const first = skillQdrantPoint(
+			skillPointPayload({ name: "a-skill", description: "one", body: "one" }),
+		);
+		const second = skillQdrantPoint(
+			skillPointPayload({ name: "a-skill", description: "two", body: "two" }),
+		);
+
+		// The content may change, the identity of the point may not.
+		expect(second.id).toBe(first.id);
+	});
+
+	test("different names address different points", () => {
+		const first = skillQdrantPoint(
+			skillPointPayload({ name: "a-skill", description: "d", body: "b" }),
+		);
+		const second = skillQdrantPoint(
+			skillPointPayload({ name: "b-skill", description: "d", body: "b" }),
+		);
+
+		expect(second.id).not.toBe(first.id);
+	});
+
+	test("the payload carries the shared fields and nothing else", () => {
+		const payload = skillPointPayload({
+			name: "a-skill",
+			title: "A Skill",
+			description: "does something",
+			body: "the body",
+		});
+
+		expect(Object.keys(payload).sort()).toEqual([
+			"body",
+			"description",
+			"name",
+			"text",
+		]);
+	});
+
+	test("the payload and the text are truncated", () => {
+		const payload = skillPointPayload({
+			name: "a-skill",
+			description: "d",
+			body: "x".repeat(SKILL_PAYLOAD_BODY_LIMIT + 500),
+		});
+
+		expect(payload.body.length).toBe(SKILL_PAYLOAD_BODY_LIMIT);
+		expect(payload.text.length).toBeLessThanOrEqual(
+			SKILL_PAYLOAD_TEXT_LIMIT + payload.name.length + payload.description.length + 10,
+		);
+	});
+
+	test("the title is part of the text when it differs from the name", () => {
+		const withTitle = skillPointPayload({
+			name: "a-skill",
+			title: "A Human Title",
+			description: "d",
+			body: "b",
+		});
+		const withoutTitle = skillPointPayload({
+			name: "a-skill",
+			description: "d",
+			body: "b",
+		});
+
+		expect(withTitle.text).toContain("A Human Title");
+		expect(withoutTitle.text).not.toContain("A Human Title");
+	});
+});
+
+describe("the runtime writes the points of the shared schema", () => {
+	test("the indexed point carries the shared payload and id", async () => {
+		const indexer = new RecordingIndexer();
+		const registry = createSkillRegistry({ indexer });
+
+		registry.register(makeSkill("alpha"));
+		await settle(registry);
+
+		const points = indexer.upsertCalls.flat();
+		expect(points.length).toBe(1);
+
+		const point = points[0];
+		expect(point.id).toBe(stableHash(point.payload.name));
+		expect(point.payload.name).toBe("alpha");
+		expect(Object.keys(point.payload).sort()).toEqual([
+			"body",
+			"description",
+			"name",
+			"text",
+		]);
+	});
+});

package/packages/sprint-planner/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/sprint-planner",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "type": "module",
   "description": "Sprint planning engine for pi-harness",
   "main": "dist/index.js",

package/packages/subscription-engine/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/subscription-engine",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "type": "module",
   "main": "src/index.ts",
   "exports": {

package/packages/task-compiler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/task-compiler",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "Task Compiler for pi-harness-runtime — produces deterministic task DAG from CompiledRequirement",
   "type": "module",
   "main": "./dist/index.js",

package/packages/telegram/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/telegram",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "pi-harness-runtime — Telegram inbound/outbound",
   "type": "module",
   "main": "./src/index.js",

package/packages/tencentdb-memory/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/tencentdb-memory",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "TencentDB Agent Memory client - sync skills to centralized knowledge base",
   "type": "module",
   "main": "src/index.ts",

package/packages/tencentdb-sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/tencentdb-sync",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "Sync SKILL.md skills to TencentDB-Agent-Memory server (RFC-0105/0106)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/test-data-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi/test-data-generator",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "Test data generation for pi-harness-runtime",
   "type": "module",
   "main": "./dist/index.js",

package/packages/test-generator/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/test-generator",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "Auto-Test Generator — generate unit and E2E tests from source code (RFC-0013)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/todo-bd-sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moocoding/todo-bd-sync",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "Two-way sync between rpiv-todo overlay and bd issue tracker",
   "type": "module",
   "main": "src/index.ts",

package/packages/token-estimation/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/token-estimation",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "pi-harness-runtime — token-estimation",
   "type": "module",
   "main": "./dist/index.js",

package/packages/token-optimizer/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/token-optimizer",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "Token Budget Optimizer — maximize prompt value within token limits (RFC-0011)",
   "type": "module",
   "main": "./dist/index.js",

package/packages/tooling-preference/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-harness/tooling-preference",
-  "version": "1.1.412",
+  "version": "1.1.414",
   "description": "pi-harness-runtime — JS/TS tooling preference (Biome vs ESLint/Prettier) proposal",
   "type": "module",
   "main": "./dist/index.js",