pi-harness-runtime 1.1.410 → 1.1.412

package/packages/skills/src/remote-source.ts

@@ -0,0 +1,148 @@
+/**
+ * The server-held skills.
+ *
+ * The vector store holds an INDEX: the name, the description and a
+ * truncated body. That is enough to FIND a skill, but not to use it - a
+ * complete SKILL.md (with its references and examples) is larger than a
+ * vector store payload should be.
+ *
+ * The remote source is therefore the authoritative store of the complete
+ * skill content. It is reached through an authenticated service, so the
+ * SERVER decides which skills this user may see. A client never decides
+ * that on its own.
+ */
+
+import type { Skill } from "./types.js";
+
+/** A skill the server holds. */
+export interface RemoteSkill {
+	/** The id the server knows the skill by. */
+	id: string;
+	/** The name of the skill. */
+	name: string;
+	/** What the skill does. */
+	description: string;
+	/** The version of the content. */
+	version?: string;
+	/** When the content changed the last time. */
+	updatedAt?: string;
+	/** The content of the skill (the body of SKILL.md). */
+	content?: string;
+	/**
+	 * Whether the content is the COMPLETE skill.
+	 *
+	 * A vector store payload carries a truncated body - that is not a
+	 * complete skill. The service returns the whole thing, so it sets
+	 * this flag.
+	 */
+	contentComplete?: boolean;
+}
+
+/** Where the server-held skills come from. */
+export interface RemoteSkillSource {
+	/**
+	 * The skills the server holds for this user and workspace.
+	 *
+	 * @param options The search term and the number of skills to return.
+	 */
+	list(options?: { search?: string; limit?: number }): Promise<RemoteSkill[]>;
+	/** The complete content of one skill. */
+	get(id: string): Promise<RemoteSkill | null>;
+}
+
+/** The reasons a call to the remote source can fail. */
+export type RemoteSkillFailure =
+	/** The service refused the call - the token is missing or wrong. */
+	| "denied"
+	/** The service could not be reached. */
+	| "unavailable";
+
+/** The state of the server-held skills. */
+export type RemoteSkillState =
+	/** No service is configured. */
+	| "not-configured"
+	/** The service refused the call. */
+	| "denied"
+	/** The service could not be reached. */
+	| "unavailable"
+	/** The service holds no skill for this user. */
+	| "empty"
+	/** The service holds skills. */
+	| "populated";
+
+/** What the remote source knows. */
+export interface RemoteSkillStats {
+	/** The state of the source. */
+	state: RemoteSkillState;
+	/** The number of skills the source holds. */
+	count: number;
+	/** The reason of a failed call. */
+	error?: string;
+}
+
+/**
+ * A skill from a remote record.
+ *
+ * The record carries the content, so the skill is usable without a local
+ * copy. The path stays empty: a server skill has no directory on this
+ * machine.
+ */
+export function skillFromRemote(record: RemoteSkill): Skill {
+	const version = record.version;
+	return {
+		id: record.id,
+		path: "",
+		frontmatter: {
+			name: record.name,
+			description: record.description,
+			...(version ? { version } : {}),
+		},
+		body: record.content ?? "",
+		references: [],
+		examples: [],
+		metadata: {
+			name: record.name,
+			description: record.description,
+			...(version ? { version } : {}),
+			...(record.updatedAt ? { updated_at: record.updatedAt } : {}),
+		},
+		loaded: true,
+	};
+}
+
+/**
+ * The version of a cached remote skill.
+ *
+ * A skill without a version cannot be compared - the cache keeps it.
+ */
+export function remoteVersion(skill: Skill | undefined): string | undefined {
+	return skill?.frontmatter.version ?? skill?.metadata.version;
+}
+
+/**
+ * The safe reason of a failed call.
+ *
+ * The message never carries a credential: a token that a service echoed
+ * back is removed before the message leaves the process.
+ */
+export function describeRemoteFailure(error: unknown): string {
+  const raw = error instanceof Error ? error.message : String(error ?? "");
+  const safe = raw
+    .replace(/bearer\s+\S+/gi, "bearer <redacted>")
+    .replace(/\b[A-Za-z0-9_-]{32,}\b/g, "<redacted>")
+    .replace(/\s+/g, " ")
+    .trim();
+  return safe.slice(0, 200) || "the service could not be reached";
+}
+
+/**
+ * Whether the cached skill is older than the record.
+ *
+ * A record without a version never replaces a cached skill: an unknown
+ * version is not evidence of a change.
+ */
+export function isStale(cached: Skill | undefined, record: RemoteSkill): boolean {
+	if (!cached) return true;
+	if (!record.version) return false;
+	return remoteVersion(cached) !== record.version;
+}

package/packages/skills/src/vector-schema.ts

@@ -0,0 +1,72 @@
+/**
+ * The vector schema of the skill collection.
+ *
+ * A Qdrant collection holds vectors of exactly ONE size. Two different
+ * embedding models produce two different vector sizes, so they must never
+ * write into the same collection: the second writer is rejected by the
+ * server with a dimension error, and the data of the first writer becomes
+ * unsearchable by the second.
+ *
+ * The collection name carries the model and a version, so a change of the
+ * model leads to a NEW collection instead of a silent mix of two vector
+ * spaces.
+ *
+ * The model itself lives in the Qdrant package (the embedder owns it). This
+ * module owns the collection - that is a decision of the skills domain, not
+ * of the vector store.
+ */
+
+/**
+ * The collection the runtime indexes into and searches.
+ *
+ * `QDRANT_COLLECTION` overrides it, so a cluster that already holds a
+ * collection under another name keeps working.
+ */
+export const SKILL_COLLECTION = "pi-harness-skills-openai-v1";
+
+/**
+ * The collection that was used before the schema was versioned.
+ *
+ * It holds 384-dimensional vectors - the model that
+ * `scripts/sync-to-qdrant.ts` used - and is NOT compatible with the
+ * runtime. It is named here so a mismatch can name the real cause instead
+ * of only the numbers.
+ */
+export const LEGACY_SKILL_COLLECTION = "pi-harness-skills";
+
+/** The model of the legacy collection. */
+export const LEGACY_SKILL_COLLECTION_MODEL = "sentence-transformers/all-MiniLM-L6-v2";
+
+/** The vector size of the legacy collection. */
+export const LEGACY_SKILL_COLLECTION_DIMENSIONS = 384;
+
+/** The schema a skill collection is expected to have. */
+export interface SkillVectorSchema {
+	/** The vector size. */
+	dimensions: number;
+	/** The distance metric. */
+	distance: string;
+	/** The embedding model, when it is known. */
+	model?: string;
+}
+
+/**
+ * Why the collection does not match the expected schema - or null when it
+ * does.
+ *
+ * The message names the numbers, so the reader can tell WHICH schema the
+ * collection was built with.
+ */
+export function describeSkillSchemaMismatch(
+	actual: { dimensions?: number; distance?: string },
+	expected: SkillVectorSchema,
+): string | null {
+	if (actual.dimensions !== undefined && actual.dimensions !== expected.dimensions) {
+		const model = expected.model ? ` (${expected.model})` : "";
+		return `the collection holds ${actual.dimensions}-dimensional vectors, the runtime embeds with ${expected.dimensions}${model}`;
+	}
+	if (actual.distance !== undefined && actual.distance !== expected.distance) {
+		return `the collection uses the ${actual.distance} distance, the runtime expects ${expected.distance}`;
+	}
+	return null;
+}

package/packages/skills/test/init-skills.test.ts

@@ -1,118 +1,114 @@
 /**
- * Skills Initialization E2E Test
- * 
- * Tests the initSkills function to ensure:
- * 1. initSkills() with no args uses default directories
- * 2. initSkills({ locations }) with explicit locations works
- * 3. initSkills({ locations: undefined }) falls back to defaults (the bug case)
- * 
- * Bug: Previously, passing undefined for locations would override defaults
- * with undefined, causing "directories is not iterable" error.
- * 
- * Usage:
- *   bun packages/skills/test/init-skills.test.ts
+ * The initialization of the skills system.
+ *
+ * The test used to be a script that ran `initSkills()` while the module was
+ * imported. That scanned the real skill directories of the machine and
+ * started the vector indexing of every skill it found, which opened a
+ * connection per skill and starved every other test of the process.
+ *
+ * The test now uses a temporary directory and a registry without a vector
+ * index, so it neither reads the machine nor reaches the network.
  */
 
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
 import { initSkills } from "../src/index.js";
+import { createSkillRegistry } from "../src/loader.js";
+
+let tempDir: string;
 
-async function runTests() {
-  console.log("=== initSkills E2E Test ===\n");
-  
-  let passed = 0;
-  let failed = 0;
-
-  // Test 1: no args
-  console.log("[Test 1] initSkills() with no arguments...");
-  try {
-    const result = initSkills();
-    if (!result || !Array.isArray(result.skills) || !Array.isArray(result.errors)) {
-      throw new Error("Invalid result structure");
-    }
-    console.log(`  PASS: Returned ${result.skills.length} skills, ${result.errors.length} errors\n`);
-    passed++;
-  } catch (error) {
-    console.error(`  FAIL: ${error}\n`);
-    failed++;
-  }
-
-  // Test 2: explicit locations
-  console.log("[Test 2] initSkills({ locations: [...] })...");
-  try {
-    const result = initSkills({ locations: ["/tmp/test"] });
-    if (!result) {
-      throw new Error("No result returned");
-    }
-    console.log(`  PASS: Returned ${result.skills.length} skills\n`);
-    passed++;
-  } catch (error) {
-    console.error(`  FAIL: ${error}\n`);
-    failed++;
-  }
-
-  // Test 3: undefined locations (the bug case)
-  console.log("[Test 3] initSkills({ locations: undefined }) - THE BUG CASE...");
-  try {
-    const result = initSkills({ locations: undefined });
-    if (!result || !Array.isArray(result.skills)) {
-      throw new Error("Invalid result structure");
-    }
-    console.log(`  PASS: Returned ${result.skills.length} skills\n`);
-    passed++;
-  } catch (error) {
-    console.error(`  FAIL: ${error}\n`);
-    failed++;
-  }
-
-  // Test 4: partial config with only maxDepth
-  console.log("[Test 4] initSkills({ maxDepth: 5 })...");
-  try {
-    const result = initSkills({ maxDepth: 5 });
-    if (!result || !Array.isArray(result.skills)) {
-      throw new Error("Invalid result structure");
-    }
-    console.log(`  PASS: Returned ${result.skills.length} skills\n`);
-    passed++;
-  } catch (error) {
-    console.error(`  FAIL: ${error}\n`);
-    failed++;
-  }
-
-  // Test 5: partial config with only includeHidden
-  console.log("[Test 5] initSkills({ includeHidden: true })...");
-  try {
-    const result = initSkills({ includeHidden: true });
-    if (!result || !Array.isArray(result.skills)) {
-      throw new Error("Invalid result structure");
-    }
-    console.log(`  PASS: Returned ${result.skills.length} skills\n`);
-    passed++;
-  } catch (error) {
-    console.error(`  FAIL: ${error}\n`);
-    failed++;
-  }
-
-  // Test 6: empty locations array
-  console.log("[Test 6] initSkills({ locations: [] })...");
-  try {
-    const result = initSkills({ locations: [] });
-    if (!result) {
-      throw new Error("No result returned");
-    }
-    console.log(`  PASS: Returned ${result.skills.length} skills\n`);
-    passed++;
-  } catch (error) {
-    console.error(`  FAIL: ${error}\n`);
-    failed++;
-  }
-
-  // Summary
-  console.log("=== Summary ===");
-  console.log(`Passed: ${passed}`);
-  console.log(`Failed: ${failed}`);
-  
-  if (failed > 0) {
-    process.exit(1);
-  }
+/** Write a skill into a directory of the temporary home. */
+function writeSkill(directory: string, name: string): void {
+	const skillDir = join(tempDir, directory, name);
+	mkdirSync(skillDir, { recursive: true });
+	writeFileSync(
+		join(skillDir, "SKILL.md"),
+		[
+			"---",
+			`name: ${name}`,
+			`description: The ${name} skill.`,
+			"version: 1.0.0",
+			"---",
+			"",
+			`The body of the ${name} skill.`,
+			"",
+		].join("\n"),
+		"utf8",
+	);
 }
 
-runTests();
+beforeEach(() => {
+	tempDir = mkdtempSync(join(tmpdir(), "test-init-skills-"));
+	mkdirSync(join(tempDir, "skills"), { recursive: true });
+	writeSkill("skills", "alpha");
+	writeSkill("skills", "beta");
+});
+
+afterEach(() => {
+	rmSync(tempDir, { recursive: true, force: true });
+});
+
+describe("the initialization of the skills system", () => {
+	it("scans the given locations", () => {
+		const result = initSkills({
+			locations: [join(tempDir, "skills")],
+			registry: createSkillRegistry({ indexer: null }),
+		});
+
+		expect(result.skills.length).toBe(2);
+		expect(result.skills.map((skill) => skill.id).sort()).toEqual(["alpha", "beta"]);
+		expect(result.errors).toEqual([]);
+	});
+
+	it("falls back to the default locations when none are given", () => {
+		// The default locations point at the machine of the test, so the
+		// result is only checked for its shape.
+		const result = initSkills({
+			registry: createSkillRegistry({ indexer: null }),
+		});
+
+		expect(Array.isArray(result.skills)).toBe(true);
+		expect(Array.isArray(result.errors)).toBe(true);
+	});
+
+	it("falls back to the default locations when they are undefined", () => {
+		// The bug case: an undefined value must not override the defaults.
+		const result = initSkills({
+			locations: undefined,
+			registry: createSkillRegistry({ indexer: null }),
+		});
+
+		expect(Array.isArray(result.skills)).toBe(true);
+		expect(Array.isArray(result.errors)).toBe(true);
+	});
+
+	it("accepts a partial configuration", () => {
+		const result = initSkills({
+			maxDepth: 5,
+			includeHidden: true,
+			registry: createSkillRegistry({ indexer: null }),
+		});
+
+		expect(Array.isArray(result.skills)).toBe(true);
+	});
+
+	it("accepts an empty list of locations", () => {
+		const result = initSkills({
+			locations: [],
+			registry: createSkillRegistry({ indexer: null }),
+		});
+
+		expect(result.skills).toEqual([]);
+	});
+
+	it("registers the scanned skills in the registry", () => {
+		const registry = createSkillRegistry({ indexer: null });
+
+		initSkills({ locations: [join(tempDir, "skills")], registry });
+
+		expect(registry.list().length).toBe(2);
+	});
+});

package/packages/skills/test/remote-skills.test.ts

@@ -0,0 +1,242 @@
+/**
+ * The skills that only exist in the vector store.
+ *
+ * The vector store holds the skills of a team. A client that has no local
+ * copy of a skill must still be able to find it - and to use it: the
+ * payload carries the name, the description and the body.
+ *
+ * The registry used to DROP every hit whose skill was not registered
+ * locally, so a shared skill was invisible on every machine that had no
+ * local copy of it - which is exactly the case the vector store exists for.
+ *
+ * The client and the embedder are injected, so the test neither reaches the
+ * network nor replaces a process-global module.
+ */
+
+import { describe, expect, it } from "bun:test";
+
+import {
+	createSkillRegistry,
+	type QdrantClientLike,
+	type QdrantEmbedderLike,
+} from "../src/loader.js";
+import type { Skill } from "../src/types.js";
+
+/** A point of the vector store. */
+interface VectorPoint {
+	id: number;
+	score: number;
+	payload?: Record<string, unknown>;
+}
+
+/** A client that answers a query with the given points. */
+function fakeClient(
+	points: VectorPoint[],
+	description?: Record<string, unknown>,
+): QdrantClientLike & { queries: number } {
+	const client = {
+		queries: 0,
+		async upsert(): Promise<void> {},
+		async query(): Promise<unknown> {
+			client.queries += 1;
+			return { points };
+		},
+		async getCollection(): Promise<unknown> {
+			if (!description) throw new Error("Not found: Collection not found");
+			return description;
+		},
+	};
+	return client;
+}
+
+/** An embedder that returns a fixed vector. */
+const fakeEmbedder: QdrantEmbedderLike = {
+	async createEmbedding() {
+		return { embedding: [0.1, 0.2, 0.3], tokens: 3 };
+	},
+};
+
+/** A local skill, as the scanner would register it. */
+function localSkill(id: string): Skill {
+	return {
+		id,
+		path: `/tmp/skills/${id}`,
+		frontmatter: { name: id, description: `the local ${id}` },
+		body: `the local body of ${id}`,
+		references: [],
+		examples: [],
+		metadata: { name: id, description: `the local ${id}` },
+		loaded: false,
+	};
+}
+
+describe("a skill that only exists in the vector store", () => {
+	it("is found instead of being dropped", async () => {
+		const client = fakeClient([
+			{
+				id: 19,
+				score: 0.91,
+				payload: {
+					name: "frappe-doctype-field-order",
+					description: "the shared skill",
+					body: "the shared body",
+				},
+			},
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+
+		const results = await registry.findVector("field order");
+
+		expect(results).toHaveLength(1);
+		expect(results[0].source).toBe("remote");
+		expect(results[0].skill.id).toBe("frappe-doctype-field-order");
+		// The payload is the skill - the body travels with it.
+		expect(results[0].skill.body).toBe("the shared body");
+		expect(results[0].skill.frontmatter.description).toBe("the shared skill");
+	});
+
+	it("does not count as a local skill", async () => {
+		const client = fakeClient([
+			{ id: 19, score: 0.9, payload: { name: "shared", description: "d", body: "b" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+		registry.register(localSkill("local-one"));
+
+		await registry.findVector("shared");
+
+		// The local count stays local - that is what the startup line shows.
+		expect(registry.list()).toHaveLength(1);
+		expect(registry.list().map((skill) => skill.id)).toEqual(["local-one"]);
+		expect(registry.listRemote().map((skill) => skill.id)).toEqual(["shared"]);
+	});
+
+	it("can be loaded without a local directory", async () => {
+		const client = fakeClient([
+			{ id: 19, score: 0.9, payload: { name: "shared", description: "d", body: "the body" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+
+		await registry.findVector("shared");
+		const loaded = await registry.load("shared");
+
+		expect(loaded).not.toBeNull();
+		expect(loaded?.body).toBe("the body");
+		expect(loaded?.path).toBe("");
+	});
+
+	it("marks a local hit as local", async () => {
+		const client = fakeClient([
+			{ id: 1, score: 0.9, payload: { name: "local-one", description: "d", body: "b" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+		registry.register(localSkill("local-one"));
+
+		const results = await registry.findVector("local");
+
+		expect(results).toHaveLength(1);
+		expect(results[0].source).toBe("local");
+		expect(results[0].skill.path).toBe("/tmp/skills/local-one");
+	});
+
+	it("skips a hit without a name", async () => {
+		const client = fakeClient([
+			{ id: 1, score: 0.9, payload: { description: "no name" } },
+			{ id: 2, score: 0.8, payload: { name: "", description: "empty name" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+
+		expect(await registry.findVector("anything")).toHaveLength(0);
+	});
+});
+
+describe("the state of the vector store", () => {
+	it("reports the number of skills", async () => {
+		const client = fakeClient([], {
+			points_count: 79,
+			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("populated");
+		expect(stats.count).toBe(79);
+		expect(stats.dimensions).toBe(1536);
+		expect(stats.schemaMismatch).toBeUndefined();
+	});
+
+	it("names the real cause of an incompatible collection", async () => {
+		const client = fakeClient([], {
+			points_count: 79,
+			config: { params: { vectors: { size: 384, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		// The mismatch names the numbers, so the reader knows WHICH schema
+		// the collection was built with.
+		expect(stats.schemaMismatch).toContain("384");
+		expect(stats.schemaMismatch).toContain("1536");
+		expect(stats.schemaMismatch).toContain("sync-to-qdrant.ts");
+	});
+
+	it("reports an empty collection", async () => {
+		const client = fakeClient([], {
+			points_count: 0,
+			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("empty");
+		expect(stats.count).toBe(0);
+	});
+
+	it("reports a collection that does not exist", async () => {
+		const client = fakeClient([]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("missing");
+		expect(stats.count).toBe(0);
+	});
+});