pi-harness-runtime 1.1.411 → 1.1.412

package/packages/skills/src/remote-source.ts

@@ -0,0 +1,148 @@
+/**
+ * The server-held skills.
+ *
+ * The vector store holds an INDEX: the name, the description and a
+ * truncated body. That is enough to FIND a skill, but not to use it - a
+ * complete SKILL.md (with its references and examples) is larger than a
+ * vector store payload should be.
+ *
+ * The remote source is therefore the authoritative store of the complete
+ * skill content. It is reached through an authenticated service, so the
+ * SERVER decides which skills this user may see. A client never decides
+ * that on its own.
+ */
+
+import type { Skill } from "./types.js";
+
+/** A skill the server holds. */
+export interface RemoteSkill {
+	/** The id the server knows the skill by. */
+	id: string;
+	/** The name of the skill. */
+	name: string;
+	/** What the skill does. */
+	description: string;
+	/** The version of the content. */
+	version?: string;
+	/** When the content changed the last time. */
+	updatedAt?: string;
+	/** The content of the skill (the body of SKILL.md). */
+	content?: string;
+	/**
+	 * Whether the content is the COMPLETE skill.
+	 *
+	 * A vector store payload carries a truncated body - that is not a
+	 * complete skill. The service returns the whole thing, so it sets
+	 * this flag.
+	 */
+	contentComplete?: boolean;
+}
+
+/** Where the server-held skills come from. */
+export interface RemoteSkillSource {
+	/**
+	 * The skills the server holds for this user and workspace.
+	 *
+	 * @param options The search term and the number of skills to return.
+	 */
+	list(options?: { search?: string; limit?: number }): Promise<RemoteSkill[]>;
+	/** The complete content of one skill. */
+	get(id: string): Promise<RemoteSkill | null>;
+}
+
+/** The reasons a call to the remote source can fail. */
+export type RemoteSkillFailure =
+	/** The service refused the call - the token is missing or wrong. */
+	| "denied"
+	/** The service could not be reached. */
+	| "unavailable";
+
+/** The state of the server-held skills. */
+export type RemoteSkillState =
+	/** No service is configured. */
+	| "not-configured"
+	/** The service refused the call. */
+	| "denied"
+	/** The service could not be reached. */
+	| "unavailable"
+	/** The service holds no skill for this user. */
+	| "empty"
+	/** The service holds skills. */
+	| "populated";
+
+/** What the remote source knows. */
+export interface RemoteSkillStats {
+	/** The state of the source. */
+	state: RemoteSkillState;
+	/** The number of skills the source holds. */
+	count: number;
+	/** The reason of a failed call. */
+	error?: string;
+}
+
+/**
+ * A skill from a remote record.
+ *
+ * The record carries the content, so the skill is usable without a local
+ * copy. The path stays empty: a server skill has no directory on this
+ * machine.
+ */
+export function skillFromRemote(record: RemoteSkill): Skill {
+	const version = record.version;
+	return {
+		id: record.id,
+		path: "",
+		frontmatter: {
+			name: record.name,
+			description: record.description,
+			...(version ? { version } : {}),
+		},
+		body: record.content ?? "",
+		references: [],
+		examples: [],
+		metadata: {
+			name: record.name,
+			description: record.description,
+			...(version ? { version } : {}),
+			...(record.updatedAt ? { updated_at: record.updatedAt } : {}),
+		},
+		loaded: true,
+	};
+}
+
+/**
+ * The version of a cached remote skill.
+ *
+ * A skill without a version cannot be compared - the cache keeps it.
+ */
+export function remoteVersion(skill: Skill | undefined): string | undefined {
+	return skill?.frontmatter.version ?? skill?.metadata.version;
+}
+
+/**
+ * The safe reason of a failed call.
+ *
+ * The message never carries a credential: a token that a service echoed
+ * back is removed before the message leaves the process.
+ */
+export function describeRemoteFailure(error: unknown): string {
+  const raw = error instanceof Error ? error.message : String(error ?? "");
+  const safe = raw
+    .replace(/bearer\s+\S+/gi, "bearer <redacted>")
+    .replace(/\b[A-Za-z0-9_-]{32,}\b/g, "<redacted>")
+    .replace(/\s+/g, " ")
+    .trim();
+  return safe.slice(0, 200) || "the service could not be reached";
+}
+
+/**
+ * Whether the cached skill is older than the record.
+ *
+ * A record without a version never replaces a cached skill: an unknown
+ * version is not evidence of a change.
+ */
+export function isStale(cached: Skill | undefined, record: RemoteSkill): boolean {
+	if (!cached) return true;
+	if (!record.version) return false;
+	return remoteVersion(cached) !== record.version;
+}

package/packages/skills/src/vector-schema.ts

@@ -0,0 +1,72 @@
+/**
+ * The vector schema of the skill collection.
+ *
+ * A Qdrant collection holds vectors of exactly ONE size. Two different
+ * embedding models produce two different vector sizes, so they must never
+ * write into the same collection: the second writer is rejected by the
+ * server with a dimension error, and the data of the first writer becomes
+ * unsearchable by the second.
+ *
+ * The collection name carries the model and a version, so a change of the
+ * model leads to a NEW collection instead of a silent mix of two vector
+ * spaces.
+ *
+ * The model itself lives in the Qdrant package (the embedder owns it). This
+ * module owns the collection - that is a decision of the skills domain, not
+ * of the vector store.
+ */
+
+/**
+ * The collection the runtime indexes into and searches.
+ *
+ * `QDRANT_COLLECTION` overrides it, so a cluster that already holds a
+ * collection under another name keeps working.
+ */
+export const SKILL_COLLECTION = "pi-harness-skills-openai-v1";
+
+/**
+ * The collection that was used before the schema was versioned.
+ *
+ * It holds 384-dimensional vectors - the model that
+ * `scripts/sync-to-qdrant.ts` used - and is NOT compatible with the
+ * runtime. It is named here so a mismatch can name the real cause instead
+ * of only the numbers.
+ */
+export const LEGACY_SKILL_COLLECTION = "pi-harness-skills";
+
+/** The model of the legacy collection. */
+export const LEGACY_SKILL_COLLECTION_MODEL = "sentence-transformers/all-MiniLM-L6-v2";
+
+/** The vector size of the legacy collection. */
+export const LEGACY_SKILL_COLLECTION_DIMENSIONS = 384;
+
+/** The schema a skill collection is expected to have. */
+export interface SkillVectorSchema {
+	/** The vector size. */
+	dimensions: number;
+	/** The distance metric. */
+	distance: string;
+	/** The embedding model, when it is known. */
+	model?: string;
+}
+
+/**
+ * Why the collection does not match the expected schema - or null when it
+ * does.
+ *
+ * The message names the numbers, so the reader can tell WHICH schema the
+ * collection was built with.
+ */
+export function describeSkillSchemaMismatch(
+	actual: { dimensions?: number; distance?: string },
+	expected: SkillVectorSchema,
+): string | null {
+	if (actual.dimensions !== undefined && actual.dimensions !== expected.dimensions) {
+		const model = expected.model ? ` (${expected.model})` : "";
+		return `the collection holds ${actual.dimensions}-dimensional vectors, the runtime embeds with ${expected.dimensions}${model}`;
+	}
+	if (actual.distance !== undefined && actual.distance !== expected.distance) {
+		return `the collection uses the ${actual.distance} distance, the runtime expects ${expected.distance}`;
+	}
+	return null;
+}

package/packages/skills/test/remote-skills.test.ts

@@ -0,0 +1,242 @@
+/**
+ * The skills that only exist in the vector store.
+ *
+ * The vector store holds the skills of a team. A client that has no local
+ * copy of a skill must still be able to find it - and to use it: the
+ * payload carries the name, the description and the body.
+ *
+ * The registry used to DROP every hit whose skill was not registered
+ * locally, so a shared skill was invisible on every machine that had no
+ * local copy of it - which is exactly the case the vector store exists for.
+ *
+ * The client and the embedder are injected, so the test neither reaches the
+ * network nor replaces a process-global module.
+ */
+
+import { describe, expect, it } from "bun:test";
+
+import {
+	createSkillRegistry,
+	type QdrantClientLike,
+	type QdrantEmbedderLike,
+} from "../src/loader.js";
+import type { Skill } from "../src/types.js";
+
+/** A point of the vector store. */
+interface VectorPoint {
+	id: number;
+	score: number;
+	payload?: Record<string, unknown>;
+}
+
+/** A client that answers a query with the given points. */
+function fakeClient(
+	points: VectorPoint[],
+	description?: Record<string, unknown>,
+): QdrantClientLike & { queries: number } {
+	const client = {
+		queries: 0,
+		async upsert(): Promise<void> {},
+		async query(): Promise<unknown> {
+			client.queries += 1;
+			return { points };
+		},
+		async getCollection(): Promise<unknown> {
+			if (!description) throw new Error("Not found: Collection not found");
+			return description;
+		},
+	};
+	return client;
+}
+
+/** An embedder that returns a fixed vector. */
+const fakeEmbedder: QdrantEmbedderLike = {
+	async createEmbedding() {
+		return { embedding: [0.1, 0.2, 0.3], tokens: 3 };
+	},
+};
+
+/** A local skill, as the scanner would register it. */
+function localSkill(id: string): Skill {
+	return {
+		id,
+		path: `/tmp/skills/${id}`,
+		frontmatter: { name: id, description: `the local ${id}` },
+		body: `the local body of ${id}`,
+		references: [],
+		examples: [],
+		metadata: { name: id, description: `the local ${id}` },
+		loaded: false,
+	};
+}
+
+describe("a skill that only exists in the vector store", () => {
+	it("is found instead of being dropped", async () => {
+		const client = fakeClient([
+			{
+				id: 19,
+				score: 0.91,
+				payload: {
+					name: "frappe-doctype-field-order",
+					description: "the shared skill",
+					body: "the shared body",
+				},
+			},
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+
+		const results = await registry.findVector("field order");
+
+		expect(results).toHaveLength(1);
+		expect(results[0].source).toBe("remote");
+		expect(results[0].skill.id).toBe("frappe-doctype-field-order");
+		// The payload is the skill - the body travels with it.
+		expect(results[0].skill.body).toBe("the shared body");
+		expect(results[0].skill.frontmatter.description).toBe("the shared skill");
+	});
+
+	it("does not count as a local skill", async () => {
+		const client = fakeClient([
+			{ id: 19, score: 0.9, payload: { name: "shared", description: "d", body: "b" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+		registry.register(localSkill("local-one"));
+
+		await registry.findVector("shared");
+
+		// The local count stays local - that is what the startup line shows.
+		expect(registry.list()).toHaveLength(1);
+		expect(registry.list().map((skill) => skill.id)).toEqual(["local-one"]);
+		expect(registry.listRemote().map((skill) => skill.id)).toEqual(["shared"]);
+	});
+
+	it("can be loaded without a local directory", async () => {
+		const client = fakeClient([
+			{ id: 19, score: 0.9, payload: { name: "shared", description: "d", body: "the body" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+
+		await registry.findVector("shared");
+		const loaded = await registry.load("shared");
+
+		expect(loaded).not.toBeNull();
+		expect(loaded?.body).toBe("the body");
+		expect(loaded?.path).toBe("");
+	});
+
+	it("marks a local hit as local", async () => {
+		const client = fakeClient([
+			{ id: 1, score: 0.9, payload: { name: "local-one", description: "d", body: "b" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+		registry.register(localSkill("local-one"));
+
+		const results = await registry.findVector("local");
+
+		expect(results).toHaveLength(1);
+		expect(results[0].source).toBe("local");
+		expect(results[0].skill.path).toBe("/tmp/skills/local-one");
+	});
+
+	it("skips a hit without a name", async () => {
+		const client = fakeClient([
+			{ id: 1, score: 0.9, payload: { description: "no name" } },
+			{ id: 2, score: 0.8, payload: { name: "", description: "empty name" } },
+		]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-test",
+		});
+
+		expect(await registry.findVector("anything")).toHaveLength(0);
+	});
+});
+
+describe("the state of the vector store", () => {
+	it("reports the number of skills", async () => {
+		const client = fakeClient([], {
+			points_count: 79,
+			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("populated");
+		expect(stats.count).toBe(79);
+		expect(stats.dimensions).toBe(1536);
+		expect(stats.schemaMismatch).toBeUndefined();
+	});
+
+	it("names the real cause of an incompatible collection", async () => {
+		const client = fakeClient([], {
+			points_count: 79,
+			config: { params: { vectors: { size: 384, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		// The mismatch names the numbers, so the reader knows WHICH schema
+		// the collection was built with.
+		expect(stats.schemaMismatch).toContain("384");
+		expect(stats.schemaMismatch).toContain("1536");
+		expect(stats.schemaMismatch).toContain("sync-to-qdrant.ts");
+	});
+
+	it("reports an empty collection", async () => {
+		const client = fakeClient([], {
+			points_count: 0,
+			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
+		});
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("empty");
+		expect(stats.count).toBe(0);
+	});
+
+	it("reports a collection that does not exist", async () => {
+		const client = fakeClient([]);
+		const registry = createSkillRegistry({
+			qdrantClient: client,
+			embedder: fakeEmbedder,
+			collection: "pi-harness-skills-openai-v1",
+		});
+
+		const stats = await registry.getQdrantStats();
+
+		expect(stats.state).toBe("missing");
+		expect(stats.count).toBe(0);
+	});
+});

package/packages/skills/test/remote-source.test.ts

@@ -0,0 +1,248 @@
+/**
+ * The skills that live on the server.
+ *
+ * The vector store is an index - it says WHICH skill fits a question. The
+ * service holds the authoritative files, so a machine without a local copy
+ * can still use a skill.
+ *
+ * The states must stay apart: a service that refused the call needs a
+ * different fix than a service that could not be reached, and both differ
+ * from a service that holds nothing.
+ *
+ * The source is injected, so the test neither reaches the network nor
+ * replaces a process-global module.
+ */
+
+import { describe, expect, it } from "bun:test";
+
+import { createSkillRegistry } from "../src/loader.js";
+import type { RemoteSkill, RemoteSkillSource } from "../src/remote-source.js";
+
+/** A source that holds the given skills. */
+class FakeSource implements RemoteSkillSource {
+	/** The skills the source holds. */
+	readonly skills: RemoteSkill[];
+	/** The ids that were asked for. */
+	readonly requested: string[] = [];
+	/** The error the next call raises. */
+	failure: { message: string; code: "denied" | "unavailable" } | null = null;
+
+	constructor(skills: RemoteSkill[] = []) {
+		this.skills = skills;
+	}
+
+	async list(): Promise<RemoteSkill[]> {
+		if (this.failure) {
+			const error = new Error(this.failure.message) as Error & {
+				code: string;
+			};
+			error.code = this.failure.code;
+			throw error;
+		}
+		return this.skills;
+	}
+
+	async get(id: string): Promise<RemoteSkill | null> {
+		this.requested.push(id);
+		if (this.failure) {
+			const error = new Error(this.failure.message) as Error & {
+				code: string;
+			};
+			error.code = this.failure.code;
+			throw error;
+		}
+		return this.skills.find((skill) => skill.id === id) ?? null;
+	}
+}
+
+describe("the skills of the server", () => {
+	it("are listed by the registry", async () => {
+		const source = new FakeSource([
+			{
+				id: "shared-skill",
+				name: "shared-skill",
+				description: "the shared skill",
+				version: "1.2.0",
+			},
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+
+		const stats = await registry.refreshRemoteSkills();
+
+		expect(stats.state).toBe("populated");
+		expect(stats.count).toBe(1);
+		expect(registry.listRemote().map((skill) => skill.id)).toEqual(["shared-skill"]);
+		// The local count stays local.
+		expect(registry.list()).toHaveLength(0);
+	});
+
+	it("are not local skills", async () => {
+		const source = new FakeSource([
+			{ id: "shared-skill", name: "shared-skill", description: "d", version: "1.0.0" },
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+		await registry.refreshRemoteSkills();
+
+		expect(registry.list()).toHaveLength(0);
+		expect(registry.listRemote()).toHaveLength(1);
+	});
+
+	it("replace the cached skill when the version changed", async () => {
+		const source = new FakeSource([
+			{ id: "shared-skill", name: "shared-skill", description: "old", version: "1.0.0" },
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+		await registry.refreshRemoteSkills();
+		expect(registry.listRemote()[0].frontmatter.description).toBe("old");
+
+		// The server publishes a new version.
+		source.skills[0] = {
+			id: "shared-skill",
+			name: "shared-skill",
+			description: "new",
+			version: "1.1.0",
+		};
+		await registry.refreshRemoteSkills();
+
+		expect(registry.listRemote()[0].frontmatter.description).toBe("new");
+		expect(registry.listRemote()[0].frontmatter.version).toBe("1.1.0");
+	});
+
+	it("stay when the version is unknown", async () => {
+		const source = new FakeSource([
+			{ id: "shared-skill", name: "shared-skill", description: "known", version: "1.0.0" },
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+		await registry.refreshRemoteSkills();
+
+		// A record without a version is not evidence of a change.
+		source.skills[0] = {
+			id: "shared-skill",
+			name: "shared-skill",
+			description: "changed without a version",
+		};
+		await registry.refreshRemoteSkills();
+
+		expect(registry.listRemote()[0].frontmatter.description).toBe("known");
+	});
+
+	it("are loaded completely from the service", async () => {
+		const source = new FakeSource([
+			{
+				id: "shared-skill",
+				name: "shared-skill",
+				description: "the shared skill",
+				version: "1.0.0",
+				content: "the complete body of the skill",
+				contentComplete: true,
+			},
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+
+		const loaded = await registry.load("shared-skill");
+
+		expect(loaded).not.toBeNull();
+		expect(loaded?.body).toBe("the complete body of the skill");
+		expect(source.requested).toContain("shared-skill");
+	});
+
+	it("are not asked for again once the content is complete", async () => {
+		const source = new FakeSource([
+			{
+				id: "shared-skill",
+				name: "shared-skill",
+				description: "d",
+				version: "1.0.0",
+				content: "the body",
+				contentComplete: true,
+			},
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+
+		await registry.load("shared-skill");
+		const afterFirst = source.requested.length;
+		await registry.load("shared-skill");
+
+		expect(source.requested.length).toBe(afterFirst);
+	});
+
+	it("are reported as empty when the service holds nothing", async () => {
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(new FakeSource([]));
+
+		const stats = await registry.refreshRemoteSkills();
+
+		expect(stats.state).toBe("empty");
+		expect(stats.count).toBe(0);
+	});
+
+	it("are reported as not configured without a service", async () => {
+		const registry = createSkillRegistry();
+
+		const stats = await registry.getRemoteStats();
+
+		expect(stats.state).toBe("not-configured");
+	});
+});
+
+describe("a service that does not answer", () => {
+	it("reports a refusal as denied", async () => {
+		const source = new FakeSource();
+		source.failure = { message: "unauthorized", code: "denied" };
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+
+		const stats = await registry.refreshRemoteSkills();
+
+		expect(stats.state).toBe("denied");
+		expect(stats.error).toContain("unauthorized");
+	});
+
+	it("reports an unreachable service as unavailable", async () => {
+		const source = new FakeSource();
+		source.failure = { message: "connection refused", code: "unavailable" };
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+
+		const stats = await registry.refreshRemoteSkills();
+
+		expect(stats.state).toBe("unavailable");
+		expect(stats.error).toContain("connection refused");
+	});
+
+	it("keeps the cached skills after a failed refresh", async () => {
+		const source = new FakeSource([
+			{ id: "shared-skill", name: "shared-skill", description: "d", version: "1.0.0" },
+		]);
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+		await registry.refreshRemoteSkills();
+
+		source.failure = { message: "connection refused", code: "unavailable" };
+		const stats = await registry.refreshRemoteSkills();
+
+		expect(stats.state).toBe("unavailable");
+		// A failed refresh must not throw the cached skills away.
+		expect(registry.listRemote()).toHaveLength(1);
+	});
+
+	it("does not put a credential into the message", async () => {
+		const source = new FakeSource();
+		source.failure = {
+			message: "Bearer sk-abcdefghijklmnopqrstuvwxyz0123456789 refused",
+			code: "denied",
+		};
+		const registry = createSkillRegistry();
+		registry.setRemoteSource(source);
+
+		const stats = await registry.refreshRemoteSkills();
+
+		expect(stats.error).not.toContain("sk-abcdefghijklmnopqrstuvwxyz0123456789");
+	});
+});