pi-harness-runtime 1.1.440 → 1.1.441

package/packages/skills/test/remote-skills.test.ts

@@ -1,360 +0,0 @@
-/**
- * The skills that only exist in the vector store.
- *
- * The vector store holds the skills of a team. A client that has no local
- * copy of a skill must still be able to find it - and to use it: the
- * payload carries the name, the description and the body.
- *
- * The registry used to DROP every hit whose skill was not registered
- * locally, so a shared skill was invisible on every machine that had no
- * local copy of it - which is exactly the case the vector store exists for.
- *
- * The client and the embedder are injected, so the test neither reaches the
- * network nor replaces a process-global module.
- */
-
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import { fileURLToPath } from "node:url";
-
-import {
-	createSkillRegistry,
-	type QdrantClientLike,
-	type QdrantEmbedderLike,
-} from "../src/loader.js";
-import { resetSanitizationRulesCache } from "../src/sanitize-rules.js";
-import type { Skill } from "../src/types.js";
-
-/**
- * The rules of the test.
- *
- * The rules of a client live in a local file, outside the repository. The
- * test uses a fixture with made up names, so that the result of the test
- * does not depend on the machine it runs on.
- */
-const FIXTURE = fileURLToPath(
-	new URL("./fixtures/sanitization.json", import.meta.url),
-);
-const originalEnv = process.env.PI_SKILL_SANITIZATION_FILE;
-
-beforeEach(() => {
-	process.env.PI_SKILL_SANITIZATION_FILE = FIXTURE;
-	resetSanitizationRulesCache();
-});
-
-afterEach(() => {
-	resetSanitizationRulesCache();
-	if (originalEnv === undefined) delete process.env.PI_SKILL_SANITIZATION_FILE;
-	else process.env.PI_SKILL_SANITIZATION_FILE = originalEnv;
-});
-
-/** A point of the vector store. */
-interface VectorPoint {
-	id: number;
-	score: number;
-	payload?: Record<string, unknown>;
-}
-
-/** A client that answers a query with the given points. */
-function fakeClient(
-	points: VectorPoint[],
-	description?: Record<string, unknown>,
-): QdrantClientLike & { queries: number } {
-	const client = {
-		queries: 0,
-		async upsert(): Promise<void> {},
-		async query(): Promise<unknown> {
-			client.queries += 1;
-			return { points };
-		},
-		async getCollection(): Promise<unknown> {
-			if (!description) throw new Error("Not found: Collection not found");
-			return description;
-		},
-	};
-	return client;
-}
-
-/** An embedder that returns a fixed vector. */
-const fakeEmbedder: QdrantEmbedderLike = {
-	async createEmbedding() {
-		return { embedding: [0.1, 0.2, 0.3], tokens: 3 };
-	},
-};
-
-/** A local skill, as the scanner would register it. */
-function localSkill(id: string): Skill {
-	return {
-		id,
-		path: `/tmp/skills/${id}`,
-		frontmatter: { name: id, description: `the local ${id}` },
-		body: `the local body of ${id}`,
-		references: [],
-		examples: [],
-		metadata: { name: id, description: `the local ${id}` },
-		loaded: false,
-	};
-}
-
-describe("a skill that only exists in the vector store", () => {
-	it("is found instead of being dropped", async () => {
-		const client = fakeClient([
-			{
-				id: 19,
-				score: 0.91,
-				payload: {
-					name: "frappe-doctype-field-order",
-					description: "the shared skill",
-					body: "the shared body",
-				},
-			},
-		]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-test",
-		});
-
-		const results = await registry.findVector("field order");
-
-		expect(results).toHaveLength(1);
-		expect(results[0].source).toBe("remote");
-		expect(results[0].skill.id).toBe("frappe-doctype-field-order");
-		// The payload is the skill - the body travels with it.
-		expect(results[0].skill.body).toBe("the shared body");
-		expect(results[0].skill.frontmatter.description).toBe("the shared skill");
-	});
-
-	it("does not count as a local skill", async () => {
-		const client = fakeClient([
-			{ id: 19, score: 0.9, payload: { name: "shared", description: "d", body: "b" } },
-		]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-test",
-		});
-		registry.register(localSkill("local-one"));
-
-		await registry.findVector("shared");
-
-		// The local count stays local - that is what the startup line shows.
-		expect(registry.list()).toHaveLength(1);
-		expect(registry.list().map((skill) => skill.id)).toEqual(["local-one"]);
-		expect(registry.listRemote().map((skill) => skill.id)).toEqual(["shared"]);
-	});
-
-	it("can be loaded without a local directory", async () => {
-		const client = fakeClient([
-			{ id: 19, score: 0.9, payload: { name: "shared", description: "d", body: "the body" } },
-		]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-test",
-		});
-
-		await registry.findVector("shared");
-		const loaded = await registry.load("shared");
-
-		expect(loaded).not.toBeNull();
-		expect(loaded?.body).toBe("the body");
-		expect(loaded?.path).toBe("");
-	});
-
-	it("marks a local hit as local", async () => {
-		const client = fakeClient([
-			{ id: 1, score: 0.9, payload: { name: "local-one", description: "d", body: "b" } },
-		]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-test",
-		});
-		registry.register(localSkill("local-one"));
-
-		const results = await registry.findVector("local");
-
-		expect(results).toHaveLength(1);
-		expect(results[0].source).toBe("local");
-		expect(results[0].skill.path).toBe("/tmp/skills/local-one");
-	});
-
-	it("skips a hit without a name", async () => {
-		const client = fakeClient([
-			{ id: 1, score: 0.9, payload: { description: "no name" } },
-			{ id: 2, score: 0.8, payload: { name: "", description: "empty name" } },
-		]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-test",
-		});
-
-		expect(await registry.findVector("anything")).toHaveLength(0);
-	});
-});
-
-describe("a hit that carries the shared name of a local skill", () => {
-	it("resolves to the local skill", async () => {
-		// The collection is shared, so the point carries the SANITIZED name.
-		const client = fakeClient([
-			{
-				id: 1,
-				score: 0.9,
-				payload: {
-					name: "client-a-pe-approval-workflow",
-					description: "the workflow",
-					body: "the body",
-				},
-			},
-		]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-test",
-		});
-		registry.register(localSkill("acmecorp-pe-approval-workflow"));
-
-		const results = await registry.findVector("workflow");
-
-		expect(results).toHaveLength(1);
-		// The skill of this machine stays a local skill - with its own
-		// name and its own body, not the sanitized copy.
-		expect(results[0].source).toBe("local");
-		expect(results[0].skill.id).toBe("acmecorp-pe-approval-workflow");
-		expect(results[0].skill.body).toBe(
-			"the local body of acmecorp-pe-approval-workflow",
-		);
-	});
-
-	it("resolves the shared name of a local skill in a lookup", () => {
-		const registry = createSkillRegistry({ indexer: null });
-		registry.register(localSkill("acmecorp-pe-approval-workflow"));
-
-		expect(registry.get("client-a-pe-approval-workflow")?.id).toBe(
-			"acmecorp-pe-approval-workflow",
-		);
-	});
-});
-
-describe("the state of the vector store", () => {
-	it("reports the number of skills", async () => {
-		const client = fakeClient([], {
-			points_count: 79,
-			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
-		});
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-openai-v1",
-		});
-
-		const stats = await registry.getQdrantStats();
-
-		expect(stats.state).toBe("populated");
-		expect(stats.count).toBe(79);
-		expect(stats.dimensions).toBe(1536);
-		expect(stats.schemaMismatch).toBeUndefined();
-	});
-
-	it("names the real cause of an incompatible collection", async () => {
-		const client = fakeClient([], {
-			points_count: 79,
-			config: { params: { vectors: { size: 384, distance: "Cosine" } } },
-		});
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills",
-		});
-
-		const stats = await registry.getQdrantStats();
-
-		// The mismatch names the numbers, so the reader knows WHICH schema
-		// the collection was built with.
-		expect(stats.schemaMismatch).toContain("384");
-		expect(stats.schemaMismatch).toContain("1536");
-		expect(stats.schemaMismatch).toContain("sync-to-qdrant.ts");
-	});
-
-	it("reports an empty collection", async () => {
-		const client = fakeClient([], {
-			points_count: 0,
-			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
-		});
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-openai-v1",
-		});
-
-		const stats = await registry.getQdrantStats();
-
-		expect(stats.state).toBe("empty");
-		expect(stats.count).toBe(0);
-	});
-
-	it("reports a collection that does not exist", async () => {
-		const client = fakeClient([]);
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-openai-v1",
-		});
-
-		const stats = await registry.getQdrantStats();
-
-		expect(stats.state).toBe("missing");
-		expect(stats.count).toBe(0);
-	});
-
-	it("names the skills of the legacy collection", async () => {
-		// The runtime collection does not exist yet, but the collection of
-		// the previous version still holds the skills. They are not lost -
-		// the reader has to know where they are.
-		const client = {
-			async upsert(): Promise<void> {},
-			async query(): Promise<unknown> {
-				return { points: [] };
-			},
-			async getCollection(name: string): Promise<unknown> {
-				if (name === "pi-harness-skills") {
-					return {
-						points_count: 79,
-						config: { params: { vectors: { size: 384, distance: "Cosine" } } },
-					};
-				}
-				throw new Error("Not Found");
-			},
-		};
-		const registry = createSkillRegistry({
-			qdrantClient: client as never,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills-openai-v1",
-		});
-
-		const stats = await registry.getQdrantStats();
-
-		expect(stats.state).toBe("missing");
-		expect(stats.legacy?.collection).toBe("pi-harness-skills");
-		expect(stats.legacy?.count).toBe(79);
-		expect(stats.legacy?.dimensions).toBe(384);
-		expect(stats.legacy?.model).toBe("sentence-transformers/all-MiniLM-L6-v2");
-	});
-
-	it("does not look for a second collection when the legacy one is the runtime one", async () => {
-		const client = fakeClient([], {
-			points_count: 0,
-			config: { params: { vectors: { size: 1536, distance: "Cosine" } } },
-		});
-		const registry = createSkillRegistry({
-			qdrantClient: client,
-			embedder: fakeEmbedder,
-			collection: "pi-harness-skills",
-		});
-
-		const stats = await registry.getQdrantStats();
-
-		expect(stats.legacy).toBeUndefined();
-	});
-});

package/packages/skills/test/remote-source.test.ts

@@ -1,248 +0,0 @@
-/**
- * The skills that live on the server.
- *
- * The vector store is an index - it says WHICH skill fits a question. The
- * service holds the authoritative files, so a machine without a local copy
- * can still use a skill.
- *
- * The states must stay apart: a service that refused the call needs a
- * different fix than a service that could not be reached, and both differ
- * from a service that holds nothing.
- *
- * The source is injected, so the test neither reaches the network nor
- * replaces a process-global module.
- */
-
-import { describe, expect, it } from "bun:test";
-
-import { createSkillRegistry } from "../src/loader.js";
-import type { RemoteSkill, RemoteSkillSource } from "../src/remote-source.js";
-
-/** A source that holds the given skills. */
-class FakeSource implements RemoteSkillSource {
-	/** The skills the source holds. */
-	readonly skills: RemoteSkill[];
-	/** The ids that were asked for. */
-	readonly requested: string[] = [];
-	/** The error the next call raises. */
-	failure: { message: string; code: "denied" | "unavailable" } | null = null;
-
-	constructor(skills: RemoteSkill[] = []) {
-		this.skills = skills;
-	}
-
-	async list(): Promise<RemoteSkill[]> {
-		if (this.failure) {
-			const error = new Error(this.failure.message) as Error & {
-				code: string;
-			};
-			error.code = this.failure.code;
-			throw error;
-		}
-		return this.skills;
-	}
-
-	async get(id: string): Promise<RemoteSkill | null> {
-		this.requested.push(id);
-		if (this.failure) {
-			const error = new Error(this.failure.message) as Error & {
-				code: string;
-			};
-			error.code = this.failure.code;
-			throw error;
-		}
-		return this.skills.find((skill) => skill.id === id) ?? null;
-	}
-}
-
-describe("the skills of the server", () => {
-	it("are listed by the registry", async () => {
-		const source = new FakeSource([
-			{
-				id: "shared-skill",
-				name: "shared-skill",
-				description: "the shared skill",
-				version: "1.2.0",
-			},
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-
-		const stats = await registry.refreshRemoteSkills();
-
-		expect(stats.state).toBe("populated");
-		expect(stats.count).toBe(1);
-		expect(registry.listRemote().map((skill) => skill.id)).toEqual(["shared-skill"]);
-		// The local count stays local.
-		expect(registry.list()).toHaveLength(0);
-	});
-
-	it("are not local skills", async () => {
-		const source = new FakeSource([
-			{ id: "shared-skill", name: "shared-skill", description: "d", version: "1.0.0" },
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-		await registry.refreshRemoteSkills();
-
-		expect(registry.list()).toHaveLength(0);
-		expect(registry.listRemote()).toHaveLength(1);
-	});
-
-	it("replace the cached skill when the version changed", async () => {
-		const source = new FakeSource([
-			{ id: "shared-skill", name: "shared-skill", description: "old", version: "1.0.0" },
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-		await registry.refreshRemoteSkills();
-		expect(registry.listRemote()[0].frontmatter.description).toBe("old");
-
-		// The server publishes a new version.
-		source.skills[0] = {
-			id: "shared-skill",
-			name: "shared-skill",
-			description: "new",
-			version: "1.1.0",
-		};
-		await registry.refreshRemoteSkills();
-
-		expect(registry.listRemote()[0].frontmatter.description).toBe("new");
-		expect(registry.listRemote()[0].frontmatter.version).toBe("1.1.0");
-	});
-
-	it("stay when the version is unknown", async () => {
-		const source = new FakeSource([
-			{ id: "shared-skill", name: "shared-skill", description: "known", version: "1.0.0" },
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-		await registry.refreshRemoteSkills();
-
-		// A record without a version is not evidence of a change.
-		source.skills[0] = {
-			id: "shared-skill",
-			name: "shared-skill",
-			description: "changed without a version",
-		};
-		await registry.refreshRemoteSkills();
-
-		expect(registry.listRemote()[0].frontmatter.description).toBe("known");
-	});
-
-	it("are loaded completely from the service", async () => {
-		const source = new FakeSource([
-			{
-				id: "shared-skill",
-				name: "shared-skill",
-				description: "the shared skill",
-				version: "1.0.0",
-				content: "the complete body of the skill",
-				contentComplete: true,
-			},
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-
-		const loaded = await registry.load("shared-skill");
-
-		expect(loaded).not.toBeNull();
-		expect(loaded?.body).toBe("the complete body of the skill");
-		expect(source.requested).toContain("shared-skill");
-	});
-
-	it("are not asked for again once the content is complete", async () => {
-		const source = new FakeSource([
-			{
-				id: "shared-skill",
-				name: "shared-skill",
-				description: "d",
-				version: "1.0.0",
-				content: "the body",
-				contentComplete: true,
-			},
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-
-		await registry.load("shared-skill");
-		const afterFirst = source.requested.length;
-		await registry.load("shared-skill");
-
-		expect(source.requested.length).toBe(afterFirst);
-	});
-
-	it("are reported as empty when the service holds nothing", async () => {
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(new FakeSource([]));
-
-		const stats = await registry.refreshRemoteSkills();
-
-		expect(stats.state).toBe("empty");
-		expect(stats.count).toBe(0);
-	});
-
-	it("are reported as not configured without a service", async () => {
-		const registry = createSkillRegistry();
-
-		const stats = await registry.getRemoteStats();
-
-		expect(stats.state).toBe("not-configured");
-	});
-});
-
-describe("a service that does not answer", () => {
-	it("reports a refusal as denied", async () => {
-		const source = new FakeSource();
-		source.failure = { message: "unauthorized", code: "denied" };
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-
-		const stats = await registry.refreshRemoteSkills();
-
-		expect(stats.state).toBe("denied");
-		expect(stats.error).toContain("unauthorized");
-	});
-
-	it("reports an unreachable service as unavailable", async () => {
-		const source = new FakeSource();
-		source.failure = { message: "connection refused", code: "unavailable" };
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-
-		const stats = await registry.refreshRemoteSkills();
-
-		expect(stats.state).toBe("unavailable");
-		expect(stats.error).toContain("connection refused");
-	});
-
-	it("keeps the cached skills after a failed refresh", async () => {
-		const source = new FakeSource([
-			{ id: "shared-skill", name: "shared-skill", description: "d", version: "1.0.0" },
-		]);
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-		await registry.refreshRemoteSkills();
-
-		source.failure = { message: "connection refused", code: "unavailable" };
-		const stats = await registry.refreshRemoteSkills();
-
-		expect(stats.state).toBe("unavailable");
-		// A failed refresh must not throw the cached skills away.
-		expect(registry.listRemote()).toHaveLength(1);
-	});
-
-	it("does not put a credential into the message", async () => {
-		const source = new FakeSource();
-		source.failure = {
-			message: "Bearer sk-abcdefghijklmnopqrstuvwxyz0123456789 refused",
-			code: "denied",
-		};
-		const registry = createSkillRegistry();
-		registry.setRemoteSource(source);
-
-		const stats = await registry.refreshRemoteSkills();
-
-		expect(stats.error).not.toContain("sk-abcdefghijklmnopqrstuvwxyz0123456789");
-	});
-});