pi-harness-runtime 1.1.439 → 1.1.441

package/packages/skills/test/sanitize.test.ts

@@ -1,250 +0,0 @@
-/**
- * Tests for the sanitization of a skill for the shared collection.
- *
- * The collection is shared, so the client names of a user must not travel
- * into it. The rules used to live in `scripts/sync-to-qdrant.ts` - the
- * only writer that could reach them. The runtime registry writes into the
- * same collection, so it needs them too.
- *
- * The rules of the clients are **not** in this repository: it is public,
- * and a list of client names in it is a list of client names in the world.
- * They live in a local file, so the tests use a fixture with made up
- * names and pass the rules in. A test must never read the rules of the
- * machine it runs on - that would make the result of the test depend on
- * the machine.
- */
-
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import { mkdtempSync, writeFileSync } from "node:fs";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { fileURLToPath } from "node:url";
-
-import { sanitizeText, skillPointName } from "../src/sanitize.js";
-import {
-  loadSanitizationRules,
-  resetSanitizationRulesCache,
-  sanitizationStatus,
-} from "../src/sanitize-rules.js";
-import { skillPointPayload } from "../src/vector-schema.js";
-
-const FIXTURE = fileURLToPath(new URL("./fixtures/sanitization.json", import.meta.url));
-const MISSING = join(tmpdir(), "pi-harness-no-such-sanitization-file.json");
-
-const originalEnv = process.env.PI_SKILL_SANITIZATION_FILE;
-
-/** The rules of the fixture, as the mechanism reads them. */
-function fixtureRules() {
-  resetSanitizationRulesCache();
-  process.env.PI_SKILL_SANITIZATION_FILE = FIXTURE;
-  const set = loadSanitizationRules();
-  resetSanitizationRulesCache();
-  return set;
-}
-
-beforeEach(() => {
-  resetSanitizationRulesCache();
-  process.env.PI_SKILL_SANITIZATION_FILE = FIXTURE;
-});
-
-afterEach(() => {
-  resetSanitizationRulesCache();
-  if (originalEnv === undefined) delete process.env.PI_SKILL_SANITIZATION_FILE;
-  else process.env.PI_SKILL_SANITIZATION_FILE = originalEnv;
-});
-
-describe("the sanitization of a text", () => {
-	test("the name of a client is replaced", () => {
-		const rules = fixtureRules();
-		expect(sanitizeText("the acmecorp workflow", rules)).toBe(
-			"the [Client A] workflow",
-		);
-		expect(sanitizeText("for globex", rules)).toBe("for [Client B]");
-		expect(sanitizeText("initech step", rules)).toBe("[Insurance Project] step");
-	});
-
-	test("the name of a client is replaced in any case", () => {
-		const rules = fixtureRules();
-		expect(sanitizeText("ACMEcorp and Acmecorp", rules)).toBe(
-			"[Client A] and [Client A]",
-		);
-	});
-
-	test("a text without a client name stays the same", () => {
-		const rules = fixtureRules();
-		expect(sanitizeText("a plain skill", rules)).toBe("a plain skill");
-	});
-
-	test("the generic phrases are replaced", () => {
-		// The generic rules hold for every installation, so they work
-		// without a local file.
-		process.env.PI_SKILL_SANITIZATION_FILE = MISSING;
-		resetSanitizationRulesCache();
-		expect(sanitizeText("Company Name")).toBe("[Client Name]");
-		expect(sanitizeText("your company")).toBe("[Client]");
-		expect(sanitizeText("our client")).toBe("[Client]");
-	});
-
-	test("a client name inside an identifier is replaced", () => {
-		// The name of a client rarely stands alone. A word boundary would
-		// miss every one of these - and an underscore is a word character,
-		// so a boundary never follows one.
-		const rules = fixtureRules();
-		expect(sanitizeText("acmecorp_pharma", rules)).toBe("[Client A]_pharma");
-		expect(sanitizeText("th_globex_erp_chart.json", rules)).toBe(
-			"th_[Client B]_erp_chart.json",
-		);
-		expect(sanitizeText("apps/acmecorp_pharma/setup", rules)).toBe(
-			"apps/[Client A]_pharma/setup",
-		);
-	});
-
-	test("a client name in a sentence is replaced", () => {
-		const rules = fixtureRules();
-		expect(sanitizeText("integrated with Globex", rules)).toBe(
-			"integrated with [Client B]",
-		);
-		expect(sanitizeText("Initech secrets", rules)).toBe(
-			"[Insurance Project] secrets",
-		);
-	});
-});
-
-describe("the name of a skill in the shared collection", () => {
-	test("a name of the map is renamed", () => {
-		const rules = fixtureRules();
-		expect(skillPointName("acmecorp-pe-approval-workflow", rules)).toBe(
-			"client-a-pe-approval-workflow",
-		);
-		expect(skillPointName("initech-step5-slot-filter", rules)).toBe(
-			"insurance-project-step-filter",
-		);
-	});
-
-	test("a name without a client name stays the same", () => {
-		const rules = fixtureRules();
-		expect(skillPointName("code-review", rules)).toBe("code-review");
-	});
-
-	test("the name never carries a client name", () => {
-		const rules = fixtureRules();
-		for (const name of [
-			"acmecorp-pe-approval-workflow",
-			"initech-step5-slot-filter",
-			"globex-channel",
-		]) {
-			expect(skillPointName(name, rules)).not.toMatch(
-				/acmecorp|initech|globex/i,
-			);
-		}
-	});
-
-	test("the name is a valid skill name", () => {
-		// The name of a point is the id of the skill that a search returns,
-		// so it has to be a name that a skill may carry.
-		const NAME = /^[a-z0-9]+(-[a-z0-9]+)*$/;
-		const rules = fixtureRules();
-		for (const name of [
-			"acmecorp-pe-approval-workflow",
-			"globex-channel",
-			"initech-step5-slot-filter",
-			"code-review",
-		]) {
-			expect(skillPointName(name, rules)).toMatch(NAME);
-		}
-	});
-
-	test("a name that needs no sanitization stays the same", () => {
-		const rules = fixtureRules();
-		expect(skillPointName("code-review", rules)).toBe("code-review");
-		expect(skillPointName("frappe-doctype-field-order", rules)).toBe(
-			"frappe-doctype-field-order",
-		);
-	});
-});
-
-describe("the rules of the installation", () => {
-	test("a missing file leaves the generic rules", () => {
-		process.env.PI_SKILL_SANITIZATION_FILE = MISSING;
-		resetSanitizationRulesCache();
-		const status = sanitizationStatus();
-		expect(status.configured).toBe(false);
-		expect(status.ruleCount).toBeGreaterThan(0);
-	});
-
-	test("a broken file is reported and does not take the rules down", () => {
-		const dir = mkdtempSync(join(tmpdir(), "pi-harness-sanitization-"));
-		const path = join(dir, "rules.json");
-		writeFileSync(path, "{ not json");
-		process.env.PI_SKILL_SANITIZATION_FILE = path;
-		resetSanitizationRulesCache();
-		const status = sanitizationStatus();
-		expect(status.configured).toBe(false);
-		expect(status.error).toContain("not valid JSON");
-		// The generic rules still hold.
-		expect(sanitizeText("Company Name")).toBe("[Client Name]");
-	});
-
-	test("a broken pattern does not take the other rules down", () => {
-		const dir = mkdtempSync(join(tmpdir(), "pi-harness-sanitization-"));
-		const path = join(dir, "rules.json");
-		writeFileSync(
-			path,
-			JSON.stringify({
-				rules: [
-					{ pattern: "acmecorp", replacement: "[Client A]" },
-					{ pattern: "([unclosed", replacement: "[Broken]" },
-					{ pattern: "globex", replacement: "[Client B]" },
-				],
-			}),
-		);
-		process.env.PI_SKILL_SANITIZATION_FILE = path;
-		resetSanitizationRulesCache();
-		const rules = loadSanitizationRules();
-		expect(sanitizeText("acmecorp and globex", rules)).toBe(
-			"[Client A] and [Client B]",
-		);
-	});
-
-	test("the local file is read once", () => {
-		process.env.PI_SKILL_SANITIZATION_FILE = FIXTURE;
-		resetSanitizationRulesCache();
-		const first = loadSanitizationRules();
-		const second = loadSanitizationRules();
-		expect(second).toBe(first);
-	});
-});
-
-describe("the payload of a point is sanitized", () => {
-	test("the client names of the body do not travel into the collection", () => {
-		const rules = fixtureRules();
-		const payload = skillPointPayload(
-			{
-				name: "acmecorp-pe-approval-workflow",
-				description: "the workflow of acmecorp",
-				body: "the approval of acmecorp for the client",
-			},
-			rules,
-		);
-
-		expect(payload.name).toBe("client-a-pe-approval-workflow");
-		expect(payload.description).not.toMatch(/acmecorp/i);
-		expect(payload.body).not.toMatch(/acmecorp/i);
-		expect(payload.text).not.toMatch(/acmecorp/i);
-	});
-
-	test("the text is built from the sanitized parts", () => {
-		const rules = fixtureRules();
-		const payload = skillPointPayload(
-			{
-				name: "acmecorp-pe-approval-workflow",
-				description: "the workflow",
-				body: "the body",
-			},
-			rules,
-		);
-
-		expect(payload.text).toContain("client-a-pe-approval-workflow");
-		expect(payload.text).not.toContain("acmecorp-pe-approval-workflow");
-	});
-});

package/packages/skills/test/vector-indexing.test.ts

@@ -1,205 +0,0 @@
-/**
- * Regression tests for the vector indexing of the skill registry.
- *
- * The registry used to start one embedding request per registered skill, all
- * at the same time. A registry with a few hundred skills opened a few hundred
- * connections at once, which starved every other request of the process.
- *
- * These tests use an injected indexer, so they neither reach the network nor
- * touch a process-global module.
- */
-
-import { describe, expect, test } from "bun:test";
-
-import {
-	createSkillRegistry,
-	type SkillVectorIndexer,
-	type SkillVectorPoint,
-} from "../src/loader.js";
-import type { Skill } from "../src/types.js";
-
-/** An indexer that counts the requests it receives. */
-class CountingIndexer implements SkillVectorIndexer {
-	/** The number of texts of every embed call. */
-	readonly embedCalls: string[][] = [];
-	/** The points of every upsert call. */
-	readonly upsertCalls: SkillVectorPoint[][] = [];
-	/** The number of embed calls that were in flight at the same time. */
-	maxConcurrentEmbeds = 0;
-	private inFlight = 0;
-	/** When set, every embed call waits for this gate. */
-	gate: { promise: Promise<void>; release: () => void } | null = null;
-
-	async embed(texts: string[]): Promise<number[][]> {
-		this.inFlight += 1;
-		this.maxConcurrentEmbeds = Math.max(this.maxConcurrentEmbeds, this.inFlight);
-		this.embedCalls.push(texts);
-		try {
-			if (this.gate) await this.gate.promise;
-			return texts.map((text) => [text.length, 1, 2]);
-		} finally {
-			this.inFlight -= 1;
-		}
-	}
-
-	async upsert(points: SkillVectorPoint[]): Promise<void> {
-		this.upsertCalls.push(points);
-	}
-
-	/** The number of texts that were embedded. */
-	get embeddedCount(): number {
-		return this.embedCalls.reduce((total, texts) => total + texts.length, 0);
-	}
-}
-
-/** A skill with the given id and body. */
-function makeSkill(id: string, body = `body of ${id}`): Skill {
-	return {
-		id,
-		path: `/tmp/skills/${id}`,
-		frontmatter: {
-			name: id,
-			description: `description of ${id}`,
-			version: "1.0.0",
-		},
-		body,
-		references: [],
-		examples: [],
-		metadata: {},
-		loaded: false,
-	};
-}
-
-/** Wait until the registry has no queued skill left. */
-async function settle(registry: { isIndexing(): boolean }): Promise<void> {
-	for (let attempt = 0; attempt < 200 && registry.isIndexing(); attempt++) {
-		await new Promise((resolve) => setTimeout(resolve, 5));
-	}
-}
-
-describe("skill vector indexing", () => {
-	test("one embedding request carries many skills", async () => {
-		const indexer = new CountingIndexer();
-		const registry = createSkillRegistry({ indexer });
-
-		for (let i = 0; i < 100; i++) {
-			registry.register(makeSkill(`skill-${i}`));
-		}
-		await settle(registry);
-
-		expect(indexer.embeddedCount).toBe(100);
-		expect(indexer.embedCalls.length).toBeLessThanOrEqual(4);
-		expect(indexer.maxConcurrentEmbeds).toBe(1);
-		expect(indexer.upsertCalls.length).toBe(indexer.embedCalls.length);
-	});
-
-	test("the number of requests does not grow with the number of skills", async () => {
-		const small = new CountingIndexer();
-		const smallRegistry = createSkillRegistry({ indexer: small, batchSize: 8 });
-		for (let i = 0; i < 8; i++) smallRegistry.register(makeSkill(`small-${i}`));
-		await settle(smallRegistry);
-
-		const large = new CountingIndexer();
-		const largeRegistry = createSkillRegistry({ indexer: large, batchSize: 8 });
-		for (let i = 0; i < 80; i++) largeRegistry.register(makeSkill(`large-${i}`));
-		await settle(largeRegistry);
-
-		// Ten times the skills, ten times the requests - never ten times the
-		// skills in a single request.
-		expect(large.embedCalls.length).toBe(10);
-		expect(large.embedCalls.length).toBe(small.embedCalls.length * 10);
-	});
-
-	test("only one embedding request is in flight at a time", async () => {
-		const indexer = new CountingIndexer();
-		const registry = createSkillRegistry({ indexer, batchSize: 4 });
-
-		let release!: () => void;
-		indexer.gate = {
-			promise: new Promise<void>((resolve) => {
-				release = resolve;
-			}),
-			release: () => {},
-		};
-
-		for (let i = 0; i < 40; i++) {
-			registry.register(makeSkill(`gated-${i}`));
-		}
-		await new Promise((resolve) => setTimeout(resolve, 50));
-
-		// The queue waits for the first batch, so no second request started.
-		expect(indexer.embedCalls.length).toBe(1);
-
-		indexer.gate.release = release;
-		indexer.gate = null;
-		release();
-		await settle(registry);
-
-		expect(indexer.embeddedCount).toBe(40);
-		expect(indexer.maxConcurrentEmbeds).toBe(1);
-	});
-
-	test("a skill whose text did not change is not indexed again", async () => {
-		const indexer = new CountingIndexer();
-		const registry = createSkillRegistry({ indexer });
-
-		registry.register(makeSkill("stable"));
-		await settle(registry);
-		expect(indexer.embeddedCount).toBe(1);
-
-		// The same skill, registered again with the same text.
-		registry.register(makeSkill("stable"));
-		await settle(registry);
-		expect(indexer.embeddedCount).toBe(1);
-
-		// The same skill with a different body is indexed again.
-		registry.register(makeSkill("stable", "a different body"));
-		await settle(registry);
-		expect(indexer.embeddedCount).toBe(2);
-	});
-
-	test("the points carry the vector of their skill", async () => {
-		const indexer = new CountingIndexer();
-		const registry = createSkillRegistry({ indexer, batchSize: 2 });
-
-		registry.register(makeSkill("alpha"));
-		registry.register(makeSkill("beta"));
-		await settle(registry);
-
-		const points = indexer.upsertCalls.flat();
-		expect(points.length).toBe(2);
-		expect(points.map((point) => point.payload.name).sort()).toEqual(["alpha", "beta"]);
-		for (const point of points) {
-			expect(point.vector.length).toBe(3);
-		}
-	});
-
-	test("a registry without an index keeps no vectors", async () => {
-		const registry = createSkillRegistry({ indexer: null });
-
-		registry.register(makeSkill("unindexed"));
-		await settle(registry);
-
-		expect(registry.list().length).toBe(1);
-		expect(registry.isQdrantFailed()).toBe(false);
-	});
-
-	test("a failing index does not stop the other skills", async () => {
-		let calls = 0;
-		const indexer: SkillVectorIndexer = {
-			embed: async (texts: string[]) => {
-				calls += 1;
-				if (calls === 1) throw new Error("the index is unavailable");
-				return texts.map(() => [1, 2, 3]);
-			},
-			upsert: async () => {},
-		};
-		const registry = createSkillRegistry({ indexer, batchSize: 2 });
-
-		registry.register(makeSkill("first"));
-		registry.register(makeSkill("second"));
-		await settle(registry);
-
-		expect(registry.list().length).toBe(2);
-	});
-});

package/packages/skills/test/vector-schema.test.ts

@@ -1,180 +0,0 @@
-/**
- * Regression tests for the point identity of the skill collection.
- *
- * Two writers fill the same collection: the runtime registry and
- * `scripts/sync-to-qdrant.ts`. They used to disagree about the identity of
- * a point - the runtime hashed the name of the skill, the script used a
- * running number. The same skill then existed twice, and a running number
- * could overwrite a point of an unrelated skill.
- *
- * The identity now comes from one shared place, and these tests hold it
- * there.
- */
-
-import { describe, expect, test } from "bun:test";
-
-import {
-	createSkillRegistry,
-	type SkillVectorIndexer,
-	type SkillVectorPoint,
-} from "../src/loader.js";
-import {
-	SKILL_PAYLOAD_BODY_LIMIT,
-	SKILL_PAYLOAD_TEXT_LIMIT,
-	skillPointPayload,
-	skillQdrantPoint,
-	stableHash,
-} from "../src/vector-schema.js";
-import type { Skill } from "../src/types.js";
-
-/** Wait until the registry has no index work left. */
-async function settle(registry: { isIndexing(): boolean }): Promise<void> {
-	for (let attempt = 0; attempt < 200 && registry.isIndexing(); attempt++) {
-		await new Promise((resolve) => setTimeout(resolve, 5));
-	}
-}
-
-/** A skill with the given id and body. */
-function makeSkill(id: string, body = `body of ${id}`): Skill {
-	return {
-		id,
-		path: `/tmp/skills/${id}`,
-		frontmatter: {
-			name: id,
-			description: `the ${id} skill`,
-		},
-		body,
-		metadata: {} as Skill["metadata"],
-	} as Skill;
-}
-
-/** An indexer that keeps the points it receives. */
-class RecordingIndexer implements SkillVectorIndexer {
-	readonly upsertCalls: SkillVectorPoint[][] = [];
-
-	async embed(texts: string[]): Promise<number[][]> {
-		return texts.map((text) => [text.length, 1, 2]);
-	}
-
-	async upsert(points: SkillVectorPoint[]): Promise<void> {
-		this.upsertCalls.push(points);
-	}
-}
-
-describe("the identity of a skill point", () => {
-	test("the id of the point is the hash of the name in the payload", () => {
-		const payload = skillPointPayload({
-			name: "a-skill",
-			description: "does something",
-			body: "the body",
-		});
-		const point = skillQdrantPoint(payload);
-
-		expect(point.id).toBe(stableHash(payload.name));
-	});
-
-	test("the same name always addresses the same point", () => {
-		const first = skillQdrantPoint(
-			skillPointPayload({ name: "a-skill", description: "one", body: "one" }),
-		);
-		const second = skillQdrantPoint(
-			skillPointPayload({ name: "a-skill", description: "two", body: "two" }),
-		);
-
-		// The content may change, the identity of the point may not.
-		expect(second.id).toBe(first.id);
-	});
-
-	test("different names address different points", () => {
-		const first = skillQdrantPoint(
-			skillPointPayload({ name: "a-skill", description: "d", body: "b" }),
-		);
-		const second = skillQdrantPoint(
-			skillPointPayload({ name: "b-skill", description: "d", body: "b" }),
-		);
-
-		expect(second.id).not.toBe(first.id);
-	});
-
-	test("the payload carries the shared fields and nothing else", () => {
-		const payload = skillPointPayload({
-			name: "a-skill",
-			title: "A Skill",
-			description: "does something",
-			body: "the body",
-		});
-
-		expect(Object.keys(payload).sort()).toEqual([
-			"body",
-			"description",
-			"name",
-			"text",
-		]);
-	});
-
-	test("the payload and the text are truncated", () => {
-		const payload = skillPointPayload({
-			name: "a-skill",
-			description: "d",
-			body: "x".repeat(SKILL_PAYLOAD_BODY_LIMIT + 500),
-		});
-
-		expect(payload.body.length).toBe(SKILL_PAYLOAD_BODY_LIMIT);
-		expect(payload.text.length).toBeLessThanOrEqual(
-			SKILL_PAYLOAD_TEXT_LIMIT + payload.name.length + payload.description.length + 10,
-		);
-	});
-
-	test("the title is part of the text when it differs from the name", () => {
-		const withTitle = skillPointPayload({
-			name: "a-skill",
-			title: "A Human Title",
-			description: "d",
-			body: "b",
-		});
-		const withoutTitle = skillPointPayload({
-			name: "a-skill",
-			description: "d",
-			body: "b",
-		});
-
-		// The title is treated like the name: it is sanitized and it is
-		// brought into the shape of a skill name.
-		expect(withTitle.text).toContain("a-human-title");
-		expect(withoutTitle.text).not.toContain("a-human-title");
-	});
-
-	test("a title that is the name is not repeated in the text", () => {
-		const payload = skillPointPayload({
-			name: "a-skill",
-			title: "a-skill",
-			description: "d",
-			body: "b",
-		});
-
-		expect(payload.text.match(/a-skill/g)?.length).toBe(1);
-	});
-});
-
-describe("the runtime writes the points of the shared schema", () => {
-	test("the indexed point carries the shared payload and id", async () => {
-		const indexer = new RecordingIndexer();
-		const registry = createSkillRegistry({ indexer });
-
-		registry.register(makeSkill("alpha"));
-		await settle(registry);
-
-		const points = indexer.upsertCalls.flat();
-		expect(points.length).toBe(1);
-
-		const point = points[0];
-		expect(point.id).toBe(stableHash(point.payload.name));
-		expect(point.payload.name).toBe("alpha");
-		expect(Object.keys(point.payload).sort()).toEqual([
-			"body",
-			"description",
-			"name",
-			"text",
-		]);
-	});
-});