skilld-harness 3.6.3 → 3.6.5

package/README.md

@@ -48,11 +48,14 @@ Both `Ok` and `Err` carry a `report`, so a failed run still reports its cost.
 A token count is `undefined` when the adapter does not report it.
 If the run fails before the Agent starts, `steps` is 0.
 `report.warnings` names each source file the Harness left out for size, each `onEvent` failure, and each cleanup problem.
-An `InvalidSkill` error lists each failed output check in `issues`.
+If the output fails a check, the Harness sends the issues back to the Agent in the same session, up to two times.
+Each repair turn adds its steps and usage to the report, and a warning names the issues.
+An `InvalidSkill` error lists each check that still fails in `issues`.
 
 The Harness checks the rules that the generation Skills state:
 
 - The frontmatter contains only `name` and `description`.
+- The description is one plain YAML line without double quotes, backticks, or `%`.
 - `SKILL.md` links every file under `references/`, by a Markdown link or an inline code path.
 - A package Skill keeps `SKILL.md` under 500 lines and writes at most eight reference files.
 - A project Skill points only at project files and gives at least one search command.

package/dist/index.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/harness.ts"],"mappings":";;UAGiB;WACN;WACA;;KAGC;WAEC;WACA;;WAGA;WACA;WACA;;KAGD;WAEC;WACA,QAAQ;WACR,aAAa;;WAGb;WACA;WACA,aAAa;;WAGb;WACA;;UAGI;WACN;WACA;WACA;WACA;WACA;WACA;;;UAIM;;WAEN;;WAEA;WACA;;;KAIC;WACK;WAA4B;;WAE9B;WACA;WACA;WACA;WACA;;WAGA;WACA;WACA;WACA,OAAO;;UAGL;WACN,SAAS;;;;;WAKT,WAAW,OAAO;;UAGZ;WACN;WACA;;UAGM;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA,OAAO,cAAc;WACrB,gBAAgB,cAAc;;UAGxB;WACN;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA,UAAU,cAAc;;KAGvB;WACK;WAA+B;;WAEjC;WACA;WACA,UAAU,cAAc;WACxB;;WAEE;WAA8B;WAA0B;;WACxD;WAA+B;WAA0B,QAAQ;;WACjE;WAAmC;WAA0B;;WAC7D;WAA6B;WAA0B;;WACvD;WAAkC;WAA0B;WAAuB;;WACnF;WAA4B;;;UAG5B;WACN,cAAc;WACd,cAAc;WACd,aAAa;;;;;;UAOP;;WAEN,OAAO;;WAEP;;;;;WAKA,UAAU;;KAGT,eAAe,YAAY,mBAAmB;WACzC;WAAqB,OAAO,eAAe;WAAe,QAAQ;;WAClE;WAAsB,OAAO;WAAwB,QAAQ;;UAE7D;WACN,MAAM,YAAY,UAAU,OAAO,KAAK,UAAU,oBAAoB,QAAQ,eAAe;;UAGvF;WACN,SAAS;;WAET,SAAS;;WAET,gBAAgB;WAChB,eAAe,QAAQ;;WAEvB,SAAS,gBAAgB,MAAM,SAAS,OAAO,gBAAgB,QAAQ;;wBC0GlE,mBAAmB,SAAS,4BAA4B"}
+{"version":3,"file":"index.d.mts","names":[],"sources":["../src/types.ts","../src/harness.ts"],"mappings":";;UAGiB;WACN;WACA;;KAGC;WAEC;WACA;;WAGA;WACA;WACA;;KAGD;WAEC;WACA,QAAQ;WACR,aAAa;;WAGb;WACA;WACA,aAAa;;WAGb;WACA;;UAGI;WACN;WACA;WACA;WACA;WACA;WACA;;;UAIM;;WAEN;;WAEA;WACA;;;KAIC;WACK;WAA4B;;WAE9B;WACA;WACA;WACA;WACA;;WAGA;WACA;WACA;WACA,OAAO;;UAGL;WACN,SAAS;;;;;WAKT,WAAW,OAAO;;UAGZ;WACN;WACA;;UAGM;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA,OAAO,cAAc;WACrB,gBAAgB,cAAc;;UAGxB;WACN;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA,UAAU,cAAc;;KAGvB;WACK;WAA+B;;WAEjC;WACA;WACA,UAAU,cAAc;WACxB;;WAEE;WAA8B;WAA0B;;WACxD;WAA+B;WAA0B,QAAQ;;WACjE;WAAmC;WAA0B;;WAC7D;WAA6B;WAA0B;;WACvD;WAAkC;WAA0B;WAAuB;;WACnF;WAA4B;;;UAG5B;WACN,cAAc;WACd,cAAc;WACd,aAAa;;;;;;UAOP;;WAEN,OAAO;;WAEP;;;;;WAKA,UAAU;;KAGT,eAAe,YAAY,mBAAmB;WACzC;WAAqB,OAAO,eAAe;WAAe,QAAQ;;WAClE;WAAsB,OAAO;WAAwB,QAAQ;;UAE7D;WACN,MAAM,YAAY,UAAU,OAAO,KAAK,UAAU,oBAAoB,QAAQ,eAAe;;UAGvF;WACN,SAAS;;WAET,SAAS;;WAET,gBAAgB;WAChB,eAAe,QAAQ;;WAEvB,SAAS,gBAAgB,MAAM,SAAS,OAAO,gBAAgB,QAAQ;;wBC6HlE,mBAAmB,SAAS,4BAA4B"}

package/dist/index.mjs

@@ -3,7 +3,7 @@ import { lstat, mkdir, open, opendir, realpath, rename, rm, rmdir, unlink, write
 import { basename, dirname, join, posix, relative, resolve, sep } from "node:path";
 import { HarnessAgent } from "@ai-sdk/harness/agent";
 import { createHash, randomUUID } from "node:crypto";
-import { parseDocument } from "yaml";
+import { Scalar, isScalar, parseDocument } from "yaml";
 import { constants } from "node:fs";
 import { Readable } from "node:stream";
 import { createGunzip } from "node:zlib";
@@ -362,6 +362,7 @@ function checkProjectSkill(input) {
 const allowedFrontmatter = /* @__PURE__ */ new Set(["name", "description"]);
 const packageSkillLineLimit = 500;
 const packageSkillReferenceLimit = 8;
+const riskyDescription = /["`%]/;
 const invalid = (issues) => err({
 	_tag: "InvalidSkill",
 	message: "Skill output failed deterministic checks.",
@@ -428,6 +429,8 @@ const validateGeneratedSkill = (name, files, contract) => {
 	if (frontmatter.name !== name) issues.push("Frontmatter name must match the Skill directory name.");
 	if (typeof frontmatter.name !== "string" || !isSkillName(frontmatter.name)) issues.push("Frontmatter name is invalid.");
 	if (typeof frontmatter.description !== "string" || frontmatter.description.trim().length === 0 || frontmatter.description.length > 1024) issues.push("Frontmatter description must contain 1 to 1024 characters.");
+	const description = document.get("description", true);
+	if (typeof frontmatter.description === "string" && (!isScalar(description) || description.type !== Scalar.PLAIN || riskyDescription.test(frontmatter.description))) issues.push("Frontmatter description must be one plain line without double quotes, backticks, or %. Name symptoms in plain words.");
 	issues.push(...structureIssues(source, files, contract));
 	return issues.length === 0 ? ok(void 0) : invalid(issues);
 };
@@ -1146,11 +1149,23 @@ function toUsage(usage) {
 		outputTokens: usage.outputTokens
 	};
 }
-function cancelled() {
-	return err({
-		_tag: "Cancelled",
-		message: "Skill run was cancelled."
-	});
+function sumUsage(a, b) {
+	const sum = (x, y) => x === void 0 || y === void 0 ? void 0 : x + y;
+	return {
+		inputTokens: sum(a.inputTokens, b.inputTokens),
+		cachedInputTokens: sum(a.cachedInputTokens, b.cachedInputTokens),
+		outputTokens: sum(a.outputTokens, b.outputTokens)
+	};
+}
+const OUTPUT_REPAIR_TURNS = 2;
+function repairRequest(outputPath, issues) {
+	return [
+		`The Harness checked the output at \`${outputPath}\` and found these problems:`,
+		"",
+		...issues.map((issue) => `- ${issue}`),
+		"",
+		`Fix each problem in place. Write no files outside \`${outputPath}\`. Finish when the output passes these checks.`
+	].join("\n");
 }
 async function prepareCurrentSkill(destination, policy, signal) {
 	const root = resolve(destination.rootDir);
@@ -1301,12 +1316,18 @@ async function writePreparedSource(active, prepared, signal) {
 		abortSignal: signal
 	});
 }
-function toAgentError(cause, signal) {
-	return signal?.aborted ? cancelled() : err({
+function agentError(cause, signal) {
+	return signal?.aborted ? {
+		_tag: "Cancelled",
+		message: "Skill run was cancelled."
+	} : {
 		_tag: "AgentFailed",
 		message: "Harness Agent failed during the Skill run.",
 		cause
-	});
+	};
+}
+function toAgentError(cause, signal) {
+	return err(agentError(cause, signal));
 }
 function createSkillHarness(options) {
 	const policy = parseOutputPolicy({
@@ -1380,47 +1401,71 @@ function createSkillHarness(options) {
 			const currentSkillPath = posix.join(active.workDir, "input/current-skill");
 			const outputPath = posix.join(active.workDir, "skilld-output", prepared.outputName);
 			const prompt = renderRequest(requestContent(skill), sourcePath, currentSkillPath, outputPath, prepared.outputName);
+			const outputSandbox = active.sandbox;
 			let step = 0;
-			const generated = await agent.generate({
-				session,
-				prompt,
-				abortSignal: signal,
-				onStepStart: (event) => {
-					step = event.stepNumber;
-					log.emit({
-						_tag: "StepStart",
-						step
-					});
-				},
-				onToolExecutionStart: ({ toolCall }) => {
-					log.emit({
-						_tag: "ToolCall",
-						step,
-						toolName: toolCall.toolName,
-						toolCallId: toolCall.toolCallId,
-						input: toolCall.input
-					});
-				},
-				onStepEnd: (result) => {
-					log.emit({
-						_tag: "StepFinish",
-						step: result.stepNumber,
-						finishReason: result.finishReason,
-						usage: toUsage(result.usage)
-					});
-				}
-			}).then(ok, (cause) => err(cause));
-			if (generated._tag === "Err") return toAgentError(generated.error, signal);
-			log.settle(toUsage(generated.value.totalUsage), generated.value.steps.length);
-			if (signal?.aborted) return cancelled();
-			const collected = await collectSandboxOutput(active.sandbox, outputPath, policy, signal);
-			if (collected._tag === "Err") return collected;
+			let stepOffset = 0;
+			let usage;
+			const turn = async (text) => {
+				const generated = await agent.generate({
+					session,
+					prompt: text,
+					abortSignal: signal,
+					onStepStart: (event) => {
+						step = stepOffset + event.stepNumber;
+						log.emit({
+							_tag: "StepStart",
+							step
+						});
+					},
+					onToolExecutionStart: ({ toolCall }) => {
+						log.emit({
+							_tag: "ToolCall",
+							step,
+							toolName: toolCall.toolName,
+							toolCallId: toolCall.toolCallId,
+							input: toolCall.input
+						});
+					},
+					onStepEnd: (result) => {
+						log.emit({
+							_tag: "StepFinish",
+							step: stepOffset + result.stepNumber,
+							finishReason: result.finishReason,
+							usage: toUsage(result.usage)
+						});
+					}
+				}).then(ok, (cause) => err(cause));
+				if (generated._tag === "Err") return err(agentError(generated.error, signal));
+				const turnUsage = toUsage(generated.value.totalUsage);
+				usage = usage ? sumUsage(usage, turnUsage) : turnUsage;
+				stepOffset += generated.value.steps.length;
+				log.settle(usage, stepOffset);
+				return signal?.aborted ? err({
+					_tag: "Cancelled",
+					message: "Skill run was cancelled."
+				}) : ok(void 0);
+			};
+			const check = async () => {
+				const collected = await collectSandboxOutput(outputSandbox, outputPath, policy, signal);
+				if (collected._tag === "Err") return collected;
+				const validated = prepared.skillName === "review-skill" ? validateSkillReview(collected.value) : validateGeneratedSkill(prepared.outputName, collected.value, prepared.skillName === "generate-project-skill" ? {
+					_tag: "ProjectSkill",
+					projectPaths: prepared.source.files.map((file) => file.path)
+				} : { _tag: "PackageSkill" });
+				return validated._tag === "Err" ? validated : ok(collected.value);
+			};
+			const first = await turn(prompt);
+			if (first._tag === "Err") return first;
+			let checked = await check();
+			for (let repair = 1; repair <= OUTPUT_REPAIR_TURNS && checked._tag === "Err" && checked.error._tag === "InvalidSkill"; repair++) {
+				log.warn(`Output checks failed; repair turn ${repair} of ${OUTPUT_REPAIR_TURNS}: ${checked.error.issues.join(" ")}`);
+				const repaired = await turn(repairRequest(outputPath, checked.error.issues));
+				if (repaired._tag === "Err") return repaired;
+				checked = await check();
+			}
+			if (checked._tag === "Err") return checked;
+			const collected = checked;
 			if (prepared.skillName === "review-skill") return validateSkillReview(collected.value);
-			const validated = validateGeneratedSkill(prepared.outputName, collected.value, prepared.skillName === "generate-project-skill" ? {
-				_tag: "ProjectSkill",
-				projectPaths: prepared.source.files.map((file) => file.path)
-			} : { _tag: "PackageSkill" });
-			if (validated._tag === "Err") return validated;
 			if (!prepared.destination) return err({
 				_tag: "InvalidInput",
 				message: "Skill destination is required."