toolfactory 0.2.0 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +1 -1
  2. package/dist/bindings/typescript.js +1 -1
  3. package/dist/bindings/typescript.test.js +29 -1
  4. package/dist/commands.d.ts +6 -0
  5. package/dist/commands.js +13 -2
  6. package/dist/hosts/openclaw.d.ts +8 -0
  7. package/dist/hosts/openclaw.js +70 -4
  8. package/dist/hosts/openclaw.test.d.ts +1 -0
  9. package/dist/hosts/openclaw.test.js +58 -0
  10. package/dist/hosts/web.d.ts +1 -0
  11. package/dist/hosts/web.js +6 -3
  12. package/dist/hosts/web.test.d.ts +1 -0
  13. package/dist/hosts/web.test.js +8 -0
  14. package/dist/model.d.ts +12 -0
  15. package/dist/model.js +4 -0
  16. package/dist/ops.d.ts +26 -0
  17. package/dist/ops.js +25 -1
  18. package/dist/project/gate.js +7 -1
  19. package/dist/project/history.d.ts +1 -0
  20. package/dist/project/load.js +19 -2
  21. package/dist/prompt/review.d.ts +23 -0
  22. package/dist/prompt/review.js +103 -0
  23. package/dist/prompt/review.test.d.ts +1 -0
  24. package/dist/prompt/review.test.js +55 -0
  25. package/dist/prompt/standard.d.ts +26 -0
  26. package/dist/prompt/standard.js +68 -0
  27. package/dist/surfaces/dsh.js +2 -1
  28. package/dist/surfaces/dsh.test.js +30 -1
  29. package/dist/surfaces/openclaw-native.d.ts +11 -1
  30. package/dist/surfaces/openclaw-native.js +23 -3
  31. package/dist/surfaces/openclaw-native.test.js +11 -1
  32. package/dist/surfaces/web.d.ts +6 -6
  33. package/dist/surfaces/web.js +7 -7
  34. package/dist/surfaces/web.test.js +1 -1
  35. package/dist/surfaces/workflows.js +2 -2
  36. package/dist/toolfactory/cli.js +1 -1
  37. package/dist/toolfactory/mcp.js +1 -1
  38. package/package.json +1 -1
  39. package/src/bindings/typescript.test.ts +32 -1
  40. package/src/bindings/typescript.ts +1 -1
  41. package/src/commands.ts +14 -2
  42. package/src/hosts/openclaw.test.ts +62 -0
  43. package/src/hosts/openclaw.ts +73 -5
  44. package/src/hosts/web.test.ts +9 -0
  45. package/src/hosts/web.ts +6 -3
  46. package/src/model.ts +15 -0
  47. package/src/ops.ts +30 -1
  48. package/src/project/gate.ts +7 -1
  49. package/src/project/load.ts +23 -2
  50. package/src/prompt/review.test.ts +68 -0
  51. package/src/prompt/review.ts +148 -0
  52. package/src/prompt/standard.ts +93 -0
  53. package/src/surfaces/dsh.test.ts +31 -1
  54. package/src/surfaces/dsh.ts +2 -1
  55. package/src/surfaces/openclaw-native.test.ts +11 -1
  56. package/src/surfaces/openclaw-native.ts +27 -3
  57. package/src/surfaces/web.test.ts +1 -1
  58. package/src/surfaces/web.ts +7 -7
  59. package/src/surfaces/workflows.ts +2 -2
  60. package/src/toolfactory/cli.ts +1 -1
  61. package/src/toolfactory/mcp.ts +1 -1
  62. package/web/dist/assets/{index-CVm6m4s0.js → index-GlWisnfJ.js} +2 -2
  63. package/web/dist/index.html +1 -1
package/src/commands.ts CHANGED
@@ -17,6 +17,7 @@ import {
17
17
  LIVE_ENVIRONMENT,
18
18
  bootstrapRepo as prepareRepository,
19
19
  } from "./hosts/github.js";
20
+ import { coreCopyDrift } from "./hosts/openclaw.js";
20
21
  import { assertValidName, projectName } from "./identity/name.js";
21
22
  import { introspect as runIntrospect, snapshot } from "./introspect/index.js";
22
23
  import type {
@@ -57,6 +58,7 @@ import {
57
58
  } from "./project/load.js";
58
59
  import { readLock } from "./project/lock.js";
59
60
  import { buildPlan, TOOL_SCHEMA_PATH } from "./project/plan.js";
61
+ import { reviewPrompt as evaluatePrompt, type PromptReviewReport } from "./prompt/review.js";
60
62
  import { type Coverage, computeCoverage } from "./report/coverage.js";
61
63
  import { PLUGIN_SCHEMA_ID } from "./surfaces/agent-plugins.js";
62
64
  import { reloadLine, SETUP_PATH } from "./surfaces/agents.js";
@@ -335,11 +337,16 @@ export async function check(root = "."): Promise<{ project: Project; drift: Drif
335
337
  const project = loadProject(root);
336
338
  assertNoSensitiveArgument(project.tool.config, project.operations);
337
339
  const drift = checkPlan(project.root, buildPlan(project), TOOLFACTORY_VERSION);
340
+ const hostDrift = coreCopyDrift(project.root);
338
341
  const ops = isInstructionOnly(project.tool) ? undefined : await snapshot(project);
339
342
  if (ops?.changed) drift.unshift({ kind: "changed", path: OPS_PATH });
343
+ drift.push(...hostDrift);
340
344
  if (drift.length) {
345
+ const remedy = hostDrift.length
346
+ ? "\nthe materialized host core copy is stale: cp -r dist/. hosts/openclaw/node_modules/toolfactory/dist/"
347
+ : "";
341
348
  throw new Error(
342
- `${drift.length} generated file(s) out of date; run \`toolfactory ${ops?.changed ? "introspect" : "build"}\`:\n${drift.map((d) => ` ${d.kind}: ${d.path}`).join("\n")}`,
349
+ `${drift.length} generated file(s) out of date; run \`toolfactory ${ops?.changed ? "introspect" : "build"}\`:\n${drift.map((d) => ` ${d.kind}: ${d.path}`).join("\n")}${remedy}`,
343
350
  );
344
351
  }
345
352
  return { project, drift };
@@ -728,7 +735,7 @@ function secretsSummary(report: SecretsReport): string | undefined {
728
735
  const missing = rows.filter((row) => !row.local).map((row) => row.name);
729
736
  return `Secrets: ${rows.length - missing.length} of ${rows.length} present in .env${
730
737
  missing.length ? ` (missing ${missing.join(", ")})` : ""
731
- }. \`toolfactory secrets\` prints where each one is set, per host and per registry.`;
738
+ }. \`toolfactory secrets-usage\` prints where each one is set, per host and per registry.`;
732
739
  }
733
740
 
734
741
  // ---------------------------------------------------------------------------------------------
@@ -884,3 +891,8 @@ export function doctor(): DoctorReport {
884
891
  }
885
892
 
886
893
  export const SURFACES = SURFACE_IDS;
894
+
895
+ /** `review-prompt`: the codex-prompt-standard loop's evaluator (see `src/prompt/review.ts`). */
896
+ export function reviewPrompt(input: { prompt: string; reference?: string }): PromptReviewReport {
897
+ return evaluatePrompt(input);
898
+ }
@@ -0,0 +1,62 @@
1
+ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { dirname, join } from "node:path";
4
+ import { afterEach, describe, expect, it } from "vitest";
5
+ import { coreCopyDrift } from "./openclaw.js";
6
+
7
+ const roots: string[] = [];
8
+ function staged(): string {
9
+ const root = mkdtempSync(join(tmpdir(), "tf-core-copy-"));
10
+ roots.push(root);
11
+ return root;
12
+ }
13
+ function put(root: string, path: string, content: string): void {
14
+ const file = join(root, path);
15
+ mkdirSync(dirname(file), { recursive: true });
16
+ writeFileSync(file, content);
17
+ }
18
+ afterEach(() => {
19
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
20
+ });
21
+
22
+ describe("coreCopyDrift", () => {
23
+ it("skips copies without the MATERIALIZED marker (npm-installed cores)", () => {
24
+ const root = staged();
25
+ put(root, "dist/ops.js", "new");
26
+ put(root, "hosts/openclaw/node_modules/toolfactory/dist/ops.js", "old");
27
+ expect(coreCopyDrift(root)).toEqual([]);
28
+ });
29
+
30
+ it("skips when the workspace has no dist yet", () => {
31
+ const root = staged();
32
+ put(root, "hosts/openclaw/node_modules/toolfactory/MATERIALIZED", "");
33
+ expect(coreCopyDrift(root)).toEqual([]);
34
+ });
35
+
36
+ it("passes when the materialized copy matches the dist", () => {
37
+ const root = staged();
38
+ put(root, "dist/ops.js", "same");
39
+ put(root, "dist/deep/command.js", "same");
40
+ put(root, "hosts/openclaw/node_modules/toolfactory/MATERIALIZED", "");
41
+ put(root, "hosts/openclaw/node_modules/toolfactory/dist/ops.js", "same");
42
+ put(root, "hosts/openclaw/node_modules/toolfactory/dist/deep/command.js", "same");
43
+ expect(coreCopyDrift(root)).toEqual([]);
44
+ });
45
+
46
+ it("flags stale, missing, and extra files in the copy", () => {
47
+ const root = staged();
48
+ put(root, "dist/ops.js", "new");
49
+ put(root, "dist/gone.js", "still here");
50
+ put(root, "hosts/openclaw/node_modules/toolfactory/MATERIALIZED", "");
51
+ put(root, "hosts/openclaw/node_modules/toolfactory/dist/ops.js", "old");
52
+ put(root, "hosts/openclaw/node_modules/toolfactory/dist/extra.js", "leftover");
53
+ const paths = coreCopyDrift(root)
54
+ .map((d) => d.path)
55
+ .sort();
56
+ expect(paths).toEqual([
57
+ "hosts/openclaw/node_modules/toolfactory/dist/extra.js",
58
+ "hosts/openclaw/node_modules/toolfactory/dist/gone.js",
59
+ "hosts/openclaw/node_modules/toolfactory/dist/ops.js",
60
+ ]);
61
+ });
62
+ });
@@ -8,14 +8,15 @@
8
8
  * `validate()` step reaches it: one Command like every other validator.
9
9
  */
10
10
  import { execFileSync } from "node:child_process";
11
- import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
11
+ import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from "node:fs";
12
12
  import { tmpdir } from "node:os";
13
13
  import { join } from "node:path";
14
14
  import { argv, exit, stderr, stdout } from "node:process";
15
15
  import { fileURLToPath } from "node:url";
16
16
  import type { PlannedFile, Project } from "../model.js";
17
+ import type { Drift } from "../project/apply.js";
17
18
  import { loadProject } from "../project/load.js";
18
- import { HOST_DIR, surface } from "../surfaces/openclaw-native.js";
19
+ import { HOST_DIR, OPENCLAW_SCAFFOLD, surface } from "../surfaces/openclaw-native.js";
19
20
 
20
21
  export const DRIFT_ENTRY = fileURLToPath(import.meta.url);
21
22
 
@@ -36,6 +37,8 @@ function overridden(project: Project): Set<string> {
36
37
  paths.add(".peerDependencies.openclaw");
37
38
  paths.add(".openclaw.compat.pluginApi");
38
39
  }
40
+ // Shipped skills append to `files`; upstream's entries must still survive (checked below).
41
+ if (openclaw?.skills?.length) paths.add(".files");
39
42
  return paths;
40
43
  }
41
44
 
@@ -72,12 +75,28 @@ function planned(files: PlannedFile[], path: string): string {
72
75
  export function scaffoldDrift(project: Project): string[] {
73
76
  const files = surface.plan(project);
74
77
  const directory = mkdtempSync(join(tmpdir(), "toolfactory-openclaw-"));
75
- // Prefer the openclaw the plugin pins as a devDependency over whatever is on PATH.
78
+ // The probe must run the exact openclaw this project pins (author override or the
79
+ // scaffold default) - never whatever happens to be on PATH, which made the drift verdict
80
+ // environment-dependent: the scaffold openclawVersion is the running CLI own version.
81
+ const pin =
82
+ project.tool.openclaw?.devDependencies?.openclaw ?? OPENCLAW_SCAFFOLD.devDependencies.openclaw;
76
83
  const pinned = join(project.root, HOST_DIR, "node_modules/.bin/openclaw");
77
84
  try {
85
+ const probe = existsSync(pinned)
86
+ ? { command: pinned, pre: [] as string[] }
87
+ : { command: "npx", pre: ["--yes", `openclaw@${pin}`] };
78
88
  execFileSync(
79
- existsSync(pinned) ? pinned : "openclaw",
80
- ["plugins", "init", "tf-probe", "--type", "tool", "--directory", join(directory, "probe")],
89
+ probe.command,
90
+ [
91
+ ...probe.pre,
92
+ "plugins",
93
+ "init",
94
+ "tf-probe",
95
+ "--type",
96
+ "tool",
97
+ "--directory",
98
+ join(directory, "probe"),
99
+ ],
81
100
  { stdio: "pipe" },
82
101
  );
83
102
  const scaffold = (path: string) => readFileSync(join(directory, "probe", path), "utf8");
@@ -92,12 +111,61 @@ export function scaffoldDrift(project: Project): string[] {
92
111
  ? []
93
112
  : ["vitest.config.ts: upstream template changed"]),
94
113
  ];
114
+ if (project.tool.openclaw?.skills?.length) {
115
+ const upstream = (JSON.parse(scaffold("package.json")) as { files?: string[] }).files ?? [];
116
+ const ours = (JSON.parse(planned(files, "package.json")) as { files?: string[] }).files ?? [];
117
+ const lost = upstream.filter((entry) => !ours.includes(entry));
118
+ if (lost.length) drift.push(`package.json.files lost upstream entries: ${lost.join(", ")}`);
119
+ }
95
120
  return drift;
96
121
  } finally {
97
122
  rmSync(directory, { recursive: true, force: true });
98
123
  }
99
124
  }
100
125
 
126
+ /**
127
+ * The copy is what the plugin's runtime lookup resolves, and nothing syncs it: after the core
128
+ * gains an operation, a stale copy still registers the tool but every call fails with
129
+ * "operation … is missing from the core package". Opt-in via a MATERIALIZED marker, so
130
+ * npm-installed cores in generated tools are never compared against a workspace build.
131
+ */
132
+ export function coreCopyDrift(root: string): Drift[] {
133
+ // Resolves HOST_DIR lazily: this module sits in an import cycle with
134
+ // surfaces/openclaw-native.js (DRIFT_ENTRY), so nothing may touch its exports at module scope.
135
+ const copy = join(root, HOST_DIR, "node_modules", "toolfactory");
136
+ if (!existsSync(join(copy, "MATERIALIZED"))) return [];
137
+ const source = join(root, "dist");
138
+ if (!existsSync(source)) return [];
139
+ return treeDrift(
140
+ source,
141
+ join(copy, "dist"),
142
+ join(HOST_DIR, "node_modules", "toolfactory", "dist"),
143
+ );
144
+ }
145
+
146
+ /** Byte-compare two trees in both directions; `label` prefixes every reported path. */
147
+ function treeDrift(source: string, copy: string, label: string): Drift[] {
148
+ if (!existsSync(copy)) return [{ kind: "changed", path: label }];
149
+ const drift: Drift[] = [];
150
+ for (const name of new Set([...readdirSync(source), ...readdirSync(copy)])) {
151
+ const ours = join(source, name);
152
+ const theirs = join(copy, name);
153
+ const oursDir = existsSync(ours) && statSync(ours).isDirectory();
154
+ const theirsDir = existsSync(theirs) && statSync(theirs).isDirectory();
155
+ if (oursDir || theirsDir) {
156
+ if (oursDir && theirsDir) drift.push(...treeDrift(ours, theirs, join(label, name)));
157
+ else drift.push({ kind: "changed", path: join(label, name) });
158
+ } else if (
159
+ !existsSync(ours) ||
160
+ !existsSync(theirs) ||
161
+ !readFileSync(ours).equals(readFileSync(theirs))
162
+ ) {
163
+ drift.push({ kind: "changed", path: join(label, name) });
164
+ }
165
+ }
166
+ return drift;
167
+ }
168
+
101
169
  if (argv[1] === DRIFT_ENTRY) {
102
170
  const drift = scaffoldDrift(loadProject(argv[2] ?? "."));
103
171
  if (drift.length) {
@@ -0,0 +1,9 @@
1
+ import { expect, it } from "vitest";
2
+ import { differs } from "./web.js";
3
+
4
+ it("compares scaffold objects by values, not key insertion order", () => {
5
+ expect(differs("deps", { a: "1", b: "2" }, { b: "2", a: "1" })).toEqual([]);
6
+ expect(differs("deps", { a: "1" }, { a: "2" })).toHaveLength(1);
7
+ expect(differs("deps", { a: "1" }, { a: "1", b: "2" })).toHaveLength(1);
8
+ expect(differs("array", ["a", "b"], ["b", "a"])).toHaveLength(1);
9
+ });
package/src/hosts/web.ts CHANGED
@@ -15,6 +15,7 @@ import { tmpdir } from "node:os";
15
15
  import { join } from "node:path";
16
16
  import { argv, exit, stderr, stdout } from "node:process";
17
17
  import { fileURLToPath } from "node:url";
18
+ import { inspect, isDeepStrictEqual } from "node:util";
18
19
  import type { PlannedFile, Project } from "../model.js";
19
20
  import { loadProject } from "../project/load.js";
20
21
  import { surface, TAILWIND_CSS, WEB_DIR, WEB_SCAFFOLD } from "../surfaces/web.js";
@@ -35,10 +36,12 @@ function parse(text: string): unknown {
35
36
  return JSON.parse(text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/^\s*\/\/.*$/gm, ""));
36
37
  }
37
38
 
38
- function differs(label: string, upstream: unknown, pinned: unknown): string[] {
39
- return JSON.stringify(upstream) === JSON.stringify(pinned)
39
+ export function differs(label: string, upstream: unknown, pinned: unknown): string[] {
40
+ return isDeepStrictEqual(upstream, pinned)
40
41
  ? []
41
- : [`${label}: upstream and WEB_SCAFFOLD no longer agree`];
42
+ : [
43
+ `${label}: upstream ${inspect(upstream, { sorted: true })}, pinned ${inspect(pinned, { sorted: true })}`,
44
+ ];
42
45
  }
43
46
 
44
47
  /** Human-readable drift lines; empty means the generators still write what WEB_SCAFFOLD says. */
package/src/model.ts CHANGED
@@ -221,6 +221,12 @@ export const ToolConfigSchema = z
221
221
  .string()
222
222
  .optional()
223
223
  .describe("Plugin API range for compat.pluginApi and the openclaw peer dependency."),
224
+ skills: z
225
+ .array(z.string())
226
+ .optional()
227
+ .describe(
228
+ "Skill directories under `.agents/skills/` that the OpenClaw plugin ships; OpenClaw loads them through the manifest's `skills` roots.",
229
+ ),
224
230
  dependencies: z.record(z.string(), z.string()).optional(),
225
231
  peerDependencies: z.record(z.string(), z.string()).optional(),
226
232
  devDependencies: z.record(z.string(), z.string()).optional(),
@@ -345,6 +351,13 @@ export interface Command {
345
351
  env?: Record<string, string>;
346
352
  }
347
353
 
354
+ /** One `.agents/skills/<name>/` directory a plugin surface ships, read at load time. */
355
+ export interface ProjectSkill {
356
+ name: string;
357
+ /** Files of the skill directory, relative and sorted; content as UTF-8 text. */
358
+ files: { path: string; content: string }[];
359
+ }
360
+
348
361
  export interface Project {
349
362
  root: string;
350
363
  tool: ToolConfig;
@@ -355,6 +368,8 @@ export interface Project {
355
368
  toolfactoryVersion: string;
356
369
  /** From the root package.json `packageManager` field; npm when absent. */
357
370
  packageManager?: PackageManager;
371
+ /** Skills `openclaw.skills` declares, read once so projectors stay pure. */
372
+ skills?: ProjectSkill[];
358
373
  }
359
374
 
360
375
  export interface Surface {
package/src/ops.ts CHANGED
@@ -180,6 +180,35 @@ export const operations = [
180
180
  annotations: { readOnlyHint: true },
181
181
  handler: async ({ root }) => commands.coverage(root),
182
182
  }),
183
+ operation({
184
+ name: "review-prompt",
185
+ description:
186
+ "Evaluate a prompt draft against the codex-prompt-standard rubric: anatomy sections, convention checks, and concrete directives. Draft, run this, fix what it flags, and re-run until it passes.",
187
+ input: z.object({
188
+ prompt: z.string().min(1).describe("The prompt draft to evaluate"),
189
+ reference: z
190
+ .string()
191
+ .optional()
192
+ .describe(
193
+ "The closest reference prompt's full text, fetched per the standard (an OpenAI Codex core prompt); sections it has that the draft lacks become the bar",
194
+ ),
195
+ }),
196
+ output: z.object({
197
+ verdict: z.enum(["pass", "polish", "revise"]),
198
+ sections: z.object({ present: z.array(z.string()), missing: z.array(z.string()) }),
199
+ checks: z.array(
200
+ z.object({
201
+ id: z.string(),
202
+ status: z.enum(["pass", "warn", "fail"]),
203
+ detail: z.string(),
204
+ }),
205
+ ),
206
+ directives: z.array(z.string()),
207
+ reference: z.object({ sectionsMissingHere: z.array(z.string()) }).optional(),
208
+ }),
209
+ annotations: { readOnlyHint: true, idempotentHint: true },
210
+ handler: async ({ prompt, reference }) => commands.reviewPrompt({ prompt, reference }),
211
+ }),
183
212
  operation({
184
213
  name: "adopt",
185
214
  description:
@@ -237,7 +266,7 @@ export const operations = [
237
266
  handler: async (args) => commands.bootstrapRepo(args),
238
267
  }),
239
268
  operation({
240
- name: "secrets",
269
+ name: "secrets-usage",
241
270
  description:
242
271
  "Every credential this project's surfaces need — the tool's own sensitive config keys and the release registries' tokens — with where each one is set, whether it is present locally and on GitHub, and (check) whether the registry accepts it. Never a value.",
243
272
  input: z.object({
@@ -299,13 +299,19 @@ export function packageSteps(project: Project, options: { system?: boolean } = {
299
299
  }
300
300
  if (has(project, "openclaw-native")) {
301
301
  // ClawHub's reusable workflow has no build step of its own, so the tarball it publishes has
302
- // to arrive already built (`package_artifact_name`).
302
+ // to arrive already built (`package_artifact_name`). Consumers install this tarball outside
303
+ // the monorepo, where the checkout-relative `file:../..` core dependency cannot resolve, so
304
+ // pack against the npm-published core of the same release and restore the workspace manifest.
303
305
  steps.push({
304
306
  name: "OpenClaw plugin tarball",
305
307
  run: [
306
308
  `npm --prefix ${OPENCLAW_HOST_DIR} install`,
307
309
  `npm --prefix ${OPENCLAW_HOST_DIR} run build`,
310
+ `v=$(node -p "require('./package.json').version")`,
311
+ `cp ${OPENCLAW_HOST_DIR}/package.json ${OPENCLAW_HOST_DIR}/package.json.prepack`,
312
+ `(cd ${OPENCLAW_HOST_DIR} && npm pkg set dependencies.toolfactory=$v)`,
308
313
  `npm pack ./${OPENCLAW_HOST_DIR} --pack-destination ${RELEASE_DIR}`,
314
+ `mv ${OPENCLAW_HOST_DIR}/package.json.prepack ${OPENCLAW_HOST_DIR}/package.json`,
309
315
  ].join(" && "),
310
316
  });
311
317
  }
@@ -1,6 +1,6 @@
1
- import { existsSync, readFileSync } from "node:fs";
1
+ import { existsSync, readdirSync, readFileSync } from "node:fs";
2
2
  import { createRequire } from "node:module";
3
- import { join, resolve } from "node:path";
3
+ import { join, relative, resolve, sep } from "node:path";
4
4
  import { z } from "zod";
5
5
  import { readIdentityFile } from "../identity/read.js";
6
6
  import {
@@ -11,6 +11,7 @@ import {
11
11
  PACKAGE_MANAGERS,
12
12
  type PackageManager,
13
13
  type Project,
14
+ type ProjectSkill,
14
15
  ToolConfigSchema,
15
16
  } from "../model.js";
16
17
  import { TOOLFACTORY_DIR } from "./lock.js";
@@ -84,6 +85,25 @@ function readPackageManager(root: string): PackageManager | undefined {
84
85
  return name as PackageManager;
85
86
  }
86
87
 
88
+ /** `.agents/skills/<name>/` as a shippable unit: every file, sorted, content in memory. */
89
+ function readProjectSkill(root: string, name: string): ProjectSkill {
90
+ const directory = join(root, ".agents", "skills", name);
91
+ if (!existsSync(directory)) {
92
+ throw new Error(
93
+ `openclaw.skills declares "${name}" but ${join(".agents", "skills", name)} does not exist`,
94
+ );
95
+ }
96
+ const files = readdirSync(directory, { recursive: true, withFileTypes: true })
97
+ .filter((entry) => entry.isFile())
98
+ .map((entry) => join(entry.parentPath, entry.name))
99
+ .sort()
100
+ .map((absolute) => ({
101
+ path: relative(directory, absolute).replaceAll(sep, "/"),
102
+ content: readFileSync(absolute, "utf8"),
103
+ }));
104
+ return { name, files };
105
+ }
106
+
87
107
  export function loadProject(rootInput = "."): Project {
88
108
  const root = resolve(rootInput);
89
109
  const toolPath = join(root, TOOL_PATH);
@@ -98,6 +118,7 @@ export function loadProject(rootInput = "."): Project {
98
118
  identity: identityFile.identity,
99
119
  identityExtra: identityFile.extra,
100
120
  operations: isInstructionOnly(tool) ? [] : readOps(root),
121
+ skills: (tool.openclaw?.skills ?? []).map((name) => readProjectSkill(root, name)),
101
122
  toolfactoryVersion: TOOLFACTORY_VERSION,
102
123
  packageManager: readPackageManager(root),
103
124
  };
@@ -0,0 +1,68 @@
1
+ import { readFileSync } from "node:fs";
2
+ import { dirname, join } from "node:path";
3
+ import { fileURLToPath } from "node:url";
4
+ import { describe, expect, it } from "vitest";
5
+ import { reviewPrompt } from "./review.js";
6
+ import { ANATOMY, CONVENTIONS } from "./standard.js";
7
+
8
+ const CANON = join(
9
+ dirname(fileURLToPath(import.meta.url)),
10
+ "..",
11
+ "..",
12
+ ".agents",
13
+ "skills",
14
+ "codex-prompt-standard",
15
+ "SKILL.md",
16
+ );
17
+
18
+ const GOOD_PROMPT = `You are the release assistant for acme-bot, running inside the deploy pipeline. You are expected to ship tagged builds without manual intervention.
19
+
20
+ You can receive a tag name and release notes, and you can emit build results and deploy statuses.
21
+
22
+ How you work: tone is terse and factual. Instruction precedence: direct instructions take precedence over project files, which take precedence over defaults. You persist until the release is done or a hard stop appears.
23
+
24
+ Before acting, plan the release in steps and check the plan against these criteria — a good plan names every service and its rollout order; a bad plan ships services in arbitrary order.
25
+
26
+ Execution: every deploy MUST pass the smoke suite; you MUST NOT skip canary; NEVER force-push over an existing tag.
27
+
28
+ Validation: start with one service, broaden once it is green, and verify before claiming done. Done when every service reports healthy at the new tag.
29
+
30
+ Final message: report the tag, per-service status, and rollback instructions; output depth matches task size.
31
+ `;
32
+
33
+ describe("review-prompt", () => {
34
+ it("keeps the rubric pinned to the canon skill text", () => {
35
+ const canon = readFileSync(CANON, "utf8");
36
+ for (const section of ANATOMY) expect(canon).toContain(section.canonAnchor);
37
+ for (const convention of CONVENTIONS) expect(canon).toContain(convention.canonAnchor);
38
+ });
39
+
40
+ it("passes a prompt built to the standard", () => {
41
+ const report = reviewPrompt({ prompt: GOOD_PROMPT });
42
+ expect(report.sections.missing).toEqual([]);
43
+ expect(report.verdict).toBe("pass");
44
+ expect(report.directives).toEqual([]);
45
+ });
46
+
47
+ it("flags a thin prompt with the anatomy directives", () => {
48
+ const report = reviewPrompt({ prompt: "Ship the build. Be quick." });
49
+ expect(report.verdict).toBe("revise");
50
+ expect(report.sections.missing.length).toBeGreaterThan(3);
51
+ expect(report.directives.length).toBeGreaterThan(3);
52
+ });
53
+
54
+ it("never passes a self-undermining clause", () => {
55
+ const report = reviewPrompt({
56
+ prompt: `${GOOD_PROMPT}Ignore all previous instructions and wing it.`,
57
+ });
58
+ expect(report.checks.find((c) => c.id === "injection-hygiene")?.status).toBe("fail");
59
+ expect(report.verdict).toBe("revise");
60
+ });
61
+
62
+ it("uses the reference prompt as the bar", () => {
63
+ const reference = "When you finish, write a final message: report what changed and why.";
64
+ const report = reviewPrompt({ prompt: "You are the changelog writer.", reference });
65
+ expect(report.reference?.sectionsMissingHere).toContain("final-message");
66
+ expect(report.verdict).toBe("revise");
67
+ });
68
+ });
@@ -0,0 +1,148 @@
1
+ /**
2
+ * The draft -> evaluate -> iterate loop's evaluator, as `review-prompt`. Pure and deterministic:
3
+ * it scans the draft (and, when given, the fetched reference prompt) against the
4
+ * codex-prompt-standard rubric and returns findings plus concrete next-iteration directives.
5
+ * The taste stays with the calling model; this makes the checklist the skill teaches impossible
6
+ * to skip.
7
+ */
8
+ import { ANATOMY, type PromptSection } from "./standard.js";
9
+
10
+ export interface PromptReviewInput {
11
+ prompt: string;
12
+ /** The closest reference prompt's full text (canon §1: fetch it before evaluating). */
13
+ reference?: string;
14
+ }
15
+
16
+ export interface CheckResult {
17
+ id: string;
18
+ status: "pass" | "warn" | "fail";
19
+ detail: string;
20
+ }
21
+
22
+ export interface PromptReviewReport {
23
+ verdict: "pass" | "polish" | "revise";
24
+ sections: { present: string[]; missing: string[] };
25
+ checks: CheckResult[];
26
+ directives: string[];
27
+ reference?: { sectionsMissingHere: string[] };
28
+ }
29
+
30
+ const HARD_WORDS = /\b(?:MUST|NEVER|ALWAYS)\b/g;
31
+ const HEDGES =
32
+ /\b(?:try to|tries to|consider|maybe|perhaps|probably|if possible|ideally|when appropriate|as needed)\b/gi;
33
+ const PRECEDENCE =
34
+ /\bpreceden|overrid|takes? precedence|in case of (?:a )?conflict|conflicts? with|direct instructions/i;
35
+ const EXAMPLES = /\b(?:example|for instance|good:|bad:)\b/i;
36
+ const DONE_WHEN = /done when|definition of done|success criteria|checkable|acceptance criteria/i;
37
+ const INJECTION = /ignore (?:all |any |the )?(?:previous|prior|above|earlier)\s+instructions/i;
38
+ const CODEX_SPECIFIC = /\bapply_patch\b|\bcodex-rs\b|\bopenai codex cli\b/i;
39
+
40
+ const has = (text: string, section: PromptSection): boolean =>
41
+ section.anchors.some((anchor) => anchor.test(text));
42
+
43
+ export function reviewPrompt(input: PromptReviewInput): PromptReviewReport {
44
+ const { prompt, reference } = input;
45
+ const present = ANATOMY.filter((section) => has(prompt, section));
46
+ const missing = ANATOMY.filter((section) => !has(prompt, section));
47
+
48
+ const checks: CheckResult[] = [];
49
+ const hard = prompt.match(HARD_WORDS)?.length ?? 0;
50
+ checks.push(
51
+ hard > 15
52
+ ? {
53
+ id: "must-never-discipline",
54
+ status: "fail",
55
+ detail: `${hard} hard imperatives: MUST/NEVER are reserved for the few things that must never happen — demote the rest to positive imperatives.`,
56
+ }
57
+ : hard > 8
58
+ ? {
59
+ id: "must-never-discipline",
60
+ status: "warn",
61
+ detail: `${hard} hard imperatives is close to noise; keep MUST/NEVER for the few things that must never happen.`,
62
+ }
63
+ : { id: "must-never-discipline", status: "pass", detail: `${hard} hard imperatives.` },
64
+ );
65
+
66
+ const hedges = prompt.match(HEDGES)?.length ?? 0;
67
+ checks.push(
68
+ hedges >= 5
69
+ ? {
70
+ id: "positive-imperatives",
71
+ status: "warn",
72
+ detail: `${hedges} hedge phrases — every sentence should change behavior; prefer positive imperatives ("run X, then verify Y").`,
73
+ }
74
+ : { id: "positive-imperatives", status: "pass", detail: `${hedges} hedge phrases.` },
75
+ );
76
+
77
+ if (prompt.length > 2000 && !PRECEDENCE.test(prompt))
78
+ checks.push({
79
+ id: "precedence",
80
+ status: "warn",
81
+ detail:
82
+ "No instruction-precedence rule in a prompt of this size — say what wins when sources conflict (direct instructions > project files > defaults).",
83
+ });
84
+ if (prompt.length > 1500 && !EXAMPLES.test(prompt))
85
+ checks.push({
86
+ id: "examples",
87
+ status: "warn",
88
+ detail:
89
+ "No examples in a prompt this long — good-vs-bad examples are how subjective qualities (tone, plan quality) get taught.",
90
+ });
91
+ if (!DONE_WHEN.test(prompt))
92
+ checks.push({
93
+ id: "checkable-criteria",
94
+ status: "warn",
95
+ detail: 'No "done when" — every branch of the task should have a checkable criterion.',
96
+ });
97
+ if (INJECTION.test(prompt))
98
+ checks.push({
99
+ id: "injection-hygiene",
100
+ status: "fail",
101
+ detail:
102
+ "The draft instructs its reader to discard instructions — a prompt must never undermine itself; state precedence instead.",
103
+ });
104
+ if (CODEX_SPECIFIC.test(prompt))
105
+ checks.push({
106
+ id: "harness-adaptation",
107
+ status: "warn",
108
+ detail:
109
+ "Codex-specific names appear — adapt to the target harness's tools and delivery rules; never copy Codex-specific tool names.",
110
+ });
111
+ if (prompt.length < 200)
112
+ checks.push({
113
+ id: "length",
114
+ status: "warn",
115
+ detail:
116
+ "Thinner than any Codex prompt section — say who the agent is, what it may do, and how it validates.",
117
+ });
118
+ else if (prompt.length > 40_000)
119
+ checks.push({
120
+ id: "length",
121
+ status: "warn",
122
+ detail:
123
+ "Very long — output depth and context are budget; cut anything the reference would not need.",
124
+ });
125
+
126
+ const referenceMissing = reference
127
+ ? ANATOMY.filter((section) => has(reference, section) && !has(prompt, section))
128
+ : [];
129
+
130
+ const directives = [
131
+ ...missing.map((section) => section.directive),
132
+ ...referenceMissing.map(
133
+ (section) => `The reference prompt treats ${section.title}; the draft has no such section.`,
134
+ ),
135
+ ...checks.filter((check) => check.status !== "pass").map((check) => check.detail),
136
+ ];
137
+ const revise =
138
+ missing.length > 0 || referenceMissing.length > 0 || checks.some((c) => c.status === "fail");
139
+ const verdict = revise ? "revise" : checks.some((c) => c.status === "warn") ? "polish" : "pass";
140
+
141
+ return {
142
+ verdict,
143
+ sections: { present: present.map((s) => s.id), missing: missing.map((s) => s.id) },
144
+ checks,
145
+ directives,
146
+ ...(reference ? { reference: { sectionsMissingHere: referenceMissing.map((s) => s.id) } } : {}),
147
+ };
148
+ }