simt-dev 0.2.0 → 0.2.2

package/src/commands/skills.mjs

@@ -3,7 +3,7 @@
 // Implementation sequence steps 4 and 5.
 //
 // Output design (plan 0002 section 6): one fact per line, fixed field order, no
-// wrapping, no colour when piped, bounded by --max-bytes with an explicit
+// wrapping, no ANSI on any stream, bounded by --max-bytes with an explicit
 // TRUNCATED line. The NOT SELECTED block is as important as the resolved block -
 // the agent can see what exists and why it was excluded, and override in one
 // token instead of guessing. Suppressing it is how a discovery tool becomes a
@@ -14,9 +14,10 @@ import { join } from 'node:path';
 import { existsSync } from 'node:fs';
 
 import { EXIT, emit, emitJson, fail, isJson, note as emitNote } from '../output/index.mjs';
-import { loadRegistry, REGISTRY_PATH } from '../graph/load.mjs';
+import { loadRegistry } from '../graph/load.mjs';
 import { locateSource, sourceHeldLocally } from '../resolve/source.mjs';
-import { resolveWorkspace, WORKSPACE_REMEDY } from '../workspace.mjs';
+import { unknownSkills, scoreCorpus } from '../resolve/eval.mjs';
+import { PACKAGE_ROOT, resolveWorkspace, WORKSPACE_REMEDY } from '../workspace.mjs';
 import { fingerprint } from '../resolve/fingerprint.mjs';
 import { tokenize, rankSkills, closure, selectWithinBudget, resolveAlias, scoreSkill, selectionVerdict, stackCompatible } from '../resolve/resolve.mjs';
 import { nearMatches } from '../resolve/near.mjs';
@@ -48,6 +49,22 @@ export function parsePathList(value) {
     .filter(Boolean);
 }
 
+/**
+ * Read a list flag in both the forms this tool ships.
+ *
+ * `--skill` accepts `--skill a --skill b` and `--skill a,b`, and arrives here as an
+ * array in the first case and a string in the second. `String(['a','b'])` happens to
+ * produce `'a,b'`, so the old one-line reader would have kept working by accident — and
+ * would have started throwing the moment a skill name contained the character the
+ * join inserts. Both shapes are handled rather than one being relied upon, because
+ * relying on it is the same class of bug as the one being fixed here.
+ */
+export function listFlag(value) {
+  return (Array.isArray(value) ? value : [value])
+    .flatMap((v) => parsePathList(v))
+    .filter(Boolean);
+}
+
 /** Compact stack label: the tags that actually discriminate, not all of them. */
 function stackLabel(tags = []) {
   const DISCRIMINATING = [
@@ -75,29 +92,24 @@ const kb = (bytes) => `${(bytes / 1000).toFixed(1)}k`;
 // --- tier 0: index ------------------------------------------------------------
 
 export async function indexCommand(args, flags) {
-  const { registry, errors, ok } = await loadRegistry(REGISTRY_PATH, flags);
-  if (!ok) return fail(errors[0], EXIT.REGISTRY);
+  const { registry, errors, ok } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
   const workspace = await workspaceLine(flags);
 
   if (isJson(flags)) {
-    emit(
-      `${JSON.stringify(
-        {
-          ok: true,
-          workspace,
-          skills: registry.skills.map((s) => ({
-            name: s.name,
-            summary: s.summary,
-            capabilities: s.capabilities,
-            stack: s.stack,
-            bytes: s.bytes,
-            deferred: Boolean(s.deferred),
-          })),
-        },
-        null,
-        2,
-      )}\n`,
-      { maxBytes: flags.maxBytes ?? 0 },
+    emitJson(
+      {
+                ok: true,
+                workspace,
+                skills: registry.skills.map((s) => ({
+                  name: s.name,
+                  summary: s.summary,
+                  capabilities: s.capabilities,
+                  stack: s.stack,
+                  bytes: s.bytes,
+                  deferred: Boolean(s.deferred),
+                })),
+              },
     );
     return EXIT.OK;
   }
@@ -151,20 +163,15 @@ async function emitAll(registry, repoStack, flags, warning) {
   const inline = usable.filter((s) => !s.deferred);
 
   if (isJson(flags)) {
-    emit(
-      `${JSON.stringify(
-        {
-          ok: true,
-          all: true,
-          repoStack,
-          inline: inline.map((s) => s.name),
-          deferred: deferred.map((s) => s.name),
-          ...(warning ? { warning } : {}),
-        },
-        null,
-        2,
-      )}\n`,
-      { maxBytes: flags.maxBytes ?? 0 },
+    emitJson(
+      {
+                ok: true,
+                all: true,
+                repoStack,
+                inline: inline.map((s) => s.name),
+                deferred: deferred.map((s) => s.name),
+                ...(warning ? { warning } : {}),
+              },
     );
     return EXIT.OK;
   }
@@ -189,8 +196,8 @@ export async function findCommand(args, flags) {
   const query = args.join(' ').trim();
   if (!query) return fail('find needs a query', EXIT.USAGE, `usage: ${CLI} skills find <query>`);
 
-  const { registry, errors, ok } = await loadRegistry(REGISTRY_PATH, flags);
-  if (!ok) return fail(errors[0], EXIT.REGISTRY);
+  const { registry, errors, ok } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
 
   const tokens = tokenize(query);
   const workspace = await workspaceLine(flags);
@@ -200,7 +207,10 @@ export async function findCommand(args, flags) {
     .sort((a, b) => b.score - a.score || a.skill.name.localeCompare(b.skill.name));
 
   if (isJson(flags)) {
-    emit(`${JSON.stringify({ ok: true, query, workspace, results: results.map((r) => ({ name: r.skill.name, score: r.score, reasons: r.reasons })) }, null, 2)}\n`, { maxBytes: flags.maxBytes ?? 0 });
+    emitJson(
+      { ok: true, query, workspace, results: results.map((r) => ({ name: r.skill.name, score: r.score, reasons: r.reasons })) },
+      { maxBytes: flags.maxBytes ?? 0 },
+    );
     return EXIT.OK;
   }
 
@@ -227,8 +237,8 @@ export async function findCommand(args, flags) {
 // --- brief: tier 1 ------------------------------------------------------------
 
 export async function briefCommand(args, flags) {
-  const { registry, errors, ok } = await loadRegistry(REGISTRY_PATH, flags);
-  if (!ok) return fail(errors[0], EXIT.REGISTRY);
+  const { registry, errors, ok } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
 
   const root = flags.root ?? process.cwd();
   const repo = await fingerprint(root);
@@ -237,7 +247,7 @@ export async function briefCommand(args, flags) {
   const repoStack = repo.signals;
   const paths = flags.paths ? parsePathList(flags.paths) : await dirtyPaths(root);
   const taskTokens = tokenize(flags.task ?? '');
-  const forced = parsePathList(flags.skill);
+  const forced = listFlag(flags.skill);
   const budget = flags.budget ?? DEFAULT_BUDGET;
 
   // A brief with no task and no paths resolves on stack alone, which produces a
@@ -258,24 +268,19 @@ export async function briefCommand(args, flags) {
       // is what `warning` carries. Failing here would tell a caller the tool
       // broke when in fact it is telling them their query was.
       if (flags.all) return emitAll(registry, repoStack, flags, warning);
-      emit(
-        `${JSON.stringify(
-          {
-            ok: true,
-            repo: { root, stack: repoStack },
-            workspace,
-            task: null,
-            budget,
-            resolved: [],
-            deferred: [],
-            excluded: [],
-            warning,
-          },
-          null,
-          2,
-        )}\n`,
-        { maxBytes: flags.maxBytes ?? 0 },
-      );
+      emitJson(
+        {
+                    ok: true,
+                    repo: { root, stack: repoStack },
+                    workspace,
+                    task: null,
+                    budget,
+                    resolved: [],
+                    deferred: [],
+                    excluded: [],
+                    warning,
+                  },
+    );
       return EXIT.OK;
     }
     emit(
@@ -316,22 +321,17 @@ export async function briefCommand(args, flags) {
   const lines = [];
 
   if (isJson(flags)) {
-    emit(
-      `${JSON.stringify(
-        {
-          ok: true,
-          repo: { root, stack: repoStack },
-          workspace,
-          task: flags.task ?? null,
-          budget,
-          resolved: picked.map((e) => ({ name: e.skill.name, bytes: e.skill.bytes, why: [...e.reasons, ...[...e.via].map((v) => `requires:${v}`)] })),
-          deferred: deferred.map((e) => ({ name: e.skill.name, bytes: e.skill.bytes })),
-          excluded: excluded.filter((e) => !closureNames.has(e.skill.name)).map((e) => ({ name: e.skill.name, reason: e.reason })),
-        },
-        null,
-        2,
-      )}\n`,
-      { maxBytes: flags.maxBytes ?? 0 },
+    emitJson(
+      {
+                ok: true,
+                repo: { root, stack: repoStack },
+                workspace,
+                task: flags.task ?? null,
+                budget,
+                resolved: picked.map((e) => ({ name: e.skill.name, bytes: e.skill.bytes, why: [...e.reasons, ...[...e.via].map((v) => `requires:${v}`)] })),
+                deferred: deferred.map((e) => ({ name: e.skill.name, bytes: e.skill.bytes })),
+                excluded: excluded.filter((e) => !closureNames.has(e.skill.name)).map((e) => ({ name: e.skill.name, reason: e.reason })),
+              },
     );
     return EXIT.OK;
   }
@@ -432,8 +432,8 @@ export async function showCommand(args, flags) {
   const name = args[0];
   if (!name) return fail('show needs a skill name', EXIT.USAGE, `usage: ${CLI} skills show <name>`);
 
-  const { registry, errors, ok } = await loadRegistry(REGISTRY_PATH, flags);
-  if (!ok) return fail(errors[0], EXIT.REGISTRY);
+  const { registry, errors, ok } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
 
   const skill = registry.skills.find((s) => s.name === name);
   if (!skill) {
@@ -453,10 +453,7 @@ export async function showCommand(args, flags) {
   const body = await sourcePath(skill, workspaceRoot, root);
   const bodyFrom = body === null ? 'none' : body.startsWith(`${root}/`) ? 'repo' : 'workspace';
   if (isJson(flags)) {
-    emit(
-      `${JSON.stringify({ ok: true, ...skill, workspace, body, bodyFrom }, null, 2)}\n`,
-      { maxBytes: flags.maxBytes ?? 0 },
-    );
+    emitJson({ ok: true, ...skill, workspace, body, bodyFrom }, { maxBytes: flags.maxBytes ?? 0 });
     return EXIT.OK;
   }
   emit(
@@ -490,8 +487,8 @@ export async function whyCommand(args, flags) {
   const name = args[0];
   if (!name) return fail('why needs a skill name', EXIT.USAGE, `usage: ${CLI} skills why <name>`);
 
-  const { registry, errors, ok } = await loadRegistry(REGISTRY_PATH, flags);
-  if (!ok) return fail(errors[0], EXIT.REGISTRY);
+  const { registry, errors, ok } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
 
   const skill = registry.skills.find((s) => s.name === name);
   if (!skill) {
@@ -565,6 +562,112 @@ export async function whyCommand(args, flags) {
   return EXIT.OK;
 }
 
+// --- eval ---------------------------------------------------------------------
+
+/**
+ * The gold corpus when the reader names none. A checkout holds one at this fixed path;
+ * `test/` is not in the published files, so an installed copy does not. That absence is
+ * a fact about where this copy runs, reported as such (ADR-061's tiered resolution is
+ * the same shape: explicit path, then the package's own file, then an honest absence).
+ */
+const DEFAULT_CORPUS = join(PACKAGE_ROOT, 'test', 'fixtures', 'eval-corpus.json');
+
+/**
+ * `simt skills eval [corpus]` - measure the resolver against the gold corpus.
+ *
+ * Plan 0002 section 9: "Reports precision and recall and fails on regression." The
+ * corpus states, per case, which skills must come back (`select`) and which must not
+ * (`reject`); a case that names no selection is exhaustive, and anything it returns is
+ * a miss. The exit code rests on `clean`, because the two ratios can both read 1.0
+ * while an exhaustive case returns a skill neither list named (see src/resolve/eval.mjs).
+ */
+export async function evalCommand(positional, flags) {
+  const corpusPath = positional[0] ?? (existsSync(DEFAULT_CORPUS) ? DEFAULT_CORPUS : null);
+
+  if (corpusPath === null) {
+    return fail(
+      `no eval corpus: ${DEFAULT_CORPUS} is not on this machine (test data does not ship in the package)`,
+      EXIT.FAIL,
+      `run: ${CLI} skills eval <path-to-corpus>  (a file of cases with select and reject lists)`,
+    );
+  }
+
+  let corpus;
+  try {
+    corpus = JSON.parse(await readFile(corpusPath, 'utf8'));
+  } catch (error) {
+    return fail(
+      `${corpusPath}: unreadable or invalid JSON (${error.message})`,
+      EXIT.FAIL,
+      `run: ${CLI} skills eval <path-to-corpus>`,
+    );
+  }
+  if (!Array.isArray(corpus.cases) || corpus.cases.length === 0) {
+    return fail(
+      `${corpusPath}: no cases (a corpus is {"cases": [{"name", "repo", "task", "select", "reject"}]})`,
+      EXIT.FAIL,
+      `run: ${CLI} skills eval <path-to-corpus>`,
+    );
+  }
+
+  // The corpus is measured against the registry it cites, so it needs the same
+  // resolution and the same failure as every other skills command.
+  const { registry, errors, ok, path } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
+
+  const unknown = unknownSkills(registry, corpus);
+  const score = scoreCorpus(registry, corpus);
+  const okRun = unknown.length === 0 && score.clean;
+
+  if (isJson(flags)) {
+    emitJson(
+      {
+                ok: okRun,
+                command: 'skills eval',
+                corpus: corpusPath,
+                registry: path,
+                cases: score.cases.map((c) => ({
+                  name: c.name,
+                  selected: c.selected,
+                  missing: c.missing,
+                  unexpected: c.unexpected,
+                })),
+                unknownSkills: unknown,
+                precision: score.precision,
+                recall: score.recall,
+              },
+    );
+    return okRun ? EXIT.OK : EXIT.FAIL;
+  }
+
+  const failed = score.cases.filter((c) => c.missing.length > 0 || c.unexpected.length > 0);
+  const lines = [
+    `corpus    ${corpusPath} (${score.cases.length} cases)`,
+    `registry  ${path}`,
+    `${score.cases.length - failed.length} pass, ${failed.length} fail`,
+    `precision ${score.precision.toFixed(3)}  recall ${score.recall.toFixed(3)}`,
+    '',
+  ];
+
+  for (const c of score.cases) {
+    if (c.missing.length === 0 && c.unexpected.length === 0) {
+      lines.push(`PASS  ${c.name}`);
+      continue;
+    }
+    lines.push(`FAIL  ${c.name}`);
+    for (const name of c.missing) lines.push(`      missed     ${name}`);
+    for (const name of c.unexpected) lines.push(`      unexpected ${name}`);
+  }
+
+  if (unknown.length > 0) {
+    lines.push('', `FAIL  corpus cites skills the registry does not declare: ${unknown.join(', ')}`);
+  }
+
+  lines.push('', okRun ? 'OK the resolver meets the gold corpus' : 'REGRESSION: the resolver no longer meets the gold corpus');
+  emit(lines.join('\n'), { maxBytes: flags.maxBytes ?? 0 });
+  return okRun ? EXIT.OK : EXIT.FAIL;
+}
+
 // --- bodies -------------------------------------------------------------------
 
 /**
@@ -581,7 +684,7 @@ export async function whyCommand(args, flags) {
  * location for a declared source, matched on bytes rather than on hope (ADR-015).
  * The canonical source is still tried first, so a workspace body always wins.
  */
-export async function sourcePath(skill, workspaceRoot, root) {
+async function sourcePath(skill, workspaceRoot, root) {
   const canonical = skill.sources?.find((s) => s.role === 'canonical') ?? skill.sources?.[0];
 
   if (canonical) {

package/src/commands/skillsinstall.mjs

@@ -24,8 +24,8 @@ import { cp, mkdir, readFile, stat, writeFile } from 'node:fs/promises';
 import { existsSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 
-import { EXIT, emit, fail, isJson, note, writeMode } from '../output/index.mjs';
-import { loadRegistry, REGISTRY_PATH, measureSkill } from '../graph/load.mjs';
+import { EXIT, emit, emitJson, fail, isJson, note, writeMode } from '../output/index.mjs';
+import { loadRegistry, measureSkill } from '../graph/load.mjs';
 import { fingerprint } from '../resolve/fingerprint.mjs';
 import { readCapability } from '../hooks/capability.mjs';
 import { MANIFEST, buildManifest, readManifest, writeManifest } from '../projection/manifest.mjs';
@@ -35,6 +35,7 @@ import { nearMatches } from '../resolve/near.mjs';
 import { resolveWorkspace, WORKSPACE_REMEDY } from '../workspace.mjs';
 import { CLI } from '../name.mjs';
 import { MANIFEST_FILE } from '../name.mjs';
+import { nameProblems } from '../graph/spec.mjs';
 
 /**
  * Where skills live in this repo. The estate already has a convention - a
@@ -59,8 +60,8 @@ export async function installCommand(args, flags) {
     );
   }
 
-  const { registry, errors, ok } = await loadRegistry(REGISTRY_PATH, flags);
-  if (!ok) return fail(errors[0], EXIT.REGISTRY);
+  const { registry, errors, ok } = await loadRegistry(null, flags);
+  if (!ok) return fail(errors[0], EXIT.REGISTRY, WORKSPACE_REMEDY);
 
   const skill = registry.skills.find((s) => s.name === name);
   if (!skill) {
@@ -112,6 +113,20 @@ export async function installCommand(args, flags) {
   }
 
   const dir = skillsDir(root, await dirEntries(root));
+  // The Agent Skills specification requires `name` to equal the directory it lives in,
+  // and the directory is named after the skill. So a name that breaks the naming rules
+  // cannot be installed at all: the client resolves by name, finds a directory the name
+  // does not match, and never reads the body this command just copied. Refused here,
+  // before anything is written, with the rule broken - because the alternative is a
+  // successful install that no client can load.
+  const broken = nameProblems(name);
+  if (broken !== null) {
+    return fail(
+      `${name}: ${broken}`,
+      EXIT.REGISTRY,
+      `rename it in ${skill.source ?? 'the registry'} to lowercase letters, digits and single hyphens, then install again`,
+    );
+  }
   const dest = join(root, dir, name);
   const inplace = await samePath(src, dest);
   const action = inplace ? 'in-place' : await compare(dest, src, measured.flat, name);
@@ -147,7 +162,7 @@ export async function installCommand(args, flags) {
   // combination a script writes. The payload is now emitted once, at the end,
   // after the write it describes.
   const report = () => {
-    if (isJson(flags)) emit(`${JSON.stringify(payload, null, 2)}\n`, { maxBytes: flags.maxBytes ?? 0 });
+    if (isJson(flags)) emitJson(payload);
   };
   const mode = writeMode(flags);