skillrepo 4.3.0 → 4.5.0

package/src/lib/placement-walk.mjs

@@ -0,0 +1,285 @@
+/**
+ * Walk on-disk skill placements and compute SHAs (#1555).
+ *
+ * Read-side counterpart of `file-write.mjs`: given a set of detected
+ * vendor keys, enumerate the skill directories each vendor's
+ * placement target contains and compute the same SHA pair that
+ * `runSync` persists in `.last-sync` v2. The output feeds `drift.mjs`
+ * for per-skill state classification.
+ *
+ * Design notes
+ * ------------
+ *
+ * - **One walk per unique target.** Cohort vendors (cursor, windsurf,
+ *   gemini, codex, cline, copilot) all share `agentsProject`. Walking
+ *   the parent dir N times — once per detected cohort vendor — would
+ *   read every file N times. Instead, we walk each unique target
+ *   ONCE and expand the result to one entry per (vendor, scope,
+ *   skill) combination.
+ *
+ * - **POSIX path normalization for hashing.** `filesSha256` is the
+ *   SHA-256 of a sorted `<path>|<sha>` projection. The path component
+ *   MUST be forward-slash-separated to match what the server-side
+ *   hashing produces (and what `crypto-shas.mjs` documents). Disk
+ *   paths on Windows have backslashes; we normalize at the
+ *   computation boundary, not in the data shape.
+ *
+ * - **Project scope only in v1.** The spec defers `--global` to a
+ *   future flag. This module walks each detected vendor's
+ *   `projectTarget` exclusively. When the global flag lands, add a
+ *   `scopes` parameter that defaults to `["project"]`.
+ *
+ * - **Errors degrade gracefully.** A skill directory we can't read
+ *   (permission error, broken symlink, partial file) produces an
+ *   entry with `present: true` and `skillMdSha256: null` /
+ *   `filesSha256: null`. The drift classifier treats null SHAs as
+ *   `edited` — conservative, surfaces the issue to the user without
+ *   crashing the command.
+ *
+ * - **`.tmp/` and `.old/` are skipped.** These are crashed-write
+ *   artifacts handled by `cleanupOrphans` in `file-write.mjs`. They
+ *   are not skill directories.
+ */
+
+import { existsSync, readdirSync, readFileSync, lstatSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+
+import { computeSkillShas } from "./crypto-shas.mjs";
+import { resolvePlacementRoot } from "./file-write.mjs";
+import { getAgentByKey } from "./agent-registry.mjs";
+
+/**
+ * @typedef {Object} LocalPlacement
+ * @property {string} vendorKey         - Canonical agent key from agent-registry.
+ * @property {"project"} scope          - Always "project" in v1.
+ * @property {string} target            - PlacementTarget enum value.
+ * @property {string} skillName         - Directory name on disk.
+ * @property {boolean} present          - Always true for entries returned here.
+ * @property {string | null} skillMdSha256
+ * @property {string | null} filesSha256
+ */
+
+/**
+ * Walk every detected vendor's project placement and return a Map
+ * keyed by `"<vendorKey>::project::<skillName>"` for O(1) lookup
+ * from `list.mjs`.
+ *
+ * Each unique placement TARGET is walked exactly once. Detected
+ * vendors that map to the same target share the walk's results.
+ *
+ * @param {string[]} detectedVendorKeys
+ * @returns {Map<string, LocalPlacement>}
+ */
+export function walkDetectedPlacements(detectedVendorKeys) {
+  if (!Array.isArray(detectedVendorKeys) || detectedVendorKeys.length === 0) {
+    return new Map();
+  }
+
+  // Group detected vendors by their projectTarget. A vendor with a
+  // null projectTarget (no documented project-scope placement) is
+  // dropped from this map — there's nothing to walk.
+  /** @type {Map<string, string[]>} */
+  const targetToVendors = new Map();
+  for (const key of detectedVendorKeys) {
+    const entry = getAgentByKey(key);
+    if (!entry || !entry.projectTarget) continue;
+    const target = entry.projectTarget;
+    if (!targetToVendors.has(target)) targetToVendors.set(target, []);
+    targetToVendors.get(target).push(key);
+  }
+
+  /** @type {Map<string, LocalPlacement>} */
+  const result = new Map();
+  for (const [target, vendorKeys] of targetToVendors) {
+    const skillsAtTarget = walkPlacementTarget(target);
+    for (const skill of skillsAtTarget) {
+      for (const vendorKey of vendorKeys) {
+        const key = `${vendorKey}::project::${skill.skillName}`;
+        result.set(key, {
+          vendorKey,
+          scope: "project",
+          target,
+          skillName: skill.skillName,
+          present: true,
+          skillMdSha256: skill.skillMdSha256,
+          filesSha256: skill.filesSha256,
+        });
+      }
+    }
+  }
+  return result;
+}
+
+/**
+ * Walk a single placement target's PARENT directory and return one
+ * entry per skill subdirectory found. Exposed for tests and for
+ * callers that need a single-target view.
+ *
+ * Output shape: `{ skillName, skillMdSha256, filesSha256 }[]`. The
+ * vendor/scope expansion happens in `walkDetectedPlacements`.
+ *
+ * @param {string} target - PlacementTarget enum value.
+ * @returns {{ skillName: string, skillMdSha256: string | null, filesSha256: string | null }[]}
+ */
+export function walkPlacementTarget(target) {
+  let parentDir;
+  try {
+    parentDir = resolvePlacementRoot(target);
+  } catch {
+    // Unknown target — surface as no placements rather than throwing.
+    // Defensive: detect-agents + agent-registry should never produce
+    // an unknown target, but if they ever do we don't want `list` to
+    // crash.
+    return [];
+  }
+
+  if (!existsSync(parentDir)) return [];
+
+  /** @type {string[]} */
+  let entries;
+  try {
+    entries = readdirSync(parentDir);
+  } catch {
+    return [];
+  }
+
+  /** @type {{ skillName: string, skillMdSha256: string | null, filesSha256: string | null }[]} */
+  const out = [];
+  for (const entry of entries) {
+    // Skip crashed-write artifacts handled by cleanupOrphans.
+    if (entry.endsWith(".tmp") || entry.endsWith(".old")) continue;
+
+    const skillDir = join(parentDir, entry);
+    let stat;
+    try {
+      // lstatSync (not statSync) so a symlink-to-directory at the
+      // placements root is rejected here, not recursed into. A
+      // circular symlink would otherwise crash `list` with a stack
+      // overflow from infinite recursion in walkSkillDirRecursive.
+      stat = lstatSync(skillDir);
+    } catch {
+      continue;
+    }
+    if (stat.isSymbolicLink()) continue;
+    if (!stat.isDirectory()) continue;
+
+    const shas = computeShasFromDir(skillDir);
+    out.push({
+      skillName: entry,
+      skillMdSha256: shas.skillMdSha256,
+      filesSha256: shas.filesSha256,
+    });
+  }
+  return out;
+}
+
+/**
+ * Read every file in `skillDir` and compute the same SHA pair that
+ * `runSync` persists. On ANY file-read failure (permission, broken
+ * symlink, truncated file), returns null SHAs — the caller treats
+ * this as `edited` (conservative). Returning null is intentional:
+ * we cannot honestly claim `current` on data we couldn't read.
+ *
+ * @param {string} skillDir
+ * @returns {{ skillMdSha256: string | null, filesSha256: string | null }}
+ */
+function computeShasFromDir(skillDir) {
+  /** @type {{ path: string, content: string }[]} */
+  const files = [];
+  let readError = false;
+  walkSkillDirRecursive(skillDir, skillDir, files, (err) => {
+    if (err) readError = true;
+  });
+
+  if (readError || files.length === 0) {
+    return { skillMdSha256: null, filesSha256: null };
+  }
+
+  try {
+    return computeSkillShas(files);
+  } catch {
+    // computeSkillShas only throws on bad input shape. We constructed
+    // the input ourselves, so a throw indicates a programming error
+    // we can't recover from — but treat as null SHAs rather than
+    // crashing the user's `list` command.
+    return { skillMdSha256: null, filesSha256: null };
+  }
+}
+
+/**
+ * Recursive directory walk that populates a flat `{path, content}[]`
+ * array, with paths POSIX-normalized relative to `skillDir`. Errors
+ * on individual entries are reported through the `onError` callback
+ * (the caller flips a `readError` boolean to discard the partial
+ * accum and return null SHAs) but the walk CONTINUES with the
+ * remaining siblings.
+ *
+ * The continue-not-return choice matters: a `readdirSync` listing of
+ * [unreadable, readable1, readable2] would previously bail on entry
+ * 0 and never see entries 1 and 2. Setting `readError` once and
+ * still walking the rest gives the caller correct behavior (null
+ * SHAs from the gate at `computeShasFromDir`) and produces a
+ * deterministic outcome that doesn't depend on `readdirSync`
+ * ordering. The accum is partially populated when `readError` is
+ * true but the gate in `computeShasFromDir` discards it.
+ *
+ * Depth is not capped — agentskills.io spec mandates files at most
+ * one level deep from SKILL.md (`scripts/foo.sh`, `references/bar.md`)
+ * but the CLI doesn't reject a deeper tree. If the user has hand-
+ * authored a deeper structure, we hash it faithfully.
+ */
+function walkSkillDirRecursive(rootDir, currentDir, accum, onError) {
+  let entries;
+  try {
+    entries = readdirSync(currentDir);
+  } catch (err) {
+    // Can't list this directory at all — flag the error and bail
+    // out of this level (no entries to walk). The outer caller
+    // continues with whatever siblings the parent dir contains.
+    onError(err);
+    return;
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(currentDir, entry);
+    let stat;
+    try {
+      // lstatSync (not statSync) is load-bearing here. statSync
+      // follows symlinks, which means:
+      //   1. A symlink-to-directory passes isDirectory() and
+      //      walkSkillDirRecursive recurses into it. A circular
+      //      symlink (e.g. `references/loop -> .`) infinite-loops
+      //      and crashes the process with a stack overflow.
+      //   2. A symlink-to-file passes isFile() and readFileSync
+      //      reads the target — potentially OUTSIDE the skill dir.
+      //      Not a privilege escalation (we only read what the user
+      //      can read), but a principle-of-least-surprise violation.
+      // The agentskills.io spec does not define symlinks as valid
+      // skill content. Skip them.
+      stat = lstatSync(fullPath);
+    } catch (err) {
+      onError(err);
+      continue;
+    }
+    if (stat.isSymbolicLink()) continue;
+    if (stat.isDirectory()) {
+      walkSkillDirRecursive(rootDir, fullPath, accum, onError);
+      continue;
+    }
+    if (!stat.isFile()) continue; // skip sockets / pipes / etc.
+
+    let content;
+    try {
+      content = readFileSync(fullPath, "utf8");
+    } catch (err) {
+      onError(err);
+      continue;
+    }
+    // POSIX-normalize the path so SHA computation matches what the
+    // server produces. On POSIX, `sep` is "/" and this is a no-op;
+    // on Windows, `sep` is "\\" and we convert.
+    const posixRelPath =
+      sep === "/" ? relative(rootDir, fullPath) : relative(rootDir, fullPath).split(sep).join("/");
+    accum.push({ path: posixRelPath, content });
+  }
+}

package/src/lib/sync.mjs

@@ -22,6 +22,32 @@
  * can evolve it without breaking older CLIs catastrophically. Future
  * versions should bump the schema when fields are added/removed.
  *
+ * Schema history
+ * --------------
+ *   v1 — `{ schemaVersion, etag, syncedAt }`. Library-level state only;
+ *        no per-skill information persisted. Sufficient for ETag-based
+ *        short-circuit, insufficient for per-skill drift detection.
+ *   v2 — Adds a `skills` map keyed by `"<owner>/<name>"` carrying the
+ *        version + content SHAs of every skill written in the last
+ *        successful sync. Unblocks per-skill drift detection in `list`
+ *        (#1555) and any future `status`-style command.
+ *
+ * Forward/backward compatibility
+ * ------------------------------
+ *   • Old CLI reading a v2 file → unknown schemaVersion → null → full
+ *     sync → writes v1 file. User loses per-skill metadata until they
+ *     upgrade again.
+ *   • New CLI reading a v1 file → in-memory migration to a v2 shape
+ *     with the existing `etag` + `syncedAt` preserved and an EMPTY
+ *     `skills` map. The next successful sync writes a v2 file on
+ *     disk. This is safe because an empty map means "we don't know
+ *     per-skill state from this sync" — callers must treat absent
+ *     entries as unknown, not as "missing from library." Preserving
+ *     the etag also keeps the 304 short-circuit working, so an
+ *     upgrade-then-no-changes path stays cheap.
+ *   • Future unknown schemaVersion → null → full sync. Forward-compat
+ *     without crashing.
+ *
  * @typedef {Object} SyncSummary
  * @property {number} added       - Skills newly written that were not on disk
  * @property {number} updated     - Skills overwritten on disk
@@ -45,14 +71,20 @@
  * @property {string} [syncedAt]
  * @property {string} syncedAt    - ISO timestamp from the server response
  *
+ * @typedef {Object} SyncedSkillEntry
+ * @property {string} version       - Version of the skill as synced (e.g. "1.4.0").
+ * @property {string} skillMdSha256 - Hex SHA-256 of SKILL.md content as written.
+ * @property {string} filesSha256   - Hex SHA-256 of the full file projection.
+ * @property {string} syncedAt      - ISO timestamp when this skill was synced.
+ *
  * @typedef {Object} SyncStateFile
  * @property {number} schemaVersion
  * @property {string|null} etag
  * @property {string} syncedAt
+ * @property {Record<string, SyncedSkillEntry>} skills - v2+. Keyed by `"<owner>/<name>"`.
  */
 
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname } from "node:path";
+import { existsSync, readFileSync } from "node:fs";
 
 import { getLibrary } from "./http.mjs";
 import {
@@ -62,16 +94,21 @@ import {
   placementTargetsFor,
   resolvePlacementDir,
 } from "./file-write.mjs";
+import { writeFileAtomic } from "./fs-utils.mjs";
 import { globalLastSyncPath } from "./paths.mjs";
 import { diskError, validationError } from "./errors.mjs";
+import { computeSkillShas } from "./crypto-shas.mjs";
 
 /**
  * Current schema version of the .last-sync file. Bump on any
  * structural change. The reader treats unknown future versions as
  * "ignore the cache and do a full sync" — forward-compat without
  * crashing.
+ *
+ * v2 (#1553): adds `skills` map keyed by `"<owner>/<name>"` →
+ * `{ version, skillMdSha256, filesSha256, syncedAt }`.
  */
-export const LAST_SYNC_SCHEMA_VERSION = 1;
+export const LAST_SYNC_SCHEMA_VERSION = 2;
 
 /**
  * Read the persisted last-sync state from ~/.claude/skillrepo/.last-sync.
@@ -99,12 +136,42 @@ export function readLastSync() {
     return null;
   }
 
+  if (!parsed || typeof parsed !== "object") {
+    return null;
+  }
+
+  // v1 → v2 in-memory migration. Preserve the etag and syncedAt so
+  // delta sync still benefits from the prior state; start with an
+  // empty `skills` map. The next successful runSync writes v2 to
+  // disk. We do NOT write here — readers should be pure with
+  // respect to the filesystem.
+  if (parsed.schemaVersion === 1) {
+    return {
+      schemaVersion: LAST_SYNC_SCHEMA_VERSION,
+      etag: typeof parsed.etag === "string" ? parsed.etag : null,
+      syncedAt: typeof parsed.syncedAt === "string" ? parsed.syncedAt : null,
+      skills: {},
+    };
+  }
+
+  if (parsed.schemaVersion !== LAST_SYNC_SCHEMA_VERSION) {
+    return null;
+  }
+
+  // Defense: coerce a malformed `skills` field to {}. Both an array
+  // (`typeof === "object"` but not a real map) and a missing/null
+  // field can otherwise sneak past callers that only check
+  // `typeof === "object"`. Specifically: `list.mjs` does direct
+  // dictionary access on `lastSync.skills`, and `lastSync.skills[key]`
+  // on an array returns `undefined`, which would render every skill
+  // as `missing` — wrong, not safe-fail. Normalizing here means every
+  // caller sees a real Record<string, ...>.
   if (
-    !parsed ||
-    typeof parsed !== "object" ||
-    parsed.schemaVersion !== LAST_SYNC_SCHEMA_VERSION
+    !parsed.skills ||
+    typeof parsed.skills !== "object" ||
+    Array.isArray(parsed.skills)
   ) {
-    return null;
+    parsed.skills = {};
   }
 
   return parsed;
@@ -116,22 +183,36 @@ export function readLastSync() {
  * error. Most callers should treat a state-file write failure as
  * non-fatal because the actual skill files were already written
  * successfully.
+ *
+ * v2 (#1553): `skills` is the per-skill version+SHA map. Omitting it
+ * persists an empty `{}` — the schema bump is structural even when
+ * the caller has no skills to record (e.g. a 304 short-circuit before
+ * runSync ran any writes). An empty map means "we know the library
+ * was up-to-date at syncedAt but have no per-skill details from this
+ * sync"; consumers should treat absent entries as "unknown" rather
+ * than as "missing from library."
+ *
+ * @param {object} state
+ * @param {string | null} [state.etag]
+ * @param {string} [state.syncedAt]
+ * @param {Record<string, SyncedSkillEntry>} [state.skills]
  */
-export function writeLastSync({ etag, syncedAt }) {
+export function writeLastSync({ etag, syncedAt, skills } = {}) {
   const path = globalLastSyncPath();
-  const dir = dirname(path);
-  try {
-    if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-  } catch (err) {
-    throw diskError(`Cannot create ${dir}: ${err.message}`, { cause: err });
-  }
   const body = {
     schemaVersion: LAST_SYNC_SCHEMA_VERSION,
     etag: etag ?? null,
     syncedAt: syncedAt ?? new Date().toISOString(),
+    skills: skills && typeof skills === "object" && !Array.isArray(skills) ? skills : {},
   };
   try {
-    writeFileSync(path, JSON.stringify(body, null, 2) + "\n", "utf-8");
+    // writeFileAtomic handles parent-dir creation and uses a
+    // per-process unique temp suffix so two concurrent writers
+    // (e.g., a SessionStart hook running `skillrepo update` while
+    // the user runs another command) can't collide on the same
+    // .tmp path and produce a torn final file. The cohort hooks
+    // (#1240) made this race observable in production.
+    writeFileAtomic(path, JSON.stringify(body, null, 2) + "\n");
   } catch (err) {
     throw diskError(`Cannot write ${path}: ${err.message}`, { cause: err });
   }
@@ -270,7 +351,50 @@ export async function runSync(options) {
   // a partial skill would leave the user with broken resource
   // references (the SKILL.md body would point to files that don't
   // exist on disk).
+  //
+  // v2 of `.last-sync` carries a per-skill version + SHA map. We
+  // compute SHAs from the in-memory `skill.files` array — same bytes
+  // we're about to write to disk via writeSkillDir — so the persisted
+  // SHA exactly matches what landed on the user's filesystem. SHAs are
+  // recorded ONLY for skills we actually wrote (skipped on
+  // filesIncomplete, never persisted on a write failure since the
+  // throw propagates and the whole state file write is bypassed).
   let anyIncomplete = false;
+  /** @type {Record<string, SyncedSkillEntry>} */
+  const skillsMap = {};
+  // Carry forward entries from the previous sync for skills we did
+  // NOT re-fetch this round. A delta sync that touched 2 of 50
+  // skills must keep the other 48's metadata — otherwise the
+  // persisted map shrinks every delta sync until only the most
+  // recently-touched skills remain, defeating `list`'s drift
+  // detection. The previous ETag is only persisted again when no
+  // filesIncomplete fired, so this carry-forward is safe even on
+  // partial syncs.
+  //
+  // We reuse `lastSync` (captured before the network call) rather
+  // than re-reading from disk. No write happens between line ~330
+  // and here, so the values are identical — but re-reading would
+  // be a footgun if a future change ever persisted partial state
+  // between those points (the carry-forward would silently pick up
+  // the freshly-written data instead of the truly-prior state).
+  if (lastSync && lastSync.skills && typeof lastSync.skills === "object") {
+    for (const [key, entry] of Object.entries(lastSync.skills)) {
+      if (entry && typeof entry === "object") {
+        skillsMap[key] = entry;
+      }
+    }
+  }
+  // Tombstones invalidate any prior map entry: a skill that was
+  // removed from the library should not linger in our local
+  // version/SHA cache. Apply this BEFORE re-adding entries from the
+  // current response so a re-add in the same sync window (handled
+  // earlier under "removals applied first") still ends up in the
+  // map with fresh data.
+  for (const tombstone of result.removals) {
+    const key = `${tombstone.owner}/${tombstone.name}`;
+    delete skillsMap[key];
+  }
+
   for (const skill of result.skills) {
     if (skill.filesIncomplete) {
       anyIncomplete = true;
@@ -283,12 +407,34 @@ export async function runSync(options) {
     } else {
       summary.added++;
     }
+
+    // Compute SHAs from the same content writeSkillDir just persisted
+    // to disk. If the skill has no SKILL.md (malformed payload that
+    // somehow passed validateSkill), `skillMdSha256` is null and we
+    // skip the map entry — recording a partial entry would poison
+    // future drift detection. validateSkill in file-write.mjs already
+    // rejects skills without SKILL.md, so this branch is defense in
+    // depth; if it ever fires, our cache is correct (no entry > bad
+    // entry).
+    const shas = computeSkillShas(skill.files);
+    if (shas.skillMdSha256 !== null) {
+      skillsMap[`${skill.owner}/${skill.name}`] = {
+        version: typeof skill.version === "string" ? skill.version : "",
+        skillMdSha256: shas.skillMdSha256,
+        filesSha256: shas.filesSha256,
+        syncedAt: result.syncedAt,
+      };
+    }
   }
 
-  // Step 7: persist new ETag (only if the response was complete)
+  // Step 7: persist new ETag + skills map (only if the response was complete)
   if (!anyIncomplete && result.etag) {
     try {
-      writeLastSync({ etag: result.etag, syncedAt: result.syncedAt });
+      writeLastSync({
+        etag: result.etag,
+        syncedAt: result.syncedAt,
+        skills: skillsMap,
+      });
     } catch (err) {
       // Non-fatal — the skills are on disk, we just won't get the
       // 304 short-circuit on the next run. Surface a warning via