skillshelf 0.1.0

package/src/core/crawl.ts

@@ -0,0 +1,267 @@
+// Crawl skill roots into Skill[] applying the crawl rules (see docs/ARCHITECTURE.md §6).
+//
+// Rules:
+//   - Dedupe by realpath (aliased mounts like cloud-sync mirror locations).
+//   - Treat `.agents/skills` as bridge mirrors of `.claude/skills`: set mirrorOf,
+//     do not double-count as an independent skill.
+//   - Skip `_retired/`: tag `retired: true`, do not activate.
+//   - Ignore any path containing `node_modules`.
+//   - Support both `name/SKILL.md` and `skills/name/SKILL.md` layouts.
+
+import { createHash } from "node:crypto";
+import { join, basename, dirname, sep } from "node:path";
+import { readdir } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import type { Skill } from "../types.ts";
+import { parseFrontmatter } from "../lib/frontmatter.ts";
+import { realpathOrSelf, isDirectory } from "../lib/fs.ts";
+
+const SKILL_FILE = "SKILL.md";
+const RETIRED_DIR = "_retired";
+
+export function hashContent(content: string): string {
+  return createHash("sha256").update(content, "utf8").digest("hex");
+}
+
+function pathHasSegment(p: string, seg: string): boolean {
+  return p.split(sep).includes(seg);
+}
+
+/** Pull effective domains from frontmatter (`domains` or `primaryDomain`). */
+function readDomains(data: Record<string, unknown>): string[] {
+  const out: string[] = [];
+  const primary = data.primaryDomain ?? data.primary_domain;
+  if (typeof primary === "string" && primary.trim() !== "") out.push(primary.trim());
+  const d = data.domains;
+  if (Array.isArray(d)) {
+    for (const x of d) {
+      const s = String(x).trim();
+      if (s !== "") out.push(s);
+    }
+  } else if (typeof d === "string" && d.trim() !== "") {
+    out.push(d.trim());
+  }
+  return [...new Set(out)];
+}
+
+/** True if a directory looks like a skill dir (contains SKILL.md). */
+function isSkillDir(dir: string): boolean {
+  return existsSync(join(dir, SKILL_FILE));
+}
+
+async function listRefFiles(skillDir: string, skillName: string): Promise<string[]> {
+  const out: string[] = [];
+  let entries: Awaited<ReturnType<typeof readdir>>;
+  try {
+    entries = await readdir(skillDir, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+  for (const e of entries) {
+    if (e.name === SKILL_FILE) continue;
+    if (e.name === `${skillName}.shelf.json`) continue;
+    if (e.name === "shelf.lock.json") continue;
+    if (e.name === ".DS_Store") continue;
+    out.push(join(skillDir, e.name));
+  }
+  return out.sort();
+}
+
+async function buildSkill(
+  skillDir: string,
+  opts: { retired: boolean; mirrorOf: string | null; primaryDomain: string | null },
+): Promise<Skill | null> {
+  const bodyPath = join(skillDir, SKILL_FILE);
+  if (!existsSync(bodyPath)) return null;
+  let raw: string;
+  try {
+    raw = await Bun.file(bodyPath).text();
+  } catch {
+    return null;
+  }
+  const { data, body } = parseFrontmatter(raw);
+  const dirName = basename(skillDir);
+  const name =
+    typeof data.name === "string" && data.name.trim() !== ""
+      ? data.name.trim()
+      : dirName;
+  const description =
+    typeof data.description === "string" ? data.description.trim() : "";
+  const domains = readDomains(data);
+  const primaryDomain =
+    opts.primaryDomain ?? (domains.length > 0 ? domains[0]! : null);
+  const refFiles = await listRefFiles(skillDir, name);
+
+  return {
+    name,
+    description,
+    primaryDomain,
+    domains: domains.length > 0 ? domains : primaryDomain ? [primaryDomain] : [],
+    path: skillDir,
+    bodyPath,
+    refFiles,
+    source: null,
+    retired: opts.retired,
+    mirrorOf: opts.mirrorOf,
+    contentHash: hashContent(body),
+  };
+}
+
+/**
+ * Find every skill dir under a root, supporting:
+ *   - <root>/<name>/SKILL.md
+ *   - <root>/skills/<name>/SKILL.md
+ *   - <root>/_retired/<name>/SKILL.md  (retired)
+ * Returns absolute skill dirs paired with whether they're retired.
+ */
+async function discoverSkillDirs(
+  root: string,
+): Promise<Array<{ dir: string; retired: boolean }>> {
+  const out: Array<{ dir: string; retired: boolean }> = [];
+  if (!existsSync(root)) return out;
+
+  // Bounded recursive descent. A directory containing SKILL.md is a skill leaf.
+  // Otherwise it's a grouping dir (domain folder, `skills/`, `.agents/skills/`,
+  // `_retired/`, project root) and we recurse. `_retired` taints everything below.
+  const SKIP = new Set(["node_modules", ".git"]);
+  const MAX_DEPTH = 8;
+
+  async function recurse(dir: string, depth: number, retired: boolean): Promise<void> {
+    if (depth > MAX_DEPTH) return;
+    if (isSkillDir(dir)) {
+      out.push({ dir, retired });
+      return; // do not descend into a skill's own subtree (reference/ etc.)
+    }
+    let entries: Awaited<ReturnType<typeof readdir>>;
+    try {
+      entries = await readdir(dir, { withFileTypes: true });
+    } catch {
+      return;
+    }
+    for (const e of entries) {
+      if (SKIP.has(e.name)) continue;
+      const full = join(dir, e.name);
+      if (pathHasSegment(full, "node_modules")) continue;
+      const isDir =
+        e.isDirectory() || (e.isSymbolicLink() && (await isDirectory(full)));
+      if (!isDir) continue;
+      const childRetired = retired || e.name === RETIRED_DIR;
+      await recurse(full, depth + 1, childRetired);
+    }
+  }
+
+  await recurse(root, 0, false);
+  return out;
+}
+
+export interface CrawlOptions {
+  /** primary-domain hint applied to all skills found under this root (library mode) */
+  primaryDomainOf?: (skillDir: string) => string | null;
+}
+
+export interface CrawlResult {
+  skills: Skill[];
+  /** roots that were skipped because they realpath-dedupe to an earlier root */
+  dedupedRoots: string[];
+}
+
+/**
+ * Crawl a set of roots into Skill[]. Applies realpath-dedupe across roots and
+ * across individual skill dirs, marks `.agents/skills` mirrors, tags retired,
+ * skips node_modules.
+ */
+export async function crawl(
+  roots: string[],
+  opts: CrawlOptions = {},
+): Promise<CrawlResult> {
+  const dedupedRoots: string[] = [];
+  const seenRootReal = new Set<string>();
+  const effectiveRoots: string[] = [];
+  for (const r of roots) {
+    const rp = realpathOrSelf(r);
+    if (seenRootReal.has(rp)) {
+      dedupedRoots.push(r);
+      continue;
+    }
+    seenRootReal.add(rp);
+    effectiveRoots.push(r);
+  }
+
+  // Map realpath(skillDir) -> Skill, so aliased copies collapse.
+  const byReal = new Map<string, Skill>();
+  // Track canonical (.claude / non-.agents) skill dirs by realpath of body,
+  // so .agents mirrors can point mirrorOf at them.
+  const claudeByName = new Map<string, string>(); // name -> canonical skill dir path
+
+  // First pass: collect all skill dirs with their root + agents flag.
+  interface Found {
+    dir: string;
+    retired: boolean;
+    isAgents: boolean;
+    root: string;
+  }
+  const found: Found[] = [];
+  for (const root of effectiveRoots) {
+    const dirs = await discoverSkillDirs(root);
+    for (const d of dirs) {
+      // A skill dir is a bridge mirror if it lives under a `.agents` path.
+      const isAgents = pathHasSegment(d.dir, ".agents");
+      found.push({ dir: d.dir, retired: d.retired, isAgents, root });
+    }
+  }
+
+  // Record canonical (non-.agents) names first, so mirrors can point at them.
+  for (const f of found) {
+    if (f.isAgents) continue;
+    const name = basename(f.dir);
+    if (!claudeByName.has(name)) claudeByName.set(name, f.dir);
+  }
+
+  for (const f of found) {
+    const real = realpathOrSelf(f.dir);
+    if (byReal.has(real)) continue; // aliased duplicate dir
+
+    let mirrorOf: string | null = null;
+    if (f.isAgents) {
+      const name = basename(f.dir);
+      mirrorOf = claudeByName.get(name) ?? null;
+    }
+    const primaryDomain = opts.primaryDomainOf?.(f.dir) ?? null;
+    const skill = await buildSkill(f.dir, {
+      retired: f.retired,
+      mirrorOf,
+      primaryDomain,
+    });
+    if (skill) byReal.set(real, skill);
+  }
+
+  return { skills: [...byReal.values()], dedupedRoots };
+}
+
+/** Expand a parent dir (like ~/Documents/GitHub) into candidate skill roots. */
+export async function expandProjectRoots(parent: string): Promise<string[]> {
+  const out: string[] = [];
+  if (!existsSync(parent)) return out;
+  let entries: Awaited<ReturnType<typeof readdir>>;
+  try {
+    entries = await readdir(parent, { withFileTypes: true });
+  } catch {
+    return out;
+  }
+  for (const e of entries) {
+    if (!e.isDirectory()) continue;
+    if (e.name === "node_modules") continue;
+    const proj = join(parent, e.name);
+    for (const sub of [
+      join(proj, ".claude", "skills"),
+      join(proj, ".agents", "skills"),
+      join(proj, "skills"),
+      join(proj, "skill"),
+    ]) {
+      if (existsSync(sub)) out.push(sub);
+    }
+  }
+  return out;
+}
+
+export { dirname as _dirname };

package/src/core/dedupe.ts

@@ -0,0 +1,67 @@
+// Group duplicate / drifted skills by name + content hash.
+// Classify a canonical copy vs divergent (drifted) copies vs exact duplicates.
+
+import type { DuplicateGroup, Skill } from "../types.ts";
+
+/**
+ * Rank a skill as a canonical candidate. Higher is more canonical.
+ * Preference: non-mirror > non-retired > has-domains > has-provenance(third-party kept) > path length (shorter).
+ */
+function canonicalScore(s: Skill): number {
+  let score = 0;
+  if (!s.mirrorOf) score += 1000; // real file, not a bridge mirror
+  if (!s.retired) score += 500; // active over retired
+  if (s.domains.length > 0) score += 100; // tagged
+  score += Math.max(0, 50 - Math.min(50, s.path.length / 4)); // prefer shorter path
+  return score;
+}
+
+/**
+ * Group skills that share a `name`. For each group:
+ *   - pick `canonical` by canonicalScore
+ *   - `duplicates` = other copies with the SAME contentHash as canonical
+ *   - `divergent` = copies with a DIFFERENT contentHash (drift)
+ *   - `identical` = every copy shares one hash
+ * Single-copy names are not returned (no duplication).
+ */
+export function findDuplicates(skills: Skill[]): DuplicateGroup[] {
+  const byName = new Map<string, Skill[]>();
+  for (const s of skills) {
+    const arr = byName.get(s.name);
+    if (arr) arr.push(s);
+    else byName.set(s.name, [s]);
+  }
+
+  const groups: DuplicateGroup[] = [];
+  for (const [name, copies] of byName) {
+    if (copies.length < 2) continue;
+    const sorted = [...copies].sort((a, b) => canonicalScore(b) - canonicalScore(a));
+    const canonical = sorted[0]!;
+    const rest = sorted.slice(1);
+    const duplicates = rest.filter((s) => s.contentHash === canonical.contentHash);
+    const divergent = rest.filter((s) => s.contentHash !== canonical.contentHash);
+    const hashes = new Set(copies.map((s) => s.contentHash));
+    groups.push({
+      name,
+      canonical,
+      duplicates,
+      divergent,
+      identical: hashes.size === 1,
+    });
+  }
+  groups.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+  return groups;
+}
+
+/**
+ * Groups that have at least one divergent (drifted) copy — the ones needing
+ * human review during migration.
+ */
+export function driftedGroups(groups: DuplicateGroup[]): DuplicateGroup[] {
+  return groups.filter((g) => g.divergent.length > 0);
+}
+
+/** Exact-duplicate groups (identical content in multiple non-mirror locations). */
+export function exactDuplicateGroups(groups: DuplicateGroup[]): DuplicateGroup[] {
+  return groups.filter((g) => g.identical && g.duplicates.length > 0);
+}