skillshelf 0.1.0

package/src/adapters/inference/api.ts

@@ -0,0 +1,309 @@
+// API inference adapter — closes the loop automatically against any
+// OpenAI-compatible chat/completions endpoint. The LLM-FREE core lives in
+// agent.ts; this file is the ONLY place a network LLM call happens.
+//
+// Configuration (base URL, API key, model) resolves from, highest precedence
+// first:
+//   1. CLI flags        (--base-url, --model; --provider for a base-URL preset)
+//   2. Environment vars  (SKILLSHELF_LLM_* , then OPENAI_* fallbacks)
+//   3. Optional dotenv   ($SKILLSHELF_ENV_FILE, default ./.env if present)
+//
+// Nothing here is provider-specific: any server that speaks the OpenAI
+// chat/completions schema works (OpenAI, OpenRouter, Groq, Ollama, vLLM,
+// a local gateway, …).
+
+import { existsSync } from "node:fs";
+import type { InferenceCorpus } from "../../types.ts";
+import {
+  INFER_INSTRUCTION,
+  PROPOSAL_SCHEMA,
+  normalizeProposal,
+  type InferenceProposal,
+} from "./agent.ts";
+
+export interface ProviderConfig {
+  /** OpenAI-compatible base, including /v1 */
+  base: string;
+  /** bearer key */
+  apiKey: string;
+  /** chat model id */
+  model: string;
+  /** provider label */
+  name: string;
+}
+
+/** Sensible default endpoint + model when nothing else is configured. */
+const DEFAULT_BASE = "https://api.openai.com/v1";
+const DEFAULT_MODEL = "gpt-4o-mini"; // placeholder default; override with --model / *_MODEL
+
+/**
+ * Generic public provider presets. These set ONLY a default base URL — the API
+ * key always comes from the environment (or dotenv). `custom` means "use
+ * --base-url / SKILLSHELF_LLM_BASE_URL". No private/institutional presets.
+ */
+const PROVIDER_BASES: Record<string, string> = {
+  openai: "https://api.openai.com/v1",
+  openrouter: "https://openrouter.ai/api/v1",
+  groq: "https://api.groq.com/openai/v1",
+  ollama: "http://localhost:11434/v1",
+  custom: "", // resolved entirely from flags/env
+};
+
+export function knownProviders(): string[] {
+  return Object.keys(PROVIDER_BASES);
+}
+
+/**
+ * Parse `export KEY=value` / `KEY=value` lines from a dotenv file.
+ * Strips surrounding quotes; ignores comments. Never throws.
+ */
+async function readEnvFile(file: string): Promise<Record<string, string>> {
+  const out: Record<string, string> = {};
+  if (!existsSync(file)) return out;
+  let text = "";
+  try {
+    text = await Bun.file(file).text();
+  } catch {
+    return out;
+  }
+  for (const lineRaw of text.split("\n")) {
+    const line = lineRaw.trim();
+    if (line === "" || line.startsWith("#")) continue;
+    const m = line.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/);
+    if (!m) continue;
+    let val = m[2]!.trim();
+    if (
+      val.length >= 2 &&
+      ((val.startsWith('"') && val.endsWith('"')) ||
+        (val.startsWith("'") && val.endsWith("'")))
+    ) {
+      val = val.slice(1, -1);
+    }
+    out[m[1]!] = val;
+  }
+  return out;
+}
+
+/** Resolve the dotenv path: explicit $SKILLSHELF_ENV_FILE, else ./.env if present. */
+function resolveEnvFilePath(env: NodeJS.ProcessEnv, override?: string): string | null {
+  const explicit = override?.trim() || env.SKILLSHELF_ENV_FILE?.trim();
+  if (explicit) return explicit;
+  return existsSync(".env") ? ".env" : null;
+}
+
+/** First non-empty trimmed value, or "". */
+function firstNonEmpty(...vals: (string | undefined)[]): string {
+  for (const v of vals) {
+    if (v && v.trim() !== "") return v.trim();
+  }
+  return "";
+}
+
+/**
+ * Resolve an LLM config for an OpenAI-compatible endpoint.
+ *
+ * Precedence (highest first): explicit opts (from CLI flags) > env vars
+ * (SKILLSHELF_LLM_* then OPENAI_*) > dotenv file. `opts.provider` only seeds a
+ * default base URL; the key is always resolved from env/dotenv. Returns an
+ * `error` string instead of throwing when no API key can be resolved.
+ */
+export async function resolveProvider(
+  provider: string | undefined,
+  opts: {
+    base?: string;
+    model?: string;
+    env?: NodeJS.ProcessEnv;
+    envFile?: string;
+  } = {},
+): Promise<{ config: ProviderConfig } | { error: string }> {
+  const env = opts.env ?? process.env;
+
+  let presetBase = "";
+  let providerName = "custom";
+  if (provider !== undefined && provider !== "") {
+    if (!(provider in PROVIDER_BASES)) {
+      return {
+        error: `unknown provider "${provider}". known: ${knownProviders().join(", ")}`,
+      };
+    }
+    providerName = provider;
+    presetBase = PROVIDER_BASES[provider]!;
+  }
+
+  const filePath = resolveEnvFilePath(env, opts.envFile);
+  const fileEnv = filePath ? await readEnvFile(filePath) : {};
+
+  // base: flag > SKILLSHELF_LLM_BASE_URL > OPENAI_BASE_URL > provider preset > default
+  const base = firstNonEmpty(
+    opts.base,
+    env.SKILLSHELF_LLM_BASE_URL,
+    fileEnv.SKILLSHELF_LLM_BASE_URL,
+    env.OPENAI_BASE_URL,
+    fileEnv.OPENAI_BASE_URL,
+    presetBase,
+    DEFAULT_BASE,
+  );
+
+  // key: SKILLSHELF_LLM_API_KEY > OPENAI_API_KEY (env then dotenv)
+  const apiKey = firstNonEmpty(
+    env.SKILLSHELF_LLM_API_KEY,
+    fileEnv.SKILLSHELF_LLM_API_KEY,
+    env.OPENAI_API_KEY,
+    fileEnv.OPENAI_API_KEY,
+  );
+
+  // model: flag > SKILLSHELF_LLM_MODEL > OPENAI_MODEL > default
+  const model = firstNonEmpty(
+    opts.model,
+    env.SKILLSHELF_LLM_MODEL,
+    fileEnv.SKILLSHELF_LLM_MODEL,
+    env.OPENAI_MODEL,
+    fileEnv.OPENAI_MODEL,
+    DEFAULT_MODEL,
+  );
+
+  if (apiKey === "") {
+    return {
+      error:
+        "missing API key. Set SKILLSHELF_LLM_API_KEY (or OPENAI_API_KEY) in the " +
+        "environment or a dotenv file ($SKILLSHELF_ENV_FILE, default ./.env).",
+    };
+  }
+
+  return { config: { base, apiKey, model, name: providerName } };
+}
+
+/** Extract the first balanced JSON object from a possibly-fenced string. */
+function extractJsonObject(text: string): string | null {
+  const trimmed = text.trim();
+  // strip ```json fences if present
+  const fenced = trimmed.match(/```(?:json)?\s*([\s\S]*?)```/);
+  const candidate = fenced ? fenced[1]!.trim() : trimmed;
+  const start = candidate.indexOf("{");
+  if (start === -1) return null;
+  let depth = 0;
+  let inStr = false;
+  let esc = false;
+  for (let i = start; i < candidate.length; i++) {
+    const c = candidate[i]!;
+    if (inStr) {
+      if (esc) esc = false;
+      else if (c === "\\") esc = true;
+      else if (c === '"') inStr = false;
+      continue;
+    }
+    if (c === '"') inStr = true;
+    else if (c === "{") depth++;
+    else if (c === "}") {
+      depth--;
+      if (depth === 0) return candidate.slice(start, i + 1);
+    }
+  }
+  return null;
+}
+
+/**
+ * POST the corpus to the endpoint and parse a strict-JSON proposal back.
+ * Requests JSON via response_format; falls back to brace-extraction if the
+ * model wraps the JSON in prose/fences.
+ */
+export async function inferViaApi(
+  corpus: InferenceCorpus,
+  config: ProviderConfig,
+  opts: { timeoutMs?: number; fetchImpl?: typeof fetch } = {},
+): Promise<{ proposal: InferenceProposal } | { error: string }> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const userPayload = JSON.stringify({ schema: PROPOSAL_SCHEMA, corpus });
+  const body = {
+    model: config.model,
+    temperature: 0,
+    response_format: { type: "json_object" },
+    messages: [
+      {
+        role: "system",
+        content:
+          INFER_INSTRUCTION +
+          " Respond with ONLY the JSON object — no markdown, no commentary.",
+      },
+      { role: "user", content: userPayload },
+    ],
+  };
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), opts.timeoutMs ?? 120_000);
+  let res: Response;
+  try {
+    res = await fetchImpl(`${config.base.replace(/\/+$/, "")}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        authorization: `Bearer ${config.apiKey}`,
+      },
+      body: JSON.stringify(body),
+      signal: controller.signal,
+    });
+  } catch (e) {
+    clearTimeout(timeout);
+    const msg = e instanceof Error ? e.message : String(e);
+    return { error: `LLM request failed: ${msg}` };
+  }
+  clearTimeout(timeout);
+
+  if (!res.ok) {
+    let detail = "";
+    try {
+      detail = (await res.text()).slice(0, 500);
+    } catch {
+      /* ignore */
+    }
+    return { error: `LLM endpoint returned ${res.status} ${res.statusText}: ${detail}` };
+  }
+
+  let data: unknown;
+  try {
+    data = await res.json();
+  } catch (e) {
+    const msg = e instanceof Error ? e.message : String(e);
+    return { error: `LLM response was not JSON: ${msg}` };
+  }
+
+  const content = extractContent(data);
+  if (content === null) {
+    return { error: "LLM response had no message content" };
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    const obj = extractJsonObject(content);
+    if (!obj) return { error: "could not parse JSON from LLM content" };
+    try {
+      parsed = JSON.parse(obj);
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e);
+      return { error: `could not parse extracted JSON: ${msg}` };
+    }
+  }
+
+  return { proposal: normalizeProposal(parsed) };
+}
+
+/** Pull `choices[0].message.content` from an OpenAI-compatible response. */
+function extractContent(data: unknown): string | null {
+  if (!data || typeof data !== "object") return null;
+  const choices = (data as Record<string, unknown>).choices;
+  if (!Array.isArray(choices) || choices.length === 0) return null;
+  const first = choices[0] as Record<string, unknown>;
+  const message = first.message as Record<string, unknown> | undefined;
+  const content = message?.content;
+  if (typeof content === "string") return content;
+  // some endpoints return content as an array of parts
+  if (Array.isArray(content)) {
+    const text = content
+      .map((p) => (p && typeof p === "object" ? String((p as Record<string, unknown>).text ?? "") : ""))
+      .join("");
+    return text || null;
+  }
+  return null;
+}

package/src/cli.ts

@@ -0,0 +1,127 @@
+#!/usr/bin/env bun
+// skillshelf CLI entry / router.
+//
+// Responsibilities:
+//   - parse argv[0] as the subcommand (`skl <cmd> ...`)
+//   - build the execution Ctx once via config.loadContext()
+//   - dispatch to the matching command module's run(restArgv, ctx)
+//   - `skl` (no args) / `skl help` -> print help listing every command's meta
+//   - clean non-zero exit on unknown command
+//
+// Both `bun run src/cli.ts <cmd>` and the installed bin `skl <cmd>` route here
+// (package.json bin -> ./src/cli.ts).
+
+import type { CommandModule, Ctx } from "./types.ts";
+import { loadContext } from "./config.ts";
+
+import * as search from "./commands/search.ts";
+import * as ls from "./commands/ls.ts";
+import * as status from "./commands/status.ts";
+import * as show from "./commands/show.ts";
+import * as index from "./commands/index.ts";
+import * as use from "./commands/use.ts";
+import * as drop from "./commands/drop.ts";
+import * as init from "./commands/init.ts";
+import * as add from "./commands/add.ts";
+import * as outdated from "./commands/outdated.ts";
+import * as update from "./commands/update.ts";
+import * as infer from "./commands/infer.ts";
+import * as newCmd from "./commands/new.ts";
+
+// Registration order = display order in help.
+const MODULES: CommandModule[] = [
+  search,
+  ls,
+  status,
+  show,
+  use,
+  drop,
+  add,
+  outdated,
+  update,
+  init,
+  newCmd,
+  index,
+  infer,
+];
+
+const COMMANDS = new Map<string, CommandModule>();
+for (const mod of MODULES) {
+  COMMANDS.set(mod.meta.name, mod);
+}
+
+function helpText(): string {
+  const lines: string[] = [];
+  lines.push("skl — skillshelf: agent-first skill registry + manager");
+  lines.push("");
+  lines.push("Usage: skl <command> [args] [--json]");
+  lines.push("");
+  lines.push("Commands:");
+  const width = Math.max(...MODULES.map((m) => m.meta.name.length));
+  for (const mod of MODULES) {
+    lines.push(`  ${mod.meta.name.padEnd(width)}  ${mod.meta.summary}`);
+  }
+  lines.push("");
+  lines.push("Run `skl help <command>` for command-specific usage.");
+  return lines.join("\n");
+}
+
+function commandHelp(mod: CommandModule): string {
+  return [`${mod.meta.name} — ${mod.meta.summary}`, `Usage: ${mod.meta.usage}`].join("\n");
+}
+
+async function main(rawArgv: string[]): Promise<number> {
+  // No subcommand -> help.
+  const first = rawArgv[0];
+  if (first === undefined || first === "--help" || first === "-h") {
+    console.log(helpText());
+    return 0;
+  }
+
+  if (first === "help") {
+    const target = rawArgv[1];
+    if (target !== undefined) {
+      const mod = COMMANDS.get(target);
+      if (mod) {
+        console.log(commandHelp(mod));
+        return 0;
+      }
+      console.error(`Unknown command: ${target}`);
+      console.error("");
+      console.error(helpText());
+      return 1;
+    }
+    console.log(helpText());
+    return 0;
+  }
+
+  const mod = COMMANDS.get(first);
+  if (!mod) {
+    console.error(`Unknown command: ${first}`);
+    console.error("");
+    console.error(helpText());
+    return 1;
+  }
+
+  let ctx: Ctx;
+  try {
+    ctx = await loadContext();
+  } catch (err) {
+    console.error(`Failed to initialize skillshelf: ${err instanceof Error ? err.message : String(err)}`);
+    return 1;
+  }
+
+  // argv passed to the command = everything AFTER the subcommand.
+  const rest = rawArgv.slice(1);
+  try {
+    const code = await mod.run(rest, ctx);
+    return typeof code === "number" ? code : 0;
+  } catch (err) {
+    // Commands are contracted not to throw, but guard the router regardless.
+    ctx.error(`skl ${first}: ${err instanceof Error ? err.message : String(err)}`);
+    return 1;
+  }
+}
+
+const code = await main(process.argv.slice(2));
+process.exit(code);

package/src/commands/add.ts

@@ -0,0 +1,222 @@
+// skl add <src> — install a third-party skill into the library.
+//
+// Flow:
+//   1. parse <src> (github:owner/repo[/path] or a bare registry name)
+//   2. shell out (git / `skills`) to DOWNLOAD only — never reinvent fetching
+//   3. copy the skill dir into the library under its primary-domain folder
+//   4. write a provenance lockfile entry (source + ref + channel + installedAt)
+//   5. create an empty overlay (<name>.shelf.json) so taxonomy survives updates
+//   6. call the inference tagging hook if one is available, else leave untagged
+//
+// Read-only commands take --json; add is a write, but still emits a --json
+// summary on success for agent consumption.
+
+import { join, basename } from "node:path";
+import { existsSync } from "node:fs";
+import type { Ctx, Skill, LockEntry } from "../types.ts";
+import {
+  parseSource,
+  fetchSource,
+  copySkillDir,
+  cleanupStaging,
+  readSkillBody,
+} from "../core/fetch.ts";
+import { parseFrontmatter } from "../lib/frontmatter.ts";
+import { hashContent } from "../core/crawl.ts";
+import { recordEntry } from "../core/provenance.ts";
+import { writeOverlay } from "../core/overlay.ts";
+import { ensureDir } from "../lib/fs.ts";
+
+export const meta = {
+  name: "add",
+  summary: "Install a third-party skill (github:/registry), record provenance, tag",
+  usage: "skl add <src> [--domain <d>] [--name <slug>] [--no-infer] [--force] [--json]",
+} as const;
+
+interface Flags {
+  json: boolean;
+  domain: string | null;
+  name: string | null;
+  infer: boolean;
+  force: boolean;
+  src: string | null;
+}
+
+function parseFlags(argv: string[]): Flags {
+  const f: Flags = { json: false, domain: null, name: null, infer: true, force: false, src: null };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]!;
+    if (a === "--json") f.json = true;
+    else if (a === "--no-infer") f.infer = false;
+    else if (a === "--force") f.force = true;
+    else if (a === "--domain") f.domain = argv[++i] ?? null;
+    else if (a === "--name") f.name = argv[++i] ?? null;
+    else if (a === "--domain=" || a.startsWith("--domain=")) f.domain = a.slice("--domain=".length);
+    else if (a.startsWith("--name=")) f.name = a.slice("--name=".length);
+    else if (!a.startsWith("-") && f.src === null) f.src = a;
+  }
+  return f;
+}
+
+/** Slug from frontmatter `name`, else the source dir name. */
+async function deriveName(skillDir: string, override: string | null): Promise<string> {
+  if (override && override.trim() !== "") return override.trim();
+  const body = await readSkillBody(skillDir);
+  const { data } = parseFrontmatter(body);
+  if (typeof data.name === "string" && data.name.trim() !== "") return data.name.trim();
+  return basename(skillDir);
+}
+
+/**
+ * Optionally run an AI inference tagging pass over the freshly-installed skill.
+ *
+ * The taxonomy pass (`skl infer`) is corpus-based and lives in the inference
+ * adapters; there is no committed single-skill tagging hook in the public API.
+ * To stay decoupled (and to leave the skill *untagged* rather than fail when no
+ * hook is present), we look for an OPTIONAL convention module that may export a
+ * `tagSkill(skill) => string[]`. The specifier is built at runtime so a missing
+ * module degrades gracefully instead of becoming a static import error.
+ *
+ * On any failure the skill stays untagged (empty overlay) — a valid state.
+ * Returns the domains written, if any.
+ */
+async function maybeInferTags(skill: Skill): Promise<string[] | null> {
+  const candidates = ["../core/infer.ts", "../adapters/inference/tag.ts"];
+  for (const rel of candidates) {
+    try {
+      // Non-literal specifier: keeps a missing optional module from becoming a
+      // static import/resolution error; degrades to "untagged" at runtime.
+      const spec: string = rel;
+      const mod: unknown = await import(spec).catch(() => null);
+      if (!mod || typeof mod !== "object") continue;
+      const hook = (mod as Record<string, unknown>).tagSkill;
+      if (typeof hook !== "function") continue;
+      const result = await (hook as (s: Skill) => Promise<string[] | null>)(skill);
+      if (Array.isArray(result)) {
+        return result.filter((d) => typeof d === "string" && d.trim() !== "");
+      }
+      return null;
+    } catch {
+      /* try next candidate / leave untagged */
+    }
+  }
+  return null;
+}
+
+export async function run(argv: string[], ctx: Ctx): Promise<number> {
+  const flags = parseFlags(argv);
+  if (!flags.src) {
+    ctx.error("usage:", meta.usage);
+    return 1;
+  }
+
+  const parsed = parseSource(flags.src);
+
+  // 1+2. DOWNLOAD into a staging dir (shell out only).
+  const fetched = await fetchSource(parsed);
+  if (!fetched.ok) {
+    await cleanupStaging(fetched.staging);
+    ctx.error("add: download failed:", fetched.error);
+    return 1;
+  }
+
+  try {
+    // 3. Determine destination in the library.
+    const name = await deriveName(fetched.skillDir, flags.name);
+    const domainFolder = flags.domain && flags.domain.trim() !== "" ? flags.domain.trim() : null;
+    const destDir = domainFolder
+      ? join(ctx.config.libraryPath, domainFolder, name)
+      : join(ctx.config.libraryPath, name);
+
+    if (existsSync(destDir) && !flags.force) {
+      ctx.error(
+        `add: ${name} already exists at ${destDir} (use --force to overwrite, or skl update ${name} to re-pull)`,
+      );
+      return 1;
+    }
+
+    await ensureDir(domainFolder ? join(ctx.config.libraryPath, domainFolder) : ctx.config.libraryPath);
+    await copySkillDir(fetched.skillDir, destDir);
+
+    // 4. Provenance lockfile entry. Record the installed body hash so a later
+    //    `skl update` can tell a user hand-edit apart from upstream moving forward.
+    const installedBody = parseFrontmatter(await readSkillBody(fetched.skillDir)).body;
+    // git: sources already encode their subpath as `#subpath` in fetched.source;
+    // only github sources use the `@subpath` suffix convention here.
+    const subSuffix = parsed.subpath && parsed.channel !== "git" ? `@${parsed.subpath}` : "";
+    const entry: LockEntry = {
+      name,
+      source: `${fetched.source}${subSuffix}`,
+      ref: fetched.ref,
+      channel: fetched.channel,
+      installedAt: new Date().toISOString(),
+      localEdits: false,
+      installedHash: hashContent(installedBody),
+    };
+    await recordEntry(ctx.config.libraryPath, entry);
+
+    // 5. Empty overlay so taxonomy survives future updates.
+    const installed: Skill = {
+      name,
+      description: "",
+      primaryDomain: domainFolder,
+      domains: domainFolder ? [domainFolder] : [],
+      path: destDir,
+      bodyPath: join(destDir, "SKILL.md"),
+      refFiles: [],
+      source: {
+        source: entry.source,
+        ref: entry.ref,
+        channel: entry.channel,
+        installedAt: entry.installedAt,
+        localEdits: false,
+      },
+      retired: false,
+      mirrorOf: null,
+      contentHash: "",
+    };
+    const overlayPathStr = join(destDir, `${name}.shelf.json`);
+    if (!existsSync(overlayPathStr)) {
+      await writeOverlay(installed, domainFolder ? { domains: [domainFolder] } : {});
+    }
+
+    // 6. Inference tagging hook (best-effort, leaves untagged if unavailable).
+    let inferredDomains: string[] | null = null;
+    if (flags.infer) {
+      inferredDomains = await maybeInferTags(installed);
+      if (inferredDomains && inferredDomains.length > 0) {
+        await writeOverlay(installed, { domains: inferredDomains });
+      }
+    }
+
+    const summary = {
+      ok: true,
+      name,
+      path: destDir,
+      source: entry.source,
+      ref: entry.ref,
+      channel: entry.channel,
+      installedAt: entry.installedAt,
+      tagged: Boolean(inferredDomains && inferredDomains.length > 0),
+      domains: inferredDomains ?? (domainFolder ? [domainFolder] : []),
+    };
+
+    if (flags.json) {
+      ctx.json(summary);
+    } else {
+      ctx.log(`added ${name}`);
+      ctx.log(`  path:    ${destDir}`);
+      ctx.log(`  source:  ${entry.source}`);
+      ctx.log(`  ref:     ${entry.ref || "(unknown)"}`);
+      ctx.log(`  channel: ${entry.channel}`);
+      if (summary.tagged) ctx.log(`  domains: ${summary.domains.join(", ")}`);
+      else ctx.log(`  domains: (untagged — run \`skl infer\` to assign)`);
+    }
+    return 0;
+  } catch (err) {
+    ctx.error("add: failed:", err instanceof Error ? err.message : String(err));
+    return 1;
+  } finally {
+    await cleanupStaging(fetched.staging);
+  }
+}