rovecode 0.4.0-beta.3 → 0.4.0

package/src/market/clone.ts

@@ -0,0 +1,72 @@
+/** Cloning at a ref, and the one subtle decision in it.
+ *
+ *  `--ref` takes a branch, a tag or a commit, and git needs a DIFFERENT command for the last one:
+ *  `clone --depth 1 --branch <x>` works for a branch or a tag and fails for a bare commit, so a commit has
+ *  to go through `init` + `fetch --depth 1 <url> <sha>` + `checkout FETCH_HEAD`.
+ *
+ *  We do not decide which one `<x>` is by looking at it. `/^[0-9a-f]{7,40}$/` is a guess, and a wrong guess
+ *  here installs the wrong tree silently: `abcdef` is a perfectly legal branch name, and a repository that
+ *  has one would be pinned to a commit that merely shares the spelling. Git's answer is a fact, so we ask
+ *  git: try the branch form, and use the commit form only when that fails.
+ *
+ *  ORDER IS A DECISION, not a side effect. When a name is BOTH a branch and the short id of some commit,
+ *  the branch wins, because someone typing `--ref` almost always means a name. That is why the branch form
+ *  is tried first, and the manifest records `resolvedBy` so the question "was this a branch or a commit?"
+ *  has an answer afterwards rather than a shrug.
+ *
+ *  One deliberate refusal: fetching a bare SHA needs `uploadpack.allowReachableSHA1InWant` on the server,
+ *  and plenty of hosts leave it off. When that is refused we STOP and say so. The alternative — quietly
+ *  dropping `--depth` and cloning the entire history — turns a pin into a very long download the person
+ *  did not ask for, and they would have no idea why. */
+
+export type Spawn = (cmd: string[], cwd: string) => Promise<{ code: number; stderr: string }>;
+
+export type ResolvedBy = "branch" | "commit" | "default";
+
+export interface CloneResult {
+  ok: true;
+  /** which form of the command actually produced the tree */
+  resolvedBy: ResolvedBy;
+}
+export type CloneOutcome = CloneResult | { ok: false; error: string };
+
+/** the last line of git's complaint, which is the part a person can act on */
+const lastLine = (stderr: string): string => stderr.trim().split("\n").at(-1)?.trim() ?? "";
+
+/** a server that will not serve a bare commit says so in one of a few ways; all of them mean the same
+ *  thing to us, and none of them should be answered by fetching more */
+function refusesBareSha(stderr: string): boolean {
+  return /allow.*sha1.*in.?want|not our ref|unadvertised object|Server does not allow request for unadvertised object/i.test(stderr);
+}
+
+/** Clone `url` into `dir` (which must exist and be empty), optionally at `ref`. */
+/** `--` before every positional, on purpose. git parses options wherever it finds them, so a value that
+ *  begins with a dash is read as an option no matter which slot it sits in: a ref or a URL spelled
+ *  `--upload-pack=...` becomes an instruction rather than a name. Catalog data cannot reach here in that
+ *  shape (`url()` filters it), but `market install <git-url>` and `--ref` come straight from a command
+ *  line. The marker costs one argument and says "what follows is a value", which is the claim we mean. */
+export async function cloneAtRef(spawn: Spawn, url: string, dir: string, ref?: string): Promise<CloneOutcome> {
+  if (ref === undefined || ref.trim() === "") {
+    const r = await spawn(["git", "clone", "--depth", "1", "--quiet", "--", url, "."], dir);
+    return r.code === 0 ? { ok: true, resolvedBy: "default" } : { ok: false, error: `git clone failed (exit ${r.code})${lastLine(r.stderr) ? `: ${lastLine(r.stderr)}` : ""}` };
+  }
+
+  // 1. a branch or a tag — the common case, and the winner when a name is both
+  const branch = await spawn(["git", "clone", "--depth", "1", "--quiet", "--branch", ref, "--", url, "."], dir);
+  if (branch.code === 0) return { ok: true, resolvedBy: "branch" };
+
+  // 2. a commit. Not a fallback for "anything went wrong": if the failure was the network or auth, the
+  //    fetch will fail the same way and we report THAT, not a misleading "no such ref".
+  const init = await spawn(["git", "init", "--quiet"], dir);
+  if (init.code !== 0) return { ok: false, error: `git init failed (exit ${init.code})${lastLine(init.stderr) ? `: ${lastLine(init.stderr)}` : ""}` };
+  const fetch = await spawn(["git", "fetch", "--depth", "1", "--quiet", "--", url, ref], dir);
+  if (fetch.code !== 0) {
+    if (refusesBareSha(fetch.stderr)) {
+      return { ok: false, error: `${url} will not serve the single commit ${ref} (the server has uploadpack.allowReachableSHA1InWant off) — give a branch or tag instead, or ask the host to enable it` };
+    }
+    return { ok: false, error: `"${ref}" is not a branch or tag there, and fetching it as a commit failed (exit ${fetch.code})${lastLine(fetch.stderr) ? `: ${lastLine(fetch.stderr)}` : ""}` };
+  }
+  const checkout = await spawn(["git", "checkout", "--quiet", "FETCH_HEAD"], dir);
+  if (checkout.code !== 0) return { ok: false, error: `fetched ${ref} but could not check it out (exit ${checkout.code})${lastLine(checkout.stderr) ? `: ${lastLine(checkout.stderr)}` : ""}` };
+  return { ok: true, resolvedBy: "commit" };
+}

package/src/market/context-cost.ts

@@ -0,0 +1,121 @@
+/** What installing this item costs you in context, said before the yes.
+ *
+ *  THE NUMBER IS TWO NUMBERS, and collapsing them would be the whole feature's undoing. rovecode's skills
+ *  are progressive disclosure: `buildSkillsIndex` puts one line per skill in the system prompt — the name,
+ *  the version and the clipped description — and the SKILL.md body enters context only when the model calls
+ *  `skill_view`. Measured across the 19 skills that ship: the index line averages 79 tokens, the body
+ *  averages 2,288. A single "costs ~2.3k tokens per turn" would be 29× the truth for a skill nobody opens,
+ *  and a reader who checked once would never trust the line again.
+ *
+ *  So: `perTurn` is what every request carries, `whenUsed` is what arrives if the model opens it.
+ *
+ *  WHAT CANNOT BE KNOWN is said instead of guessed:
+ *    - an MCP server publishes its tool schemas after it connects. Before that there is no number, and
+ *      inventing one would spend this line's credibility on the day it shipped.
+ *    - a plugin's entry module contributes tools whose schemas we would have to LOAD ITS CODE to read,
+ *      which is exactly what a plan must not do. Its skills and commands directories are files, so those
+ *      are measured.
+ *
+ *  THE ESTIMATE IS SCALED, NOT CAVEATED. `countTokens` is gpt-tokenizer's o200k_base and Anthropic's
+ *  tokenizer is not public, so for a Claude model the raw count is simply wrong — measured against
+ *  Anthropic's own count_tokens endpoint over all 19 skill bodies that ship, it reads 1.41×–1.79× low
+ *  (mean 1.59×). Rather than describe that in prose, the numbers are multiplied by `tokenScaleFor`, which
+ *  holds a measured factor per model generation. An earlier draft of this file said "reads roughly a third
+ *  low on Anthropic models"; that was already wrong for Claude 4.5, where the factor is 1.21×, and a
+ *  sentence like it is wrong again every time a model ships.
+ *
+ *  When no model is named the numbers are left unscaled and the line says so. When the provider's own
+ *  tokenizer IS o200k — OpenAI — there is nothing to scale and the line says that instead. A model nobody
+ *  has measured is never given a neighbour's factor.
+ */
+
+import { tokenScaleFor, type TokenScale } from "../core/token-scale.ts";
+import { countTokens } from "../core/usage.ts";
+import { INDEX_PROMPT_LIMIT } from "../skills/tools.ts";
+import type { MarketItem } from "./types.ts";
+
+export interface ContextCost {
+  /** tokens added to EVERY request while this is installed */
+  perTurn: number;
+  /** tokens added only when the model opens it; absent when there is no such body */
+  whenUsed?: number;
+  /** the tokenizer the numbers came from — named because it is not the model's */
+  tokenizer: "o200k_base";
+  /** a part of the cost that cannot be known before installing, and why */
+  unknown?: string;
+  /** installing this would push the skill count past the index limit; see `INDEX_PROMPT_LIMIT` */
+  overIndexLimit?: boolean;
+  /** how wrong the estimate is for the model that will carry it, when a measurement exists for it */
+  scale?: TokenScale;
+}
+
+export interface ContextCostOptions {
+  /** how many skills are already installed, so the index-limit warning can be right */
+  installedSkills?: number;
+  /** the model that will actually carry this, so the estimate can be scaled by a MEASURED factor rather
+   *  than described with a hand-written one. Omit it and the line says the number is unscaled. */
+  model?: { provider: string; model: string };
+}
+
+/** The system-prompt line `buildSkillsIndex` renders, rebuilt from catalog fields so the count describes
+ *  what would actually be in the prompt rather than an approximation of it (skills/tools.ts
+ *  renderIndexLine). The description is already clipped to 60 characters by the skill reader. */
+const indexLine = (item: MarketItem): string =>
+  `- ${item.id}${item.version ? ` (v${item.version})` : ""}: ${item.description}`;
+
+export function contextCostOf(item: MarketItem, opts: ContextCostOptions = {}): ContextCost | undefined {
+  const install = item.install;
+
+  if (install.kind === "mcp") {
+    // Deliberately no numbers. A server's tools are whatever it reports on connect.
+    return { perTurn: 0, tokenizer: "o200k_base",
+             unknown: "an MCP server's tools are only known once it connects, so its context cost cannot be measured before you install it" };
+  }
+
+  if (install.kind === "skill") {
+    const body = item.docs?.body;
+    const cost: ContextCost = { perTurn: countTokens(indexLine(item)), tokenizer: "o200k_base" };
+    if (opts.model !== undefined) cost.scale = tokenScaleFor(opts.model);
+    if (body !== undefined) cost.whenUsed = countTokens(body);
+    else cost.unknown = "the SKILL.md is not carried in the catalog, so only the index line can be measured here";
+    const installed = opts.installedSkills;
+    if (installed !== undefined && installed + 1 > INDEX_PROMPT_LIMIT) cost.overIndexLimit = true;
+    return cost;
+  }
+
+  // A plugin, and the answer is nothing measurable — for two separate reasons, both worth stating plainly
+  // rather than papering over with a number:
+  //   its entry module registers tools whose schemas need the code LOADED to read, which a plan must not do;
+  //   its commands and skills are files inside the repository, and the catalog carries the plugin's README,
+  //   not those files — measuring them would mean cloning during a preview.
+  return { perTurn: 0, tokenizer: "o200k_base",
+           unknown: "a plugin's context cost cannot be measured from the catalog: its tools need the code loaded to read, and its commands and skills are files that are only fetched when you install it" };
+}
+
+const n = (t: number): string => (t >= 1000 ? `${(t / 1000).toFixed(1)}k` : String(t));
+
+/** The plan's line(s). Returns [] when there is nothing worth saying. */
+export function contextCostLines(c: ContextCost | undefined): string[] {
+  if (c === undefined) return [];
+  const lines: string[] = [];
+
+  if (c.perTurn > 0 || c.whenUsed !== undefined) {
+    // When a measurement exists for the model in play, the numbers are SCALED BY IT rather than described
+    // with a hand-written caveat. "Reads roughly a third low on Anthropic models" was wrong the moment
+    // token-scale.ts had two Anthropic generations in it: 1.59x on Claude 5 and 1.21x on 4.5. A sentence
+    // that has to be rewritten every time a model ships is a sentence that will be wrong between ships.
+    const k = c.scale?.measured === true ? c.scale.scale : 1;
+    const turn = `~${n(Math.round(c.perTurn * k))} tokens every turn`;
+    const used = c.whenUsed !== undefined ? `, ~${n(Math.round(c.whenUsed * k))} more when the model opens it` : "";
+    lines.push(`${turn}${used}`);
+    if (c.scale === undefined) lines.push(`counted with o200k_base, which is not every model's tokenizer — unscaled here because no model was named`);
+    else if (!c.scale.measured) lines.push(`counted with o200k_base — ${c.scale.note}`);
+    else if (k === 1) lines.push(`counted with o200k_base — ${c.scale.note}`);
+    else lines.push(`o200k_base scaled ${k}× for this model — ${c.scale.note}`);
+  }
+  if (c.overIndexLimit) {
+    lines.push(`past ${INDEX_PROMPT_LIMIT} skills the index leaves the prompt entirely and the model lists skills with a tool instead — this changes the cost of every skill you have, not just this one`);
+  }
+  if (c.unknown !== undefined) lines.push(c.unknown);
+  return lines;
+}

package/src/market/digest.ts

@@ -0,0 +1,106 @@
+/** What landed on disk, as one number — so "has this changed since I installed it?" has an answer.
+ *
+ *  `--ref` pins what we ASK for; a digest is what we GOT. The two answer different questions and the second
+ *  is the one that survives: a commit id says the remote pointed there at fetch time, a digest says the
+ *  bytes on this machine are still the bytes that arrived. Nobody in this space signs anything, so a
+ *  recorded digest is the honest ceiling — it detects drift, it does not prove provenance, and the wording
+ *  everywhere says the former.
+ *
+ *  Over a FOLDER, the hash covers the relative path and the content of every file, in a sorted order, so it
+ *  is stable across platforms and independent of the order a directory happens to list. `.git` is excluded:
+ *  it is fetch bookkeeping, it differs between a clone and a copy of the same tree, and including it would
+ *  make every digest disagree with itself for no reason a person could act on.
+ *
+ *  Line endings are NOT normalised. A file that arrives with CRLF and later has LF is genuinely different
+ *  content, and on a repository that has been through a Windows checkout that difference is exactly the
+ *  kind of thing someone would want reported rather than hidden. */
+
+import { createHash } from "node:crypto";
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+
+export interface Digest {
+  algo: "sha256";
+  /** hex */
+  value: string;
+  /** how many files went into it — a cheap sanity check when two digests differ */
+  files: number;
+}
+
+/** Every file under `dir`, relative and slash-separated, sorted. Excludes `.git`. */
+function walk(dir: string): string[] {
+  const out: string[] = [];
+  const stack = [dir];
+  while (stack.length > 0) {
+    const here = stack.pop()!;
+    let names: string[];
+    try { names = readdirSync(here); } catch { continue; }
+    for (const name of names) {
+      if (name === ".git") continue;
+      const full = join(here, name);
+      let st;
+      try { st = statSync(full); } catch { continue; }
+      // a symlink is never followed: install refuses them, and following one here would hash something
+      // outside the folder we claim to be describing
+      if (st.isDirectory()) { stack.push(full); continue; }
+      if (st.isFile()) out.push(relative(dir, full).split(sep).join("/"));
+    }
+  }
+  return out.sort();
+}
+
+/** The digest of a file or a folder. Undefined when the path is not there — a missing thing has no
+ *  digest, and pretending it has one of an empty folder would make "deleted" and "emptied" look alike. */
+export function digestOf(path: string): Digest | undefined {
+  if (!existsSync(path)) return undefined;
+  const hash = createHash("sha256");
+  let files = 0;
+  try {
+    if (statSync(path).isFile()) {
+      hash.update(readFileSync(path));
+      files = 1;
+    } else {
+      for (const rel of walk(path)) {
+        // the PATH goes in too: moving a file's content to another name is a change
+        hash.update(rel);
+        hash.update("\0");
+        hash.update(readFileSync(join(path, rel)));
+        hash.update("\0");
+        files += 1;
+      }
+    }
+  } catch { return undefined; }
+  return { algo: "sha256", value: hash.digest("hex"), files };
+}
+
+export type VerifyState =
+  /** the bytes are what they were */
+  | { state: "unchanged"; digest: Digest }
+  /** it is still installed, and it is not what it was */
+  | { state: "changed"; recorded: Digest; now: Digest }
+  /** it was installed and is no longer there */
+  | { state: "missing"; recorded: Digest }
+  /** there is nothing recorded to compare against; not a failure */
+  | { state: "unrecorded"; now?: Digest }
+  /** the kind carries no digest by design (an MCP entry is a line in a shared file) */
+  | { state: "not-applicable"; why: string };
+
+/** Compare what is on disk with what was recorded. Pure over its inputs apart from reading the path. */
+export function verifyDigest(recorded: Digest | undefined, path: string | undefined): VerifyState {
+  if (path === undefined) return { state: "not-applicable", why: "this kind of install has no folder of its own to hash" };
+  const now = digestOf(path);
+  if (recorded === undefined) return now === undefined ? { state: "unrecorded" } : { state: "unrecorded", now };
+  if (now === undefined) return { state: "missing", recorded };
+  return now.value === recorded.value ? { state: "unchanged", digest: now } : { state: "changed", recorded, now };
+}
+
+/** One line for `market verify`, in the words the state deserves. */
+export function verifyLine(id: string, v: VerifyState): string {
+  switch (v.state) {
+    case "unchanged": return `${id}  unchanged (${v.digest.files} file${v.digest.files === 1 ? "" : "s"})`;
+    case "changed": return `${id}  CHANGED since install — recorded ${v.recorded.value.slice(0, 12)} (${v.recorded.files} files), now ${v.now.value.slice(0, 12)} (${v.now.files} files)`;
+    case "missing": return `${id}  gone from disk, but a record remains`;
+    case "unrecorded": return `${id}  no digest recorded — installed before rovecode kept one, or by hand`;
+    case "not-applicable": return `${id}  ${v.why}`;
+  }
+}

package/src/market/index.ts

@@ -0,0 +1,22 @@
+/** The market's public face. Every surface — the CLI, the sextant's /market, the site's build step —
+ *  imports from here and from nowhere deeper, so the internals can move without touching them.
+ *
+ *  Importing this module pulls in `registry.ts` and `install.ts`, which reach the MCP and plugin modules;
+ *  it is NOT free, so keep it off the boot path (cli/main.ts loads `market-cmd.ts` with a dynamic import
+ *  inside `case "market"`, and the TUI should reach it the same way, on first use). `types.ts` alone is
+ *  free — import that directly when all you need is a shape. */
+
+export type {
+  MarketKind, MarketSource, MarketScope, MarketEnv, InstallSpec, MarketItem,
+  InstalledState, MarketRow, SourceStatus, MarketResult, InstallPlanView, InstallOutcome, ItemDocs,
+} from "./types.ts";
+export { LIMITS, parseQualifiedId, qualify, itemLine } from "./types.ts";
+
+export type { RegistryDeps } from "./registry.ts";
+export { searchMarket, findItem, allItems, itemFromCatalog, itemFromMcp, CATALOG_FILES, CATALOG_DIR, MCP_DOCS_FILE } from "./registry.ts";
+
+export type { Resolution } from "./resolve.ts";
+export { resolveTarget, resolveDirect } from "./resolve.ts";
+
+export type { PlanOptions, RunDeps } from "./install.ts";
+export { planInstall, runInstall, installedState, withInstalled, removeItem, skillDir } from "./install.ts";