pinnace 0.0.0 → 0.2.0

package/src/site/site-management.ts

@@ -0,0 +1,241 @@
+/**
+ * First-class **site management** — the `site` namespace (spec user stories 4,
+ * 15; CONTEXT.md `gateway warming`). Sites are auto-discovered from MFS
+ * `/sites/*` (that is how `gateway warming`, IPNS republish, and `status` find
+ * them), so managing the sites a node serves = managing those MFS entries + the
+ * pins that back them. This module gives the operator explicit verbs for that
+ * lifecycle instead of leaving it an implicit side-effect of `deploy`:
+ *
+ *   - **list**   — enumerate `/sites/*` with each site's current CID and, when a
+ *                  same-`id` keystore key exists, its IPNS id.
+ *   - **remove** — `files/rm /sites/<id>` (the entry drops out of
+ *                  warm/republish/status auto-discovery) AND `pin/rm <cid>` so
+ *                  the content stops being served/announced and its storage is
+ *                  reclaimed.
+ *   - **add**    — place an already-imported `/ipfs/<cid>` into MFS
+ *                  `/sites/<id>` (mkdir parents / rm old / cp). See the
+ *                  DESIGN NOTE below on its relationship to `deploy`.
+ *
+ * Every verb speaks ONLY the Kubo RPC seam (MFS + pin endpoints), so the same
+ * core is usable both as a TypeScript API and behind the thin `pinnace site
+ * <verb>` CLI (CONTEXT.md `core vs cli`). Tests drive it through the recording
+ * `MockKuboApi`, never a live daemon (spec Testing Decisions).
+ *
+ * DESIGN NOTE (add vs deploy) — recorded per the task's "decide during build".
+ * `add` is a DISTINCT verb, not an alias over `deploy`. `deploy` (see
+ * `deploy-multi-target`) builds a fresh CAR, imports+pins it on every node, and
+ * THEN performs this MFS-placement step. `add` is exactly that final placement
+ * step in isolation: it takes an EXISTING CID (already imported/pinned, e.g. a
+ * known-good historical CID, or one imported out of band) and makes the node
+ * serve it under a site name, WITHOUT building or importing a CAR. Adding a site
+ * from an existing CID is a meaningful operation deploy does not cover (no fresh
+ * artifact), so it earns its own verb. Deploy is expected to REUSE
+ * {@link placeInMfs} for its placement step when it lands, keeping a single
+ * implementation of the mkdir/rm/cp sequence rather than forking it. This is
+ * NOT a new domain concept: `add` sits at the same layer as the other explicit
+ * client verbs and reuses the existing `/sites/<name>` MFS convention.
+ */
+import type {KuboRpcClient} from '../rpc/kubo-rpc-client.js';
+
+/** The three site-management verbs, under the `site` namespace. */
+export type SiteVerb = 'list' | 'remove' | 'add';
+
+/** The verbs, in a stable order (for help text / iteration). */
+export const SITE_VERBS: readonly SiteVerb[] = ['list', 'remove', 'add'];
+
+/** The MFS directory sites live under. */
+const DEFAULT_SITES_DIR = '/sites';
+
+/** One site as listed: its MFS `id`, current CID, and IPNS id if a key exists. */
+export interface SiteListing {
+	/** The site's single `id` (its MFS entry under `/sites/`). */
+	id: string;
+	/** The current content root CID (`files/stat --hash`). */
+	cid: string;
+	/** The IPNS id, if a same-`id` keystore key exists (ipfs-mode sites lack one). */
+	ipns?: string;
+}
+
+/** Inputs shared by every site-management verb. */
+interface SiteBaseInput {
+	/** The Kubo RPC client for the node whose sites are being managed. */
+	client: KuboRpcClient;
+	/** The MFS directory sites live under (default `/sites`). */
+	sitesDir?: string;
+}
+
+/** Inputs to {@link listSites}. */
+export type ListSitesInput = SiteBaseInput;
+
+/** Inputs to {@link removeSite}. */
+export interface RemoveSiteInput extends SiteBaseInput {
+	/** The site `id` (its MFS entry `/sites/<id>`) to remove. */
+	id: string;
+}
+
+/** The outcome of {@link removeSite}. */
+export interface RemoveSiteResult {
+	/** The site `id` that was removed. */
+	id: string;
+	/** The CID that backed it (undefined if it had no resolvable content). */
+	cid?: string;
+	/** True when the content was unpinned (storage reclaimed); false if unpin failed. */
+	unpinned: boolean;
+}
+
+/** Inputs to {@link addSite}. */
+export interface AddSiteInput extends SiteBaseInput {
+	/** The site `id` to expose it under (its MFS entry `/sites/<id>`). */
+	id: string;
+	/** The already-imported content root CID to place into MFS. */
+	cid: string;
+}
+
+/** The outcome of {@link addSite}. */
+export interface AddSiteResult {
+	/** The site `id` that was placed. */
+	id: string;
+	/** The CID that now backs it in MFS. */
+	cid: string;
+}
+
+/**
+ * **list** — enumerate the sites the node currently serves (MFS `/sites/*`),
+ * annotating each with its current CID (from `files/stat`) and, when a keystore
+ * key of the same name exists, its IPNS id (from `key/list -l`). A fresh box
+ * with no `/sites` dir yields an empty list, not an error.
+ */
+export async function listSites(input: ListSitesInput): Promise<SiteListing[]> {
+	const sitesDir = input.sitesDir ?? DEFAULT_SITES_DIR;
+	const entries = await lsSites(input.client, sitesDir);
+	if (entries.length === 0) return [];
+
+	const keys = await listKeys(input.client);
+	const sites: SiteListing[] = [];
+	for (const id of entries) {
+		const cid = await statCid(input.client, `${sitesDir}/${id}`);
+		if (!cid) continue; // an entry with no resolvable CID is skipped, not fatal.
+		sites.push({id, cid, ipns: keys.get(id)});
+	}
+	return sites;
+}
+
+/**
+ * **remove** — delete a site: `files/rm /sites/<name>` FIRST (so it immediately
+ * drops out of MFS auto-discovery and stops being served/announced/warmed),
+ * THEN `pin/rm <cid>` to unpin the content so its storage is reclaimed. The CID
+ * is resolved up-front with `files/stat` so we know what to unpin after the
+ * entry is gone.
+ *
+ * The MFS removal is the load-bearing step and is always attempted. The unpin
+ * is best-effort: if the content was never pinned (or is pinned indirectly),
+ * `pin/rm` errors, which is REPORTED (`unpinned: false`) rather than thrown, so
+ * a partially-pinned site still removes cleanly.
+ */
+export async function removeSite(
+	input: RemoveSiteInput,
+): Promise<RemoveSiteResult> {
+	const sitesDir = input.sitesDir ?? DEFAULT_SITES_DIR;
+	const path = `${sitesDir}/${input.id}`;
+
+	// Resolve the CID before we remove the entry (afterwards it is gone).
+	const cid = await statCid(input.client, path);
+
+	// Remove the MFS entry first: it stops being discovered/served/announced.
+	await input.client.filesRm(path, {recursive: true, force: true});
+
+	// Then reclaim storage by unpinning the content. Best-effort: a site that
+	// was never pinned (or pinned indirectly) must not fail the removal.
+	let unpinned = false;
+	if (cid) {
+		try {
+			await input.client.pinRm(cid);
+			unpinned = true;
+		} catch {
+			unpinned = false;
+		}
+	}
+
+	return {id: input.id, cid, unpinned};
+}
+
+/**
+ * **add** — expose an existing CID as a served site by placing it into MFS at
+ * `/sites/<name>` (the discoverable location warm/republish/status read). This
+ * is deploy's MFS-placement step in isolation (see the module DESIGN NOTE): it
+ * does NOT build or import a CAR and does NOT pin (the CID is assumed already
+ * imported/pinned on the node).
+ */
+export async function addSite(input: AddSiteInput): Promise<AddSiteResult> {
+	const sitesDir = input.sitesDir ?? DEFAULT_SITES_DIR;
+	await placeInMfs(input.client, sitesDir, input.id, input.cid);
+	return {id: input.id, cid: input.cid};
+}
+
+/**
+ * The MFS-placement step: `files/mkdir /sites --parents`, `files/rm
+ * /sites/<id> --recursive --force` (clear any prior content), then `files/cp
+ * /ipfs/<cid> /sites/<id>`. Ported from the reference prototype's deploy MFS
+ * placement (`~/searches/ipfs-hetzner/deploy-car.mjs`). Exported so `deploy`
+ * can REUSE this exact sequence rather than forking it.
+ */
+export async function placeInMfs(
+	client: KuboRpcClient,
+	sitesDir: string,
+	id: string,
+	cid: string,
+): Promise<void> {
+	await client.filesMkdir(sitesDir, {parents: true});
+	await client.filesRm(`${sitesDir}/${id}`, {recursive: true, force: true});
+	await client.filesCp(`/ipfs/${cid}`, `${sitesDir}/${id}`);
+}
+
+// ---------------------------------------------------------------------------
+// Small shared Kubo reads (fail-soft where a missing dir is not an error).
+// ---------------------------------------------------------------------------
+
+/** List the entry NAMES under the sites dir; empty on a fresh box (no dir). */
+async function lsSites(
+	client: KuboRpcClient,
+	sitesDir: string,
+): Promise<string[]> {
+	let listing: {Entries?: Array<{Name?: string}> | null};
+	try {
+		listing = await client.filesLs<{Entries?: Array<{Name?: string}> | null}>(
+			sitesDir,
+		);
+	} catch {
+		// No /sites dir yet (fresh box) — nothing to list, not an error.
+		return [];
+	}
+	const names: string[] = [];
+	for (const entry of listing.Entries ?? []) {
+		if (entry?.Name) names.push(entry.Name);
+	}
+	return names;
+}
+
+/** Stat an MFS path for its current CID, or undefined if it cannot be resolved. */
+async function statCid(
+	client: KuboRpcClient,
+	path: string,
+): Promise<string | undefined> {
+	try {
+		const stat = await client.filesStat<{Hash?: string}>(path);
+		return stat?.Hash;
+	} catch {
+		return undefined;
+	}
+}
+
+/** Map site/key name -> IPNS id from `key/list -l` (same shape as node-commands). */
+async function listKeys(client: KuboRpcClient): Promise<Map<string, string>> {
+	const res = await client.keyList<{
+		Keys?: Array<{Name?: string; Id?: string}> | null;
+	}>();
+	const map = new Map<string, string>();
+	for (const k of res.Keys ?? []) {
+		if (k?.Name && k?.Id) map.set(k.Name, k.Id);
+	}
+	return map;
+}

package/src/status/status-report.ts

@@ -0,0 +1,291 @@
+/**
+ * The `status` core operation — pinnace's discoverability VERIFICATION surface
+ * (CONTEXT.md `gateway warming`; spec user story 18).
+ *
+ * For every site auto-discovered from MFS `/sites/*` (over the Kubo RPC seam) it
+ * reports FOUR fields so an operator can confirm a deploy actually landed
+ * everywhere:
+ *
+ *   1. current CID        — `files/stat --hash` (via {@link discoverSites}).
+ *   2. IPNS id            — from `key/list -l` IF a key of the SAME name exists
+ *                           (ipfs-mode sites have none).
+ *   3. network-announce   — does the NETWORK announce THIS node for the CID? The
+ *                           external delegated-routing providers list for the
+ *                           CID contains this node's PeerID (`id`).
+ *   4. gateway-serves     — does a COLD public gateway serve it? An HTTP
+ *                           range/HEAD result from a public gateway.
+ *
+ * The two checks in (3) and (4) reach OUTSIDE the node. They are INJECTABLE
+ * ({@link ProvidersLookup}, {@link GatewayProbe}) so tests run against a fake
+ * HTTP layer and never the live network; production supplies the default
+ * `fetch`-backed implementations below.
+ *
+ * Behaviour ported (NOT copied) from the reference prototype
+ * `~/searches/ipfs-hetzner/status.sh`: the delegated-routing providers lookup at
+ * `https://delegated-ipfs.dev/routing/v1/providers/<cid>` (does
+ * `.Providers[].ID` include our PeerID?) and the
+ * `https://<cid>.ipfs.dweb.link/` range-request cold-gateway probe. The shell's
+ * `jq .Providers[].ID == $PEERID` and `curl -r 0-0 -w %{http_code}` shapes
+ * become the two TS predicates here.
+ *
+ * SEAM NOTE: this module OWNS the per-site status checks; the on-box `status`
+ * verb (`../node/node-commands.ts`) consumes them via its injectable
+ * `NodeCommandOps.status` seam — {@link makeStatusOp} adapts this core into that
+ * seam. The dashboard persistence (`status.json`) is the command layer's job,
+ * not this module's: this module PRODUCES the report and stops.
+ *
+ * A failed external check is a REPORTED outcome, never a throw: a
+ * delegated-routing lookup that errors reports `announced=false` (not findable
+ * right now) and a gateway probe that errors reports `gatewayServes=false`. One
+ * cold gateway or one flaky routing endpoint must not fail the whole report.
+ */
+import {discoverSites, type DiscoveredSite} from '../node/node-commands.js';
+import type {KuboRpcClient} from '../rpc/kubo-rpc-client.js';
+import type {NodeCommandContext, NodeOpResult} from '../node/node-commands.js';
+
+/** The delegated-routing providers endpoint (path takes the CID). */
+const DELEGATED_ROUTING_BASE =
+	'https://delegated-ipfs.dev/routing/v1/providers';
+
+/** The public gateway used for the cold-serve probe (`<cid>.ipfs.<host>`). */
+const DWEB_LINK_HOST = 'ipfs.dweb.link';
+
+/**
+ * The subset of the delegated-routing `/routing/v1/providers/<cid>` JSON we
+ * read: the `Providers` array, each with an `ID` (a PeerID). Other fields are
+ * ignored.
+ */
+export interface ProvidersResponse {
+	Providers?: Array<{ID?: string}> | null;
+}
+
+/**
+ * An injectable delegated-routing providers lookup: given a CID, return the
+ * providers response (whose `.Providers[].ID` we scan for our PeerID). Tests
+ * inject a fake; production uses {@link defaultProvidersLookup}. Throwing is
+ * treated as "not findable" (announced=false), never propagated.
+ */
+export type ProvidersLookup = (cid: string) => Promise<ProvidersResponse>;
+
+/**
+ * An injectable cold-gateway probe: given a CID, return the HTTP status a
+ * public gateway responds with (a 2xx/206 means it served). Tests inject a
+ * fake; production uses {@link defaultGatewayProbe}. Throwing is treated as
+ * "did not serve" (gatewayServes=false), never propagated.
+ */
+export type GatewayProbe = (cid: string) => Promise<number>;
+
+/** The status of one site: the four report fields plus its `id`. */
+export interface SiteStatus {
+	/** The site's single `id` (its MFS entry under `/sites/`). */
+	id: string;
+	/** The current content root CID (`files/stat --hash`). */
+	cid: string;
+	/** The IPNS id, if a same-named keystore key exists (else undefined). */
+	ipns?: string;
+	/** True when the delegated-routing providers list for the CID has our PeerID. */
+	announced: boolean;
+	/** The HTTP status the cold public gateway probe returned (undefined on error). */
+	gatewayHttp?: number;
+	/** True when the gateway probe status indicates the CID was served (2xx/206). */
+	gatewayServes: boolean;
+}
+
+/** The full status report: this node's PeerID plus a per-site status line. */
+export interface StatusReport {
+	/** This node's PeerID (from Kubo `id`); the announce check looks for it. */
+	peerId: string;
+	/** One {@link SiteStatus} per site discovered under MFS `/sites/*`. */
+	sites: SiteStatus[];
+}
+
+/** Inputs to {@link statusReport}. */
+export interface StatusReportInput {
+	/** The Kubo RPC client for THIS node (files/ls, files/stat, key/list, id). */
+	client: KuboRpcClient;
+	/** The MFS directory sites live under (default `/sites`). */
+	sitesDir?: string;
+	/**
+	 * The delegated-routing providers lookup (injectable; defaults to the live
+	 * `delegated-ipfs.dev` fetch via {@link defaultProvidersLookup}).
+	 */
+	providersLookup?: ProvidersLookup;
+	/**
+	 * The cold-gateway probe (injectable; defaults to the live `dweb.link`
+	 * range-request via {@link defaultGatewayProbe}).
+	 */
+	gatewayProbe?: GatewayProbe;
+	/**
+	 * Pre-discovered sites (optional). When set, discovery is SKIPPED and these
+	 * are used as-is — the node-command layer already walked MFS, so the adapter
+	 * ({@link makeStatusOp}) passes them straight through rather than re-listing.
+	 */
+	sites?: DiscoveredSite[];
+}
+
+/**
+ * Build the per-site status report. Discovers sites (unless supplied), reads
+ * this node's PeerID and its keystore once, then for each site runs the two
+ * external checks (through the injected/default fakes) to fill the four fields.
+ * External-check failures degrade to `false`, never throw.
+ */
+export async function statusReport(
+	input: StatusReportInput,
+): Promise<StatusReport> {
+	const sitesDir = input.sitesDir ?? '/sites';
+	const providersLookup = input.providersLookup ?? defaultProvidersLookup;
+	const gatewayProbe = input.gatewayProbe ?? defaultGatewayProbe;
+
+	const sites = input.sites ?? (await discoverSites(input.client, sitesDir));
+	const peerId = await readPeerId(input.client);
+	const keys = await listKeys(input.client);
+
+	const statuses: SiteStatus[] = [];
+	for (const site of sites) {
+		const announced = await checkAnnounced(providersLookup, site.cid, peerId);
+		const gatewayHttp = await probeGateway(gatewayProbe, site.cid);
+		statuses.push({
+			id: site.id,
+			cid: site.cid,
+			ipns: keys.get(site.id),
+			announced,
+			gatewayHttp,
+			gatewayServes: gatewayHttp !== undefined && servesStatus(gatewayHttp),
+		});
+	}
+
+	return {peerId, sites: statuses};
+}
+
+/**
+ * Adapt {@link statusReport} into a {@link NodeCommandContext} `status` op so
+ * the on-box `status` verb can inject it (its `ops.status` seam). The command
+ * layer discovers sites and passes them in; this adapter runs the four-field
+ * checks over them and flattens the result into a {@link NodeOpResult} (its
+ * per-site outcomes carry the announce + gateway outcomes too).
+ */
+export function makeStatusOp(
+	checks: {providersLookup?: ProvidersLookup; gatewayProbe?: GatewayProbe} = {},
+): (ctx: NodeCommandContext, sites: DiscoveredSite[]) => Promise<NodeOpResult> {
+	return async (ctx, sites) => {
+		const report = await statusReport({
+			client: ctx.client,
+			sitesDir: ctx.sitesDir,
+			sites,
+			providersLookup: checks.providersLookup,
+			gatewayProbe: checks.gatewayProbe,
+		});
+		return {
+			sites: report.sites.map((s) => ({
+				id: s.id,
+				cid: s.cid,
+				ipns: s.ipns ?? '',
+				announced: s.announced,
+				gatewayServes: s.gatewayServes,
+				gatewayHttp: s.gatewayHttp,
+				status: s.gatewayServes && s.announced ? 'ok' : 'unverified',
+			})),
+		};
+	};
+}
+
+// ---------------------------------------------------------------------------
+// Per-site checks (fail-soft).
+// ---------------------------------------------------------------------------
+
+/**
+ * Ask delegated routing who provides the CID and answer: is OUR PeerID among
+ * them? A lookup error is "not findable right now" -> false, never thrown (one
+ * flaky routing endpoint must not fail the report). Ports the shell's
+ * `jq '.Providers[]|select(.ID==$p)'`.
+ */
+async function checkAnnounced(
+	lookup: ProvidersLookup,
+	cid: string,
+	peerId: string,
+): Promise<boolean> {
+	if (!peerId) return false;
+	try {
+		const res = await lookup(cid);
+		return (res.Providers ?? []).some((p) => p?.ID === peerId);
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Probe a cold public gateway for the CID and return its HTTP status, or
+ * undefined if the probe itself errored (network down, DNS, ...). Ports the
+ * shell's `curl -r 0-0 -w %{http_code}`.
+ */
+async function probeGateway(
+	probe: GatewayProbe,
+	cid: string,
+): Promise<number | undefined> {
+	try {
+		return await probe(cid);
+	} catch {
+		return undefined;
+	}
+}
+
+/** A gateway HTTP status counts as "served" iff it is 2xx (including 206). */
+function servesStatus(status: number): boolean {
+	return status >= 200 && status < 300;
+}
+
+// ---------------------------------------------------------------------------
+// Kubo reads.
+// ---------------------------------------------------------------------------
+
+/** Read this node's PeerID from Kubo `id` (empty string if unavailable). */
+async function readPeerId(client: KuboRpcClient): Promise<string> {
+	try {
+		const res = await client.id<{ID?: string}>();
+		return res?.ID ?? '';
+	} catch {
+		return '';
+	}
+}
+
+/** Map site/key name -> IPNS id from `key/list -l` (same shape as node-commands). */
+async function listKeys(client: KuboRpcClient): Promise<Map<string, string>> {
+	const res = await client.keyList<{
+		Keys?: Array<{Name?: string; Id?: string}> | null;
+	}>();
+	const map = new Map<string, string>();
+	for (const k of res.Keys ?? []) {
+		if (k?.Name && k?.Id) map.set(k.Name, k.Id);
+	}
+	return map;
+}
+
+// ---------------------------------------------------------------------------
+// Default (LIVE) external checks. Production wiring; tests inject fakes instead.
+// ---------------------------------------------------------------------------
+
+/**
+ * The default delegated-routing lookup: GET
+ * `https://delegated-ipfs.dev/routing/v1/providers/<cid>` and parse its JSON.
+ * (Injected fakes replace this in tests; it is the only place the live routing
+ * endpoint is named.)
+ */
+export const defaultProvidersLookup: ProvidersLookup = async (cid) => {
+	const res = await fetch(`${DELEGATED_ROUTING_BASE}/${cid}`, {
+		headers: {accept: 'application/json'},
+	});
+	if (!res.ok) return {Providers: []};
+	return (await res.json()) as ProvidersResponse;
+};
+
+/**
+ * The default cold-gateway probe: a single-byte range request to
+ * `https://<cid>.ipfs.dweb.link/` returning the HTTP status. (Injected fakes
+ * replace this in tests; it is the only place the live gateway is named.)
+ */
+export const defaultGatewayProbe: GatewayProbe = async (cid) => {
+	const res = await fetch(`https://${cid}.${DWEB_LINK_HOST}/`, {
+		headers: {range: 'bytes=0-0'},
+	});
+	return res.status;
+};