pkgref 0.0.1

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# pkgref
+
+## 0.0.1
+
+### Patch Changes
+
+- Add interactive dependency repository cloning, indexing, and updates

package/README.md

@@ -0,0 +1,40 @@
+# pkgref
+
+[GitHub repository](https://github.com/yue4u/pkgref)
+
+```sh
+pnpm dlx pkgref
+# list package.json deps in interactive mode for select and clone repo to target dir (default to docs/pkg-reference).
+pnpm dlx pkgref --pkgs=react,vitest --dir docs/repo
+# specify pkg and dir to clone with cli param
+pnpm dlx pkgref update
+# fast-forward all clones in docs/pkg-reference
+```
+
+## Features
+
+- list docs/example dir in cloned repo to root as index markdown for search
+
+## Future
+
+- support more than package.json
+
+## Development
+
+- Install dependencies:
+
+```bash
+vp install
+```
+
+- Run the unit tests:
+
+```bash
+vp test
+```
+
+- Build the library:
+
+```bash
+vp pack
+```

package/dist/cli.d.mts

@@ -0,0 +1,31 @@
+//#region src/types.d.ts
+type RepositoryRef = {
+  cloneUrl: string;
+  key: string;
+  name: string;
+};
+type RepositoryGroup = RepositoryRef & {
+  packages: string[];
+};
+type IndexRepository = {
+  directory: string;
+  packages: string[];
+  root: string;
+};
+//#endregion
+//#region src/cli.d.ts
+type GitResult = {
+  ok: boolean;
+  output: string;
+};
+type GitRunner = (args: string[], cwd?: string) => Promise<GitResult>;
+declare function runCli(argv?: string[], cwd?: string): Promise<0 | 1>;
+declare function resolveRepository(packageName: string, cwd: string): Promise<RepositoryRef>;
+declare function processRepository(group: RepositoryGroup, target: string, runGit?: GitRunner): Promise<{
+  failed: boolean;
+  index?: IndexRepository;
+}>;
+declare function pullRepository(root: string, name: string, prune?: boolean, runGit?: GitRunner): Promise<"dirty" | "failed" | "updated">;
+declare function invalidCloneReason(root: string, expected: RepositoryGroup, runGit?: GitRunner): Promise<string | undefined>;
+//#endregion
+export { invalidCloneReason, processRepository, pullRepository, resolveRepository, runCli };

package/dist/cli.mjs

@@ -0,0 +1,464 @@
+#!/usr/bin/env node
+import { execFile } from "node:child_process";
+import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { basename, join, relative, resolve, sep } from "node:path";
+import { parseArgs, promisify } from "node:util";
+import { cancel, confirm, isCancel, multiselect, note, text } from "@clack/prompts";
+//#region package.json
+var version = "0.0.1";
+//#endregion
+//#region src/core.ts
+const INDEX_MARKER = "<!-- generated by pkgref; do not edit -->";
+const AGENTS_START = "<!-- PKGREF START -->";
+const AGENTS_END = "<!-- PKGREF END -->";
+const DEPENDENCY_FIELDS = [
+	"dependencies",
+	"devDependencies",
+	"peerDependencies",
+	"optionalDependencies"
+];
+const REFERENCE_DIRECTORIES = /* @__PURE__ */ new Set([
+	"doc",
+	"docs",
+	"example",
+	"examples"
+]);
+const byName = (a, b) => a.name.localeCompare(b.name);
+function discoverDependencies(manifest) {
+	return [...new Set(DEPENDENCY_FIELDS.flatMap((field) => Object.keys(manifest[field] ?? {})))].sort();
+}
+async function readManifest(path) {
+	try {
+		const manifest = JSON.parse(await readFile(path, "utf8"));
+		if (!manifest || typeof manifest !== "object" || Array.isArray(manifest)) throw new Error("expected a JSON object");
+		return manifest;
+	} catch (error) {
+		const action = error.code ? "Cannot read" : "Invalid";
+		throw new Error(`${action} package manifest at ${path}: ${errorMessage(error)}`);
+	}
+}
+function parsePackageList(value) {
+	return [...new Set(value.split(",").map((name) => name.trim()).filter(Boolean))];
+}
+function renderAgentsReference(contents, indexPath) {
+	const block = `${AGENTS_START}
+
+## Package references
+
+Before working with dependency APIs, consult the
+[package reference index](${indexPath}).
+It links to cloned upstream documentation and examples.
+
+${AGENTS_END}`;
+	const marked = new RegExp(`${AGENTS_START}[\\s\\S]*?${AGENTS_END}`);
+	return `${(marked.test(contents) ? contents.replace(marked, block) : contents.trim() ? `${contents.trimEnd()}\n\n${block}` : block).trimEnd()}\n`;
+}
+function repositoryFromMetadata(metadata) {
+	const repository = metadata.repository;
+	const url = typeof repository === "string" ? repository : repository?.url;
+	if (!url) throw new Error("package metadata does not declare a repository");
+	return normalizeRepository(url);
+}
+function normalizeRepository(rawUrl) {
+	let url = rawUrl.trim().replace(/^git\+/, "");
+	if (!url) throw new Error("repository URL is empty");
+	if (/^github:/i.test(url)) url = `https://github.com/${url.slice(7)}`;
+	else if (/^[\w.-]+\/[\w.-]+$/.test(url)) url = `https://github.com/${url}`;
+	else if (/^git@[^:]+:/.test(url)) url = url.replace(/^git@([^:]+):/, "ssh://git@$1/");
+	else if (url.startsWith("git://")) url = `https://${url.slice(6)}`;
+	if (!URL.canParse(url)) throw new Error(`unsupported repository URL: ${rawUrl}`);
+	const parsed = new URL(url);
+	if (![
+		"http:",
+		"https:",
+		"ssh:"
+	].includes(parsed.protocol)) throw new Error(`unsupported repository protocol: ${parsed.protocol}`);
+	const path = parsed.pathname.replace(/^\/+|\/+$/g, "").replace(/\.git$/i, "");
+	const name = basename(path);
+	if (!parsed.hostname || !path || !name || name === "." || name === "..") throw new Error(`invalid repository URL: ${rawUrl}`);
+	const user = parsed.protocol === "ssh:" && parsed.username ? `${parsed.username}@` : "";
+	return {
+		cloneUrl: parsed.protocol === "ssh:" ? `ssh://${user}${parsed.hostname}/${path}.git` : `${parsed.protocol}//${parsed.host}/${path}.git`,
+		key: `${parsed.hostname}/${path}`.toLowerCase(),
+		name
+	};
+}
+function groupRepositories(packages) {
+	const groups = /* @__PURE__ */ new Map();
+	const folders = /* @__PURE__ */ new Map();
+	for (const { name, repository } of packages) {
+		const owner = folders.get(repository.name);
+		if (owner && owner !== repository.key) throw new Error(`repository folder collision: different repositories resolve to "${repository.name}"`);
+		folders.set(repository.name, repository.key);
+		const group = groups.get(repository.key);
+		if (group) group.packages.push(name);
+		else groups.set(repository.key, {
+			...repository,
+			packages: [name]
+		});
+	}
+	return [...groups.values()].map((group) => ({
+		...group,
+		packages: [...new Set(group.packages)].sort()
+	})).sort(byName);
+}
+async function findReferenceDirectories(repositoryRoot, targetRoot) {
+	const found = [];
+	async function walk(directory) {
+		for (const entry of await readdir(directory, { withFileTypes: true })) {
+			if (!entry.isDirectory() || entry.name.startsWith(".")) continue;
+			const path = join(directory, entry.name);
+			if (REFERENCE_DIRECTORIES.has(entry.name.toLowerCase())) found.push({
+				label: posixPath(relative(repositoryRoot, path)),
+				path: posixPath(relative(targetRoot, path))
+			});
+			else await walk(path);
+		}
+	}
+	await walk(repositoryRoot);
+	return found.sort((a, b) => a.path.localeCompare(b.path));
+}
+async function renderIndex(repositories, targetRoot) {
+	const sections = await Promise.all([...repositories].sort((a, b) => a.directory.localeCompare(b.directory)).map(async ({ directory, packages, root }) => {
+		const references = await findReferenceDirectories(root, targetRoot);
+		const links = references.length ? references.map((item) => `- [${escapeLabel(item.label)}](./${encodePath(item.path)})`) : [`- [Repository root](./${encodePath(posixPath(relative(targetRoot, root)))})`];
+		return `## ${directory}
+
+Packages: ${[...packages].sort().map((name) => `\`${name}\``).join(", ")}
+
+${links.join("\n")}`;
+	}));
+	return `${INDEX_MARKER}\n\n# Package references\n\n${sections.join("\n\n")}\n`;
+}
+function mergeIndexes(existing, generated) {
+	const oldIndex = splitIndex(existing);
+	const newIndex = splitIndex(generated);
+	const sections = new Map([...oldIndex.sections, ...newIndex.sections]);
+	return `${[oldIndex.preamble, ...[...sections].sort(([a], [b]) => a.localeCompare(b)).map(([, value]) => value)].filter(Boolean).join("\n\n")}\n`;
+}
+function errorMessage(error) {
+	return error instanceof Error ? error.message : String(error);
+}
+const posixPath = (path) => path.split(sep).join("/");
+const encodePath = (path) => path.split("/").map(encodeURIComponent).join("/");
+const escapeLabel = (label) => label.replaceAll("\\", "\\\\").replaceAll("[", "\\[").replaceAll("]", "\\]");
+function splitIndex(contents) {
+	const matches = [...contents.matchAll(/^## (.+)$/gm)];
+	const sections = /* @__PURE__ */ new Map();
+	matches.forEach((match, index) => {
+		sections.set(match[1].trim(), contents.slice(match.index, matches[index + 1]?.index).trimEnd());
+	});
+	return {
+		preamble: contents.slice(0, matches[0]?.index ?? contents.length).trimEnd(),
+		sections
+	};
+}
+//#endregion
+//#region src/cli.ts
+const exec = promisify(execFile);
+const DEFAULT_DIR = "docs/pkg-reference";
+const CANCELLED = Symbol("cancelled");
+const OPTIONS = {
+	dir: { type: "string" },
+	help: {
+		type: "boolean",
+		short: "h"
+	},
+	pkgs: { type: "string" },
+	version: {
+		type: "boolean",
+		short: "v"
+	}
+};
+const HELP = `pkgref - clone package source references
+
+Usage:
+  pkgref [--pkgs=<name,...>] [--dir=<path>]
+  pkgref update [--dir=<path>]
+
+Options:
+  --pkgs  Comma-separated packages. Undeclared packages are allowed with a warning.
+  --dir   Target directory (default: ${DEFAULT_DIR}).
+  -h, --help  Show this help.
+  -v, --version  Show the installed version.
+
+Commands:
+  update  Fast-forward all cloned repositories to their latest upstream revision.
+`;
+async function runCli(argv = process.argv.slice(2), cwd = process.cwd()) {
+	try {
+		const { values, positionals } = parseArgs({
+			args: argv,
+			allowPositionals: true,
+			options: OPTIONS,
+			strict: true
+		});
+		if (values.help) {
+			console.log(HELP);
+			return 0;
+		}
+		if (values.version) {
+			console.log(version);
+			return 0;
+		}
+		if (positionals.length > 1 || positionals[0] && positionals[0] !== "update") throw new Error(`unknown command: ${positionals.join(" ")}`);
+		if (positionals[0] === "update") {
+			if (values.pkgs !== void 0) throw new Error("--pkgs cannot be used with the update command");
+			return await updateRepositories(resolve(cwd, values.dir ?? DEFAULT_DIR));
+		}
+		return await cloneRepositories(cwd, values.pkgs, values.dir);
+	} catch (error) {
+		if (error === CANCELLED) return 0;
+		console.error(`Error: ${errorMessage(error)}`);
+		return 1;
+	}
+}
+async function updateRepositories(target) {
+	if (!await exists(target)) {
+		console.log(`No package reference directory found at ${target}.`);
+		return 0;
+	}
+	const repositories = [];
+	for (const entry of await readdir(target, { withFileTypes: true })) if (entry.isDirectory() && await exists(join(target, entry.name, ".git"))) repositories.push(entry.name);
+	repositories.sort();
+	if (!repositories.length) {
+		console.log(`No cloned repositories found in ${target}.`);
+		return 0;
+	}
+	const updated = (await Promise.all(repositories.map((name) => pullRepository(join(target, name), name, true)))).filter((result) => result === "updated").length;
+	const failed = updated !== repositories.length;
+	console.log(`Finished: ${updated} of ${repositories.length} repositories updated${failed ? " with errors" : ""}.`);
+	return failed ? 1 : 0;
+}
+async function cloneRepositories(cwd, packageArgument, directory) {
+	const manifestPath = join(cwd, "package.json");
+	const declared = discoverDependencies(await readManifest(manifestPath));
+	const interactive = packageArgument === void 0;
+	let addToAgents = false;
+	const packages = interactive ? await selectPackages(declared) : parsePackageList(packageArgument);
+	if (!interactive) {
+		if (!packages.length) throw new Error("--pkgs must contain at least one package name");
+		for (const name of packages.filter((name) => !declared.includes(name))) console.warn(`Warning: "${name}" is not declared in ${manifestPath}; resolving it anyway.`);
+	}
+	const resolved = (await Promise.all(packages.map(async (name) => {
+		try {
+			return {
+				name,
+				repository: await resolveRepository(name, cwd)
+			};
+		} catch (error) {
+			console.error(`Failed to resolve ${name}: ${errorMessage(error)}`);
+			return null;
+		}
+	}))).filter((item) => item !== null);
+	let failed = resolved.length !== packages.length;
+	const groups = groupRepositories(resolved);
+	if (!groups.length) return failed ? 1 : 0;
+	if (interactive) {
+		await confirmRepositories(groups);
+		directory = (directory ?? await ask(text({
+			message: "Where should package references be stored?",
+			initialValue: DEFAULT_DIR,
+			validate: (value) => !value?.trim() ? "Target directory is required." : void 0
+		}))).trim();
+		addToAgents = await confirmChoice("Add the package reference index to AGENTS.md?");
+	}
+	const target = resolve(cwd, directory ?? DEFAULT_DIR);
+	const indexPath = join(target, "INDEX.md");
+	const existingIndex = await readOwnedIndex(indexPath);
+	await mkdir(target, { recursive: true });
+	const indexed = [];
+	for (const group of groups) {
+		const result = await processRepository(group, target);
+		failed ||= result.failed;
+		if (result.index) indexed.push(result.index);
+	}
+	const generated = await renderIndex(indexed, target);
+	await writeFile(indexPath, existingIndex ? mergeIndexes(existingIndex, generated) : generated);
+	console.log(`Wrote ${indexPath}`);
+	if (addToAgents) await addAgentsReference(cwd, indexPath);
+	console.log(`Finished: ${indexed.length} repositories indexed, ${failed ? "with errors" : "successfully"}.`);
+	return failed ? 1 : 0;
+}
+async function resolveRepository(packageName, cwd) {
+	const installedManifest = join(cwd, "node_modules", packageName, "package.json");
+	if (await exists(installedManifest)) return repositoryFromMetadata(await readManifest(installedManifest));
+	const response = await fetch(`https://registry.npmjs.org/${encodeURIComponent(packageName)}/latest`, { headers: { accept: "application/json" } });
+	if (!response.ok) throw new Error(`npm registry returned ${response.status} ${response.statusText}`);
+	return repositoryFromMetadata(await response.json());
+}
+async function processRepository(group, target, runGit = git) {
+	const root = join(target, group.name);
+	const index = {
+		directory: group.name,
+		packages: group.packages,
+		root
+	};
+	if (!await exists(root)) {
+		console.log(`Cloning ${group.name} (${group.packages.join(", ")})...`);
+		const result = await runGit([
+			"clone",
+			"--depth",
+			"1",
+			"--",
+			group.cloneUrl,
+			root
+		]);
+		if (!result.ok) {
+			console.error(`Failed to clone ${group.name}: ${result.output}`);
+			return { failed: true };
+		}
+		return {
+			failed: false,
+			index
+		};
+	}
+	const invalidReason = await invalidCloneReason(root, group, runGit);
+	if (invalidReason) {
+		console.warn(`Skipping ${group.name}: ${invalidReason}`);
+		return { failed: false };
+	}
+	if (!process.stdin.isTTY || !process.stdout.isTTY || !await confirmUpdate(group.name)) {
+		console.log(`Skipped update for ${group.name}.`);
+		return {
+			failed: false,
+			index
+		};
+	}
+	return {
+		failed: await pullRepository(root, group.name, false, runGit) === "failed",
+		index
+	};
+}
+async function pullRepository(root, name, prune = false, runGit = git) {
+	const status = await runGit(["status", "--porcelain"], root);
+	if (!status.ok) {
+		console.error(`Failed to inspect ${name}: ${status.output}`);
+		return "failed";
+	}
+	if (status.output) {
+		console.warn(`Skipped ${name}: the worktree has local changes.`);
+		return "dirty";
+	}
+	console.log(`Updating ${name}...`);
+	const pull = await runGit([
+		"pull",
+		"--ff-only",
+		...prune ? ["--prune"] : []
+	], root);
+	if (!pull.ok) {
+		console.error(`Failed to update ${name}: ${pull.output}`);
+		return "failed";
+	}
+	console.log(`Updated ${name}.`);
+	return "updated";
+}
+async function invalidCloneReason(root, expected, runGit = git) {
+	const origin = await runGit([
+		"remote",
+		"get-url",
+		"origin"
+	], root);
+	if (!origin.ok) return "the destination is not a Git clone with an origin";
+	try {
+		return normalizeRepository(origin.output).key === expected.key ? void 0 : "the existing clone has a different origin";
+	} catch (error) {
+		return `cannot read its origin: ${errorMessage(error)}`;
+	}
+}
+async function readOwnedIndex(path) {
+	const contents = await readOptional(path);
+	if (contents === void 0) return;
+	if (!contents.startsWith("<!-- generated by pkgref; do not edit -->")) throw new Error(`refusing to overwrite user-owned index at ${path}`);
+	return contents;
+}
+async function selectPackages(packages) {
+	if (!process.stdin.isTTY || !process.stdout.isTTY) throw new Error("--pkgs is required when pkgref is not running in an interactive terminal");
+	if (!packages.length) stop("No dependencies found in package.json.");
+	const selected = await ask(multiselect({
+		message: "Select packages to clone",
+		options: packages.map((value) => ({
+			label: value,
+			value
+		})),
+		required: false,
+		maxItems: 10
+	}));
+	if (!selected.length) stop("No packages selected.");
+	return selected;
+}
+async function confirmRepositories(repositories) {
+	note(repositories.map(({ name, packages, cloneUrl }) => `${name} (${packages.join(", ")})\n${cloneUrl}`).join("\n\n"), "Git repositories");
+	if (!await confirmChoice("Clone these Git repositories?", "Operation cancelled. No repositories were cloned.")) abort("Operation cancelled. No repositories were cloned.");
+}
+const confirmChoice = (message, cancelMessage) => ask(confirm({
+	message,
+	initialValue: true
+}), cancelMessage);
+async function confirmUpdate(name) {
+	const value = await confirm({
+		message: `${name} already exists. Update it?`,
+		initialValue: false
+	});
+	if (isCancel(value)) {
+		cancel("Update skipped.");
+		return false;
+	}
+	return value;
+}
+async function ask(answer, message = "Operation cancelled.") {
+	const value = await answer;
+	if (isCancel(value)) abort(message);
+	return value;
+}
+function abort(message) {
+	cancel(message);
+	throw CANCELLED;
+}
+function stop(message) {
+	console.log(message);
+	throw CANCELLED;
+}
+async function addAgentsReference(cwd, indexPath) {
+	const path = join(cwd, "AGENTS.md");
+	const link = relative(cwd, indexPath).split(sep).map(encodeURIComponent).join("/");
+	await writeFile(path, renderAgentsReference(await readOptional(path) ?? "", link));
+	console.log(`Updated ${path}`);
+}
+async function git(args, cwd) {
+	try {
+		const { stdout, stderr } = await exec("git", args, {
+			cwd,
+			encoding: "utf8"
+		});
+		return {
+			ok: true,
+			output: (stderr || stdout).trim()
+		};
+	} catch (error) {
+		const result = error;
+		return {
+			ok: false,
+			output: (result.stderr || result.stdout || result.message).trim()
+		};
+	}
+}
+async function readOptional(path) {
+	try {
+		return await readFile(path, "utf8");
+	} catch (error) {
+		if (error.code === "ENOENT") return;
+		throw error;
+	}
+}
+async function exists(path) {
+	try {
+		await access(path);
+		return true;
+	} catch {
+		return false;
+	}
+}
+if (import.meta.url === `file://${process.argv[1]}`) process.exitCode = await runCli();
+//#endregion
+export { invalidCloneReason, processRepository, pullRepository, resolveRepository, runCli };

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "pkgref",
+  "version": "0.0.1",
+  "description": "Clone dependency source repositories and index their docs and examples.",
+  "homepage": "https://github.com/yue4u/pkgref#readme",
+  "bugs": {
+    "url": "https://github.com/yue4u/pkgref/issues"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/yue4u/pkgref.git"
+  },
+  "bin": {
+    "pkgref": "./dist/cli.mjs"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "exports": {
+    "./package.json": "./package.json"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@clack/prompts": "^1.7.0"
+  },
+  "devDependencies": {
+    "@types/node": "^26.1.1",
+    "bumpp": "^11.1.0",
+    "typescript": "^7.0.2",
+    "vite": "npm:@voidzero-dev/vite-plus-core@0.2.5",
+    "vite-plus": "0.2.5"
+  },
+  "devEngines": {
+    "packageManager": {
+      "name": "pnpm",
+      "version": "11.17.0",
+      "onFail": "download"
+    }
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "vp pack",
+    "dev": "vp pack --watch",
+    "test": "vp test",
+    "check": "vp check"
+  }
+}