tryscript 0.1.0 → 0.1.2

package/README.md

@@ -1,71 +1,150 @@
 # tryscript
 
+[![CI](https://github.com/jlevy/tryscript/actions/workflows/ci.yml/badge.svg)](https://github.com/jlevy/tryscript/actions/workflows/ci.yml)
+[![Coverage](https://raw.githubusercontent.com/jlevy/tryscript/main/badges/packages/tryscript/coverage-total.svg)](https://raw.githubusercontent.com/jlevy/tryscript/main/badges/packages/tryscript/coverage-total.svg)
+[![npm version](https://img.shields.io/npm/v/tryscript)](https://www.npmjs.com/package/tryscript)
+[![X Follow](https://img.shields.io/twitter/follow/ojoshe)](https://x.com/ojoshe)
+
 Golden testing for CLI applications - a TypeScript port of [trycmd](https://github.com/assert-rs/trycmd).
 
-## Overview
+> [!NOTE]
+> 100% of the code and specs in this repository were written by Claude Code.
+> The design and management and prompting was by me ([jlevy](https://github.com/jlevy)) supported by  the workflows, agent rules,
+> and other research docs in [Speculate](https://github.com/jlevy/speculate).
+> 
+> You can see what you think, but I find the code quality higher than most agent-written code I've
+> seen because of the spec-driven process.
+> You can review the architecture doc and all of the specs all of the specs in [docs/project](docs/project).
+> The general research, guideline, and rules docs I use are in [docs/general](docs/general).
+
+## What It Does
 
-tryscript enables golden testing for any CLI application. Write test cases in Markdown with console code blocks, and tryscript runs the commands, compares output, and reports differences.
+Write CLI tests as Markdown. tryscript runs commands, captures output, and compares against expected results. Tests become documentation; documentation becomes tests.
 
 ````markdown
-# Test: Hello World
+# Test: Basic echo
 
 ```console
 $ echo "hello world"
 hello world
 ? 0
 ```
-````
 
-## Features
+# Test: Grep with pattern matching
 
-- **Markdown test format** - Tests are readable documentation
-- **Elision patterns** - Match dynamic output with `[..]`, `...`, `[EXE]`, `[ROOT]`, `[CWD]`
-- **Custom patterns** - Define regex patterns for timestamps, versions, UUIDs
-- **Update mode** - Regenerate golden files with `--update`
-- **Self-bootstrapping** - tryscript tests itself
+```console
+$ ls -la | grep ".md"
+[..]README.md
+...
+? 0
+```
+````
+
+The `[..]` matches any text on that line. The `...` matches zero or more lines. These "elision patterns" let tests handle dynamic output gracefully.
 
 ## Quick Start
 
 ```bash
 # Install
-pnpm add tryscript
+pnpm add -D tryscript
+
+# For coverage support (optional)
+pnpm add -D c8
+
+# For accurate line counts when merging with vitest (optional)
+pnpm add -D c8 monocart-coverage-reports
 
 # Run tests
-npx tryscript tests/
+npx tryscript run tests/
 
-# Update golden files
-npx tryscript --update
+# Update expected output when behavior changes
+npx tryscript run --update tests/
 ```
 
-## Documentation
+## Features
+
+- **Markdown format** - Tests are readable documentation
+- **Elision patterns** - Handle variable output: `[..]`, `...`, `[CWD]`, `[ROOT]`, `[EXE]`
+- **Custom patterns** - Define regex patterns for timestamps, versions, UUIDs
+- **Update mode** - Regenerate expected output with `--update`
+- **Sandbox mode** - Isolate tests in temp directories
+- **Code coverage** - Track coverage from subprocess execution with `--coverage`
+
+## Example Test File
 
-See [packages/tryscript/README.md](packages/tryscript/README.md) for full documentation.
+````markdown
+---
+env:
+  NO_COLOR: "1"
+sandbox: true
+---
 
-## Project Structure
+# Test: CLI help
 
+```console
+$ my-cli --help
+Usage: my-cli [options] <command>
+
+Options:
+  --version  Show version
+  --help     Show this help
+...
+? 0
 ```
-tryscript/
-├── packages/
-│   └── tryscript/     # Main package
-│       ├── src/       # TypeScript source
-│       └── tests/     # Self-tests
-└── docs/              # Documentation
+
+# Test: Version output
+
+```console
+$ my-cli --version
+my-cli v[..]
+? 0
+```
+
+# Test: Error handling
+
+```console
+$ my-cli unknown-command 2>&1
+Error: unknown command 'unknown-command'
+? 1
 ```
+````
+
+## CLI Reference
+
+```bash
+tryscript run [files...]          # Run golden tests
+tryscript coverage <commands...>  # Run commands with merged coverage
+tryscript docs                    # Show syntax quick reference
+tryscript readme                  # Show this documentation
+tryscript --help                  # Show all options
+```
+
+For complete syntax reference, run `tryscript docs` or see the [reference documentation](https://github.com/jlevy/tryscript/blob/main/docs/tryscript-reference.md).
+
+### Common Options
+
+| Option | Description |
+| --- | --- |
+| `--update` | Update test files with actual output |
+| `--fail-fast` | Stop on first failure |
+| `--filter <regex>` | Filter tests by name |
+| `--verbose` | Show detailed output |
+| `--coverage` | Collect code coverage (requires c8) |
+| `--coverage-monocart` | Use monocart for accurate line counts (requires monocart-coverage-reports) |
+| `--coverage-exclude-node-modules` | Exclude node_modules from coverage (default: true) |
+| `--coverage-exclude <pattern>` | Exclude patterns from coverage |
 
 ## Development
 
 ```bash
-# Install dependencies
+# Clone and install
+git clone https://github.com/jlevy/tryscript.git
+cd tryscript
 pnpm install
 
-# Build
+# Build and test
 pnpm build
-
-# Run tests
 pnpm test
-
-# Run self-tests
-pnpm -r tryscript tests/
 ```
 
 ## License

package/dist/bin.cjs

@@ -1,11 +1,13 @@
 #!/usr/bin/env node
 
 
-const require_src = require('./src-CP4-Q-U5.cjs');
+const require_src = require('./src-BR3CZ3tc.cjs');
 let node_url = require("node:url");
 let node_fs = require("node:fs");
 let node_path = require("node:path");
+let node_child_process = require("node:child_process");
 let node_fs_promises = require("node:fs/promises");
+let node_os = require("node:os");
 let commander = require("commander");
 let fast_glob = require("fast-glob");
 fast_glob = require_src.__toESM(fast_glob);
@@ -21,10 +23,8 @@ let atomically = require("atomically");
 const colors = {
 	success: (s) => picocolors.default.green(s),
 	error: (s) => picocolors.default.red(s),
-	info: (s) => picocolors.default.cyan(s),
 	warn: (s) => picocolors.default.yellow(s),
-	muted: (s) => picocolors.default.gray(s),
-	bold: (s) => picocolors.default.bold(s)
+	info: (s) => picocolors.default.cyan(s)
 };
 /**
 * Status indicators with emoji.
@@ -170,15 +170,121 @@ function buildUpdatedBlock(block, result) {
 	return lines.join("\n");
 }
 
+//#endregion
+//#region src/lib/coverage.ts
+/**
+* Coverage collection for CLI subprocess testing.
+*
+* Uses c8 and NODE_V8_COVERAGE to collect coverage from spawned processes.
+*/
+/**
+* Find the c8 executable path.
+* Checks local node_modules/.bin first, then falls back to npx.
+*/
+function findC8Path$1() {
+	const localPaths = [(0, node_path.resolve)(process.cwd(), "node_modules", ".bin", "c8"), (0, node_path.resolve)(process.cwd(), "..", "..", "node_modules", ".bin", "c8")];
+	for (const localPath of localPaths) if ((0, node_fs.existsSync)(localPath)) return localPath;
+	return "npx c8";
+}
+/**
+* Check if c8 is available in the current environment.
+*/
+async function isC8Available() {
+	const c8Path = findC8Path$1();
+	return new Promise((resolve$2) => {
+		const isNpx = c8Path === "npx c8";
+		const proc = (0, node_child_process.spawn)(isNpx ? "npx" : c8Path, isNpx ? ["c8", "--version"] : ["--version"], {
+			shell: false,
+			stdio: "ignore"
+		});
+		proc.on("close", (code) => {
+			resolve$2(code === 0);
+		});
+		proc.on("error", () => {
+			resolve$2(false);
+		});
+	});
+}
+/**
+* Create a coverage context for collecting V8 coverage data.
+*/
+async function createCoverageContext(config) {
+	const options = require_src.resolveCoverageConfig(config);
+	return {
+		tempDir: await (0, node_fs_promises.mkdtemp)((0, node_path.join)((0, node_os.tmpdir)(), "tryscript-coverage-")),
+		options
+	};
+}
+/**
+* Get environment variables for enabling V8 coverage in spawned processes.
+*/
+function getCoverageEnv(ctx) {
+	return { NODE_V8_COVERAGE: ctx.tempDir };
+}
+/**
+* Generate coverage report from collected V8 coverage data using c8.
+* Throws an error if coverage report generation fails.
+*/
+async function generateCoverageReport(ctx) {
+	const { options, tempDir } = ctx;
+	const c8Path = findC8Path$1();
+	const reportArgs = [
+		"report",
+		"--temp-directory",
+		tempDir,
+		"--reports-dir",
+		options.reportsDir,
+		"--src",
+		options.src,
+		"--all",
+		...options.include.flatMap((pattern) => ["--include", pattern]),
+		...options.exclude.flatMap((pattern) => ["--exclude", pattern]),
+		...options.excludeNodeModules ? ["--exclude-node-modules"] : ["--no-exclude-node-modules"],
+		...options.excludeAfterRemap ? ["--exclude-after-remap"] : [],
+		...options.skipFull ? ["--skip-full"] : [],
+		...options.allowExternal ? ["--allowExternal"] : [],
+		...options.monocart ? ["--experimental-monocart"] : [],
+		...options.reporters.flatMap((reporter) => ["--reporter", reporter])
+	];
+	const isNpx = c8Path === "npx c8";
+	const command = isNpx ? "npx" : c8Path;
+	const args = isNpx ? ["c8", ...reportArgs] : reportArgs;
+	await new Promise((resolvePromise, reject) => {
+		const proc = (0, node_child_process.spawn)(command, args, {
+			shell: false,
+			stdio: "inherit"
+		});
+		proc.on("close", (code) => {
+			if (code === 0) resolvePromise();
+			else reject(/* @__PURE__ */ new Error(`c8 report exited with code ${code}`));
+		});
+		proc.on("error", (err) => {
+			reject(/* @__PURE__ */ new Error(`Failed to run c8 report: ${err.message}`));
+		});
+	});
+}
+/**
+* Clean up coverage context by removing the temporary directory.
+*/
+async function cleanupCoverageContext(ctx) {
+	try {
+		await (0, node_fs_promises.access)(ctx.tempDir);
+		await (0, node_fs_promises.rm)(ctx.tempDir, {
+			recursive: true,
+			force: true
+		});
+	} catch {}
+}
+
 //#endregion
 //#region src/cli/commands/run.ts
 /**
 * Register the run command.
 */
 function registerRunCommand(program) {
-	program.command("run").description("Run golden tests").argument("[files...]", "Test files to run (default: **/*.tryscript.md)").option("--update", "Update golden files with actual output").option("--diff", "Show diff on failure (default: true)").option("--no-diff", "Hide diff on failure").option("--fail-fast", "Stop on first failure").option("--filter <pattern>", "Filter tests by name pattern").option("--verbose", "Show detailed output including passing test output").option("--quiet", "Suppress non-essential output (only show failures)").action(runCommand);
+	program.command("run").description("Run golden tests").argument("[files...]", "Test files to run (default: **/*.tryscript.md)").option("--update", "Update golden files with actual output").option("--diff", "Show diff on failure (default: true)").option("--no-diff", "Hide diff on failure").option("--fail-fast", "Stop on first failure").option("--filter <pattern>", "Filter tests by name pattern").option("--verbose", "Show detailed output including passing test output").option("--quiet", "Suppress non-essential output (only show failures)").option("--coverage", "Enable code coverage collection (requires c8)").option("--coverage-dir <dir>", "Coverage output directory (default: coverage-tryscript)").option("--coverage-reporter <reporter...>", "Coverage reporters (default: text, html). Can be specified multiple times.").option("--coverage-exclude <pattern...>", "Patterns to exclude from coverage (c8 --exclude). Can be specified multiple times.").option("--coverage-exclude-node-modules", "Exclude node_modules from coverage (c8 --exclude-node-modules, default: true)").option("--no-coverage-exclude-node-modules", "Include node_modules in coverage (c8 --no-exclude-node-modules)").option("--coverage-exclude-after-remap", "Apply exclude logic after sourcemap remapping (c8 --exclude-after-remap)").option("--coverage-skip-full", "Hide files with 100% coverage (c8 --skip-full)").option("--coverage-allow-external", "Allow files from outside cwd (c8 --allowExternal)").option("--coverage-monocart", "Use monocart for accurate line counts, better for merging with vitest (c8 --experimental-monocart)").action(runCommand$1);
 }
-async function runCommand(files, options) {
+async function runCommand$1(files, options) {
 	const startTime = Date.now();
 	const opts = {
 		diff: options.diff !== false,
@@ -198,6 +304,26 @@ async function runCommand(files, options) {
 		process.exit(1);
 	}
 	const globalConfig = await require_src.loadConfig(process.cwd());
+	let coverageCtx;
+	let coverageEnv = {};
+	if (options.coverage) {
+		if (!await isC8Available()) {
+			logError("Coverage requires c8. Install with: npm install -D c8");
+			process.exit(1);
+		}
+		coverageCtx = await createCoverageContext({
+			...globalConfig.coverage,
+			reportsDir: options.coverageDir ?? globalConfig.coverage?.reportsDir,
+			reporters: options.coverageReporter ?? globalConfig.coverage?.reporters,
+			exclude: options.coverageExclude ?? globalConfig.coverage?.exclude,
+			excludeNodeModules: options.coverageExcludeNodeModules ?? globalConfig.coverage?.excludeNodeModules,
+			excludeAfterRemap: options.coverageExcludeAfterRemap ?? globalConfig.coverage?.excludeAfterRemap,
+			skipFull: options.coverageSkipFull ?? globalConfig.coverage?.skipFull,
+			allowExternal: options.coverageAllowExternal ?? globalConfig.coverage?.allowExternal,
+			monocart: options.coverageMonocart ?? globalConfig.coverage?.monocart
+		});
+		coverageEnv = getCoverageEnv(coverageCtx);
+	}
 	const fileResults = [];
 	let shouldStop = false;
 	for (const filePath of testFiles) {
@@ -212,7 +338,7 @@ async function runCommand(files, options) {
 		const onlyBlocks = blocksToRun.filter((b) => b.only);
 		if (onlyBlocks.length > 0) blocksToRun = onlyBlocks;
 		if (blocksToRun.length === 0) continue;
-		const ctx = await require_src.createExecutionContext(config, filePath);
+		const ctx = await require_src.createExecutionContext(config, filePath, coverageEnv);
 		const results = [];
 		try {
 			for (const block of blocksToRun) {
@@ -264,9 +390,161 @@ async function runCommand(files, options) {
 		duration: Date.now() - startTime
 	};
 	reportSummary(summary, opts);
+	if (coverageCtx) {
+		console.error("\nGenerating coverage report...");
+		try {
+			await generateCoverageReport(coverageCtx);
+			console.error(colors.success(`Coverage report written to ${coverageCtx.options.reportsDir}/`));
+		} catch (error) {
+			logError(`Failed to generate coverage report: ${error instanceof Error ? error.message : String(error)}`);
+		} finally {
+			await cleanupCoverageContext(coverageCtx);
+		}
+	}
 	process.exit(summary.totalFailed > 0 ? 1 : 0);
 }
 
+//#endregion
+//#region src/cli/commands/coverage.ts
+/**
+* Register the coverage command.
+*/
+function registerCoverageCommand(program) {
+	program.command("coverage").description("Run commands with merged V8 coverage").argument("<commands...>", "Commands to run (each will inherit coverage environment)").option("--reports-dir <dir>", "Coverage output directory (default: coverage)").option("--reporters <reporters>", "Comma-separated coverage reporters (default: text,json,json-summary,lcov,html)").option("--include <patterns>", "Comma-separated patterns to include in coverage").option("--exclude <patterns>", "Comma-separated patterns to exclude from coverage").option("--exclude-node-modules", "Exclude node_modules from coverage (default: true)", true).option("--no-exclude-node-modules", "Include node_modules in coverage").option("--exclude-after-remap", "Apply exclude logic after sourcemap remapping").option("--skip-full", "Hide files with 100% coverage").option("--allow-external", "Allow files from outside cwd").option("--monocart", "Use monocart for accurate line counts (recommended for merging)").option("--src <dir>", "Source directory for sourcemap remapping (default: src)").action(coverageCommand);
+}
+/**
+* Run a command with inherited coverage environment.
+*/
+async function runCommand(command, env) {
+	return new Promise((resolve$2) => {
+		const proc = (0, node_child_process.spawn)(command, [], {
+			stdio: "inherit",
+			env: {
+				...process.env,
+				...env
+			},
+			shell: true
+		});
+		proc.on("close", (code) => {
+			resolve$2({
+				success: code === 0,
+				code: code ?? 1
+			});
+		});
+		proc.on("error", (err) => {
+			logError(`Failed to run command: ${err.message}`);
+			resolve$2({
+				success: false,
+				code: 1
+			});
+		});
+	});
+}
+/**
+* Find the c8 executable path.
+* Can be overridden via TRYSCRIPT_C8_COMMAND env var for testing.
+*/
+function findC8Path() {
+	const override = process.env.TRYSCRIPT_C8_COMMAND;
+	if (override) return override;
+	const localPaths = [(0, node_path.resolve)(process.cwd(), "node_modules", ".bin", "c8"), (0, node_path.resolve)(process.cwd(), "..", "..", "node_modules", ".bin", "c8")];
+	for (const localPath of localPaths) if ((0, node_fs.existsSync)(localPath)) return localPath;
+	return null;
+}
+/**
+* Generate c8 coverage report.
+*/
+async function generateReport(tempDir, options) {
+	const c8Path = findC8Path();
+	if (!c8Path) {
+		logError("c8 not found. Install with: npm install -D c8");
+		return false;
+	}
+	const reporters = options.reporters ?? [
+		"text",
+		"json",
+		"json-summary",
+		"lcov",
+		"html"
+	];
+	const include = options.include ?? ["dist/**"];
+	const exclude = options.exclude ?? [];
+	const reportArgs = [
+		"report",
+		"--temp-directory",
+		tempDir,
+		"--reports-dir",
+		options.reportsDir ?? "coverage",
+		"--src",
+		options.src ?? "src",
+		"--all",
+		...include.flatMap((pattern) => ["--include", pattern]),
+		...exclude.flatMap((pattern) => ["--exclude", pattern]),
+		...options.excludeNodeModules !== false ? ["--exclude-node-modules"] : ["--no-exclude-node-modules"],
+		...options.excludeAfterRemap ? ["--exclude-after-remap"] : [],
+		...options.skipFull ? ["--skip-full"] : [],
+		...options.allowExternal ? ["--allowExternal"] : [],
+		...options.monocart ? ["--experimental-monocart"] : [],
+		...reporters.flatMap((reporter) => ["--reporter", reporter])
+	];
+	return new Promise((resolve$2) => {
+		const proc = (0, node_child_process.spawn)(c8Path, reportArgs, {
+			stdio: "inherit",
+			shell: false
+		});
+		proc.on("close", (code) => {
+			resolve$2(code === 0);
+		});
+		proc.on("error", (err) => {
+			logError(`Failed to generate coverage report: ${err.message}`);
+			resolve$2(false);
+		});
+	});
+}
+async function coverageCommand(commands, options) {
+	if (commands.length === 0) {
+		logError("No commands specified. Usage: tryscript coverage \"cmd1\" \"cmd2\"");
+		process.exit(1);
+	}
+	if (!findC8Path()) {
+		logError("Coverage requires c8. Install with: npm install -D c8");
+		process.exit(1);
+	}
+	const coverageTemp = await (0, node_fs_promises.mkdtemp)((0, node_path.join)((0, node_os.tmpdir)(), "tryscript-coverage-"));
+	const coverageEnv = { NODE_V8_COVERAGE: coverageTemp };
+	console.error(colors.info(`Collecting V8 coverage to ${coverageTemp}`));
+	let hasFailures = false;
+	try {
+		for (let i = 0; i < commands.length; i++) {
+			const command = commands[i];
+			console.error(colors.info(`\n=== Running command ${i + 1}/${commands.length}: ${command} ===`));
+			const result = await runCommand(command, coverageEnv);
+			if (!result.success) {
+				logWarn(`Command exited with code ${result.code}: ${command}`);
+				hasFailures = true;
+			}
+		}
+		console.error(colors.info("\n=== Generating merged coverage report ==="));
+		const parsedOptions = {
+			...options,
+			reporters: options.reporters ? typeof options.reporters === "string" ? options.reporters.split(",") : options.reporters : void 0,
+			include: options.include ? typeof options.include === "string" ? options.include.split(",") : options.include : void 0,
+			exclude: options.exclude ? typeof options.exclude === "string" ? options.exclude.split(",") : options.exclude : void 0
+		};
+		if (!await generateReport(coverageTemp, parsedOptions)) {
+			logError("Failed to generate coverage report");
+			process.exit(1);
+		}
+		console.error(colors.success(`\nCoverage report written to ${parsedOptions.reportsDir ?? "coverage"}/`));
+	} finally {
+		await (0, node_fs_promises.rm)(coverageTemp, {
+			recursive: true,
+			force: true
+		});
+	}
+	if (hasFailures) process.exit(1);
+}
+
 //#endregion
 //#region src/cli/commands/readme.ts
 /**
@@ -346,7 +624,7 @@ function isInteractive$1() {
 */
 function showReadme(options) {
 	try {
-		const formatted = formatMarkdown$1(loadReadme(), !options?.raw && isInteractive$1());
+		const formatted = formatMarkdown$1(loadReadme(), options?.color ?? (!options?.raw && isInteractive$1()));
 		console.log(formatted);
 	} catch (error) {
 		const message = error instanceof Error ? error.message : String(error);
@@ -358,7 +636,7 @@ function showReadme(options) {
 * Register the readme command.
 */
 function registerReadmeCommand(program) {
-	program.command("readme").description("Display README documentation").option("--raw", "Output raw markdown without formatting").action(showReadme);
+	program.command("readme").description("Display README documentation").option("--raw", "Output raw markdown without formatting").option("--color", "Force colorized output (for testing)").action(showReadme);
 }
 
 //#endregion
@@ -438,9 +716,9 @@ function isInteractive() {
 * Register the docs command.
 */
 function registerDocsCommand(program) {
-	program.command("docs").description("Display concise syntax reference").option("--raw", "Output raw markdown without formatting").action((options) => {
+	program.command("docs").description("Display concise syntax reference").option("--raw", "Output raw markdown without formatting").option("--color", "Force colorized output (for testing)").action((options) => {
 		try {
-			const formatted = formatMarkdown(loadDocs(), !options.raw && isInteractive());
+			const formatted = formatMarkdown(loadDocs(), options.color ?? (!options.raw && isInteractive()));
 			console.log(formatted);
 		} catch (error) {
 			const message = error instanceof Error ? error.message : String(error);
@@ -460,6 +738,7 @@ function registerDocsCommand(program) {
 function run(argv) {
 	const program = withColoredHelp(new commander.Command().name("tryscript").version(require_src.VERSION, "--version", "Show version number").description("Golden testing for CLI applications").showHelpAfterError("(use --help for usage)"));
 	registerRunCommand(program);
+	registerCoverageCommand(program);
 	registerReadmeCommand(program);
 	registerDocsCommand(program);
 	program.action(() => {