pi-agent-browser-native 0.9.3 → 0.9.4

package/scripts/platform-smoke/lease-evidence.mjs

@@ -0,0 +1,134 @@
+/** Lease warmup/cleanup receipts remain part of the platform's final acceptance result. */
+import { createSuiteDir, writeCommand, writeExitCode } from "./artifacts.mjs";
+import {
+	COMMON_EVIDENCE_FILES,
+	STOP_EVIDENCE_FILES,
+	finalizeSuite,
+	makeRunId,
+	secretValuesFor,
+	secretViolationsFor,
+	writeProcessReceipt,
+	writeRedacted,
+	writeTargetEvidence,
+} from "./suite-evidence.mjs";
+import { resolve } from "node:path";
+
+export function createLeaseCleanupResult({
+	config,
+	targetName,
+	leaseId,
+	stopResult,
+	staleCleanupResult = null,
+	runId = makeRunId(),
+}) {
+	const suiteName = "lease-cleanup";
+	const suiteDir = createSuiteDir(config.artifactRoot, runId, targetName, suiteName);
+	const secretValues = secretValuesFor(config);
+	writeTargetEvidence(
+		suiteDir,
+		{ config, targetName, runId, slug: `${config.packageName}-${targetName}` },
+		{ suiteName, leaseId, modelCalls: 0 },
+	);
+	writeCommand(suiteDir, `crabbox stop ${targetName} --id ${leaseId}`);
+	writeExitCode(suiteDir, stopResult.code, stopResult.signal);
+	writeProcessReceipt(suiteDir, "crabbox.stop", stopResult, secretValues);
+	const cleanupFiles = [];
+	if (staleCleanupResult) {
+		writeProcessReceipt(suiteDir, "crabbox.cleanup", staleCleanupResult, secretValues);
+		cleanupFiles.push(
+			"crabbox.cleanup.stdout.txt",
+			"crabbox.cleanup.stderr.txt",
+			"crabbox.cleanup.exit-code.txt",
+		);
+	}
+	const secretViolations = secretViolationsFor(suiteDir, stopResult, secretValues);
+	const { assertions } = finalizeSuite(
+		suiteDir,
+		[
+			{
+				id: "lease-cleanup",
+				fn: () => stopResult.code === 0,
+				error: `Crabbox stop failed with exit ${stopResult.code}`,
+			},
+			{
+				id: "stale-cleanup",
+				fn: () => !staleCleanupResult || staleCleanupResult.code === 0,
+				error: `Crabbox cleanup failed with exit ${staleCleanupResult?.code}`,
+			},
+			{
+				id: "no-secret-artifacts",
+				fn: () => secretViolations.length === 0,
+				error: secretViolations.join(", "),
+			},
+		],
+		{
+			target: targetName,
+			suite: suiteName,
+			exitCode: stopResult.code,
+			signal: stopResult.signal,
+			elapsedMs: 0,
+		},
+		[
+			"summary.json",
+			"artifact-manifest.json",
+			"target.json",
+			"suite.json",
+			"command.txt",
+			"exit-code.txt",
+			...STOP_EVIDENCE_FILES,
+			...cleanupFiles,
+			"assertions.json",
+		],
+	);
+	return { ok: assertions.ok, suiteDir, assertions };
+}
+
+export function createLeaseCleanupFailureResult(options) {
+	return createLeaseCleanupResult({ ...options, staleCleanupResult: null });
+}
+
+export function createLeaseWarmupFailureResult(
+	config,
+	targetName,
+	warmupResult,
+	runId = makeRunId(),
+) {
+	const suiteName = "lease-warmup";
+	const suiteDir = createSuiteDir(config.artifactRoot, runId, targetName, suiteName);
+	const secretValues = secretValuesFor(config);
+	writeTargetEvidence(
+		suiteDir,
+		{ config, targetName, runId, slug: `${config.packageName}-${targetName}` },
+		{ suiteName, modelCalls: 0 },
+	);
+	writeCommand(suiteDir, `crabbox warmup ${targetName}`);
+	writeExitCode(suiteDir, warmupResult.code, warmupResult.signal);
+	writeRedacted(resolve(suiteDir, "crabbox.stdout.txt"), warmupResult.stdout ?? "", secretValues);
+	writeRedacted(resolve(suiteDir, "crabbox.stderr.txt"), warmupResult.stderr ?? "", secretValues);
+	const secretViolations = secretViolationsFor(suiteDir, warmupResult, secretValues);
+	const { assertions } = finalizeSuite(
+		suiteDir,
+		[
+			{
+				id: "lease-warmup",
+				fn: () => false,
+				error: `Crabbox warmup failed with exit ${warmupResult.code}`,
+			},
+			{
+				id: "no-secret-artifacts",
+				fn: () => secretViolations.length === 0,
+				error: secretViolations.join(", "),
+			},
+		],
+		{
+			target: targetName,
+			suite: suiteName,
+			exitCode: warmupResult.code,
+			signal: warmupResult.signal,
+			elapsedMs: 0,
+			ok: false,
+		},
+		COMMON_EVIDENCE_FILES,
+	);
+	return { ok: false, suiteDir, assertions };
+}

package/scripts/platform-smoke/suite-checks.mjs

@@ -0,0 +1,98 @@
+/** Platform and browser evidence predicates; no success is inferred from process exit alone. */
+import { marker, platformFor, section } from "./suite-evidence.mjs";
+
+export function nodeVersionAtLeast(actual, minimum) {
+	return (
+		/^v?\d+\.\d+\.\d+$/.test(actual) &&
+		actual.replace(/^v/, "").localeCompare(minimum, undefined, { numeric: true }) >= 0
+	);
+}
+
+export function commonChecks(result, secretViolations, stopResult) {
+	return [
+		{ id: "command-exit-zero", fn: () => result.code === 0, error: `exit ${result.code}` },
+		{
+			id: "no-secret-artifacts",
+			fn: () => secretViolations.length === 0,
+			error: secretViolations.join(", "),
+		},
+		...(stopResult
+			? [
+					{
+						id: "lease-cleanup",
+						fn: () => stopResult.code === 0,
+						error: `stop exit ${stopResult.code}`,
+					},
+				]
+			: []),
+	];
+}
+
+export function platformBuildChecks(config, result) {
+	const stdout = result.stdout;
+	const listOutput = section(stdout, "PI_LIST_STDOUT");
+	const nodeVersion = marker(stdout, "PLATFORM_NODE_VERSION");
+	const packagePattern = config.packageName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	return [
+		{ id: "platform-marker", fn: () => stdout.includes("PLATFORM_BUILD_OK") },
+		{
+			id: "node-version",
+			fn: () => nodeVersionAtLeast(nodeVersion, config.nodeValidationVersion),
+			error: `Node ${nodeVersion || "unknown"} < ${config.nodeValidationVersion}`,
+		},
+		{ id: "npm-ci", fn: () => /PLATFORM_NPM_CI_EXIT=0/.test(stdout) },
+		{ id: "npm-run-verify", fn: () => /PLATFORM_VERIFY_EXIT=0/.test(stdout) },
+		{
+			id: "npm-pack",
+			fn: () =>
+				/PLATFORM_NPM_PACK_EXIT=0/.test(stdout) &&
+				marker(stdout, "PLATFORM_PACKED_TARBALL").length > 0,
+		},
+		{ id: "packed-node-install", fn: () => /PLATFORM_PACKED_NODE_INSTALL_EXIT=0/.test(stdout) },
+		{ id: "pi-install-local-package", fn: () => /PLATFORM_PI_INSTALL_EXIT=0/.test(stdout) },
+		{
+			id: "pi-list-local-package",
+			fn: () =>
+				/PLATFORM_PI_LIST_EXIT=0/.test(stdout) &&
+				new RegExp(`Project packages:[\\s\\S]*${packagePattern}`).test(listOutput),
+		},
+		{
+			id: "no-source-extension-shortcut",
+			fn: () => !/\bpi\s+(?:-e|--extension)\s+\./.test(stdout),
+		},
+	];
+}
+
+export function browserDogfoodChecks(targetName, result, dogfoodReport) {
+	const platform = platformFor(targetName);
+	const reportIds = new Set((dogfoodReport.reports ?? []).map((report) => report.id));
+	return [
+		{
+			id: "browser-dogfood-marker",
+			fn: () => result.stdout.includes("PLATFORM_BROWSER_DOGFOOD_OK"),
+		},
+		{ id: "npm-ci", fn: () => /PLATFORM_NPM_CI_EXIT=0/.test(result.stdout) },
+		{
+			id: "agent-browser-baseline",
+			fn: () => /PLATFORM_AGENT_BROWSER_READY_EXIT=0/.test(result.stdout),
+		},
+		{
+			id: "agent-browser-browser-cache",
+			fn: () =>
+				platform !== "powershell" ||
+				/PLATFORM_AGENT_BROWSER_BROWSER_CACHE_EXIT=0/.test(result.stdout),
+		},
+		{
+			id: "agent-browser-prewarm",
+			fn: () =>
+				platform !== "powershell" || /PLATFORM_AGENT_BROWSER_PREWARM_EXIT=0/.test(result.stdout),
+		},
+		{ id: "dogfood-exit-zero", fn: () => /PLATFORM_DOGFOOD_EXIT=0/.test(result.stdout) },
+		{
+			id: "dogfood-report",
+			fn: () => Array.isArray(dogfoodReport.reports) && dogfoodReport.reports.length >= 5,
+		},
+		{ id: "dogfood-qa", fn: () => reportIds.has("qa-url") },
+		{ id: "dogfood-session-close", fn: () => reportIds.has("close-session") },
+	];
+}

package/scripts/platform-smoke/suite-evidence.mjs

@@ -0,0 +1,281 @@
+/** Native process receipts, redacted artifacts, and fail-closed suite evidence. */
+import { execFileSync } from "node:child_process";
+import { readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import {
+	collectSecretValues,
+	redactSecrets,
+	scanArtifactTextFiles,
+	scanForSecrets,
+	writeExitCode,
+	writeManifest,
+	writeSummary,
+} from "./artifacts.mjs";
+import { crabboxBin, describeTarget } from "./crabbox-runner.mjs";
+
+export const COMMON_EVIDENCE_FILES = [
+	"summary.json",
+	"artifact-manifest.json",
+	"target.json",
+	"suite.json",
+	"command.txt",
+	"exit-code.txt",
+	"crabbox.stdout.txt",
+	"crabbox.stderr.txt",
+	"assertions.json",
+];
+export const STOP_EVIDENCE_FILES = [
+	"crabbox.stop.stdout.txt",
+	"crabbox.stop.stderr.txt",
+	"crabbox.stop.exit-code.txt",
+];
+
+export function platformFor(targetName) {
+	return targetName === "windows-native" ? "powershell" : "posix";
+}
+
+export function makeRunId() {
+	return `run-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+export function authEnvAllowList(config = {}) {
+	const raw = process.env.PLATFORM_SMOKE_AUTH_ENV;
+	const names = raw ? raw.split(",") : (config.defaultAuthEnv ?? []);
+	return names.map((name) => String(name).trim()).filter(Boolean);
+}
+
+function packageVersion() {
+	try {
+		return JSON.parse(readFileSync("package.json", "utf8")).version ?? null;
+	} catch {
+		return null;
+	}
+}
+
+function crabboxVersion() {
+	try {
+		return (
+			execFileSync(crabboxBin(), ["--version"], {
+				encoding: "utf8",
+				stdio: "pipe",
+				timeout: 10_000,
+			})
+				.trim()
+				.split(/\r?\n/)[0] ?? null
+		);
+	} catch {
+		return null;
+	}
+}
+
+export function writeTargetEvidence(suiteDir, { config, targetName, runId, slug }, suite) {
+	const target = describeTarget(targetName, config);
+	const evidence = {
+		targetName,
+		platform: platformFor(targetName),
+		runId,
+		slug,
+		packageName: config.packageName,
+		packageVersion: packageVersion(),
+		crabbox: {
+			binary: crabboxBin(),
+			version: crabboxVersion(),
+			provider: target.provider,
+			target: target.crabboxTarget,
+			workRoot: target.workRoot,
+			image: target.image,
+			windowsMode: target.windowsMode,
+			sourceVm: target.sourceVm,
+			snapshot: target.snapshot,
+		},
+	};
+	writeFileSync(resolve(suiteDir, "target.json"), JSON.stringify(evidence, null, 2));
+	writeFileSync(resolve(suiteDir, "suite.json"), JSON.stringify(suite, null, 2));
+}
+
+export function secretValuesFor(config) {
+	return collectSecretValues(authEnvAllowList(config));
+}
+
+export function writeRedacted(path, text, secretValues) {
+	writeFileSync(path, redactSecrets(text ?? "", secretValues));
+}
+
+export function writeProcessReceipt(suiteDir, prefix, result, secretValues) {
+	writeRedacted(resolve(suiteDir, `${prefix}.stdout.txt`), result.stdout ?? "", secretValues);
+	writeRedacted(resolve(suiteDir, `${prefix}.stderr.txt`), result.stderr ?? "", secretValues);
+	writeFileSync(
+		resolve(suiteDir, `${prefix}.exit-code.txt`),
+		`code=${result.code}\nsignal=${result.signal ?? "none"}\n`,
+	);
+}
+
+export function secretViolationsFor(suiteDir, result, secretValues) {
+	return [
+		...scanForSecrets(`${result.stdout ?? ""}\n${result.stderr ?? ""}`, secretValues),
+		...scanArtifactTextFiles(suiteDir, secretValues).map(
+			(finding) => `${finding.file}: ${finding.violation}`,
+		),
+	];
+}
+
+export function section(text, name) {
+	const start = `--- ${name} START ---`;
+	const end = `--- ${name} END ---`;
+	const startIndex = text.indexOf(start);
+	if (startIndex === -1) {
+		return "";
+	}
+	const contentStart = startIndex + start.length;
+	const endIndex = text.indexOf(end, contentStart);
+	return (endIndex === -1 ? text.slice(contentStart) : text.slice(contentStart, endIndex))
+		.replace(/^\r?\n/, "")
+		.replace(/\r?\n$/, "");
+}
+
+export function marker(text, name) {
+	return text.match(new RegExp(`^${name}=(.*)$`, "m"))?.[1]?.trim() ?? "";
+}
+
+function parseJsonObject(text) {
+	const trimmed = String(text ?? "").trim();
+	if (!trimmed) {
+		return {};
+	}
+	try {
+		return JSON.parse(trimmed);
+	} catch {
+		const first = trimmed.indexOf("{");
+		const last = trimmed.lastIndexOf("}");
+		if (first !== -1 && last > first) {
+			try {
+				return JSON.parse(trimmed.slice(first, last + 1));
+			} catch {
+				return {};
+			}
+		}
+		return {};
+	}
+}
+
+export function writePlatformExtracts(suiteDir, stdout, secretValues = []) {
+	writeFileSync(
+		resolve(suiteDir, "node-version.txt"),
+		`${marker(stdout, "PLATFORM_NODE_VERSION")}\n`,
+	);
+	writeRedacted(
+		resolve(suiteDir, "packed-tarball.txt"),
+		`${marker(stdout, "PLATFORM_PACKED_TARBALL")}\n`,
+		secretValues,
+	);
+	for (const [file, label] of [
+		["packed-node-install", "PACKED_NODE_INSTALL"],
+		["pi-install", "PI_INSTALL"],
+		["pi-list", "PI_LIST"],
+	]) {
+		for (const [stream, suffix] of [
+			["stdout", "STDOUT"],
+			["stderr", "STDERR"],
+		]) {
+			writeRedacted(
+				resolve(suiteDir, `${file}.${stream}.txt`),
+				section(stdout, `${label}_${suffix}`),
+				secretValues,
+			);
+		}
+	}
+}
+
+export function writeDogfoodExtracts(suiteDir, stdout, secretValues = []) {
+	writeFileSync(
+		resolve(suiteDir, "node-version.txt"),
+		`${marker(stdout, "PLATFORM_NODE_VERSION")}\n`,
+	);
+	writeRedacted(
+		resolve(suiteDir, "dogfood-artifacts.txt"),
+		`${marker(stdout, "PLATFORM_DOGFOOD_ARTIFACT_DIR")}\n`,
+		secretValues,
+	);
+	const dogfoodStdout = section(stdout, "DOGFOOD_STDOUT");
+	writeRedacted(resolve(suiteDir, "dogfood.stdout.txt"), dogfoodStdout, secretValues);
+	writeRedacted(
+		resolve(suiteDir, "dogfood.stderr.txt"),
+		section(stdout, "DOGFOOD_STDERR"),
+		secretValues,
+	);
+	const report = parseJsonObject(dogfoodStdout);
+	writeRedacted(
+		resolve(suiteDir, "dogfood-report.json"),
+		JSON.stringify(report, null, 2),
+		secretValues,
+	);
+	return report;
+}
+
+function writeAssertions(suiteDir, checks) {
+	const evaluated = checks.map((check) => {
+		let ok = false;
+		let error = check.error;
+		try {
+			ok = check.fn() === true;
+		} catch (err) {
+			error = err.message;
+		}
+		return { id: check.id, ok, ...(ok ? {} : { error: error ?? `${check.id} failed` }) };
+	});
+	const assertions = {
+		ok: evaluated.every((check) => check.ok),
+		checks: evaluated,
+		writtenAt: new Date().toISOString(),
+	};
+	writeFileSync(resolve(suiteDir, "assertions.json"), JSON.stringify(assertions, null, 2));
+	if (!assertions.ok) {
+		writeFileSync(
+			resolve(suiteDir, "failures.md"),
+			[
+				"# Platform smoke failures",
+				"",
+				...assertions.checks
+					.filter((check) => !check.ok)
+					.map((check) => `- ${check.id}: ${check.error ?? "failed"}`),
+				"",
+				"Inspect command.txt, crabbox.stdout.txt, and crabbox.stderr.txt in this suite directory.",
+				"",
+			].join("\n"),
+		);
+	}
+	return assertions;
+}
+
+export function finalizeSuite(suiteDir, checks, summary, expectedFiles) {
+	const assertions = writeAssertions(suiteDir, checks);
+	writeSummary(suiteDir, { ...summary, ok: assertions.ok });
+	const expected = assertions.ok ? expectedFiles : [...expectedFiles, "failures.md"];
+	const manifest = writeManifest(suiteDir, expected);
+	if (manifest.missing.length === 0) {
+		return { assertions, manifest };
+	}
+	const finalAssertions = writeAssertions(suiteDir, [
+		...checks,
+		{
+			id: "artifact-manifest-complete",
+			fn: () => false,
+			error: `missing required artifact(s): ${manifest.missing.join(", ")}`,
+		},
+	]);
+	writeSummary(suiteDir, { ...summary, ok: false });
+	return {
+		assertions: finalAssertions,
+		manifest: writeManifest(suiteDir, [...expectedFiles, "failures.md"]),
+	};
+}
+
+export function writeRunReceipt(suiteDir, result, elapsedMs, secretValues) {
+	writeRedacted(resolve(suiteDir, "crabbox.stdout.txt"), result.stdout, secretValues);
+	writeRedacted(resolve(suiteDir, "crabbox.stderr.txt"), result.stderr, secretValues);
+	writeFileSync(
+		resolve(suiteDir, "crabbox.timing.json"),
+		JSON.stringify({ elapsedMs, code: result.code, signal: result.signal }, null, 2),
+	);
+	writeExitCode(suiteDir, result.code, result.signal);
+}