playwright-test-agent 0.1.0

package/bin/playwright-test-agent.mjs

@@ -0,0 +1,29 @@
+#!/usr/bin/env node
+
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { initializeProject } from '../playwright-test-agent/scripts/init-playwright.mjs';
+
+const usage = `Usage: playwright-test-agent init [target-directory]
+
+Initialize Playwright Test agents, install the playwright-test-agent skill,
+and update AGENTS.md and CLAUDE.md in the target project.`;
+
+const [command, targetDirectory, ...extraArguments] = process.argv.slice(2);
+
+if (command !== 'init' || extraArguments.length > 0) {
+  console.error(usage);
+  process.exitCode = 1;
+} else {
+  const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+
+  try {
+    await initializeProject({
+      projectDir: path.resolve(targetDirectory ?? process.cwd()),
+      skillSourceDir: path.join(packageRoot, 'playwright-test-agent'),
+    });
+  } catch (error) {
+    console.error(`playwright-test-agent init failed: ${error.message}`);
+    process.exitCode = 1;
+  }
+}

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "playwright-test-agent",
+  "version": "0.1.0",
+  "description": "Initialize Playwright Test agents and install the playwright-test-agent skill.",
+  "type": "module",
+  "bin": {
+    "playwright-test-agent": "./bin/playwright-test-agent.mjs"
+  },
+  "files": [
+    "bin/",
+    "playwright-test-agent/"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "license": "UNLICENSED"
+}

package/playwright-test-agent/SKILL.md

@@ -0,0 +1,71 @@
+---
+name: playwright-test-agent
+description: Use first when a user asks to test a website, browser workflow, HTTP API, or application feature, including planning, generating, running, debugging, healing, screenshots, and reusable test evidence.
+---
+
+# Playwright Test Agent
+
+Use Playwright Test for durable automation. Browser tests use Playwright's official planner, generator, and healer subagents. Pure HTTP API tests may be written directly with Playwright `APIRequestContext` without those subagents.
+
+## Initialize once per project
+
+Inspect the target project first. If Playwright Test, its config, or either Codex/Claude agent set is missing, tell the user initialization will modify the project, then run from the target project:
+
+```bash
+npx playwright-test-agent init
+```
+
+The command installs this skill for Codex and Claude, selects the ready-to-use Playwright defaults (TypeScript, `playwright-tests/`, no GitHub Actions, Chromium with browser installation), and runs both:
+
+```text
+npx playwright init-agents --loop=codex
+npx playwright init-agents --loop=claude
+```
+
+It uses the project-local Playwright command. Never install a global/additional Playwright CLI or run `playwright init-skills`.
+
+## Understand the request
+
+Before testing, inspect the target project for relevant source, routes/API clients, environment examples, existing tests, and run instructions. The project may be empty or may contain no useful application context; in that case, do not keep searching the filesystem or invent missing details. Use the user's description and the live target as the available evidence.
+
+Establish the objective, reachable environment/base URL, allowed side effects, authentication/roles, required test data/variables, and observable success criteria before invoking the planner. Ask the user for any material fact that cannot be verified from the project or target application.
+
+If reconnaissance shows that provided information is missing, invalid, stale, or contradictory—for example, the URL is unreachable, a page or control does not exist, credentials fail, the observed role differs, a required variable is absent, or the application behavior conflicts with the stated workflow—stop exploring. Report the exact mismatch without exposing secrets and ask the user to correct or complete the information. Do not repeatedly try nearby URLs, guess credentials, wander through unrelated pages, or invoke the planner while blocked. Resume reconnaissance and invoke the planner only after the user supplies enough corrected information.
+
+Never put credentials or tokens in plans, source, screenshots, reports, or chat output. Use environment variables or an ignored secret file.
+
+## Plan and confirm
+
+Always save a plan to `specs/<feature>.plan.md` before generating or running tests. Include prerequisites, seed, test data, independent scenarios, steps or requests, expected observable results, and output test files.
+
+For UI work, invoke `playwright_test_planner` only after the required project/user information has been checked and no known mismatch is blocking exploration. Have it inspect the live application and save the plan. Reconnaissance must not create, delete, submit, purchase, message, or otherwise mutate durable/shared data without authorization.
+
+If the planner discovers a new missing or incorrect prerequisite, stop that planning attempt, summarize what was observed, and ask the user for clarification. Continue with the planner only after the prerequisite is resolved; do not let the planner keep exploring around the missing information.
+
+Choose relevant positive, negative, empty, invalid, boundary, permission, persistence, and error scenarios. For login, normally consider valid credentials, empty username, empty password, both empty, wrong password, and unknown user; add MFA, lockout, recovery, or remember-me only when in scope.
+
+Show the complete scenario list and exclusions to the user. Ask whether it is complete. Revise until confirmed. Do not invoke the generator or formally execute tests before confirmation.
+
+## Generate and run
+
+For confirmed UI scenarios, invoke `playwright_test_generator` once for the complete confirmed plan (or the complete selected set of scenarios), passing the exact plan content, seed, and destination under `playwright-tests/`. The generator processes the scenarios in plan order in that single invocation; its internal per-scenario setup does not mean starting a new generator agent for each scenario. Require one independent test per file, semantic locators, an assertion for every expected result, and environment-based secrets.
+
+For confirmed API-only scenarios, write `APIRequestContext` tests directly under `playwright-tests/`. Assert status, headers, schema, and stable business invariants; define safe setup/cleanup for mutations.
+
+Run the generated tests with the project-local command. On failure, use `playwright_test_healer` one failure at a time for UI tests; API failures may be diagnosed directly. If observed product behavior conflicts with the confirmed plan, ask whether it is a regression or intended change. Do not weaken assertions, add sleeps/`networkidle`, retry blindly, or skip/fixme tests merely to get green.
+
+## Preserve evidence
+
+All browser-agent and test artifacts must stay under:
+
+```text
+.playwright-evidence/
+├── mcp/           # page YAML snapshots, screenshots, and agent logs
+├── snapshots/     # reusable visual/ARIA baselines
+├── test-results/  # failure screenshots, traces, videos, attachments
+└── report/        # HTML report
+```
+
+Never save snapshots or screenshots in the project root. Automatic failure evidence is enabled. For key business states, save an explicit screenshot with `testInfo.outputPath('screenshots', '<meaningful-name>.png')` so it remains in `test-results/`. Avoid secrets and sensitive personal data.
+
+Finish by reporting the plan path, generated test paths, environment without secrets, pass/fail/flaky/skipped counts, and evidence paths. Classify failures as application defect, test defect, environment/data problem, or unresolved product decision.

package/playwright-test-agent/scripts/init-playwright.mjs

@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+
+import { access, cp, mkdir, readFile, rename, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+const BLOCK_START = '<!-- playwright-test-agent:start -->';
+const BLOCK_END = '<!-- playwright-test-agent:end -->';
+const PROJECT_INSTRUCTIONS = `${BLOCK_START}
+## Playwright Test Agent
+
+When asked to test a website, browser workflow, HTTP API, or application feature, load and follow the project-installed \`playwright-test-agent\` skill first. For UI tests, use its Playwright Test planner, generator, and healer workflow instead of ad hoc browser automation. Keep its confirmation, credential, side-effect, and evidence rules in force.
+${BLOCK_END}`;
+
+const exists = async (file) => {
+  try {
+    await access(file);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+const defaultRun = (projectDir, command, args) => new Promise((resolve, reject) => {
+  const executable = process.platform === 'win32' ? `${command}.cmd` : command;
+  const child = spawn(executable, args, { cwd: projectDir, stdio: 'inherit' });
+  child.on('error', reject);
+  child.on('exit', (code) => code === 0
+    ? resolve()
+    : reject(new Error(`${command} ${args.join(' ')} exited with ${code}`)));
+});
+
+function withManagedBlock(source) {
+  const start = source.indexOf(BLOCK_START);
+  const end = source.indexOf(BLOCK_END);
+
+  if ((start === -1) !== (end === -1) || (start !== -1 && end < start)) {
+    throw new Error('found an incomplete playwright-test-agent managed block');
+  }
+
+  if (start !== -1) {
+    return source.slice(0, start) + PROJECT_INSTRUCTIONS + source.slice(end + BLOCK_END.length);
+  }
+
+  if (source.length === 0) return `${PROJECT_INSTRUCTIONS}\n`;
+  const separator = source.endsWith('\n') ? '\n' : '\n\n';
+  return `${source}${separator}${PROJECT_INSTRUCTIONS}\n`;
+}
+
+async function installSkill(projectDir, skillSourceDir) {
+  for (const root of ['.agents', '.claude']) {
+    const destination = path.join(projectDir, root, 'skills', 'playwright-test-agent');
+    await mkdir(destination, { recursive: true });
+    await cp(skillSourceDir, destination, { recursive: true, force: true });
+  }
+}
+
+async function updateInstructionFiles(projectDir) {
+  for (const name of ['AGENTS.md', 'CLAUDE.md']) {
+    const file = path.join(projectDir, name);
+    const source = await exists(file) ? await readFile(file, 'utf8') : '';
+    await writeFile(file, withManagedBlock(source), 'utf8');
+  }
+}
+
+async function patchPlaywrightConfig(projectDir) {
+  const configPath = path.join(projectDir, 'playwright.config.ts');
+  let source = await readFile(configPath, 'utf8');
+  source = source.replace(/testDir:\s*['"]\.\/tests['"]/, "testDir: './playwright-tests'");
+  source = source.replace(
+    /reporter:\s*['"]html['"],?/,
+    "reporter: [['html', { outputFolder: '.playwright-evidence/report', open: 'never' }]],",
+  );
+  source = source.replace(
+    /use:\s*\{/,
+    "outputDir: '.playwright-evidence/test-results',\n  snapshotPathTemplate: '.playwright-evidence/snapshots/{testFilePath}/{arg}{ext}',\n  use: {\n    screenshot: 'only-on-failure',\n    video: 'retain-on-failure',",
+  );
+  source = source.replace(/trace:\s*['"][^'"]+['"]/, "trace: 'retain-on-failure'");
+  await writeFile(configPath, source, 'utf8');
+}
+
+async function configureClaudeMcp(projectDir) {
+  const file = path.join(projectDir, '.mcp.json');
+  if (!await exists(file)) return;
+  const config = JSON.parse(await readFile(file, 'utf8'));
+  const server = config.mcpServers?.['playwright-test'];
+  if (!server) return;
+  server.env = {
+    ...(server.env ?? {}),
+    PLAYWRIGHT_MCP_OUTPUT_DIR: '.playwright-evidence/mcp',
+  };
+  await writeFile(file, `${JSON.stringify(config, null, 2)}\n`, 'utf8');
+}
+
+async function configureCodexAgents(projectDir) {
+  for (const name of ['planner', 'generator', 'healer']) {
+    const file = path.join(projectDir, '.codex', 'agents', `playwright_test_${name}.toml`);
+    if (!await exists(file)) continue;
+    let source = await readFile(file, 'utf8');
+    if (!source.includes('[mcp_servers.playwright-test.env]')) {
+      source = `${source.trimEnd()}\n\n[mcp_servers.playwright-test.env]\n` +
+        'PLAYWRIGHT_MCP_OUTPUT_DIR = ".playwright-evidence/mcp"\n';
+      await writeFile(file, source, 'utf8');
+    }
+  }
+}
+
+async function initializePlaywright(projectDir, run) {
+  const configPath = path.join(projectDir, 'playwright.config.ts');
+  if (!await exists(configPath)) {
+    await run(projectDir, 'npm', [
+      'init',
+      'playwright@latest',
+      '--',
+      '--quiet',
+      '--lang=TypeScript',
+      '--browser=chromium',
+    ]);
+
+    const generatedExample = path.join(projectDir, 'tests', 'example.spec.ts');
+    const targetDir = path.join(projectDir, 'playwright-tests');
+    if (await exists(generatedExample)) {
+      await mkdir(targetDir, { recursive: true });
+      await rename(generatedExample, path.join(targetDir, 'example.spec.ts'));
+    }
+    await patchPlaywrightConfig(projectDir);
+  }
+
+  await run(projectDir, 'npx', ['--no-install', 'playwright', 'init-agents', '--loop=codex']);
+  await run(projectDir, 'npx', ['--no-install', 'playwright', 'init-agents', '--loop=claude']);
+  await mkdir(path.join(projectDir, '.playwright-evidence', 'mcp'), { recursive: true });
+  await configureClaudeMcp(projectDir);
+  await configureCodexAgents(projectDir);
+}
+
+async function runStage(name, action) {
+  process.stdout.write(`[playwright-test-agent] ${name}...\n`);
+  try {
+    await action();
+  } catch (error) {
+    throw new Error(`${name}: ${error.message}`, { cause: error });
+  }
+}
+
+export async function initializeProject({
+  projectDir = process.cwd(),
+  skillSourceDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'),
+  run = defaultRun,
+} = {}) {
+  const target = path.resolve(projectDir);
+  await mkdir(target, { recursive: true });
+
+  await runStage('installing skill', () => installSkill(target, skillSourceDir));
+  await runStage('initializing Playwright Test agents', () => initializePlaywright(target, run));
+  await runStage('updating project instructions', () => updateInstructionFiles(target));
+
+  process.stdout.write(
+    'Playwright Test Agent ready: skills installed for Codex and Claude, ' +
+    'tests in playwright-tests/, evidence in .playwright-evidence/.\n',
+  );
+}
+
+const isDirectInvocation = process.argv[1] &&
+  path.resolve(process.argv[1]) === path.resolve(fileURLToPath(import.meta.url));
+
+if (isDirectInvocation) {
+  initializeProject({ projectDir: process.argv[2] ?? process.cwd() }).catch((error) => {
+    console.error(`Playwright initialization failed: ${error.message}`);
+    process.exitCode = 1;
+  });
+}