retestkit 1.4.1

package/openspec/specs/webtest-tools/spec.md

@@ -0,0 +1,282 @@
+# webtest-tools Specification
+
+## Purpose
+TBD - created by archiving change add-webtest-orchestrator. Update Purpose after archive.
+## Requirements
+### Requirement: webtest_init Tool
+
+The system SHALL provide a `webtest_init` tool that initializes an analysis workspace for a target URL and focus, storing metadata in markdown format.
+
+#### Scenario: Start analysis with valid URL
+
+- **GIVEN** the tool is called with a valid URL and focus
+- **WHEN** execution completes
+- **THEN** it SHALL generate a unique `analysisId`
+- **AND** create workspace directories
+- **AND** write initial `index.md` metadata with YAML frontmatter
+- **AND** return `{ analysisId, workspaceRootPath, workspaceRootUri, statusUri }` where statusUri points to `index.md`
+
+### Requirement: webtest_crawl_app Tool
+
+The system SHALL provide a `webtest_crawl_app` tool that dynamically explores a web application, storing all crawl artifacts in markdown format.
+
+#### Scenario: Crawl captures artifacts at each checkpoint
+
+- **GIVEN** a crawl iteration completes an action
+- **WHEN** state is captured
+- **THEN** it SHALL call Playwright MCP `browser_snapshot` for accessibility tree
+- **AND** call `browser_take_screenshot` for visual evidence
+- **AND** optionally extract HTML DOM
+- **AND** store snapshot as `snapshot.md` with formatted accessibility tree and YAML frontmatter
+- **AND** store screenshot as PNG
+- **AND** store DOM as HTML
+
+#### Scenario: Crawl outputs complete results
+
+- **GIVEN** crawl has finalized
+- **WHEN** output is returned
+- **THEN** it SHALL include `crawlId`, `crawlIndexFilePath`, `crawlIndexUri` (pointing to `index.md`), `pages[]`, `summaryUri`
+
+### Requirement: webtest_analyze_app Tool
+
+The system SHALL provide a `webtest_analyze_app` tool that reverse-engineers application structure from crawl data, outputting all results in markdown format.
+
+#### Scenario: Analyze app writes markdown report
+
+- **GIVEN** analysis is complete
+- **WHEN** output is generated
+- **THEN** it SHALL write `app-analysis.md` resource to workspace
+
+#### Scenario: Analyze app writes flows
+
+- **GIVEN** analysis is complete
+- **WHEN** output is generated
+- **THEN** it SHALL write `flows.md` resource to workspace as markdown with YAML frontmatter containing structured flow definitions
+
+#### Scenario: Analyze app outputs URIs
+
+- **GIVEN** analysis is complete
+- **WHEN** tool returns
+- **THEN** it SHALL include `appAnalysisFilePath`, `appAnalysisUri`, `flowsFilePath`, `flowsUri` all pointing to `.md` files
+
+### Requirement: webtest_generate_tests Tool
+
+The system SHALL provide a `webtest_generate_tests` tool that produces test cases from application analysis in a single markdown format.
+
+#### Scenario: Generate tests outputs structured format
+
+- **GIVEN** test generation completes
+- **WHEN** results are written
+- **THEN** it SHALL produce `tests.md` with human-readable format AND YAML frontmatter containing structured test definitions
+- **AND** there SHALL NOT be a separate `tests.json` file
+
+#### Scenario: Generate tests outputs URIs
+
+- **GIVEN** generation is complete
+- **WHEN** tool returns
+- **THEN** it SHALL include `testsFilePath` and `testsUri` pointing to `tests.md`
+
+### Requirement: webtest_run_tests Tool
+
+The system SHALL provide a `webtest_run_tests` tool that executes a test case with evidence capture, storing all results in markdown format.
+
+#### Scenario: Run test case captures evidence
+
+- **GIVEN** a step is executed
+- **WHEN** evidence is captured
+- **THEN** it SHALL take screenshot after action (stored as PNG)
+- **AND** capture accessibility snapshot (stored as `snapshot.md` with formatted tree and YAML frontmatter)
+- **AND** store with step identifier
+
+#### Scenario: Run test case outputs report
+
+- **GIVEN** test execution completes
+- **WHEN** output is generated
+- **THEN** it SHALL write `report.md` with pass/fail summary, step details, evidence links, and YAML frontmatter containing structured run data
+- **AND** there SHALL NOT be a separate `index.json` or `artifacts.json` file
+
+#### Scenario: Run test case returns URIs
+
+- **GIVEN** execution is complete
+- **WHEN** tool returns
+- **THEN** it SHALL include `testRunId`, `reportFilePath`, `reportUri` pointing to `report.md`
+
+### Requirement: Playwright MCP Integration
+
+The system SHALL orchestrate an external Playwright MCP server for browser automation with dynamic tool discovery.
+
+#### Scenario: Playwright MCP is spawned on first use
+
+- **GIVEN** a webtest tool needs browser access
+- **WHEN** Playwright client is accessed
+- **THEN** it SHALL spawn Playwright MCP server as subprocess if not running
+- **AND** connect via stdio transport
+
+#### Scenario: Playwright MCP tools are discovered dynamically
+
+- **GIVEN** Playwright MCP is connected
+- **WHEN** connection is established
+- **THEN** it SHALL call `tools/list` to discover available tools
+- **AND** build a capability adapter mapping canonical operations to actual tool names
+- **AND** cache the mapping for the session lifetime
+
+#### Scenario: Capability adapter maps canonical operations
+
+- **GIVEN** Playwright MCP tools have been discovered
+- **WHEN** the adapter is queried for operation "snapshot"
+- **THEN** it SHALL return the matching tool (e.g., `browser_snapshot` or `playwright_snapshot`)
+- **AND** if multiple matches exist, prefer the most specific
+
+#### Scenario: Missing required capability logs warning
+
+- **GIVEN** Playwright MCP tools have been discovered
+- **WHEN** a required capability (snapshot, screenshot, click, type, navigate) is missing
+- **THEN** it SHALL log a warning with the missing capability
+- **AND** tools requiring that capability SHALL return an error when invoked
+
+#### Scenario: Playwright actions are executed via adapter
+
+- **GIVEN** a crawl action specifies `{ tool: "click", args: { selector: "button" } }`
+- **WHEN** action is executed
+- **THEN** it SHALL resolve "click" through the capability adapter
+- **AND** call the resolved Playwright MCP tool with appropriate arguments
+- **AND** return the result
+
+#### Scenario: Playwright MCP version differences are handled
+
+- **GIVEN** different Playwright MCP implementations may have different tool names
+- **WHEN** the adapter maps tools
+- **THEN** it SHALL check for common variants:
+  - `browser_*` prefix (Microsoft implementation)
+  - `playwright_*` prefix (alternative implementations)
+  - unprefixed names
+- **AND** log the detected implementation variant
+
+#### Scenario: Playwright MCP is terminated on shutdown
+
+- **GIVEN** the server receives shutdown signal
+- **WHEN** shutdown begins
+- **THEN** it SHALL terminate Playwright MCP subprocess
+- **AND** wait for clean exit
+
+### Requirement: Crawl Checkpointing
+
+The system SHALL implement checkpointing during crawl using markdown format to enable resumption and provide human-readable partial results on failure.
+
+#### Scenario: Checkpoint is written periodically
+
+- **GIVEN** a crawl is in progress
+- **WHEN** N steps have completed (configurable, default 5)
+- **THEN** it SHALL write a checkpoint to `webtest://{analysisId}/crawls/{crawlId}/checkpoint.md`
+- **AND** the checkpoint SHALL be markdown with YAML frontmatter including: current step, visited pages, action history, goal progress
+- **AND** the checkpoint body SHALL contain human-readable progress summary
+
+#### Scenario: Crawl can resume from checkpoint
+
+- **GIVEN** a crawl was interrupted (cancelled, error, timeout)
+- **AND** a checkpoint exists as `checkpoint.md`
+- **WHEN** `webtest_crawl_app` is called with `resume: true`
+- **THEN** it SHALL parse the checkpoint YAML frontmatter
+- **AND** continue from the last recorded state
+
+### Requirement: Crawl Loop Detection and Prevention
+
+The system SHALL detect and prevent infinite crawl loops.
+
+#### Scenario: Same page state detected consecutively
+
+- **GIVEN** crawl detects same DOM signature 3 times consecutively
+- **WHEN** loop is detected
+- **THEN** it SHALL log a warning with loop details
+- **AND** inform sampling of the loop condition
+- **AND** request alternative action with loop context
+
+#### Scenario: URL cycle detected
+
+- **GIVEN** crawl visits the same URL more than 3 times
+- **WHEN** cycle is detected
+- **THEN** it SHALL log a warning
+- **AND** exclude that URL from future navigation suggestions
+
+#### Scenario: Action repeat detected
+
+- **GIVEN** the same action (tool + args) is attempted 3 times consecutively
+- **WHEN** repeat is detected
+- **THEN** it SHALL reject the repeated action
+- **AND** request a different action from sampling with repeat context
+
+#### Scenario: Loop detection state is included in sampling prompts
+
+- **GIVEN** loop detection has flagged potential issues
+- **WHEN** the next sampling prompt is built
+- **THEN** it SHALL include:
+  - URLs visited more than once
+  - Recently repeated actions
+  - DOM signature history
+- **AND** instruct the model to avoid these patterns
+
+### Requirement: Crawl Budget Enforcement
+
+The system SHALL enforce time, step, and page limits during crawl.
+
+#### Scenario: Step limit is enforced
+
+- **GIVEN** crawl has a `maxSteps` limit
+- **WHEN** the step count reaches the limit
+- **THEN** crawl SHALL finalize with status "limits_reached"
+- **AND** include all artifacts collected up to that point
+
+#### Scenario: Time limit is enforced
+
+- **GIVEN** crawl has a `maxMinutes` limit
+- **WHEN** elapsed time reaches the limit
+- **THEN** crawl SHALL finalize with status "timeout"
+- **AND** complete current action before stopping
+- **AND** preserve all collected artifacts
+
+#### Scenario: Page limit is enforced
+
+- **GIVEN** crawl has a `maxPages` limit
+- **WHEN** the unique page count reaches the limit
+- **THEN** crawl SHALL stop discovering new pages
+- **AND** continue actions on already-visited pages until goal or step limit
+
+#### Scenario: Budget status is reported in progress
+
+- **GIVEN** crawl is running with limits
+- **WHEN** progress notification is emitted
+- **THEN** it SHALL include budget status:
+  - `stepsUsed` / `maxSteps`
+  - `minutesElapsed` / `maxMinutes`
+  - `pagesDiscovered` / `maxPages`
+
+### Requirement: Security Domain Enforcement
+
+The system SHALL enforce domain allowlists for all navigation actions.
+
+#### Scenario: Navigation to allowed domain succeeds
+
+- **GIVEN** allowedDomains includes "example.com"
+- **WHEN** Playwright action navigates to "https://example.com/page"
+- **THEN** navigation SHALL be allowed
+
+#### Scenario: Navigation to disallowed domain is blocked
+
+- **GIVEN** allowedDomains includes only "example.com"
+- **WHEN** sampling returns action to navigate to "https://malicious.com"
+- **THEN** the action SHALL be rejected
+- **AND** error logged with attempted URL
+
+#### Scenario: Subdomain matching follows rules
+
+- **GIVEN** allowedDomains includes "example.com"
+- **WHEN** navigation to "sub.example.com" is attempted
+- **THEN** it SHALL be allowed (subdomain of allowed domain)
+
+#### Scenario: Link clicks are validated
+
+- **GIVEN** a click action may navigate to external domain
+- **WHEN** click is executed
+- **THEN** resulting URL SHALL be checked post-navigation
+- **AND** if disallowed, navigate back and report error
+

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "retestkit",
+  "version": "1.4.1",
+  "description": "MCP server for web application testing",
+  "type": "module",
+  "exports": {
+    ".": "./dist/index.js"
+  },
+  "bin": {
+    "retestkit": "./dist/index.js"
+  },
+  "scripts": {
+    "dev": "tsx watch src/index.ts",
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:integration": "vitest run --config vitest.integration.config.ts"
+  },
+  "engines": {
+    "node": ">=22.18.0"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "testing",
+    "web-testing"
+  ],
+  "license": "UNLICENSED",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/jan-beranek/testing-mcp.git"
+  },
+  "bugs": {
+    "url": "https://github.com/jan-beranek/testing-mcp/issues"
+  },
+  "homepage": "https://github.com/jan-beranek/testing-mcp#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.25.1",
+    "gray-matter": "^4.0.3",
+    "zod": "^4.2.1",
+    "zod-to-json-schema": "^3.25.0"
+  },
+  "devDependencies": {
+    "@types/node": "^25.0.3",
+    "semantic-release": "^24.2.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "vitest": "^4.0.16"
+  }
+}

package/release.config.js

@@ -0,0 +1,9 @@
+export default {
+  branches: ['main'],
+  plugins: [
+    '@semantic-release/commit-analyzer',
+    '@semantic-release/release-notes-generator',
+    '@semantic-release/npm',
+    '@semantic-release/github',
+  ],
+};

package/src/config.test.ts

@@ -0,0 +1,96 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { loadConfig } from "./config.js";
+
+describe("config", () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    // Reset environment before each test
+    process.env = { ...originalEnv };
+    delete process.env.TRANSPORT;
+    delete process.env.PORT;
+    delete process.env.LOG_LEVEL;
+  });
+
+  afterEach(() => {
+    process.env = originalEnv;
+  });
+
+  describe("defaults", () => {
+    it("uses stdio transport by default", () => {
+      const config = loadConfig();
+      expect(config.transport).toBe("stdio");
+    });
+
+    it("uses port 3000 by default", () => {
+      const config = loadConfig();
+      expect(config.port).toBe(3000);
+    });
+
+    it("uses info log level by default", () => {
+      const config = loadConfig();
+      expect(config.logLevel).toBe("info");
+    });
+  });
+
+  describe("transport", () => {
+    it("accepts stdio transport", () => {
+      process.env.TRANSPORT = "stdio";
+      const config = loadConfig();
+      expect(config.transport).toBe("stdio");
+    });
+
+    it("accepts http transport", () => {
+      process.env.TRANSPORT = "http";
+      const config = loadConfig();
+      expect(config.transport).toBe("http");
+    });
+
+    it("rejects invalid transport", () => {
+      process.env.TRANSPORT = "invalid";
+      expect(() => loadConfig()).toThrow();
+    });
+  });
+
+  describe("port", () => {
+    it("accepts valid port number", () => {
+      process.env.PORT = "8080";
+      const config = loadConfig();
+      expect(config.port).toBe(8080);
+    });
+
+    it("rejects invalid port", () => {
+      process.env.PORT = "invalid";
+      expect(() => loadConfig()).toThrow();
+    });
+
+    it("rejects port below 1", () => {
+      process.env.PORT = "0";
+      expect(() => loadConfig()).toThrow();
+    });
+
+    it("rejects port above 65535", () => {
+      process.env.PORT = "70000";
+      expect(() => loadConfig()).toThrow();
+    });
+  });
+
+  describe("log level", () => {
+    it("accepts debug level", () => {
+      process.env.LOG_LEVEL = "debug";
+      const config = loadConfig();
+      expect(config.logLevel).toBe("debug");
+    });
+
+    it("accepts error level", () => {
+      process.env.LOG_LEVEL = "error";
+      const config = loadConfig();
+      expect(config.logLevel).toBe("error");
+    });
+
+    it("rejects invalid log level", () => {
+      process.env.LOG_LEVEL = "invalid";
+      expect(() => loadConfig()).toThrow();
+    });
+  });
+});

package/src/config.ts

@@ -0,0 +1,32 @@
+import { ConfigSchema, type Config } from "./schemas/config.js";
+
+export function loadConfig(): Config {
+  const rawConfig = {
+    transport: process.env.TRANSPORT,
+    port: process.env.PORT,
+    logLevel: process.env.LOG_LEVEL,
+    // Webtest configuration
+    workspaceDir: process.env.WEBTEST_WORKSPACE_DIR,
+    playwrightMcpCommand: process.env.PLAYWRIGHT_MCP_COMMAND,
+    playwrightMcpArgs: process.env.PLAYWRIGHT_MCP_ARGS,
+    checkpointInterval: process.env.WEBTEST_CHECKPOINT_INTERVAL,
+    screenshotFormat: process.env.WEBTEST_SCREENSHOT_FORMAT,
+    screenshotQuality: process.env.WEBTEST_SCREENSHOT_QUALITY,
+    defaultMaxSteps: process.env.WEBTEST_DEFAULT_MAX_STEPS,
+    defaultMaxMinutes: process.env.WEBTEST_DEFAULT_MAX_MINUTES,
+    defaultMaxPages: process.env.WEBTEST_DEFAULT_MAX_PAGES,
+  };
+
+  const result = ConfigSchema.safeParse(rawConfig);
+
+  if (!result.success) {
+    const errors = result.error.issues
+      .map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+      .join(", ");
+    throw new Error(`Invalid configuration: ${errors}`);
+  }
+
+  return result.data;
+}
+
+export type { Config };