testeiya 0.2.0

package/skills/testomatio/test-management/testomatio-mcp/SKILL.md

@@ -0,0 +1,182 @@
+---
+name: testomatio-mcp
+description: Configures and leverages the Testomat.io MCP server for test management analytics, run analysis, failure investigation, and defect triage. Use when the user needs to connect to Testomat.io via MCP, analyze test runs, cluster failures, investigate root causes.
+license: MIT
+metadata:
+  author: Testomat.io
+  version: 1.0.0
+---
+
+# Testomatio MCP
+
+Set up access to Testomat.io via MCP and run QA analysis workflows: run analysis, failure clustering, root-cause investigation, plan analysis, defect triage.
+
+## When to Use
+
+- Detect or set up the Testomat.io MCP server for an AI agent (Claude, Cursor, OpenCode, etc.).
+- Analyze test runs: recent results, success rates, failure trends.
+- Investigate failures: group similar failures, find patterns.
+- Find the root cause when many tests fail with the same issue.
+- Create defects from failed tests in the user's issue tracker (e.g. Jira).
+
+## MCP Tools
+
+- Test: `tests_list`, `tests_get`, `tests_search`
+- Test Plan: `plans_list`, `plans_get`, `plans_search`
+- Run: `runs_list`, `runs_get`, `runs_search`
+- Testrun (results within a run): `testruns_list`, `testruns_get`
+- Suite: `suites_list`, `suites_get`, `suites_search`
+- Label: `labels_list`, `labels_get`
+- Tag: `tags_list`, `tags_get`, `tags_search`
+
+**CRUD tools (create, update, delete) exist for all entities. Use them only when explicitly needed for targeted updates.**
+
+## Rules
+
+- **Prefer local tests over MCP.** If the repo contains markdown test cases (`*.test.md`) or automated test files, read them from the filesystem (or load them via the `sync-test-cases-with-tms` skill) instead of using MCP for full test discovery and analysis.
+- Use MCP for:
+  - reading and analyzing run reports
+  - analytics and reporting
+  - test plan management
+  - tests that live exclusively in Testomat.io (no local copies), or quick test case searches (`tests_list`, `tests_search`)
+  - targeted searches and point updates to remote test cases
+- Filter `runs_list`, `plans_list`, `testruns_list` with TQL (Testomat.io Query Language). Operators and examples: [MCP Setup Reference](./references/MCP_SETUP.md).
+
+## Setup
+
+### Step 1: Detect Existing Configuration
+
+Check the user's AI agent config files for a `testomatio` or `mcp.testomatio` entry:
+- `opencode.json`
+- `.cursor/mcp.json`
+- `claude_desktop_config.json`
+- other, based on the user's local agent
+
+Validate:
+- the server is enabled (when the client supports enable/disable flags)
+- `TESTOMATIO_PROJECT_TOKEN` and `TESTOMATIO_PROJECT_ID` are present
+
+Configured and enabled → go to Workflows. Missing, incomplete, or disabled → Step 2.
+
+### Step 2: Configure MCP
+
+Before configuring:
+- Prefer confirming the project contains no more than 1000 tests.
+- If project size cannot be determined, warn the user about MCP performance limits on large projects.
+- **Use only targeted or filtered operations when explicitly requested.**
+
+Credential priority for `TESTOMATIO_PROJECT_TOKEN` and `TESTOMATIO_PROJECT_ID`:
+1. Existing environment variables.
+2. Existing MCP configuration.
+3. User-provided values.
+
+Per-agent config formats (OpenCode, Cursor, Claude Desktop) and where to obtain credentials: [MCP Setup Reference](./references/MCP_SETUP.md).
+
+**Rules:**
+- **Do not overwrite unrelated MCP server configurations.** Merge changes carefully with existing config content.
+- **Do not expose project tokens, secrets, or credentials in chat responses.**
+
+After successful configuration, confirm with a summary:
+
+```text
+Testomat.io MCP configuration completed successfully.
+
+AI Agent: Cursor
+Config Updated: `.cursor/mcp.json`
+MCP Server: Enabled
+Authentication: Verified
+
+Available workflows:
+- Test run analysis
+- Failure clustering
+- Root-cause investigation
+- Plan analysis & defect triage
+```
+
+## Workflows
+
+Common patterns, not a complete list — adapt and extend to the user's needs. All assume the MCP server is enabled.
+
+### Workflow 1: Analyze Test Runs
+
+Goal: understand the health of recent test execution.
+
+1. Call `runs_list` with `tql` filters (e.g. `finished`, `failed`, `with_defect`); narrow scope by status, environment, or plan.
+2. Select a run of interest and call `runs_get` for detailed metadata.
+3. Call `testruns_list` with `run_id` and `filter_status: failed` to extract failed results.
+4. Summarize: total tests, passed, failed, skipped, environments involved.
+   - Compare success/failure ratios across runs to spot regressions.
+
+### Workflow 2: Failure Investigation & Error Clustering
+
+Goal: determine what went wrong and which areas are affected.
+
+1. Call `testruns_list` (`filter_status: failed`, `run_id`) to get failed results.
+2. For each failed testrun, call `tests_get` to map it to its suite, priority, tags, and linked issues.
+3. Cluster failures:
+   - by `suite_id` → identify the suite with the highest failure count
+   - by `message` substring → find tests failing with the same exception or assertion
+4. Present clusters ranked by count, with suite and message breakdowns and affected test titles.
+
+### Workflow 3: Plan Analysis & Root-Cause Detection
+
+Goal: when many tests fail with the same issue, explain the root cause.
+
+1. Call `plans_get` (if a plan is linked to the run) to understand the intended scope; check whether failures correlate with specific plan sections.
+2. Call `runs_list` with `tql` targeting the same plan or environment to get historical runs.
+3. Compare failures across runs:
+   - increasing failure count → new regression
+   - same failures across multiple runs → stable bug or flaky infrastructure
+   - spike in a single suite or tag → localized code change, feature regression
+   - scattered across unrelated suites → systemic issue (infrastructure, config, dependency)
+4. Call `tests_get` on affected tests to check `code`, `state`, and linked issues.
+5. Report a root-cause hypothesis.
+
+Example — user asks: "Check why 20 tests failed in the last run — one suite or a systemic issue?"
+
+```text
+1. Detecting existing Testomat.io MCP configuration:
+   - Found `.cursor/mcp.json`
+   - Found `testomatio` MCP server entry
+   - MCP server status: enabled
+
+2. Verifying MCP availability...
+   - Authentication successful
+   - Testomat.io tools available
+
+3. Analyzing latest test run...
+   - Retrieved last completed run
+   - Total failed tests: 20
+
+4. Failure clustering summary:
+   - 16/20 failures belong to suite: "Checkout / Payments"
+   - 14 failures share the same error: "Timeout waiting for payment confirmation"
+   - First failure timestamp indicates failures started simultaneously
+
+5. Probable root cause:
+   - The failures appear systemic rather than test-specific.
+   - Most failures point to instability or outage in the payment gateway service used during checkout flows.
+```
+
+### Workflow 4: Defect Creation from Failed Tests
+
+Goal: turn recurring failures into trackable defects.
+
+1. Run Workflow 2 to cluster failures by message.
+2. If a cluster contains 5+ tests with the same failure:
+   - propose creating a defect (e.g. Jira issue) to the user
+   - draft the defect summary: title, description, affected test titles, failure messages, run link, environment
+   - if a Jira MCP is available, offer to create the ticket via the Jira skill
+3. After defect creation, call `tests_issues_link` for each affected test to link it to the defect.
+
+## Quick Commands
+
+| Action              | MCP Tool        | Key Parameters |
+|---------------------|-----------------|----------------|
+| List failed results | `testruns_list` | `run_id`, `filter_status: failed` |
+| List runs           | `runs_list`     | `tql`, `page`, `per_page` |
+| Get run details     | `runs_get`      | `run_id` |
+| Get test details    | `tests_get`     | `test_id` |
+| Get plan details    | `plans_get`     | `plan_id` |
+| Link issue to test  | `tests_issues_link` | `test_id`, `url` or `jira_id` |
+| Check server status | `system_ping`   | none |

package/skills/testomatio/test-management/testomatio-mcp/references/MCP_SETUP.md

@@ -0,0 +1,169 @@
+# MCP Setup & Configuration Reference
+
+Short reference extracted from the Testomat.io MCP server documentation.
+
+## Installation
+
+```bash
+npm install -g @testomatio/mcp@latest
+```
+
+For enterprise subscriptions, install `@testomatio/mcp-enterprise@latest` instead (includes analytics).
+
+## Server Startup
+
+```bash
+npx testomatio-mcp --token <PROJECT_TOKEN> --project <PROJECT_ID>
+```
+
+Or with environment variables:
+
+```bash
+export TESTOMATIO_PROJECT_TOKEN=<PROJECT_TOKEN>
+export TESTOMATIO_PROJECT_ID=<PROJECT_ID>
+```
+
+Optional custom base URL:
+
+```bash
+export TESTOMATIO_BASE_URL=https://beta.testomat.io
+```
+
+## AI Agent Configurations
+
+### OpenCode
+
+File: `opencode.json` (project root or `~/.config/opencode/opencode.json`)
+
+```json
+{
+  "$schema": "https://opencode.ai/config.json",
+  "mcp": {
+    "testomat": {
+      "type": "local",
+      "command": [
+        "npx",
+        "-y",
+        "@testomatio/mcp@latest",
+        "--token",
+        "<TOKEN>",
+        "--project",
+        "<PROJECT_ID>"
+      ],
+      "enabled": true,
+      "environment": {
+        "TESTOMATIO_BASE_URL": "https://app.testomat.io"
+      }
+    }
+  }
+}
+```
+
+### Claude Desktop
+
+File:
+- macOS: `~/Library/Application Support/Claude/claude_desktop_config.json`
+- Windows: `%APPDATA%\Claude\claude_desktop_config.json`
+
+```json
+{
+  "mcpServers": {
+    "testomatio": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "@testomatio/mcp@latest",
+        "--token",
+        "<TOKEN>",
+        "--project",
+        "<PROJECT_ID>"
+      ],
+      "env": {
+        "TESTOMATIO_BASE_URL": "https://app.testomat.io"
+      }
+    }
+  }
+}
+```
+
+### Cursor IDE
+
+File: `.cursor/mcp.json` (project root or `~/.cursor/mcp.json`)
+
+```json
+{
+  "mcpServers": {
+    "testomatio": {
+      "type": "stdio",
+      "command": "npx",
+      "args": [
+        "-y",
+        "@testomatio/mcp@latest",
+        "--token",
+        "<TOKEN>",
+        "--project",
+        "<PROJECT_ID>"
+      ],
+      "env": {
+        "TESTOMATIO_BASE_URL": "https://app.testomat.io"
+      }
+    }
+  }
+}
+```
+
+## Credentials
+
+- **Project Token**: Get from Testomat.io at **Settings → Project → Project Reporting API key**.
+- **Project ID**: Found in the project URL: `https://app.testomat.io/projects/<project_id>`.
+
+## Environment Variables
+
+| Variable | Required | Default | Description |
+|----------|----------|---------|-------------|
+| `TESTOMATIO_PROJECT_TOKEN` | Yes* | - | Project API token (preferred) |
+| `TESTOMATIO_PROJECT_ID` | Yes | - | Project ID |
+| `TESTOMATIO_BASE_URL` | No | `https://app.testomat.io` | API base URL |
+
+*`TESTOMATIO_PROJECT_TOKEN` is required.
+
+## TQL Quick Reference
+
+TQL (Testomat.io Query Language) is used for filtering tests and runs.
+
+Supported operators: `==`, `!=`, `>`, `<`, `>=`, `<=`, `in [...]`, `%` (contains), `and`, `or`, `not`, parentheses.
+
+### Common TQL Examples
+
+**Tests:**
+- `priority == 'high'`
+- `state == 'automated'`
+- `suite % 'Checkout'`
+- `tag IN ['@smoke', '@regression']`
+
+**Runs:**
+- `title % 'Manual tests'`
+- `plan == '{PLAN_ID}'`
+- `finished and with_defect`
+- `failed and has_test_tag == 'regression'`
+
+For full TQL syntax details, see: https://docs.testomat.io/advanced/tql/
+
+## Corporate TLS / Proxy
+
+If running behind a corporate proxy or TLS inspection, Node.js may reject HTTPS requests.
+
+Use system CA support:
+
+```bash
+NODE_OPTIONS=--use-system-ca testomatio-mcp --token <TOKEN> --project <PROJECT_ID>
+```
+
+Or in MCP client config, pass `NODE_OPTIONS` in the server `env` block.
+
+## Important Notes
+
+- **Run status transitions** use `runs_update` with `status_event` values: `finish`, `finish_manual`, `launch`, `rerun`, `scheduled`, `terminate`.
+- **Search** for tests and runs uses `tql` as the single filter input; no dedicated `/search` endpoints.
+- **Issue linking** supports scoped helpers: `tests_issues_link/unlink`, `suites_issues_link/unlink`, `runs_issues_link/unlink`.
+- **API Sessions** are automatically managed by the MCP server: a session starts before the first mutating request and stops when the server shuts down.

package/skills.lock.json

@@ -0,0 +1,72 @@
+{
+  "sources": [
+    {
+      "source": "testomatio/skills",
+      "ref": "feat/setup-pr-testing-skill",
+      "sha": "99a6808422ff9e3a6bd46edab04a32d37a6ed028",
+      "folder": "testomatio",
+      "skills": [
+        "testomatio-mcp",
+        "sync-test-cases-with-tms",
+        "scan-automation-project",
+        "qa-write-test-cases",
+        "qa-thinking",
+        "qa-test-code-coverage",
+        "qa-requirement-reviewer",
+        "qa-pr-requirements-analyzer",
+        "qa-e2e-tests-reporting",
+        "pull-request-diff-analyzer",
+        "improve-test-cases",
+        "detect-duplicate-test-cases",
+        "setup-pr-testing",
+        "run-tests-with-testomatio-reporter",
+        "qa-automation-test-consolidation",
+        "debug-fix-failed-flaky-autotests",
+        "automate-manual-test-cases",
+        "testomat-allure-adapter",
+        "explorbot-setup",
+        "explorbot-plan",
+        "explorbot-fundamentals"
+      ]
+    },
+    {
+      "source": "codeceptjs/skills",
+      "ref": null,
+      "sha": "e19ba9d0f513cbc9c3e4a4c5c6dd02cfedde4afd",
+      "folder": "codeceptjs",
+      "skills": [
+        "writing-codeceptjs-tests",
+        "refactoring-codeceptjs-tests",
+        "migrate-testcafe-to-codeceptjs",
+        "migrate-selenium-java-to-codeceptjs",
+        "migrate-protractor-to-codeceptjs",
+        "migrate-cypress-to-codeceptjs",
+        "migrate-codeceptjs-4",
+        "debugging-codeceptjs-tests",
+        "codeceptjs-run-analysis",
+        "codeceptjs-fundamentals",
+        "codeceptjs-exploration",
+        "codeceptjs-auth",
+        "ci-fix-tests"
+      ]
+    },
+    {
+      "source": "currents-dev/playwright-best-practices-skill",
+      "ref": null,
+      "sha": "ef329e7e65149918e1ff0eed2cf7d2e6e6f9eb5b",
+      "folder": "playwright-best-practices-skill",
+      "skills": [
+        "playwright-best-practices"
+      ]
+    },
+    {
+      "source": "microsoft/playwright-cli/tree/main/skills/playwright-cli",
+      "ref": null,
+      "sha": "793cfb32572733cbcb401e6f28d05a7a914ce408",
+      "folder": "playwright-cli",
+      "skills": [
+        "playwright-cli"
+      ]
+    }
+  ]
+}

package/skills.yaml

@@ -0,0 +1,21 @@
+# External skills for Testeiya. ONE rule: every line is a GitHub repo.
+#
+#   owner/repo                        a skill repo (one skill, or many)
+#   owner/repo/tree/<ref>/<subdir>    pin a branch/tag/commit, or point at a subfolder
+#
+# Each source owns one vendor folder in cli/skills/ — the repo name, or the
+# owner when the repo is just "skills" (testomatio/skills -> testomatio/).
+# Updates replace only that folder; any folder this file does not own
+# (e.g. cli/skills/testeiya/) holds first-party skills and is never touched.
+#
+# Categories are decided by the repo, never by this file:
+#   - repo with a Claude-plugin marketplace (.claude-plugin/marketplace.json)
+#     -> one category subfolder per plugin  (testomatio/ -> test-management / test-automation / explorbot)
+#   - repo without one -> skills sit right under the vendor folder (the vendor is the category)
+#
+# Edit the list, then run:  bunosh skills:update [vendor]      See the result:  bunosh skills:list
+
+- testomatio/skills
+- codeceptjs/skills
+- currents-dev/playwright-best-practices-skill
+- microsoft/playwright-cli/tree/main/skills/playwright-cli

package/src/ai-debug.ts

@@ -0,0 +1,188 @@
+import { publish, type AiEventEntry } from "./debug-bus.js";
+
+/*
+ * Pluggable PI → Debug-panel recorder. `attachAiDebug(session)` adds one
+ * independent listener to a pi-coding-agent session and maps the LLM-level
+ * events the WS bridge never forwards — per-turn model responses, the errors a
+ * failed turn hides behind a plain `finish`, and auto-retry/fallback/compaction
+ * — into the panel's `ai` channel via `debug-bus`'s `publish`. The returned
+ * unsubscribe is called from `connection.ts` when the session is torn down.
+ */
+
+const MAX_DETAIL = 16000;
+
+export function attachAiDebug(session: any, conversationId?: string | null): () => void {
+  return session.subscribe((event: any) => {
+    const entry = mapAiEvent(event);
+    if (!entry) return;
+    if (conversationId) entry.summary = withSession(entry.summary, conversationId);
+    publish(entry);
+  });
+}
+
+function mapAiEvent(event: any): AiEventInput | null {
+  if (event.type === "turn_end") return turnEntry(event.message);
+  if (event.type === "auto_retry_start") return retryStart(event);
+  if (event.type === "auto_retry_end") return retryEnd(event);
+  if (event.type === "retry_fallback_applied")
+    return fallback(`${event.from} → ${event.to} (${event.role})`, event.to, event);
+  if (event.type === "retry_fallback_succeeded")
+    return fallback(`${event.model} (${event.role})`, event.model, event);
+  if (event.type === "auto_compaction_start")
+    return compaction(`${event.reason} → ${event.action}`, true, event);
+  if (event.type === "auto_compaction_end") return compactionEnd(event);
+  return null;
+}
+
+function turnEntry(message: any): AiEventInput | null {
+  if (!message || message.role !== "assistant") return null;
+  const text = assistantText(message.content);
+  const tokens = tokensOf(message.usage);
+  const entry: AiEventInput = {
+    kind: "ai",
+    channel: "ai",
+    name: "response",
+    summary:
+      `${message.model} · ${message.stopReason} · ${text.length} chars` +
+      `${tokenSummary(tokens)}${cacheSummary(message.usage)}${durSummary(message.duration)}`,
+    ok: true,
+    model: message.model ?? null,
+    durationMs: numberOrNull(message.duration),
+    tokens,
+    detail: detail({
+      model: message.model,
+      provider: message.provider,
+      stopReason: message.stopReason,
+      errorMessage: message.errorMessage,
+      usage: message.usage,
+      ttft: message.ttft,
+      duration: message.duration,
+      text,
+    }),
+  };
+  if (message.stopReason !== "error") return entry;
+  entry.name = "error";
+  entry.ok = false;
+  entry.summary = message.errorMessage ?? "error";
+  return entry;
+}
+
+function tokenSummary(tokens: AiEventEntry["tokens"]): string {
+  if (!tokens) return "";
+  return ` · ${tokens.input}/${tokens.output} tok`;
+}
+
+function cacheSummary(usage: any): string {
+  if (!usage?.cacheRead) return "";
+  return ` · cached=${usage.cacheRead}`;
+}
+
+function durSummary(duration: any): string {
+  if (typeof duration !== "number") return "";
+  return ` · dur=${(duration / 1000).toFixed(1)}s`;
+}
+
+function withSession(summary: string | null, conversationId: string): string {
+  if (!summary) return `session:${conversationId}`;
+  return `${summary} · session:${conversationId}`;
+}
+
+function retryStart(event: any): AiEventInput {
+  return {
+    kind: "ai",
+    channel: "ai",
+    name: "retry",
+    ok: false,
+    summary: `attempt ${event.attempt}/${event.maxAttempts} — ${event.errorMessage}`,
+    model: null,
+    durationMs: numberOrNull(event.delayMs),
+    tokens: null,
+    detail: detail(event),
+  };
+}
+
+function retryEnd(event: any): AiEventInput {
+  const entry: AiEventInput = {
+    kind: "ai",
+    channel: "ai",
+    name: "retry",
+    ok: true,
+    summary: `recovered on attempt ${event.attempt}`,
+    model: null,
+    durationMs: null,
+    tokens: null,
+    detail: detail(event),
+  };
+  if (event.success) return entry;
+  entry.name = "error";
+  entry.ok = false;
+  entry.summary = event.finalError ?? "retries exhausted";
+  return entry;
+}
+
+function fallback(summary: string, model: string | undefined, event: any): AiEventInput {
+  return {
+    kind: "ai",
+    channel: "ai",
+    name: "fallback",
+    ok: true,
+    summary,
+    model: model ?? null,
+    durationMs: null,
+    tokens: null,
+    detail: detail(event),
+  };
+}
+
+function compactionEnd(event: any): AiEventInput {
+  if (event.errorMessage) return compaction(event.errorMessage, false, event);
+  return compaction(`${event.action} done`, true, event);
+}
+
+function compaction(summary: string, ok: boolean, event: any): AiEventInput {
+  return {
+    kind: "ai",
+    channel: "ai",
+    name: "compaction",
+    ok,
+    summary,
+    model: null,
+    durationMs: null,
+    tokens: null,
+    detail: detail(event),
+  };
+}
+
+function tokensOf(usage: any): AiEventEntry["tokens"] {
+  if (!usage) return null;
+  return { input: usage.input ?? 0, output: usage.output ?? 0, total: usage.totalTokens ?? 0 };
+}
+
+function assistantText(content: any): string {
+  if (!Array.isArray(content)) return "";
+  const parts: string[] = [];
+  for (const part of content) {
+    if (part?.type === "text") parts.push(part.text ?? "");
+    if (part?.type === "thinking") parts.push(`[thinking] ${part.thinking ?? part.text ?? ""}`);
+    if (part?.type === "toolCall") parts.push(`[tool:${part.name}]`);
+  }
+  return parts.filter(Boolean).join("\n");
+}
+
+function numberOrNull(value: any): number | null {
+  if (typeof value === "number") return value;
+  return null;
+}
+
+function detail(value: any): string | null {
+  let json: string;
+  try {
+    json = JSON.stringify(value, null, 2);
+  } catch {
+    return null;
+  }
+  if (json.length <= MAX_DETAIL) return json;
+  return `${json.slice(0, MAX_DETAIL)}… (+${json.length - MAX_DETAIL} more chars)`;
+}
+
+type AiEventInput = Omit<AiEventEntry, "id" | "ts">;

package/src/api/agent-get.ts

@@ -0,0 +1,22 @@
+import { getSession } from "../session-store.js";
+
+export async function agentGet(sessionId: string): Promise<Response> {
+  const session = getSession(sessionId);
+
+  if (!session) {
+    return Response.json({ error: "Session not found" }, { status: 404 });
+  }
+
+  const expectedMcpServers = (session.projects ?? [])
+    .filter((p) => p.status === "ok")
+    .map((p) => `testomatio-${p.slug}`);
+
+  return Response.json({
+    sessionId: session.sessionId,
+    cwd: session.cwd,
+    projects: session.projects,
+    backendUrl: session.backendUrl,
+    expectedMcpServers,
+    createdAt: session.createdAt,
+  });
+}