qa-engineer 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/COMPATIBILITY.md +50 -26
  2. package/README.md +40 -32
  3. package/package.json +12 -20
  4. package/packages/engine/bin/qa-engine.mjs +336 -0
  5. package/packages/engine/lib/analysis/branding.mjs +187 -0
  6. package/packages/engine/lib/analysis/context.mjs +294 -0
  7. package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
  8. package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
  9. package/packages/engine/lib/analysis/discovery.mjs +220 -0
  10. package/packages/engine/lib/analysis/evidence.mjs +116 -0
  11. package/packages/engine/lib/analysis/har.mjs +90 -0
  12. package/packages/engine/lib/analysis/junit.mjs +126 -0
  13. package/packages/engine/lib/analysis/redaction.mjs +127 -0
  14. package/packages/engine/lib/analysis/report-html.mjs +763 -0
  15. package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
  16. package/packages/engine/lib/analysis/xml.mjs +153 -0
  17. package/packages/engine/lib/analysis/zip.mjs +107 -0
  18. package/packages/engine/lib/diagnostics/engine.mjs +165 -0
  19. package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
  20. package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
  21. package/packages/engine/lib/diagnostics/repair.mjs +73 -0
  22. package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
  23. package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
  24. package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
  25. package/packages/engine/lib/frameworks/playwright.mjs +158 -0
  26. package/packages/engine/package.json +14 -0
  27. package/packages/installer/lib/agents/registry.mjs +46 -0
  28. package/packages/installer/lib/commands/doctor.mjs +7 -16
  29. package/packages/installer/lib/commands/install.mjs +59 -4
  30. package/packages/installer/lib/core/bundle.mjs +69 -92
  31. package/packages/installer/lib/core/config.mjs +1 -1
  32. package/packages/installer/lib/core/lockfile.mjs +1 -1
  33. package/packages/installer/lib/core/manifest.mjs +48 -54
  34. package/packages/installer/lib/core/validate-install.mjs +12 -19
  35. package/packages/installer/package.json +1 -1
  36. package/shared/tooling/qa-tool.mjs +125 -0
  37. package/skills/qa-api/SKILL.md +5 -5
  38. package/skills/qa-api/references/deterministic-tooling.md +34 -30
  39. package/skills/qa-api/references/evidence-and-reporting.md +29 -4
  40. package/skills/qa-api/scripts/qa-tool.mjs +126 -0
  41. package/skills/qa-audit/SKILL.md +5 -5
  42. package/skills/qa-audit/references/deterministic-tooling.md +34 -30
  43. package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
  44. package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
  45. package/skills/qa-debug/SKILL.md +6 -6
  46. package/skills/qa-debug/references/deterministic-tooling.md +34 -30
  47. package/skills/qa-debug/references/diagnostic-engine.md +3 -3
  48. package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
  49. package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
  50. package/skills/qa-explore/SKILL.md +22 -5
  51. package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
  52. package/skills/qa-explore/references/deterministic-tooling.md +132 -0
  53. package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
  54. package/skills/qa-explore/references/report-pipeline.md +89 -8
  55. package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
  56. package/skills/qa-fix/SKILL.md +4 -4
  57. package/skills/qa-fix/references/deterministic-tooling.md +34 -30
  58. package/skills/qa-fix/references/diagnostic-engine.md +3 -3
  59. package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
  60. package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
  61. package/skills/qa-flaky/SKILL.md +4 -4
  62. package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
  63. package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
  64. package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
  65. package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
  66. package/skills/qa-init/SKILL.md +4 -4
  67. package/skills/qa-init/references/deterministic-tooling.md +34 -30
  68. package/skills/qa-init/references/evidence-and-reporting.md +29 -4
  69. package/skills/qa-init/scripts/qa-tool.mjs +126 -0
  70. package/skills/qa-report/SKILL.md +8 -7
  71. package/skills/qa-report/references/deterministic-tooling.md +34 -30
  72. package/skills/qa-report/references/diagnostic-engine.md +3 -3
  73. package/skills/qa-report/references/evidence-and-reporting.md +29 -4
  74. package/skills/qa-report/scripts/qa-tool.mjs +126 -0
  75. package/skills/qa-review/references/evidence-and-reporting.md +29 -4
  76. package/skills/qa-run/SKILL.md +6 -6
  77. package/skills/qa-run/references/deterministic-tooling.md +34 -30
  78. package/skills/qa-run/references/evidence-and-reporting.md +29 -4
  79. package/skills/qa-run/scripts/qa-tool.mjs +126 -0
  80. package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
  81. package/shared/analysis/lib/qa_analysis/branding.py +0 -175
  82. package/shared/analysis/lib/qa_analysis/cli.py +0 -129
  83. package/shared/analysis/lib/qa_analysis/context.py +0 -233
  84. package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
  85. package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
  86. package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
  87. package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
  88. package/shared/analysis/lib/qa_analysis/har.py +0 -58
  89. package/shared/analysis/lib/qa_analysis/junit.py +0 -80
  90. package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
  91. package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
  92. package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
  93. package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
  94. package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
  95. package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
  96. package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
  97. package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
  98. package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
  99. package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
  100. package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
  101. package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
  102. package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
  103. package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
  104. package/shared/tooling/qa_tool.py +0 -127
  105. /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
  106. /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
  107. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
  108. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
  109. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
@@ -1,101 +0,0 @@
1
- """Deterministic finding prioritization.
2
-
3
- Assigns every finding a severity, priority, the three impacts, an owner, and an
4
- estimated effort — by a fixed algorithm, so the same finding always ranks the
5
- same way. Priority is not a feeling; it is a function of severity, confidence,
6
- and business impact.
7
- """
8
-
9
- from qa_analysis import taxonomy
10
-
11
- # Base severity per classification (before confidence adjustment).
12
- _SEVERITY = {
13
- taxonomy.APPLICATION_BUG: "high",
14
- taxonomy.INFRASTRUCTURE: "high",
15
- taxonomy.NETWORK: "high",
16
- taxonomy.AUTH: "high",
17
- taxonomy.AUTHORIZATION: "high",
18
- taxonomy.ASSERTION: "medium",
19
- taxonomy.LOCATOR: "medium",
20
- taxonomy.TIMEOUT: "medium",
21
- taxonomy.TEST_DATA: "medium",
22
- taxonomy.CONFIGURATION: "medium",
23
- taxonomy.ENVIRONMENT: "medium",
24
- taxonomy.FRAMEWORK: "medium",
25
- taxonomy.FLAKY: "medium",
26
- taxonomy.UNKNOWN: "low",
27
- }
28
-
29
- # Where the impact predominantly lands, per classification.
30
- _IMPACT = {
31
- taxonomy.APPLICATION_BUG: {"business": "high", "technical": "high", "testing": "low"},
32
- taxonomy.NETWORK: {"business": "high", "technical": "high", "testing": "medium"},
33
- taxonomy.AUTH: {"business": "high", "technical": "medium", "testing": "medium"},
34
- taxonomy.AUTHORIZATION: {"business": "high", "technical": "medium", "testing": "medium"},
35
- taxonomy.INFRASTRUCTURE: {"business": "medium", "technical": "high", "testing": "high"},
36
- taxonomy.LOCATOR: {"business": "low", "technical": "low", "testing": "high"},
37
- taxonomy.ASSERTION: {"business": "medium", "technical": "medium", "testing": "medium"},
38
- taxonomy.TIMEOUT: {"business": "low", "technical": "medium", "testing": "high"},
39
- taxonomy.TEST_DATA: {"business": "low", "technical": "low", "testing": "high"},
40
- taxonomy.CONFIGURATION: {"business": "low", "technical": "medium", "testing": "high"},
41
- taxonomy.ENVIRONMENT: {"business": "low", "technical": "medium", "testing": "high"},
42
- taxonomy.FRAMEWORK: {"business": "low", "technical": "medium", "testing": "high"},
43
- taxonomy.FLAKY: {"business": "low", "technical": "low", "testing": "high"},
44
- taxonomy.UNKNOWN: {"business": "low", "technical": "low", "testing": "medium"},
45
- }
46
-
47
- # Rough effort to resolve, per classification.
48
- _EFFORT = {
49
- taxonomy.LOCATOR: "low",
50
- taxonomy.ASSERTION: "low",
51
- taxonomy.CONFIGURATION: "low",
52
- taxonomy.ENVIRONMENT: "low",
53
- taxonomy.TEST_DATA: "medium",
54
- taxonomy.TIMEOUT: "medium",
55
- taxonomy.FLAKY: "medium",
56
- taxonomy.AUTH: "medium",
57
- taxonomy.AUTHORIZATION: "medium",
58
- taxonomy.FRAMEWORK: "medium",
59
- taxonomy.NETWORK: "high",
60
- taxonomy.INFRASTRUCTURE: "high",
61
- taxonomy.APPLICATION_BUG: "external",
62
- taxonomy.UNKNOWN: "unknown",
63
- }
64
-
65
- _RANK = {"low": 1, "medium": 2, "high": 3}
66
- _PRIORITY = {1: "P3", 2: "P2", 3: "P1", 4: "P1"}
67
-
68
-
69
- def prioritize(root_cause, blocking=False):
70
- """Prioritize a root cause. Returns a dict with severity, priority, the three
71
- impacts, confidence, owner, and estimatedEffort.
72
-
73
- priority derives from severity, business impact, and confidence, and is
74
- escalated one step when the failure blocks a release (blocking=True).
75
- """
76
- classification = root_cause["classification"]
77
- confidence = root_cause.get("confidence", 0.5)
78
- severity = _SEVERITY.get(classification, "low")
79
- impact = _IMPACT.get(classification, _IMPACT[taxonomy.UNKNOWN])
80
-
81
- # Priority score: severity and business impact drive it; low confidence
82
- # holds it back (an uncertain finding should not top the queue).
83
- score = _RANK[severity]
84
- if impact["business"] == "high":
85
- score += 1
86
- if confidence < 0.5:
87
- score -= 1
88
- if blocking:
89
- score += 1
90
- score = max(1, min(4, score))
91
-
92
- return {
93
- "severity": severity,
94
- "priority": _PRIORITY[score],
95
- "businessImpact": impact["business"],
96
- "technicalImpact": impact["technical"],
97
- "testingImpact": impact["testing"],
98
- "confidence": confidence,
99
- "owner": root_cause.get("ownership", "needs-triage"),
100
- "estimatedEffort": _EFFORT.get(classification, "unknown"),
101
- }
@@ -1,72 +0,0 @@
1
- """Deterministic repair planning.
2
-
3
- Turns a root cause into a repair *plan* — never code. It decides whether the
4
- failure is test-side repairable at all, and if so proposes an abstract change,
5
- the candidate type, the risk, and a rollback. qa-fix consumes these plans; the
6
- plan is always gated by the diff guard and always requires permission before any
7
- edit is applied.
8
- """
9
-
10
- from qa_analysis import taxonomy
11
-
12
- # classification -> (repairable, candidate type, abstract change, risk).
13
- # Only test-side causes are repairable; product, network, infra, auth, and
14
- # environment failures are escalations, not repairs.
15
- _PLANS = {
16
- taxonomy.LOCATOR: (True, "locator-update",
17
- "Update the failing locator to target the same element in the current DOM.", "low"),
18
- taxonomy.ASSERTION: (True, "assertion-improvement",
19
- "Correct the assertion to match the intended behavior, or confirm a product bug first.", "medium"),
20
- taxonomy.TIMEOUT: (True, "wait-strategy",
21
- "Replace a fixed or missing wait with a web-first wait on the awaited condition.", "medium"),
22
- taxonomy.FLAKY: (True, "synchronization",
23
- "Remove the race by awaiting the real condition; add a tracked quarantine only if needed.", "medium"),
24
- taxonomy.TEST_DATA: (True, "test-data",
25
- "Repair or reseed the test data the scenario depends on.", "medium"),
26
- taxonomy.CONFIGURATION: (True, "configuration",
27
- "Correct the test configuration the run depends on.", "low"),
28
- taxonomy.ENVIRONMENT: (False, "environment",
29
- "Fix the environment (base URL, service availability); not a test-side repair.", "n/a"),
30
- taxonomy.AUTH: (True, "authentication",
31
- "Repair the test's credentials or auth setup; do not weaken the check.", "medium"),
32
- taxonomy.AUTHORIZATION: (False, "authorization",
33
- "Grant the test account permission or use an authorized role; not a code repair.", "n/a"),
34
- taxonomy.NETWORK: (False, "network",
35
- "Investigate the upstream service; not a test-side repair.", "n/a"),
36
- taxonomy.INFRASTRUCTURE: (False, "infrastructure",
37
- "Escalate to CI/infra; not a test-side repair.", "n/a"),
38
- taxonomy.APPLICATION_BUG: (False, "application-bug",
39
- "File a product bug; the test correctly caught a real defect.", "n/a"),
40
- taxonomy.FRAMEWORK: (False, "framework",
41
- "Update or pin the framework/driver; not a test-side repair.", "n/a"),
42
- taxonomy.UNKNOWN: (False, "unknown",
43
- "Investigate further before any repair.", "n/a"),
44
- }
45
-
46
-
47
- def plan_repair(root_cause, affected_files=None):
48
- """Produce a repair plan for a root cause. Returns a dict:
49
- {repairable, candidateType, proposedChanges, affectedFiles, risk,
50
- permissionRequired, rollbackStrategy, safetyReview}. Never contains code.
51
- """
52
- classification = root_cause["classification"]
53
- repairable, candidate, change, risk = _PLANS.get(classification, _PLANS[taxonomy.UNKNOWN])
54
-
55
- if repairable:
56
- safety = ("Any edit will be checked by the diff guard before it is proposed as complete; "
57
- "the guard rejects removed assertions, added skips, forced passes, and timeout inflation.")
58
- rollback = "No source is changed without approval; revert the proposed edits to roll back."
59
- else:
60
- safety = "No test-side edit is appropriate; this is an escalation, not a repair."
61
- rollback = "Not applicable — no change is proposed."
62
-
63
- return {
64
- "repairable": repairable,
65
- "candidateType": candidate,
66
- "proposedChanges": [change] if repairable else [],
67
- "affectedFiles": list(affected_files or []),
68
- "risk": risk,
69
- "permissionRequired": True,
70
- "rollbackStrategy": rollback,
71
- "safetyReview": safety,
72
- }
@@ -1,89 +0,0 @@
1
- """Deterministic root-cause analysis.
2
-
3
- Turns a failure signal into a classified root cause with the four things every
4
- classification must carry: a taxonomy class (with confidence and evidence-backed
5
- reason), a recommended action, and an owner. It reuses the analysis platform's
6
- failure taxonomy; it adds the ownership and recommendation mappings and the
7
- metadata-driven classes (flaky) the taxonomy cannot infer from a message alone.
8
-
9
- No unsupported conclusions: a signal that matches no rule is `unknown`.
10
- """
11
-
12
- from qa_analysis import taxonomy
13
-
14
- # classification -> the party that typically owns the fix.
15
- OWNERSHIP = {
16
- taxonomy.ASSERTION: "test-author-or-product",
17
- taxonomy.LOCATOR: "test-author",
18
- taxonomy.TIMEOUT: "test-author-or-environment",
19
- taxonomy.NETWORK: "backend-or-infrastructure",
20
- taxonomy.AUTH: "auth-or-test-setup",
21
- taxonomy.AUTHORIZATION: "permissions-or-test-account",
22
- taxonomy.ENVIRONMENT: "environment-owner",
23
- taxonomy.CONFIGURATION: "config-owner",
24
- taxonomy.INFRASTRUCTURE: "ci-or-infrastructure",
25
- taxonomy.TEST_DATA: "test-data-owner",
26
- taxonomy.APPLICATION_BUG: "product",
27
- taxonomy.FRAMEWORK: "framework-or-driver",
28
- taxonomy.FLAKY: "test-author",
29
- taxonomy.UNKNOWN: "needs-triage",
30
- }
31
-
32
- # classification -> the safe recommended action (implements the analysis
33
- # platform's recommendation-guidelines; never recommends forcing a pass).
34
- RECOMMENDATION = {
35
- taxonomy.ASSERTION: "Confirm whether the app or the expectation is wrong; fix whichever is genuinely incorrect.",
36
- taxonomy.LOCATOR: "Inspect the current DOM and update the locator to target the same element.",
37
- taxonomy.TIMEOUT: "Investigate the slowness; raise a wait only if the operation is legitimately slower.",
38
- taxonomy.NETWORK: "Check the upstream service and the request; retry only if the failure is genuinely transient.",
39
- taxonomy.AUTH: "Fix the credentials or auth setup; do not weaken the authentication check.",
40
- taxonomy.AUTHORIZATION: "Grant the test account the needed permission or use an authorized role; do not bypass the check.",
41
- taxonomy.ENVIRONMENT: "Fix the environment (base URL, service availability); the test is likely fine.",
42
- taxonomy.CONFIGURATION: "Correct the configuration; do not work around it in the test.",
43
- taxonomy.INFRASTRUCTURE: "Escalate to CI or infrastructure owners; add resources, do not shrink the suite.",
44
- taxonomy.TEST_DATA: "Repair or reseed the data; do not delete the assertion that caught the gap.",
45
- taxonomy.APPLICATION_BUG: "File a bug against the product; do NOT modify the test to pass.",
46
- taxonomy.FRAMEWORK: "Update or pin the framework/driver; report upstream if it is a genuine defect.",
47
- taxonomy.FLAKY: "Stabilize the test (fix the race or synchronization); quarantine only with a tracking issue.",
48
- taxonomy.UNKNOWN: "Investigate further; the evidence was insufficient to classify.",
49
- }
50
-
51
-
52
- def analyze(signal):
53
- """Classify a failure signal into a root cause.
54
-
55
- signal is a dict that may contain: message, httpStatus, retries, finalStatus,
56
- evidence (a list of evidence refs). Returns a dict:
57
- {classification, confidence, reason, ownership, recommendation, evidence}.
58
- """
59
- message = signal.get("message", "")
60
- http_status = signal.get("httpStatus")
61
- retries = signal.get("retries", 0) or 0
62
- final_status = signal.get("finalStatus")
63
-
64
- provided = signal.get("classification")
65
-
66
- # Flakiness is a metadata signal, not a message pattern: a test that needed a
67
- # retry to pass, or is explicitly flagged flaky, is nondeterministic.
68
- if final_status == "flaky" or (retries > 0 and final_status in ("passed", "flaky")):
69
- classification, confidence, reason = (
70
- taxonomy.FLAKY, 0.8,
71
- "The test passed only after a retry, indicating nondeterministic behavior.",
72
- )
73
- elif provided in taxonomy.CLASSES:
74
- # The analysis platform already classified this deterministically; trust it
75
- # rather than re-deriving from the message.
76
- classification = provided
77
- confidence = signal.get("confidence", 0.8)
78
- reason = signal.get("reason") or f"Classified {provided} by the analysis platform."
79
- else:
80
- classification, confidence, reason = taxonomy.classify(message, http_status=http_status)
81
-
82
- return {
83
- "classification": classification,
84
- "confidence": confidence,
85
- "reason": reason,
86
- "ownership": OWNERSHIP.get(classification, "needs-triage"),
87
- "recommendation": RECOMMENDATION.get(classification, RECOMMENDATION[taxonomy.UNKNOWN]),
88
- "evidence": signal.get("evidence", []),
89
- }
@@ -1,71 +0,0 @@
1
- """Deterministic timeline reconstruction.
2
-
3
- Reconstructs the ordered sequence of a run — start, browser launch, navigation,
4
- requests and responses, console errors, assertions, failure, cleanup — from the
5
- execution result and the analysis findings. It records only stages for which
6
- there is evidence; it never invents an event to fill the shape. Reusable by
7
- qa-debug and qa-report.
8
- """
9
-
10
- # Canonical phase order, used to sort events that share (or lack) a timestamp.
11
- PHASE_ORDER = [
12
- "execution-start", "browser-launch", "navigation", "request", "response",
13
- "console-error", "assertion", "failure", "cleanup", "execution-finish",
14
- ]
15
- _PHASE_INDEX = {phase: i for i, phase in enumerate(PHASE_ORDER)}
16
-
17
- # Evidence type -> the timeline phase it contributes.
18
- _EVIDENCE_PHASE = {
19
- "network": "response",
20
- "console": "console-error",
21
- "trace": "navigation",
22
- "junit": "assertion",
23
- "report": "assertion",
24
- }
25
-
26
-
27
- def build_timeline(execution_result, findings=None):
28
- """Build an ordered timeline. Returns a list of event dicts:
29
- {order, phase, detail, source, timestamp?}. Deterministic: same inputs,
30
- same timeline. Only evidenced stages appear."""
31
- events = []
32
- execution = (execution_result or {}).get("execution", {})
33
-
34
- started = execution.get("startedAt")
35
- if started or execution_result:
36
- events.append(_event("execution-start", "Run started", "execution-result", started))
37
-
38
- # Contribute a stage per evidence entry we actually have.
39
- for finding in findings or []:
40
- for ev in finding.get("evidence", []):
41
- phase = _EVIDENCE_PHASE.get(ev.get("type"))
42
- if phase:
43
- events.append(_event(phase, ev.get("description", ""), ev.get("source", "analysis"), None))
44
-
45
- # A failure stage per failed test recorded in the result.
46
- for test in (execution_result or {}).get("executed", []):
47
- if test.get("status") == "failed":
48
- events.append(_event("failure", f"Test failed: {test.get('title', '')}",
49
- test.get("file", "execution-result"), None))
50
-
51
- finished = execution.get("finishedAt")
52
- if finished:
53
- events.append(_event("cleanup", "Run cleaned up", "execution-result", None))
54
- events.append(_event("execution-finish", "Run finished", "execution-result", finished))
55
-
56
- # Deterministic order: by canonical phase first (it encodes the logical
57
- # sequence), then by timestamp within a phase, then by insertion order. Phase
58
- # order is primary because per-event wall-clock times are often unavailable.
59
- for i, event in enumerate(events):
60
- event["_seq"] = i
61
- events.sort(key=lambda e: (_PHASE_INDEX.get(e["phase"], 99), e["timestamp"] or "", e["_seq"]))
62
- for order, event in enumerate(events):
63
- event["order"] = order
64
- del event["_seq"]
65
- if event["timestamp"] is None:
66
- del event["timestamp"]
67
- return events
68
-
69
-
70
- def _event(phase, detail, source, timestamp):
71
- return {"phase": phase, "detail": detail, "source": source, "timestamp": timestamp}
@@ -1,27 +0,0 @@
1
- """Cypress analysis adapter.
2
-
3
- Cypress emits JUnit XML (via a JUnit reporter), so Cypress normalization is the
4
- framework-agnostic JUnit parser — the adapter only points at the artifact and
5
- tags provenance. Like the Selenium adapter, its thinness is the proof of the
6
- multi-framework boundary: the contract and parsing are shared; only the artifact
7
- location differs.
8
- """
9
-
10
- from qa_analysis import junit, taxonomy
11
-
12
- FRAMEWORK = "cypress"
13
-
14
- # Where Cypress projects conventionally write JUnit results (see cypress-artifacts.md).
15
- RESULT_GLOBS = ["**/results/*.xml", "**/cypress/results/*.xml", "**/junit*.xml"]
16
-
17
-
18
- def normalize(junit_path):
19
- """Normalize a Cypress run into the shared result shape via the agnostic
20
- JUnit parser. Identical output shape to any other framework."""
21
- return junit.parse_junit(junit_path)
22
-
23
-
24
- def classify_failure(message, http_status=None):
25
- """Classify a Cypress failure using the shared taxonomy — no Cypress-specific
26
- classifier, because failure classes are framework-agnostic."""
27
- return taxonomy.classify(message, http_status=http_status)
@@ -1,167 +0,0 @@
1
- """Playwright-specific analyzers: trace and JSON report.
2
-
3
- These are the Playwright *adapter* for the analysis platform. They depend on
4
- the framework-agnostic qa_analysis core for the evidence model, redaction, and
5
- the failure taxonomy; they add only what is Playwright-specific — the shape of
6
- a Playwright trace.zip and of Playwright's JSON reporter. Everything they emit
7
- is in the shared normalized shape, so nothing downstream knows it was Playwright.
8
-
9
- The adapter carries its own CLI (`python -m playwright_analysis report|trace
10
- <path>`) so a skill can reach it the same way it reaches every other
11
- deterministic tool. Framework knowledge stays inside the adapter: the core CLI
12
- never grows a `--framework` flag ([ADR-0013](../../../../docs/architecture/ADR-0013-framework-boundary.md)).
13
-
14
- Usage:
15
- python -m playwright_analysis report <results.json>
16
- python -m playwright_analysis trace <trace.zip>
17
- """
18
-
19
- import argparse
20
- import json
21
- import sys
22
- import zipfile
23
-
24
- from qa_analysis.junit import MalformedArtifact
25
- from qa_analysis.redaction import redact_text
26
- from qa_analysis import taxonomy
27
-
28
-
29
- def parse_report(path):
30
- """Parse Playwright's JSON reporter output into the normalized result shape.
31
-
32
- Mirrors qa_analysis.junit's output ({tests, executed}) so a Playwright run
33
- normalizes identically whether it emitted JSON or JUnit. Raises
34
- MalformedArtifact on unreadable input.
35
- """
36
- try:
37
- with open(path, "r", encoding="utf-8") as handle:
38
- data = json.load(handle)
39
- except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:
40
- raise MalformedArtifact(f"could not parse Playwright report at {path}: {exc}") from exc
41
-
42
- executed = []
43
- for suite in data.get("suites", []):
44
- _walk_suite(suite, executed)
45
-
46
- counts = {
47
- "total": len(executed),
48
- "passed": sum(1 for e in executed if e["status"] == "passed"),
49
- "failed": sum(1 for e in executed if e["status"] == "failed"),
50
- "skipped": sum(1 for e in executed if e["status"] == "skipped"),
51
- }
52
- return {"tests": counts, "executed": executed}
53
-
54
-
55
- def _walk_suite(suite, executed):
56
- for spec in suite.get("specs", []):
57
- for test in spec.get("tests", []):
58
- results = test.get("results", [])
59
- status = _status_of(results)
60
- message = ""
61
- if status == "failed" and results:
62
- errors = results[-1].get("errors") or []
63
- if errors:
64
- message = redact_text((errors[0].get("message") or "").strip())
65
- entry = {
66
- "title": spec.get("title", ""),
67
- "file": suite.get("file", ""),
68
- "status": status,
69
- "durationMs": results[-1].get("duration", 0) if results else 0,
70
- "retries": max(0, len(results) - 1),
71
- }
72
- if message:
73
- entry["message"] = message
74
- executed.append(entry)
75
- for child in suite.get("suites", []):
76
- _walk_suite(child, executed)
77
-
78
-
79
- def _status_of(results):
80
- if not results:
81
- return "skipped"
82
- final = results[-1].get("status")
83
- if final in ("passed", "expected"):
84
- return "flaky" if len(results) > 1 else "passed"
85
- if final in ("skipped",):
86
- return "skipped"
87
- return "failed"
88
-
89
-
90
- def analyze_trace(path):
91
- """Extract a deterministic summary from a Playwright trace.zip.
92
-
93
- A trace is a zip of newline-delimited JSON event files. This lists the
94
- actions, surfaces the last error, and counts console and network events —
95
- enough for a diagnostic skill to reason over, without decoding the full
96
- binary. Raises MalformedArtifact if the file is not a valid trace zip.
97
- """
98
- if not zipfile.is_zipfile(path):
99
- raise MalformedArtifact(f"not a valid trace zip: {path}")
100
-
101
- actions, console, network, errors = [], 0, 0, []
102
- try:
103
- with zipfile.ZipFile(path) as archive:
104
- names = [n for n in archive.namelist() if n.endswith(".trace") or n.endswith(".jsonl") or "trace" in n]
105
- for name in names or archive.namelist():
106
- with archive.open(name) as member:
107
- for raw in member.read().decode("utf-8", "replace").splitlines():
108
- raw = raw.strip()
109
- if not raw or not raw.startswith("{"):
110
- continue
111
- try:
112
- event = json.loads(raw)
113
- except json.JSONDecodeError:
114
- continue
115
- kind = event.get("type") or event.get("kind")
116
- if kind in ("action", "before"):
117
- actions.append(redact_text(str(event.get("apiName") or event.get("method") or "")))
118
- elif kind == "console":
119
- console += 1
120
- elif kind in ("resource", "network", "http"):
121
- network += 1
122
- if event.get("error") or kind == "error":
123
- msg = event.get("error", {}).get("message") if isinstance(event.get("error"), dict) else event.get("message")
124
- if msg:
125
- errors.append(redact_text(str(msg)))
126
- except (OSError, KeyError) as exc:
127
- raise MalformedArtifact(f"could not read trace {path}: {exc}") from exc
128
-
129
- last_error = errors[-1] if errors else ""
130
- classification, confidence, reason = taxonomy.classify(last_error) if last_error else (taxonomy.UNKNOWN, 0.2, "No error found in trace.")
131
- return {
132
- "actions": [a for a in actions if a],
133
- "consoleEvents": console,
134
- "networkEvents": network,
135
- "errors": errors,
136
- "classification": classification,
137
- "confidence": confidence,
138
- "reason": reason,
139
- }
140
-
141
-
142
- def main(argv=None):
143
- """CLI: JSON to stdout, exit 0 on success, 2 on a malformed artifact."""
144
- parser = argparse.ArgumentParser(
145
- prog="playwright_analysis",
146
- description="Playwright adapter: normalize a JSON report or summarize a trace",
147
- )
148
- sub = parser.add_subparsers(dest="command", required=True)
149
- p = sub.add_parser("report", help="normalize Playwright's JSON reporter output")
150
- p.add_argument("path")
151
- p = sub.add_parser("trace", help="summarize a Playwright trace.zip")
152
- p.add_argument("path")
153
- args = parser.parse_args(argv)
154
-
155
- try:
156
- result = parse_report(args.path) if args.command == "report" else analyze_trace(args.path)
157
- except MalformedArtifact as exc:
158
- json.dump({"error": "malformed-artifact", "detail": str(exc)}, sys.stdout, indent=2, sort_keys=True)
159
- sys.stdout.write("\n")
160
- return 2
161
- json.dump(result, sys.stdout, indent=2, sort_keys=True)
162
- sys.stdout.write("\n")
163
- return 0
164
-
165
-
166
- if __name__ == "__main__":
167
- sys.exit(main())
@@ -1,28 +0,0 @@
1
- """Selenium analysis adapter.
2
-
3
- Selenium's runners (JUnit, TestNG, pytest) emit JUnit XML, so Selenium
4
- normalization *is* the framework-agnostic JUnit parser — the adapter only points
5
- at the right artifact and tags provenance. This is the proof of the pack's
6
- multi-framework claim: the second framework's analysis is a few lines, because
7
- the contract and the parsing are shared, and only the artifact location differs.
8
- """
9
-
10
- from qa_analysis import junit, taxonomy
11
-
12
- FRAMEWORK = "selenium"
13
-
14
- # Where Selenium projects conventionally write results (documented in
15
- # selenium-artifacts.md); discovery uses these, the parser is shared.
16
- RESULT_GLOBS = ["**/target/surefire-reports/*.xml", "**/test-results/*.xml", "**/junit*.xml"]
17
-
18
-
19
- def normalize(junit_path):
20
- """Normalize a Selenium run into the shared result shape via the agnostic
21
- JUnit parser. Identical output shape to any other framework's normalization."""
22
- return junit.parse_junit(junit_path)
23
-
24
-
25
- def classify_failure(message, http_status=None):
26
- """Classify a Selenium failure using the shared taxonomy — no Selenium-specific
27
- classifier, because failure classes are framework-agnostic."""
28
- return taxonomy.classify(message, http_status=http_status)
@@ -1,24 +0,0 @@
1
- """WebdriverIO analysis adapter.
2
-
3
- WebdriverIO emits JUnit XML (via @wdio/junit-reporter), so its normalization is
4
- the framework-agnostic JUnit parser — the adapter points at the artifact and
5
- tags provenance. Thin by design: the fourth framework confirms the boundary.
6
- """
7
-
8
- from qa_analysis import junit, taxonomy
9
-
10
- FRAMEWORK = "webdriverio"
11
-
12
- # Where WebdriverIO projects conventionally write JUnit results.
13
- RESULT_GLOBS = ["**/junit*.xml", "**/results/*.xml", "**/test-results/*.xml"]
14
-
15
-
16
- def normalize(junit_path):
17
- """Normalize a WebdriverIO run into the shared result shape via the agnostic
18
- JUnit parser. Identical output shape to any other framework."""
19
- return junit.parse_junit(junit_path)
20
-
21
-
22
- def classify_failure(message, http_status=None):
23
- """Classify a WebdriverIO failure using the shared taxonomy."""
24
- return taxonomy.classify(message, http_status=http_status)