qa-engineer 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +50 -26
- package/README.md +40 -32
- package/package.json +12 -20
- package/packages/engine/bin/qa-engine.mjs +336 -0
- package/packages/engine/lib/analysis/branding.mjs +187 -0
- package/packages/engine/lib/analysis/context.mjs +294 -0
- package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
- package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
- package/packages/engine/lib/analysis/discovery.mjs +220 -0
- package/packages/engine/lib/analysis/evidence.mjs +116 -0
- package/packages/engine/lib/analysis/har.mjs +90 -0
- package/packages/engine/lib/analysis/junit.mjs +126 -0
- package/packages/engine/lib/analysis/redaction.mjs +127 -0
- package/packages/engine/lib/analysis/report-html.mjs +763 -0
- package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
- package/packages/engine/lib/analysis/xml.mjs +153 -0
- package/packages/engine/lib/analysis/zip.mjs +107 -0
- package/packages/engine/lib/diagnostics/engine.mjs +165 -0
- package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
- package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
- package/packages/engine/lib/diagnostics/repair.mjs +73 -0
- package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
- package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
- package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
- package/packages/engine/lib/frameworks/playwright.mjs +158 -0
- package/packages/engine/package.json +14 -0
- package/packages/installer/lib/agents/registry.mjs +46 -0
- package/packages/installer/lib/commands/doctor.mjs +7 -16
- package/packages/installer/lib/commands/install.mjs +59 -4
- package/packages/installer/lib/core/bundle.mjs +69 -92
- package/packages/installer/lib/core/config.mjs +1 -1
- package/packages/installer/lib/core/lockfile.mjs +1 -1
- package/packages/installer/lib/core/manifest.mjs +48 -54
- package/packages/installer/lib/core/validate-install.mjs +12 -19
- package/packages/installer/package.json +1 -1
- package/shared/tooling/qa-tool.mjs +125 -0
- package/skills/qa-api/SKILL.md +5 -5
- package/skills/qa-api/references/deterministic-tooling.md +34 -30
- package/skills/qa-api/references/evidence-and-reporting.md +29 -4
- package/skills/qa-api/scripts/qa-tool.mjs +126 -0
- package/skills/qa-audit/SKILL.md +5 -5
- package/skills/qa-audit/references/deterministic-tooling.md +34 -30
- package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
- package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
- package/skills/qa-debug/SKILL.md +6 -6
- package/skills/qa-debug/references/deterministic-tooling.md +34 -30
- package/skills/qa-debug/references/diagnostic-engine.md +3 -3
- package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
- package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
- package/skills/qa-explore/SKILL.md +22 -5
- package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
- package/skills/qa-explore/references/deterministic-tooling.md +132 -0
- package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
- package/skills/qa-explore/references/report-pipeline.md +89 -8
- package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
- package/skills/qa-fix/SKILL.md +4 -4
- package/skills/qa-fix/references/deterministic-tooling.md +34 -30
- package/skills/qa-fix/references/diagnostic-engine.md +3 -3
- package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
- package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
- package/skills/qa-flaky/SKILL.md +4 -4
- package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
- package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
- package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
- package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
- package/skills/qa-init/SKILL.md +4 -4
- package/skills/qa-init/references/deterministic-tooling.md +34 -30
- package/skills/qa-init/references/evidence-and-reporting.md +29 -4
- package/skills/qa-init/scripts/qa-tool.mjs +126 -0
- package/skills/qa-report/SKILL.md +8 -7
- package/skills/qa-report/references/deterministic-tooling.md +34 -30
- package/skills/qa-report/references/diagnostic-engine.md +3 -3
- package/skills/qa-report/references/evidence-and-reporting.md +29 -4
- package/skills/qa-report/scripts/qa-tool.mjs +126 -0
- package/skills/qa-review/references/evidence-and-reporting.md +29 -4
- package/skills/qa-run/SKILL.md +6 -6
- package/skills/qa-run/references/deterministic-tooling.md +34 -30
- package/skills/qa-run/references/evidence-and-reporting.md +29 -4
- package/skills/qa-run/scripts/qa-tool.mjs +126 -0
- package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
- package/shared/analysis/lib/qa_analysis/branding.py +0 -175
- package/shared/analysis/lib/qa_analysis/cli.py +0 -129
- package/shared/analysis/lib/qa_analysis/context.py +0 -233
- package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
- package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
- package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
- package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
- package/shared/analysis/lib/qa_analysis/har.py +0 -58
- package/shared/analysis/lib/qa_analysis/junit.py +0 -80
- package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
- package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
- package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
- package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
- package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
- package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
- package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
- package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
- package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
- package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
- package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
- package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
- package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
- package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
- package/shared/tooling/qa_tool.py +0 -127
- /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
- /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
"""Deterministic finding prioritization.
|
|
2
|
-
|
|
3
|
-
Assigns every finding a severity, priority, the three impacts, an owner, and an
|
|
4
|
-
estimated effort — by a fixed algorithm, so the same finding always ranks the
|
|
5
|
-
same way. Priority is not a feeling; it is a function of severity, confidence,
|
|
6
|
-
and business impact.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
from qa_analysis import taxonomy
|
|
10
|
-
|
|
11
|
-
# Base severity per classification (before confidence adjustment).
|
|
12
|
-
_SEVERITY = {
|
|
13
|
-
taxonomy.APPLICATION_BUG: "high",
|
|
14
|
-
taxonomy.INFRASTRUCTURE: "high",
|
|
15
|
-
taxonomy.NETWORK: "high",
|
|
16
|
-
taxonomy.AUTH: "high",
|
|
17
|
-
taxonomy.AUTHORIZATION: "high",
|
|
18
|
-
taxonomy.ASSERTION: "medium",
|
|
19
|
-
taxonomy.LOCATOR: "medium",
|
|
20
|
-
taxonomy.TIMEOUT: "medium",
|
|
21
|
-
taxonomy.TEST_DATA: "medium",
|
|
22
|
-
taxonomy.CONFIGURATION: "medium",
|
|
23
|
-
taxonomy.ENVIRONMENT: "medium",
|
|
24
|
-
taxonomy.FRAMEWORK: "medium",
|
|
25
|
-
taxonomy.FLAKY: "medium",
|
|
26
|
-
taxonomy.UNKNOWN: "low",
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
# Where the impact predominantly lands, per classification.
|
|
30
|
-
_IMPACT = {
|
|
31
|
-
taxonomy.APPLICATION_BUG: {"business": "high", "technical": "high", "testing": "low"},
|
|
32
|
-
taxonomy.NETWORK: {"business": "high", "technical": "high", "testing": "medium"},
|
|
33
|
-
taxonomy.AUTH: {"business": "high", "technical": "medium", "testing": "medium"},
|
|
34
|
-
taxonomy.AUTHORIZATION: {"business": "high", "technical": "medium", "testing": "medium"},
|
|
35
|
-
taxonomy.INFRASTRUCTURE: {"business": "medium", "technical": "high", "testing": "high"},
|
|
36
|
-
taxonomy.LOCATOR: {"business": "low", "technical": "low", "testing": "high"},
|
|
37
|
-
taxonomy.ASSERTION: {"business": "medium", "technical": "medium", "testing": "medium"},
|
|
38
|
-
taxonomy.TIMEOUT: {"business": "low", "technical": "medium", "testing": "high"},
|
|
39
|
-
taxonomy.TEST_DATA: {"business": "low", "technical": "low", "testing": "high"},
|
|
40
|
-
taxonomy.CONFIGURATION: {"business": "low", "technical": "medium", "testing": "high"},
|
|
41
|
-
taxonomy.ENVIRONMENT: {"business": "low", "technical": "medium", "testing": "high"},
|
|
42
|
-
taxonomy.FRAMEWORK: {"business": "low", "technical": "medium", "testing": "high"},
|
|
43
|
-
taxonomy.FLAKY: {"business": "low", "technical": "low", "testing": "high"},
|
|
44
|
-
taxonomy.UNKNOWN: {"business": "low", "technical": "low", "testing": "medium"},
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
# Rough effort to resolve, per classification.
|
|
48
|
-
_EFFORT = {
|
|
49
|
-
taxonomy.LOCATOR: "low",
|
|
50
|
-
taxonomy.ASSERTION: "low",
|
|
51
|
-
taxonomy.CONFIGURATION: "low",
|
|
52
|
-
taxonomy.ENVIRONMENT: "low",
|
|
53
|
-
taxonomy.TEST_DATA: "medium",
|
|
54
|
-
taxonomy.TIMEOUT: "medium",
|
|
55
|
-
taxonomy.FLAKY: "medium",
|
|
56
|
-
taxonomy.AUTH: "medium",
|
|
57
|
-
taxonomy.AUTHORIZATION: "medium",
|
|
58
|
-
taxonomy.FRAMEWORK: "medium",
|
|
59
|
-
taxonomy.NETWORK: "high",
|
|
60
|
-
taxonomy.INFRASTRUCTURE: "high",
|
|
61
|
-
taxonomy.APPLICATION_BUG: "external",
|
|
62
|
-
taxonomy.UNKNOWN: "unknown",
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
_RANK = {"low": 1, "medium": 2, "high": 3}
|
|
66
|
-
_PRIORITY = {1: "P3", 2: "P2", 3: "P1", 4: "P1"}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
def prioritize(root_cause, blocking=False):
|
|
70
|
-
"""Prioritize a root cause. Returns a dict with severity, priority, the three
|
|
71
|
-
impacts, confidence, owner, and estimatedEffort.
|
|
72
|
-
|
|
73
|
-
priority derives from severity, business impact, and confidence, and is
|
|
74
|
-
escalated one step when the failure blocks a release (blocking=True).
|
|
75
|
-
"""
|
|
76
|
-
classification = root_cause["classification"]
|
|
77
|
-
confidence = root_cause.get("confidence", 0.5)
|
|
78
|
-
severity = _SEVERITY.get(classification, "low")
|
|
79
|
-
impact = _IMPACT.get(classification, _IMPACT[taxonomy.UNKNOWN])
|
|
80
|
-
|
|
81
|
-
# Priority score: severity and business impact drive it; low confidence
|
|
82
|
-
# holds it back (an uncertain finding should not top the queue).
|
|
83
|
-
score = _RANK[severity]
|
|
84
|
-
if impact["business"] == "high":
|
|
85
|
-
score += 1
|
|
86
|
-
if confidence < 0.5:
|
|
87
|
-
score -= 1
|
|
88
|
-
if blocking:
|
|
89
|
-
score += 1
|
|
90
|
-
score = max(1, min(4, score))
|
|
91
|
-
|
|
92
|
-
return {
|
|
93
|
-
"severity": severity,
|
|
94
|
-
"priority": _PRIORITY[score],
|
|
95
|
-
"businessImpact": impact["business"],
|
|
96
|
-
"technicalImpact": impact["technical"],
|
|
97
|
-
"testingImpact": impact["testing"],
|
|
98
|
-
"confidence": confidence,
|
|
99
|
-
"owner": root_cause.get("ownership", "needs-triage"),
|
|
100
|
-
"estimatedEffort": _EFFORT.get(classification, "unknown"),
|
|
101
|
-
}
|
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
"""Deterministic repair planning.
|
|
2
|
-
|
|
3
|
-
Turns a root cause into a repair *plan* — never code. It decides whether the
|
|
4
|
-
failure is test-side repairable at all, and if so proposes an abstract change,
|
|
5
|
-
the candidate type, the risk, and a rollback. qa-fix consumes these plans; the
|
|
6
|
-
plan is always gated by the diff guard and always requires permission before any
|
|
7
|
-
edit is applied.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from qa_analysis import taxonomy
|
|
11
|
-
|
|
12
|
-
# classification -> (repairable, candidate type, abstract change, risk).
|
|
13
|
-
# Only test-side causes are repairable; product, network, infra, auth, and
|
|
14
|
-
# environment failures are escalations, not repairs.
|
|
15
|
-
_PLANS = {
|
|
16
|
-
taxonomy.LOCATOR: (True, "locator-update",
|
|
17
|
-
"Update the failing locator to target the same element in the current DOM.", "low"),
|
|
18
|
-
taxonomy.ASSERTION: (True, "assertion-improvement",
|
|
19
|
-
"Correct the assertion to match the intended behavior, or confirm a product bug first.", "medium"),
|
|
20
|
-
taxonomy.TIMEOUT: (True, "wait-strategy",
|
|
21
|
-
"Replace a fixed or missing wait with a web-first wait on the awaited condition.", "medium"),
|
|
22
|
-
taxonomy.FLAKY: (True, "synchronization",
|
|
23
|
-
"Remove the race by awaiting the real condition; add a tracked quarantine only if needed.", "medium"),
|
|
24
|
-
taxonomy.TEST_DATA: (True, "test-data",
|
|
25
|
-
"Repair or reseed the test data the scenario depends on.", "medium"),
|
|
26
|
-
taxonomy.CONFIGURATION: (True, "configuration",
|
|
27
|
-
"Correct the test configuration the run depends on.", "low"),
|
|
28
|
-
taxonomy.ENVIRONMENT: (False, "environment",
|
|
29
|
-
"Fix the environment (base URL, service availability); not a test-side repair.", "n/a"),
|
|
30
|
-
taxonomy.AUTH: (True, "authentication",
|
|
31
|
-
"Repair the test's credentials or auth setup; do not weaken the check.", "medium"),
|
|
32
|
-
taxonomy.AUTHORIZATION: (False, "authorization",
|
|
33
|
-
"Grant the test account permission or use an authorized role; not a code repair.", "n/a"),
|
|
34
|
-
taxonomy.NETWORK: (False, "network",
|
|
35
|
-
"Investigate the upstream service; not a test-side repair.", "n/a"),
|
|
36
|
-
taxonomy.INFRASTRUCTURE: (False, "infrastructure",
|
|
37
|
-
"Escalate to CI/infra; not a test-side repair.", "n/a"),
|
|
38
|
-
taxonomy.APPLICATION_BUG: (False, "application-bug",
|
|
39
|
-
"File a product bug; the test correctly caught a real defect.", "n/a"),
|
|
40
|
-
taxonomy.FRAMEWORK: (False, "framework",
|
|
41
|
-
"Update or pin the framework/driver; not a test-side repair.", "n/a"),
|
|
42
|
-
taxonomy.UNKNOWN: (False, "unknown",
|
|
43
|
-
"Investigate further before any repair.", "n/a"),
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
def plan_repair(root_cause, affected_files=None):
|
|
48
|
-
"""Produce a repair plan for a root cause. Returns a dict:
|
|
49
|
-
{repairable, candidateType, proposedChanges, affectedFiles, risk,
|
|
50
|
-
permissionRequired, rollbackStrategy, safetyReview}. Never contains code.
|
|
51
|
-
"""
|
|
52
|
-
classification = root_cause["classification"]
|
|
53
|
-
repairable, candidate, change, risk = _PLANS.get(classification, _PLANS[taxonomy.UNKNOWN])
|
|
54
|
-
|
|
55
|
-
if repairable:
|
|
56
|
-
safety = ("Any edit will be checked by the diff guard before it is proposed as complete; "
|
|
57
|
-
"the guard rejects removed assertions, added skips, forced passes, and timeout inflation.")
|
|
58
|
-
rollback = "No source is changed without approval; revert the proposed edits to roll back."
|
|
59
|
-
else:
|
|
60
|
-
safety = "No test-side edit is appropriate; this is an escalation, not a repair."
|
|
61
|
-
rollback = "Not applicable — no change is proposed."
|
|
62
|
-
|
|
63
|
-
return {
|
|
64
|
-
"repairable": repairable,
|
|
65
|
-
"candidateType": candidate,
|
|
66
|
-
"proposedChanges": [change] if repairable else [],
|
|
67
|
-
"affectedFiles": list(affected_files or []),
|
|
68
|
-
"risk": risk,
|
|
69
|
-
"permissionRequired": True,
|
|
70
|
-
"rollbackStrategy": rollback,
|
|
71
|
-
"safetyReview": safety,
|
|
72
|
-
}
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
"""Deterministic root-cause analysis.
|
|
2
|
-
|
|
3
|
-
Turns a failure signal into a classified root cause with the four things every
|
|
4
|
-
classification must carry: a taxonomy class (with confidence and evidence-backed
|
|
5
|
-
reason), a recommended action, and an owner. It reuses the analysis platform's
|
|
6
|
-
failure taxonomy; it adds the ownership and recommendation mappings and the
|
|
7
|
-
metadata-driven classes (flaky) the taxonomy cannot infer from a message alone.
|
|
8
|
-
|
|
9
|
-
No unsupported conclusions: a signal that matches no rule is `unknown`.
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
from qa_analysis import taxonomy
|
|
13
|
-
|
|
14
|
-
# classification -> the party that typically owns the fix.
|
|
15
|
-
OWNERSHIP = {
|
|
16
|
-
taxonomy.ASSERTION: "test-author-or-product",
|
|
17
|
-
taxonomy.LOCATOR: "test-author",
|
|
18
|
-
taxonomy.TIMEOUT: "test-author-or-environment",
|
|
19
|
-
taxonomy.NETWORK: "backend-or-infrastructure",
|
|
20
|
-
taxonomy.AUTH: "auth-or-test-setup",
|
|
21
|
-
taxonomy.AUTHORIZATION: "permissions-or-test-account",
|
|
22
|
-
taxonomy.ENVIRONMENT: "environment-owner",
|
|
23
|
-
taxonomy.CONFIGURATION: "config-owner",
|
|
24
|
-
taxonomy.INFRASTRUCTURE: "ci-or-infrastructure",
|
|
25
|
-
taxonomy.TEST_DATA: "test-data-owner",
|
|
26
|
-
taxonomy.APPLICATION_BUG: "product",
|
|
27
|
-
taxonomy.FRAMEWORK: "framework-or-driver",
|
|
28
|
-
taxonomy.FLAKY: "test-author",
|
|
29
|
-
taxonomy.UNKNOWN: "needs-triage",
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
# classification -> the safe recommended action (implements the analysis
|
|
33
|
-
# platform's recommendation-guidelines; never recommends forcing a pass).
|
|
34
|
-
RECOMMENDATION = {
|
|
35
|
-
taxonomy.ASSERTION: "Confirm whether the app or the expectation is wrong; fix whichever is genuinely incorrect.",
|
|
36
|
-
taxonomy.LOCATOR: "Inspect the current DOM and update the locator to target the same element.",
|
|
37
|
-
taxonomy.TIMEOUT: "Investigate the slowness; raise a wait only if the operation is legitimately slower.",
|
|
38
|
-
taxonomy.NETWORK: "Check the upstream service and the request; retry only if the failure is genuinely transient.",
|
|
39
|
-
taxonomy.AUTH: "Fix the credentials or auth setup; do not weaken the authentication check.",
|
|
40
|
-
taxonomy.AUTHORIZATION: "Grant the test account the needed permission or use an authorized role; do not bypass the check.",
|
|
41
|
-
taxonomy.ENVIRONMENT: "Fix the environment (base URL, service availability); the test is likely fine.",
|
|
42
|
-
taxonomy.CONFIGURATION: "Correct the configuration; do not work around it in the test.",
|
|
43
|
-
taxonomy.INFRASTRUCTURE: "Escalate to CI or infrastructure owners; add resources, do not shrink the suite.",
|
|
44
|
-
taxonomy.TEST_DATA: "Repair or reseed the data; do not delete the assertion that caught the gap.",
|
|
45
|
-
taxonomy.APPLICATION_BUG: "File a bug against the product; do NOT modify the test to pass.",
|
|
46
|
-
taxonomy.FRAMEWORK: "Update or pin the framework/driver; report upstream if it is a genuine defect.",
|
|
47
|
-
taxonomy.FLAKY: "Stabilize the test (fix the race or synchronization); quarantine only with a tracking issue.",
|
|
48
|
-
taxonomy.UNKNOWN: "Investigate further; the evidence was insufficient to classify.",
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
def analyze(signal):
|
|
53
|
-
"""Classify a failure signal into a root cause.
|
|
54
|
-
|
|
55
|
-
signal is a dict that may contain: message, httpStatus, retries, finalStatus,
|
|
56
|
-
evidence (a list of evidence refs). Returns a dict:
|
|
57
|
-
{classification, confidence, reason, ownership, recommendation, evidence}.
|
|
58
|
-
"""
|
|
59
|
-
message = signal.get("message", "")
|
|
60
|
-
http_status = signal.get("httpStatus")
|
|
61
|
-
retries = signal.get("retries", 0) or 0
|
|
62
|
-
final_status = signal.get("finalStatus")
|
|
63
|
-
|
|
64
|
-
provided = signal.get("classification")
|
|
65
|
-
|
|
66
|
-
# Flakiness is a metadata signal, not a message pattern: a test that needed a
|
|
67
|
-
# retry to pass, or is explicitly flagged flaky, is nondeterministic.
|
|
68
|
-
if final_status == "flaky" or (retries > 0 and final_status in ("passed", "flaky")):
|
|
69
|
-
classification, confidence, reason = (
|
|
70
|
-
taxonomy.FLAKY, 0.8,
|
|
71
|
-
"The test passed only after a retry, indicating nondeterministic behavior.",
|
|
72
|
-
)
|
|
73
|
-
elif provided in taxonomy.CLASSES:
|
|
74
|
-
# The analysis platform already classified this deterministically; trust it
|
|
75
|
-
# rather than re-deriving from the message.
|
|
76
|
-
classification = provided
|
|
77
|
-
confidence = signal.get("confidence", 0.8)
|
|
78
|
-
reason = signal.get("reason") or f"Classified {provided} by the analysis platform."
|
|
79
|
-
else:
|
|
80
|
-
classification, confidence, reason = taxonomy.classify(message, http_status=http_status)
|
|
81
|
-
|
|
82
|
-
return {
|
|
83
|
-
"classification": classification,
|
|
84
|
-
"confidence": confidence,
|
|
85
|
-
"reason": reason,
|
|
86
|
-
"ownership": OWNERSHIP.get(classification, "needs-triage"),
|
|
87
|
-
"recommendation": RECOMMENDATION.get(classification, RECOMMENDATION[taxonomy.UNKNOWN]),
|
|
88
|
-
"evidence": signal.get("evidence", []),
|
|
89
|
-
}
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
"""Deterministic timeline reconstruction.
|
|
2
|
-
|
|
3
|
-
Reconstructs the ordered sequence of a run — start, browser launch, navigation,
|
|
4
|
-
requests and responses, console errors, assertions, failure, cleanup — from the
|
|
5
|
-
execution result and the analysis findings. It records only stages for which
|
|
6
|
-
there is evidence; it never invents an event to fill the shape. Reusable by
|
|
7
|
-
qa-debug and qa-report.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
# Canonical phase order, used to sort events that share (or lack) a timestamp.
|
|
11
|
-
PHASE_ORDER = [
|
|
12
|
-
"execution-start", "browser-launch", "navigation", "request", "response",
|
|
13
|
-
"console-error", "assertion", "failure", "cleanup", "execution-finish",
|
|
14
|
-
]
|
|
15
|
-
_PHASE_INDEX = {phase: i for i, phase in enumerate(PHASE_ORDER)}
|
|
16
|
-
|
|
17
|
-
# Evidence type -> the timeline phase it contributes.
|
|
18
|
-
_EVIDENCE_PHASE = {
|
|
19
|
-
"network": "response",
|
|
20
|
-
"console": "console-error",
|
|
21
|
-
"trace": "navigation",
|
|
22
|
-
"junit": "assertion",
|
|
23
|
-
"report": "assertion",
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def build_timeline(execution_result, findings=None):
|
|
28
|
-
"""Build an ordered timeline. Returns a list of event dicts:
|
|
29
|
-
{order, phase, detail, source, timestamp?}. Deterministic: same inputs,
|
|
30
|
-
same timeline. Only evidenced stages appear."""
|
|
31
|
-
events = []
|
|
32
|
-
execution = (execution_result or {}).get("execution", {})
|
|
33
|
-
|
|
34
|
-
started = execution.get("startedAt")
|
|
35
|
-
if started or execution_result:
|
|
36
|
-
events.append(_event("execution-start", "Run started", "execution-result", started))
|
|
37
|
-
|
|
38
|
-
# Contribute a stage per evidence entry we actually have.
|
|
39
|
-
for finding in findings or []:
|
|
40
|
-
for ev in finding.get("evidence", []):
|
|
41
|
-
phase = _EVIDENCE_PHASE.get(ev.get("type"))
|
|
42
|
-
if phase:
|
|
43
|
-
events.append(_event(phase, ev.get("description", ""), ev.get("source", "analysis"), None))
|
|
44
|
-
|
|
45
|
-
# A failure stage per failed test recorded in the result.
|
|
46
|
-
for test in (execution_result or {}).get("executed", []):
|
|
47
|
-
if test.get("status") == "failed":
|
|
48
|
-
events.append(_event("failure", f"Test failed: {test.get('title', '')}",
|
|
49
|
-
test.get("file", "execution-result"), None))
|
|
50
|
-
|
|
51
|
-
finished = execution.get("finishedAt")
|
|
52
|
-
if finished:
|
|
53
|
-
events.append(_event("cleanup", "Run cleaned up", "execution-result", None))
|
|
54
|
-
events.append(_event("execution-finish", "Run finished", "execution-result", finished))
|
|
55
|
-
|
|
56
|
-
# Deterministic order: by canonical phase first (it encodes the logical
|
|
57
|
-
# sequence), then by timestamp within a phase, then by insertion order. Phase
|
|
58
|
-
# order is primary because per-event wall-clock times are often unavailable.
|
|
59
|
-
for i, event in enumerate(events):
|
|
60
|
-
event["_seq"] = i
|
|
61
|
-
events.sort(key=lambda e: (_PHASE_INDEX.get(e["phase"], 99), e["timestamp"] or "", e["_seq"]))
|
|
62
|
-
for order, event in enumerate(events):
|
|
63
|
-
event["order"] = order
|
|
64
|
-
del event["_seq"]
|
|
65
|
-
if event["timestamp"] is None:
|
|
66
|
-
del event["timestamp"]
|
|
67
|
-
return events
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def _event(phase, detail, source, timestamp):
|
|
71
|
-
return {"phase": phase, "detail": detail, "source": source, "timestamp": timestamp}
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
"""Cypress analysis adapter.
|
|
2
|
-
|
|
3
|
-
Cypress emits JUnit XML (via a JUnit reporter), so Cypress normalization is the
|
|
4
|
-
framework-agnostic JUnit parser — the adapter only points at the artifact and
|
|
5
|
-
tags provenance. Like the Selenium adapter, its thinness is the proof of the
|
|
6
|
-
multi-framework boundary: the contract and parsing are shared; only the artifact
|
|
7
|
-
location differs.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from qa_analysis import junit, taxonomy
|
|
11
|
-
|
|
12
|
-
FRAMEWORK = "cypress"
|
|
13
|
-
|
|
14
|
-
# Where Cypress projects conventionally write JUnit results (see cypress-artifacts.md).
|
|
15
|
-
RESULT_GLOBS = ["**/results/*.xml", "**/cypress/results/*.xml", "**/junit*.xml"]
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
def normalize(junit_path):
|
|
19
|
-
"""Normalize a Cypress run into the shared result shape via the agnostic
|
|
20
|
-
JUnit parser. Identical output shape to any other framework."""
|
|
21
|
-
return junit.parse_junit(junit_path)
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
def classify_failure(message, http_status=None):
|
|
25
|
-
"""Classify a Cypress failure using the shared taxonomy — no Cypress-specific
|
|
26
|
-
classifier, because failure classes are framework-agnostic."""
|
|
27
|
-
return taxonomy.classify(message, http_status=http_status)
|
|
@@ -1,167 +0,0 @@
|
|
|
1
|
-
"""Playwright-specific analyzers: trace and JSON report.
|
|
2
|
-
|
|
3
|
-
These are the Playwright *adapter* for the analysis platform. They depend on
|
|
4
|
-
the framework-agnostic qa_analysis core for the evidence model, redaction, and
|
|
5
|
-
the failure taxonomy; they add only what is Playwright-specific — the shape of
|
|
6
|
-
a Playwright trace.zip and of Playwright's JSON reporter. Everything they emit
|
|
7
|
-
is in the shared normalized shape, so nothing downstream knows it was Playwright.
|
|
8
|
-
|
|
9
|
-
The adapter carries its own CLI (`python -m playwright_analysis report|trace
|
|
10
|
-
<path>`) so a skill can reach it the same way it reaches every other
|
|
11
|
-
deterministic tool. Framework knowledge stays inside the adapter: the core CLI
|
|
12
|
-
never grows a `--framework` flag ([ADR-0013](../../../../docs/architecture/ADR-0013-framework-boundary.md)).
|
|
13
|
-
|
|
14
|
-
Usage:
|
|
15
|
-
python -m playwright_analysis report <results.json>
|
|
16
|
-
python -m playwright_analysis trace <trace.zip>
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
import argparse
|
|
20
|
-
import json
|
|
21
|
-
import sys
|
|
22
|
-
import zipfile
|
|
23
|
-
|
|
24
|
-
from qa_analysis.junit import MalformedArtifact
|
|
25
|
-
from qa_analysis.redaction import redact_text
|
|
26
|
-
from qa_analysis import taxonomy
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
def parse_report(path):
|
|
30
|
-
"""Parse Playwright's JSON reporter output into the normalized result shape.
|
|
31
|
-
|
|
32
|
-
Mirrors qa_analysis.junit's output ({tests, executed}) so a Playwright run
|
|
33
|
-
normalizes identically whether it emitted JSON or JUnit. Raises
|
|
34
|
-
MalformedArtifact on unreadable input.
|
|
35
|
-
"""
|
|
36
|
-
try:
|
|
37
|
-
with open(path, "r", encoding="utf-8") as handle:
|
|
38
|
-
data = json.load(handle)
|
|
39
|
-
except (json.JSONDecodeError, OSError, UnicodeDecodeError) as exc:
|
|
40
|
-
raise MalformedArtifact(f"could not parse Playwright report at {path}: {exc}") from exc
|
|
41
|
-
|
|
42
|
-
executed = []
|
|
43
|
-
for suite in data.get("suites", []):
|
|
44
|
-
_walk_suite(suite, executed)
|
|
45
|
-
|
|
46
|
-
counts = {
|
|
47
|
-
"total": len(executed),
|
|
48
|
-
"passed": sum(1 for e in executed if e["status"] == "passed"),
|
|
49
|
-
"failed": sum(1 for e in executed if e["status"] == "failed"),
|
|
50
|
-
"skipped": sum(1 for e in executed if e["status"] == "skipped"),
|
|
51
|
-
}
|
|
52
|
-
return {"tests": counts, "executed": executed}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def _walk_suite(suite, executed):
|
|
56
|
-
for spec in suite.get("specs", []):
|
|
57
|
-
for test in spec.get("tests", []):
|
|
58
|
-
results = test.get("results", [])
|
|
59
|
-
status = _status_of(results)
|
|
60
|
-
message = ""
|
|
61
|
-
if status == "failed" and results:
|
|
62
|
-
errors = results[-1].get("errors") or []
|
|
63
|
-
if errors:
|
|
64
|
-
message = redact_text((errors[0].get("message") or "").strip())
|
|
65
|
-
entry = {
|
|
66
|
-
"title": spec.get("title", ""),
|
|
67
|
-
"file": suite.get("file", ""),
|
|
68
|
-
"status": status,
|
|
69
|
-
"durationMs": results[-1].get("duration", 0) if results else 0,
|
|
70
|
-
"retries": max(0, len(results) - 1),
|
|
71
|
-
}
|
|
72
|
-
if message:
|
|
73
|
-
entry["message"] = message
|
|
74
|
-
executed.append(entry)
|
|
75
|
-
for child in suite.get("suites", []):
|
|
76
|
-
_walk_suite(child, executed)
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
def _status_of(results):
|
|
80
|
-
if not results:
|
|
81
|
-
return "skipped"
|
|
82
|
-
final = results[-1].get("status")
|
|
83
|
-
if final in ("passed", "expected"):
|
|
84
|
-
return "flaky" if len(results) > 1 else "passed"
|
|
85
|
-
if final in ("skipped",):
|
|
86
|
-
return "skipped"
|
|
87
|
-
return "failed"
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
def analyze_trace(path):
|
|
91
|
-
"""Extract a deterministic summary from a Playwright trace.zip.
|
|
92
|
-
|
|
93
|
-
A trace is a zip of newline-delimited JSON event files. This lists the
|
|
94
|
-
actions, surfaces the last error, and counts console and network events —
|
|
95
|
-
enough for a diagnostic skill to reason over, without decoding the full
|
|
96
|
-
binary. Raises MalformedArtifact if the file is not a valid trace zip.
|
|
97
|
-
"""
|
|
98
|
-
if not zipfile.is_zipfile(path):
|
|
99
|
-
raise MalformedArtifact(f"not a valid trace zip: {path}")
|
|
100
|
-
|
|
101
|
-
actions, console, network, errors = [], 0, 0, []
|
|
102
|
-
try:
|
|
103
|
-
with zipfile.ZipFile(path) as archive:
|
|
104
|
-
names = [n for n in archive.namelist() if n.endswith(".trace") or n.endswith(".jsonl") or "trace" in n]
|
|
105
|
-
for name in names or archive.namelist():
|
|
106
|
-
with archive.open(name) as member:
|
|
107
|
-
for raw in member.read().decode("utf-8", "replace").splitlines():
|
|
108
|
-
raw = raw.strip()
|
|
109
|
-
if not raw or not raw.startswith("{"):
|
|
110
|
-
continue
|
|
111
|
-
try:
|
|
112
|
-
event = json.loads(raw)
|
|
113
|
-
except json.JSONDecodeError:
|
|
114
|
-
continue
|
|
115
|
-
kind = event.get("type") or event.get("kind")
|
|
116
|
-
if kind in ("action", "before"):
|
|
117
|
-
actions.append(redact_text(str(event.get("apiName") or event.get("method") or "")))
|
|
118
|
-
elif kind == "console":
|
|
119
|
-
console += 1
|
|
120
|
-
elif kind in ("resource", "network", "http"):
|
|
121
|
-
network += 1
|
|
122
|
-
if event.get("error") or kind == "error":
|
|
123
|
-
msg = event.get("error", {}).get("message") if isinstance(event.get("error"), dict) else event.get("message")
|
|
124
|
-
if msg:
|
|
125
|
-
errors.append(redact_text(str(msg)))
|
|
126
|
-
except (OSError, KeyError) as exc:
|
|
127
|
-
raise MalformedArtifact(f"could not read trace {path}: {exc}") from exc
|
|
128
|
-
|
|
129
|
-
last_error = errors[-1] if errors else ""
|
|
130
|
-
classification, confidence, reason = taxonomy.classify(last_error) if last_error else (taxonomy.UNKNOWN, 0.2, "No error found in trace.")
|
|
131
|
-
return {
|
|
132
|
-
"actions": [a for a in actions if a],
|
|
133
|
-
"consoleEvents": console,
|
|
134
|
-
"networkEvents": network,
|
|
135
|
-
"errors": errors,
|
|
136
|
-
"classification": classification,
|
|
137
|
-
"confidence": confidence,
|
|
138
|
-
"reason": reason,
|
|
139
|
-
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
def main(argv=None):
|
|
143
|
-
"""CLI: JSON to stdout, exit 0 on success, 2 on a malformed artifact."""
|
|
144
|
-
parser = argparse.ArgumentParser(
|
|
145
|
-
prog="playwright_analysis",
|
|
146
|
-
description="Playwright adapter: normalize a JSON report or summarize a trace",
|
|
147
|
-
)
|
|
148
|
-
sub = parser.add_subparsers(dest="command", required=True)
|
|
149
|
-
p = sub.add_parser("report", help="normalize Playwright's JSON reporter output")
|
|
150
|
-
p.add_argument("path")
|
|
151
|
-
p = sub.add_parser("trace", help="summarize a Playwright trace.zip")
|
|
152
|
-
p.add_argument("path")
|
|
153
|
-
args = parser.parse_args(argv)
|
|
154
|
-
|
|
155
|
-
try:
|
|
156
|
-
result = parse_report(args.path) if args.command == "report" else analyze_trace(args.path)
|
|
157
|
-
except MalformedArtifact as exc:
|
|
158
|
-
json.dump({"error": "malformed-artifact", "detail": str(exc)}, sys.stdout, indent=2, sort_keys=True)
|
|
159
|
-
sys.stdout.write("\n")
|
|
160
|
-
return 2
|
|
161
|
-
json.dump(result, sys.stdout, indent=2, sort_keys=True)
|
|
162
|
-
sys.stdout.write("\n")
|
|
163
|
-
return 0
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
if __name__ == "__main__":
|
|
167
|
-
sys.exit(main())
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
"""Selenium analysis adapter.
|
|
2
|
-
|
|
3
|
-
Selenium's runners (JUnit, TestNG, pytest) emit JUnit XML, so Selenium
|
|
4
|
-
normalization *is* the framework-agnostic JUnit parser — the adapter only points
|
|
5
|
-
at the right artifact and tags provenance. This is the proof of the pack's
|
|
6
|
-
multi-framework claim: the second framework's analysis is a few lines, because
|
|
7
|
-
the contract and the parsing are shared, and only the artifact location differs.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
from qa_analysis import junit, taxonomy
|
|
11
|
-
|
|
12
|
-
FRAMEWORK = "selenium"
|
|
13
|
-
|
|
14
|
-
# Where Selenium projects conventionally write results (documented in
|
|
15
|
-
# selenium-artifacts.md); discovery uses these, the parser is shared.
|
|
16
|
-
RESULT_GLOBS = ["**/target/surefire-reports/*.xml", "**/test-results/*.xml", "**/junit*.xml"]
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
def normalize(junit_path):
|
|
20
|
-
"""Normalize a Selenium run into the shared result shape via the agnostic
|
|
21
|
-
JUnit parser. Identical output shape to any other framework's normalization."""
|
|
22
|
-
return junit.parse_junit(junit_path)
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
def classify_failure(message, http_status=None):
|
|
26
|
-
"""Classify a Selenium failure using the shared taxonomy — no Selenium-specific
|
|
27
|
-
classifier, because failure classes are framework-agnostic."""
|
|
28
|
-
return taxonomy.classify(message, http_status=http_status)
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
"""WebdriverIO analysis adapter.
|
|
2
|
-
|
|
3
|
-
WebdriverIO emits JUnit XML (via @wdio/junit-reporter), so its normalization is
|
|
4
|
-
the framework-agnostic JUnit parser — the adapter points at the artifact and
|
|
5
|
-
tags provenance. Thin by design: the fourth framework confirms the boundary.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
from qa_analysis import junit, taxonomy
|
|
9
|
-
|
|
10
|
-
FRAMEWORK = "webdriverio"
|
|
11
|
-
|
|
12
|
-
# Where WebdriverIO projects conventionally write JUnit results.
|
|
13
|
-
RESULT_GLOBS = ["**/junit*.xml", "**/results/*.xml", "**/test-results/*.xml"]
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def normalize(junit_path):
|
|
17
|
-
"""Normalize a WebdriverIO run into the shared result shape via the agnostic
|
|
18
|
-
JUnit parser. Identical output shape to any other framework."""
|
|
19
|
-
return junit.parse_junit(junit_path)
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def classify_failure(message, http_status=None):
|
|
23
|
-
"""Classify a WebdriverIO failure using the shared taxonomy."""
|
|
24
|
-
return taxonomy.classify(message, http_status=http_status)
|