qa-engineer 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/COMPATIBILITY.md +50 -26
- package/README.md +40 -32
- package/package.json +12 -20
- package/packages/engine/bin/qa-engine.mjs +336 -0
- package/packages/engine/lib/analysis/branding.mjs +187 -0
- package/packages/engine/lib/analysis/context.mjs +294 -0
- package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
- package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
- package/packages/engine/lib/analysis/discovery.mjs +220 -0
- package/packages/engine/lib/analysis/evidence.mjs +116 -0
- package/packages/engine/lib/analysis/har.mjs +90 -0
- package/packages/engine/lib/analysis/junit.mjs +126 -0
- package/packages/engine/lib/analysis/redaction.mjs +127 -0
- package/packages/engine/lib/analysis/report-html.mjs +763 -0
- package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
- package/packages/engine/lib/analysis/xml.mjs +153 -0
- package/packages/engine/lib/analysis/zip.mjs +107 -0
- package/packages/engine/lib/diagnostics/engine.mjs +165 -0
- package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
- package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
- package/packages/engine/lib/diagnostics/repair.mjs +73 -0
- package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
- package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
- package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
- package/packages/engine/lib/frameworks/playwright.mjs +158 -0
- package/packages/engine/package.json +14 -0
- package/packages/installer/lib/agents/registry.mjs +46 -0
- package/packages/installer/lib/commands/doctor.mjs +7 -16
- package/packages/installer/lib/commands/install.mjs +59 -4
- package/packages/installer/lib/core/bundle.mjs +69 -92
- package/packages/installer/lib/core/config.mjs +1 -1
- package/packages/installer/lib/core/lockfile.mjs +1 -1
- package/packages/installer/lib/core/manifest.mjs +48 -54
- package/packages/installer/lib/core/validate-install.mjs +12 -19
- package/packages/installer/package.json +1 -1
- package/shared/tooling/qa-tool.mjs +125 -0
- package/skills/qa-api/SKILL.md +5 -5
- package/skills/qa-api/references/deterministic-tooling.md +34 -30
- package/skills/qa-api/references/evidence-and-reporting.md +29 -4
- package/skills/qa-api/scripts/qa-tool.mjs +126 -0
- package/skills/qa-audit/SKILL.md +5 -5
- package/skills/qa-audit/references/deterministic-tooling.md +34 -30
- package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
- package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
- package/skills/qa-debug/SKILL.md +6 -6
- package/skills/qa-debug/references/deterministic-tooling.md +34 -30
- package/skills/qa-debug/references/diagnostic-engine.md +3 -3
- package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
- package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
- package/skills/qa-explore/SKILL.md +22 -5
- package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
- package/skills/qa-explore/references/deterministic-tooling.md +132 -0
- package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
- package/skills/qa-explore/references/report-pipeline.md +89 -8
- package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
- package/skills/qa-fix/SKILL.md +4 -4
- package/skills/qa-fix/references/deterministic-tooling.md +34 -30
- package/skills/qa-fix/references/diagnostic-engine.md +3 -3
- package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
- package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
- package/skills/qa-flaky/SKILL.md +4 -4
- package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
- package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
- package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
- package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
- package/skills/qa-init/SKILL.md +4 -4
- package/skills/qa-init/references/deterministic-tooling.md +34 -30
- package/skills/qa-init/references/evidence-and-reporting.md +29 -4
- package/skills/qa-init/scripts/qa-tool.mjs +126 -0
- package/skills/qa-report/SKILL.md +8 -7
- package/skills/qa-report/references/deterministic-tooling.md +34 -30
- package/skills/qa-report/references/diagnostic-engine.md +3 -3
- package/skills/qa-report/references/evidence-and-reporting.md +29 -4
- package/skills/qa-report/scripts/qa-tool.mjs +126 -0
- package/skills/qa-review/references/evidence-and-reporting.md +29 -4
- package/skills/qa-run/SKILL.md +6 -6
- package/skills/qa-run/references/deterministic-tooling.md +34 -30
- package/skills/qa-run/references/evidence-and-reporting.md +29 -4
- package/skills/qa-run/scripts/qa-tool.mjs +126 -0
- package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
- package/shared/analysis/lib/qa_analysis/branding.py +0 -175
- package/shared/analysis/lib/qa_analysis/cli.py +0 -129
- package/shared/analysis/lib/qa_analysis/context.py +0 -233
- package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
- package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
- package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
- package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
- package/shared/analysis/lib/qa_analysis/har.py +0 -58
- package/shared/analysis/lib/qa_analysis/junit.py +0 -80
- package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
- package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
- package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
- package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
- package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
- package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
- package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
- package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
- package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
- package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
- package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
- package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
- package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
- package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
- package/shared/tooling/qa_tool.py +0 -127
- /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
- /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
- /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
|
@@ -1,80 +0,0 @@
|
|
|
1
|
-
"""JUnit XML parser.
|
|
2
|
-
|
|
3
|
-
Framework-agnostic: Playwright, Selenium, Cypress, WebdriverIO, and most unit
|
|
4
|
-
runners emit JUnit XML, so this one parser normalizes them all into the pack's
|
|
5
|
-
per-test result shape. This is the concrete proof that different frameworks
|
|
6
|
-
share one contract — only where the file lives differs, not how it is read.
|
|
7
|
-
|
|
8
|
-
Parses deterministically; a malformed document raises rather than guessing.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
import xml.etree.ElementTree as ET
|
|
12
|
-
|
|
13
|
-
from .redaction import redact_text
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class MalformedArtifact(ValueError):
|
|
17
|
-
"""Raised when an artifact cannot be parsed. Never swallowed into a guess."""
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
def _int(value, default=0):
|
|
21
|
-
try:
|
|
22
|
-
return int(value)
|
|
23
|
-
except (TypeError, ValueError):
|
|
24
|
-
return default
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def parse_junit(path):
|
|
28
|
-
"""Parse a JUnit XML file into a normalized result.
|
|
29
|
-
|
|
30
|
-
Returns a dict: {tests: {total, passed, failed, skipped}, executed: [...]}.
|
|
31
|
-
Each executed entry: {title, file, status, durationMs, message?}.
|
|
32
|
-
Raises MalformedArtifact if the XML is unreadable or not JUnit-shaped.
|
|
33
|
-
"""
|
|
34
|
-
try:
|
|
35
|
-
tree = ET.parse(path)
|
|
36
|
-
except (ET.ParseError, OSError) as exc:
|
|
37
|
-
raise MalformedArtifact(f"could not parse JUnit XML at {path}: {exc}") from exc
|
|
38
|
-
|
|
39
|
-
root = tree.getroot()
|
|
40
|
-
# Accept either a <testsuites> root or a single <testsuite> root.
|
|
41
|
-
if root.tag == "testsuites":
|
|
42
|
-
suites = root.findall("testsuite")
|
|
43
|
-
elif root.tag == "testsuite":
|
|
44
|
-
suites = [root]
|
|
45
|
-
else:
|
|
46
|
-
raise MalformedArtifact(f"not a JUnit document (root <{root.tag}>) at {path}")
|
|
47
|
-
|
|
48
|
-
executed = []
|
|
49
|
-
for suite in suites:
|
|
50
|
-
for case in suite.findall("testcase"):
|
|
51
|
-
failure = case.find("failure")
|
|
52
|
-
error = case.find("error")
|
|
53
|
-
skipped = case.find("skipped")
|
|
54
|
-
if failure is not None or error is not None:
|
|
55
|
-
status = "failed"
|
|
56
|
-
node = failure if failure is not None else error
|
|
57
|
-
message = redact_text((node.get("message") or node.text or "").strip())
|
|
58
|
-
elif skipped is not None:
|
|
59
|
-
status = "skipped"
|
|
60
|
-
message = redact_text((skipped.get("message") or "").strip())
|
|
61
|
-
else:
|
|
62
|
-
status = "passed"
|
|
63
|
-
message = ""
|
|
64
|
-
entry = {
|
|
65
|
-
"title": case.get("name", ""),
|
|
66
|
-
"file": case.get("classname", ""),
|
|
67
|
-
"status": status,
|
|
68
|
-
"durationMs": int(round(float(case.get("time", "0") or "0") * 1000)),
|
|
69
|
-
}
|
|
70
|
-
if message:
|
|
71
|
-
entry["message"] = message
|
|
72
|
-
executed.append(entry)
|
|
73
|
-
|
|
74
|
-
counts = {
|
|
75
|
-
"total": len(executed),
|
|
76
|
-
"passed": sum(1 for e in executed if e["status"] == "passed"),
|
|
77
|
-
"failed": sum(1 for e in executed if e["status"] == "failed"),
|
|
78
|
-
"skipped": sum(1 for e in executed if e["status"] == "skipped"),
|
|
79
|
-
}
|
|
80
|
-
return {"tests": counts, "executed": executed}
|
|
@@ -1,99 +0,0 @@
|
|
|
1
|
-
"""Credential, secret, token, and PII redaction.
|
|
2
|
-
|
|
3
|
-
Every analyzer runs artifact text through redaction before it appears in a
|
|
4
|
-
finding, a report, or stdout. Redaction happens as evidence is captured, not
|
|
5
|
-
after — a secret must never reach a model's context or a log. Standard library
|
|
6
|
-
only; deterministic.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
import re
|
|
10
|
-
|
|
11
|
-
# Ordered (name, pattern, replacement). Order matters: match structured,
|
|
12
|
-
# high-signal secrets (JWTs, provider keys) before generic fallbacks.
|
|
13
|
-
_RULES = [
|
|
14
|
-
("jwt", re.compile(r"eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}")),
|
|
15
|
-
("aws-access-key", re.compile(r"AKIA[0-9A-Z]{16}")),
|
|
16
|
-
("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}")),
|
|
17
|
-
("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")),
|
|
18
|
-
("openai-key", re.compile(r"sk-[A-Za-z0-9]{20,}")),
|
|
19
|
-
("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]+=*")),
|
|
20
|
-
# Sensitive header lines: keep the header name, mask the value.
|
|
21
|
-
("auth-header", re.compile(r"(?im)^(\s*(?:authorization|proxy-authorization)\s*[:=]\s*).+$")),
|
|
22
|
-
("cookie-header", re.compile(r"(?im)^(\s*(?:set-cookie|cookie)\s*[:=]\s*).+$")),
|
|
23
|
-
# Secret-like assignments: key=value / "key": "value".
|
|
24
|
-
("assigned-secret", re.compile(
|
|
25
|
-
r'(?i)(\b(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|client[_-]?secret)\b\s*[:=]\s*["\']?)'
|
|
26
|
-
r'([^\s"\'&]{4,})')),
|
|
27
|
-
# Credentials in URLs and query strings.
|
|
28
|
-
("url-credential", re.compile(r"(?i)(://[^:/@\s]+:)([^@/\s]+)(@)")),
|
|
29
|
-
("query-secret", re.compile(
|
|
30
|
-
r"(?i)([?&](?:password|passwd|pwd|token|secret|api[_-]?key|access[_-]?key)=)([^&\s#]+)")),
|
|
31
|
-
# PII: email addresses.
|
|
32
|
-
("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")),
|
|
33
|
-
]
|
|
34
|
-
|
|
35
|
-
# Rules whose replacement preserves a leading captured group (the label/prefix).
|
|
36
|
-
_PREFIX_PRESERVING = {
|
|
37
|
-
"auth-header": r"\1[REDACTED:auth-header]",
|
|
38
|
-
"cookie-header": r"\1[REDACTED:cookie-header]",
|
|
39
|
-
"assigned-secret": r"\1[REDACTED:secret]",
|
|
40
|
-
"url-credential": r"\1[REDACTED:credential]\3",
|
|
41
|
-
"query-secret": r"\1[REDACTED:secret]",
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
def redact_text(text):
|
|
46
|
-
"""Return text with every recognized secret or PII value masked."""
|
|
47
|
-
if not text:
|
|
48
|
-
return text
|
|
49
|
-
result = text
|
|
50
|
-
for name, pattern in _RULES:
|
|
51
|
-
if name in _PREFIX_PRESERVING:
|
|
52
|
-
result = pattern.sub(_PREFIX_PRESERVING[name], result)
|
|
53
|
-
else:
|
|
54
|
-
result = pattern.sub(f"[REDACTED:{name}]", result)
|
|
55
|
-
return result
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def detect_secrets(text):
|
|
59
|
-
"""Return a list of {type, start, end} for secrets found — never the values.
|
|
60
|
-
|
|
61
|
-
Used to decide whether an artifact is safe to expose, without surfacing the
|
|
62
|
-
secret itself.
|
|
63
|
-
"""
|
|
64
|
-
findings = []
|
|
65
|
-
if not text:
|
|
66
|
-
return findings
|
|
67
|
-
for name, pattern in _RULES:
|
|
68
|
-
for match in pattern.finditer(text):
|
|
69
|
-
findings.append({"type": name, "start": match.start(), "end": match.end()})
|
|
70
|
-
findings.sort(key=lambda f: f["start"])
|
|
71
|
-
return findings
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
_SENSITIVE_HEADERS = {
|
|
75
|
-
"authorization", "proxy-authorization", "cookie", "set-cookie",
|
|
76
|
-
"x-api-key", "x-auth-token", "api-key", "x-csrf-token",
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
def redact_headers(headers):
|
|
81
|
-
"""Mask the values of sensitive HTTP headers.
|
|
82
|
-
|
|
83
|
-
Accepts a dict, or a list of {"name","value"} entries (HAR shape), and
|
|
84
|
-
returns the same shape with sensitive values replaced.
|
|
85
|
-
"""
|
|
86
|
-
def mask(name, value):
|
|
87
|
-
return "[REDACTED:header]" if name.strip().lower() in _SENSITIVE_HEADERS else redact_text(value)
|
|
88
|
-
|
|
89
|
-
if isinstance(headers, dict):
|
|
90
|
-
return {k: mask(k, str(v)) for k, v in headers.items()}
|
|
91
|
-
if isinstance(headers, list):
|
|
92
|
-
out = []
|
|
93
|
-
for entry in headers:
|
|
94
|
-
if isinstance(entry, dict) and "name" in entry:
|
|
95
|
-
out.append({**entry, "value": mask(str(entry["name"]), str(entry.get("value", "")))})
|
|
96
|
-
else:
|
|
97
|
-
out.append(entry)
|
|
98
|
-
return out
|
|
99
|
-
return headers
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
"""The canonical QA failure taxonomy and its deterministic classifier.
|
|
2
|
-
|
|
3
|
-
Classification maps observed signals to one of a closed set of failure classes.
|
|
4
|
-
It is rule-based and conservative: when no rule matches with sufficient signal,
|
|
5
|
-
the result is UNKNOWN. Unknown is preferable to incorrect.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
import re
|
|
9
|
-
|
|
10
|
-
# The closed set of failure classifications. Every value implies a different
|
|
11
|
-
# owner and next action; if two would lead to the same action, they are merged.
|
|
12
|
-
ASSERTION = "assertion-failure"
|
|
13
|
-
LOCATOR = "locator-failure"
|
|
14
|
-
TIMEOUT = "timeout"
|
|
15
|
-
NETWORK = "network"
|
|
16
|
-
AUTH = "authentication"
|
|
17
|
-
AUTHORIZATION = "authorization"
|
|
18
|
-
ENVIRONMENT = "environment"
|
|
19
|
-
CONFIGURATION = "configuration"
|
|
20
|
-
INFRASTRUCTURE = "infrastructure"
|
|
21
|
-
TEST_DATA = "test-data"
|
|
22
|
-
APPLICATION_BUG = "application-bug"
|
|
23
|
-
FRAMEWORK = "framework-failure"
|
|
24
|
-
FLAKY = "flaky"
|
|
25
|
-
UNKNOWN = "unknown"
|
|
26
|
-
|
|
27
|
-
CLASSES = {
|
|
28
|
-
ASSERTION, LOCATOR, TIMEOUT, NETWORK, AUTH, AUTHORIZATION, ENVIRONMENT,
|
|
29
|
-
CONFIGURATION, INFRASTRUCTURE, TEST_DATA, APPLICATION_BUG, FRAMEWORK,
|
|
30
|
-
FLAKY, UNKNOWN,
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
# Ordered rules: (classification, message-pattern, confidence, reason).
|
|
34
|
-
#
|
|
35
|
-
# Order encodes priority, and the first two rules exist because modern runners
|
|
36
|
-
# print a timeout budget in *every* assertion failure. Playwright's message for a
|
|
37
|
-
# plain text mismatch ends with "Timeout: 5000ms" even though nothing timed out,
|
|
38
|
-
# so a naive timeout rule captures assertion and locator failures alike and sends
|
|
39
|
-
# the reader to raise a timeout — the one action guaranteed not to help. The
|
|
40
|
-
# patterns below are derived from real captured runner output; see
|
|
41
|
-
# tests/test_analysis.py::RealRunnerMessageTests, which pins the exact strings.
|
|
42
|
-
#
|
|
43
|
-
# The discriminators, from that output:
|
|
44
|
-
# element missing -> "Error: element(s) not found" (no Received: value)
|
|
45
|
-
# value mismatch -> "Expected: X" + "Received: Y" (element was resolved)
|
|
46
|
-
# real timeout -> a timeout with neither of the above
|
|
47
|
-
_RULES = [
|
|
48
|
-
(LOCATOR, re.compile(r"(?i)(no such element|element(?:\(s\))?\s+not\s+(?:found|visible|attached)|locator.*(?:resolved to 0|not found)|waiting for (?:locator|selector)|unable to locate element)"), 0.8,
|
|
49
|
-
"Error indicates the target element could not be found or resolved."),
|
|
50
|
-
# A concrete expected-vs-received comparison is an assertion result, not a
|
|
51
|
-
# time budget: the runner resolved the target and compared values.
|
|
52
|
-
(ASSERTION, re.compile(r"(?is)expected:.*received:"), 0.8,
|
|
53
|
-
"Error shows a concrete expected-versus-received comparison, so the assertion did not hold."),
|
|
54
|
-
(TIMEOUT, re.compile(r"(?i)(timeout|timed out|exceeded .*ms|deadline exceeded)"), 0.75,
|
|
55
|
-
"Error indicates an operation exceeded its time budget."),
|
|
56
|
-
(AUTH, re.compile(r"(?i)(401 unauthorized|authentication failed|invalid (?:credentials|token)|login failed|not authenticated)"), 0.85,
|
|
57
|
-
"Error indicates an authentication failure — the identity could not be established."),
|
|
58
|
-
(AUTHORIZATION, re.compile(r"(?i)(403 forbidden|permission denied|access denied|not authorized|forbidden\b|insufficient (?:permission|privilege))"), 0.85,
|
|
59
|
-
"Error indicates an authorization failure — the identity lacked permission."),
|
|
60
|
-
(NETWORK, re.compile(r"(?i)(ECONNREFUSED| ENOTFOUND|net::ERR|connection (?:refused|reset)|5\d\d (?:internal server error|bad gateway|service unavailable)|fetch failed)"), 0.8,
|
|
61
|
-
"Error indicates a network or upstream service failure."),
|
|
62
|
-
(ASSERTION, re.compile(r"(?i)(assertion|expect(?:ed)?\b|toBe|toEqual|toHaveText|toBeVisible|AssertionError|expected .* (?:to|but))"), 0.7,
|
|
63
|
-
"Error indicates an assertion did not hold."),
|
|
64
|
-
(TEST_DATA, re.compile(r"(?i)(duplicate key|constraint violation|no rows|seed data|fixture .*(?:missing|not found)|invalid test data)"), 0.7,
|
|
65
|
-
"Error indicates a test-data problem."),
|
|
66
|
-
(CONFIGURATION, re.compile(r"(?i)(cannot find module|config(?:uration)? (?:error|not found)|missing (?:config|environment variable)|unknown option)"), 0.7,
|
|
67
|
-
"Error indicates a configuration problem."),
|
|
68
|
-
(ENVIRONMENT, re.compile(r"(?i)(base ?url|env(?:ironment)? .*(?:not set|missing)|ECONNREFUSED .*localhost|dev server)"), 0.6,
|
|
69
|
-
"Error indicates an environment problem such as a missing base URL or unreachable local service."),
|
|
70
|
-
(INFRASTRUCTURE, re.compile(r"(?i)(out of memory|OOM|disk (?:full|space)|browser .*(?:crash|closed unexpectedly)|worker .*(?:died|crashed))"), 0.7,
|
|
71
|
-
"Error indicates an infrastructure problem such as a crash or resource exhaustion."),
|
|
72
|
-
(FRAMEWORK, re.compile(r"(?i)(internal error|unexpected error in (?:playwright|selenium|cypress|webdriver)|driver .*mismatch)"), 0.6,
|
|
73
|
-
"Error indicates a fault in the test framework or driver itself."),
|
|
74
|
-
]
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def classify(message, http_status=None):
|
|
78
|
-
"""Classify a failure from its error message and optional HTTP status.
|
|
79
|
-
|
|
80
|
-
Returns (classification, confidence, reason). Never guesses: an
|
|
81
|
-
unrecognized signal yields UNKNOWN at low confidence with an honest reason.
|
|
82
|
-
"""
|
|
83
|
-
text = message or ""
|
|
84
|
-
|
|
85
|
-
# A concrete HTTP status is stronger evidence than message text.
|
|
86
|
-
if http_status is not None:
|
|
87
|
-
if http_status == 401:
|
|
88
|
-
return (AUTH, 0.9, "HTTP 401 indicates an authentication failure.")
|
|
89
|
-
if http_status == 403:
|
|
90
|
-
return (AUTHORIZATION, 0.9, "HTTP 403 indicates an authorization failure.")
|
|
91
|
-
if 500 <= http_status <= 599:
|
|
92
|
-
return (NETWORK, 0.85, f"HTTP {http_status} indicates an upstream server failure.")
|
|
93
|
-
|
|
94
|
-
for classification, pattern, confidence, reason in _RULES:
|
|
95
|
-
if pattern.search(text):
|
|
96
|
-
return (classification, confidence, reason)
|
|
97
|
-
|
|
98
|
-
return (UNKNOWN, 0.2, "No classification rule matched the available signals; manual review needed.")
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
"""qa_diagnostics — the pack's shared diagnostic engine.
|
|
2
|
-
|
|
3
|
-
One place where failure reasoning lives: root-cause analysis, timeline
|
|
4
|
-
reconstruction, finding prioritization, and repair planning. The three
|
|
5
|
-
diagnostic skills (qa-debug, qa-fix, qa-report) consume this engine and differ
|
|
6
|
-
only in presentation — the reasoning is not duplicated across them.
|
|
7
|
-
|
|
8
|
-
Standard library only. Builds on qa_analysis (taxonomy, evidence, diff guard);
|
|
9
|
-
adds no framework-specific logic. Deterministic: the same inputs yield the same
|
|
10
|
-
diagnosis every time.
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
__version__ = "0.1.0"
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
"""Unified CLI for the diagnostic engine.
|
|
2
|
-
|
|
3
|
-
The engine's reasoning is deterministic; this is how a skill reaches it. Every
|
|
4
|
-
subcommand reads JSON files, validates them against the internal seam contracts,
|
|
5
|
-
and writes JSON to stdout — so an agent never has to invent glue code, and the
|
|
6
|
-
input and output shapes are checked rather than assumed.
|
|
7
|
-
|
|
8
|
-
Mirrors `qa_analysis.cli` exactly: JSON to stdout, exit 0 on success, exit 2 on
|
|
9
|
-
bad input or usage. Standard library only.
|
|
10
|
-
|
|
11
|
-
Usage:
|
|
12
|
-
python -m qa_diagnostics.cli diagnose --execution-result <path> [--analysis-result <path>]
|
|
13
|
-
python -m qa_diagnostics.cli plan-repairs --diagnosis <path>
|
|
14
|
-
python -m qa_diagnostics.cli summarize --execution-result <path> --diagnosis <path>
|
|
15
|
-
python -m qa_diagnostics.cli report --execution-result <path> [--analysis-result <path>]
|
|
16
|
-
|
|
17
|
-
`report` is the one-shot path: it diagnoses, plans, and summarizes in a single
|
|
18
|
-
invocation, so the common case is one command instead of three.
|
|
19
|
-
|
|
20
|
-
Inputs
|
|
21
|
-
--execution-result a qa-run execution result (or the minimal subset:
|
|
22
|
-
`tests` counts plus `executed[]` with `status`)
|
|
23
|
-
--analysis-result an analysis result with `findings[]`; preferred over
|
|
24
|
-
`executed[]` when present
|
|
25
|
-
--diagnosis the object emitted by `diagnose`
|
|
26
|
-
|
|
27
|
-
Exit codes
|
|
28
|
-
0 success
|
|
29
|
-
2 unreadable file, malformed JSON, or a payload that fails its seam contract
|
|
30
|
-
"""
|
|
31
|
-
|
|
32
|
-
import argparse
|
|
33
|
-
import json
|
|
34
|
-
import sys
|
|
35
|
-
|
|
36
|
-
from . import engine
|
|
37
|
-
from . import internal_contracts
|
|
38
|
-
from .internal_contracts import InternalContractError
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
def _emit(obj):
|
|
42
|
-
json.dump(obj, sys.stdout, indent=2, sort_keys=True)
|
|
43
|
-
sys.stdout.write("\n")
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def _read(path):
|
|
47
|
-
with open(path, "r", encoding="utf-8") as handle:
|
|
48
|
-
return json.load(handle)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def _read_execution(path):
|
|
52
|
-
"""Read an execution result and hold it to the seam contract."""
|
|
53
|
-
return internal_contracts.validate_execution_result_min(_read(path))
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
def _read_analysis(path):
|
|
57
|
-
"""Read an analysis result and hold it to the seam contract."""
|
|
58
|
-
if path is None:
|
|
59
|
-
return None
|
|
60
|
-
return internal_contracts.validate_analysis_result(_read(path))
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
def _read_diagnosis(path):
|
|
64
|
-
"""Read a diagnosis and hold it to the seam contract."""
|
|
65
|
-
return internal_contracts.validate_diagnosis(_read(path))
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def main(argv=None):
|
|
69
|
-
parser = argparse.ArgumentParser(
|
|
70
|
-
prog="qa-diagnostics",
|
|
71
|
-
description="Deterministic diagnostic engine: root cause, timeline, priority, repairs",
|
|
72
|
-
)
|
|
73
|
-
sub = parser.add_subparsers(dest="command", required=True)
|
|
74
|
-
|
|
75
|
-
p = sub.add_parser("diagnose", help="root causes, timeline, prioritization, recommendations")
|
|
76
|
-
p.add_argument("--execution-result", required=True)
|
|
77
|
-
p.add_argument("--analysis-result")
|
|
78
|
-
|
|
79
|
-
p = sub.add_parser("plan-repairs", help="repair plans for a diagnosis (never code)")
|
|
80
|
-
p.add_argument("--diagnosis", required=True)
|
|
81
|
-
|
|
82
|
-
p = sub.add_parser("summarize", help="totals, breakdown, top priorities, release readiness")
|
|
83
|
-
p.add_argument("--execution-result", required=True)
|
|
84
|
-
p.add_argument("--diagnosis", required=True)
|
|
85
|
-
|
|
86
|
-
p = sub.add_parser("report", help="diagnose + plan-repairs + summarize in one call")
|
|
87
|
-
p.add_argument("--execution-result", required=True)
|
|
88
|
-
p.add_argument("--analysis-result")
|
|
89
|
-
|
|
90
|
-
args = parser.parse_args(argv)
|
|
91
|
-
|
|
92
|
-
try:
|
|
93
|
-
if args.command == "diagnose":
|
|
94
|
-
execution = _read_execution(args.execution_result)
|
|
95
|
-
analysis = _read_analysis(args.analysis_result)
|
|
96
|
-
_emit(engine.diagnose(execution, analysis))
|
|
97
|
-
elif args.command == "plan-repairs":
|
|
98
|
-
_emit({"plans": engine.plan_repairs(_read_diagnosis(args.diagnosis))})
|
|
99
|
-
elif args.command == "summarize":
|
|
100
|
-
_emit(engine.summarize(_read_execution(args.execution_result),
|
|
101
|
-
_read_diagnosis(args.diagnosis)))
|
|
102
|
-
elif args.command == "report":
|
|
103
|
-
execution = _read_execution(args.execution_result)
|
|
104
|
-
analysis = _read_analysis(args.analysis_result)
|
|
105
|
-
diagnosis = engine.diagnose(execution, analysis)
|
|
106
|
-
_emit({
|
|
107
|
-
"diagnosis": diagnosis,
|
|
108
|
-
"plans": engine.plan_repairs(diagnosis),
|
|
109
|
-
"summary": engine.summarize(execution, diagnosis),
|
|
110
|
-
})
|
|
111
|
-
except InternalContractError as exc:
|
|
112
|
-
_emit({"error": "invalid-payload", "detail": str(exc)})
|
|
113
|
-
return 2
|
|
114
|
-
except (OSError, json.JSONDecodeError) as exc:
|
|
115
|
-
_emit({"error": "io-error", "detail": str(exc)})
|
|
116
|
-
return 2
|
|
117
|
-
except (KeyError, TypeError) as exc:
|
|
118
|
-
_emit({"error": "unexpected-shape", "detail": f"{type(exc).__name__}: {exc}"})
|
|
119
|
-
return 2
|
|
120
|
-
return 0
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
if __name__ == "__main__":
|
|
124
|
-
sys.exit(main())
|
|
@@ -1,143 +0,0 @@
|
|
|
1
|
-
"""The diagnostic engine — the one place failure reasoning happens.
|
|
2
|
-
|
|
3
|
-
Orchestrates the analysis platform and the diagnostics modules into a single
|
|
4
|
-
diagnosis: per-failure root cause, prioritization, a reconstructed timeline, and
|
|
5
|
-
ranked recommendations. qa-debug presents the diagnosis; qa-fix turns it into
|
|
6
|
-
repair plans; qa-report aggregates diagnoses. None of them re-implements this.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
from qa_analysis import taxonomy
|
|
10
|
-
|
|
11
|
-
from . import root_cause, prioritization, repair, timeline
|
|
12
|
-
from . import internal_contracts
|
|
13
|
-
|
|
14
|
-
_PRIORITY_RANK = {"P1": 3, "P2": 2, "P3": 1}
|
|
15
|
-
|
|
16
|
-
# Classifications that block a release when they are the cause of a failure.
|
|
17
|
-
_RELEASE_BLOCKING = {
|
|
18
|
-
taxonomy.APPLICATION_BUG, taxonomy.NETWORK, taxonomy.INFRASTRUCTURE,
|
|
19
|
-
taxonomy.AUTH, taxonomy.AUTHORIZATION,
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def diagnose(execution_result, analysis_result=None, generation_result=None):
|
|
24
|
-
"""Produce a full diagnosis from the available results.
|
|
25
|
-
|
|
26
|
-
Returns {entries: [...], timeline: [...], recommendations: [...]}, where each
|
|
27
|
-
entry combines a root cause with its prioritization and affected tests.
|
|
28
|
-
"""
|
|
29
|
-
signals = _signals(execution_result, analysis_result)
|
|
30
|
-
findings = (analysis_result or {}).get("findings", [])
|
|
31
|
-
|
|
32
|
-
entries = []
|
|
33
|
-
for signal in signals:
|
|
34
|
-
rc = root_cause.analyze(signal)
|
|
35
|
-
blocking = rc["classification"] in _RELEASE_BLOCKING
|
|
36
|
-
prio = prioritization.prioritize(rc, blocking=blocking)
|
|
37
|
-
entries.append({
|
|
38
|
-
"rootCause": rc,
|
|
39
|
-
"priority": prio,
|
|
40
|
-
"affectedTests": signal.get("affectedTests", []),
|
|
41
|
-
})
|
|
42
|
-
|
|
43
|
-
entries.sort(key=lambda e: (_PRIORITY_RANK.get(e["priority"]["priority"], 0),
|
|
44
|
-
e["rootCause"]["confidence"]), reverse=True)
|
|
45
|
-
|
|
46
|
-
diagnosis = {
|
|
47
|
-
"entries": entries,
|
|
48
|
-
"timeline": timeline.build_timeline(execution_result, findings),
|
|
49
|
-
"recommendations": _recommendations(entries),
|
|
50
|
-
}
|
|
51
|
-
# Mechanical seam enforcement: diagnosis must match the internal contract.
|
|
52
|
-
return internal_contracts.validate_diagnosis(diagnosis)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
def plan_repairs(diagnosis):
|
|
56
|
-
"""Turn a diagnosis into repair plans (for qa-fix). One plan per entry;
|
|
57
|
-
non-repairable causes yield an escalation plan. Never produces code."""
|
|
58
|
-
plans = []
|
|
59
|
-
for entry in diagnosis["entries"]:
|
|
60
|
-
plan = repair.plan_repair(entry["rootCause"], affected_files=entry.get("affectedTests"))
|
|
61
|
-
plan["priority"] = entry["priority"]["priority"]
|
|
62
|
-
plans.append(plan)
|
|
63
|
-
return plans
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def summarize(execution_result, diagnosis):
|
|
67
|
-
"""Aggregate a diagnosis for qa-report: totals, breakdown by classification,
|
|
68
|
-
the top-priority findings, and a deterministic release-readiness call."""
|
|
69
|
-
tests = (execution_result or {}).get("tests", {})
|
|
70
|
-
by_class = {}
|
|
71
|
-
for entry in diagnosis["entries"]:
|
|
72
|
-
cls = entry["rootCause"]["classification"]
|
|
73
|
-
by_class[cls] = by_class.get(cls, 0) + 1
|
|
74
|
-
|
|
75
|
-
return {
|
|
76
|
-
"totals": tests,
|
|
77
|
-
"byClassification": by_class,
|
|
78
|
-
"topPriority": [e for e in diagnosis["entries"] if e["priority"]["priority"] == "P1"],
|
|
79
|
-
"releaseReadiness": _release_readiness(execution_result, diagnosis),
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
def _signals(execution_result, analysis_result):
|
|
84
|
-
"""Derive failure signals, preferring the analysis platform's findings."""
|
|
85
|
-
signals = []
|
|
86
|
-
findings = (analysis_result or {}).get("findings", [])
|
|
87
|
-
if findings:
|
|
88
|
-
for f in findings:
|
|
89
|
-
signals.append({
|
|
90
|
-
"message": f.get("reason", ""),
|
|
91
|
-
"classification": f.get("classification"),
|
|
92
|
-
"confidence": f.get("confidence"),
|
|
93
|
-
"reason": f.get("reason"),
|
|
94
|
-
"httpStatus": f.get("httpStatus"),
|
|
95
|
-
"retries": f.get("retries", 0),
|
|
96
|
-
"finalStatus": f.get("finalStatus"),
|
|
97
|
-
"evidence": f.get("evidence", []),
|
|
98
|
-
"affectedTests": f.get("affectedTests", []),
|
|
99
|
-
})
|
|
100
|
-
return signals
|
|
101
|
-
|
|
102
|
-
for test in (execution_result or {}).get("executed", []):
|
|
103
|
-
if test.get("status") in ("failed", "flaky"):
|
|
104
|
-
signals.append({
|
|
105
|
-
"message": test.get("message", ""),
|
|
106
|
-
"retries": test.get("retries", 0),
|
|
107
|
-
"finalStatus": test.get("status"),
|
|
108
|
-
"affectedTests": [test.get("title", "")],
|
|
109
|
-
"evidence": [{
|
|
110
|
-
"type": "junit",
|
|
111
|
-
"description": f"{test.get('status')}: {test.get('title', '')}",
|
|
112
|
-
"source": test.get("file", "execution-result"),
|
|
113
|
-
}],
|
|
114
|
-
})
|
|
115
|
-
return signals
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
def _recommendations(entries):
|
|
119
|
-
"""Ranked, de-duplicated recommendations — highest priority first."""
|
|
120
|
-
seen, ranked = set(), []
|
|
121
|
-
for entry in entries:
|
|
122
|
-
rec = entry["rootCause"]["recommendation"]
|
|
123
|
-
if rec not in seen:
|
|
124
|
-
seen.add(rec)
|
|
125
|
-
ranked.append({
|
|
126
|
-
"action": rec,
|
|
127
|
-
"priority": entry["priority"]["priority"],
|
|
128
|
-
"owner": entry["rootCause"]["ownership"],
|
|
129
|
-
"classification": entry["rootCause"]["classification"],
|
|
130
|
-
})
|
|
131
|
-
return ranked
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def _release_readiness(execution_result, diagnosis):
|
|
135
|
-
failed = (execution_result or {}).get("tests", {}).get("failed", 0)
|
|
136
|
-
if failed == 0 and not diagnosis["entries"]:
|
|
137
|
-
return "ready"
|
|
138
|
-
classes = {e["rootCause"]["classification"] for e in diagnosis["entries"]}
|
|
139
|
-
if classes & _RELEASE_BLOCKING:
|
|
140
|
-
return "not-ready"
|
|
141
|
-
if classes <= {taxonomy.UNKNOWN}:
|
|
142
|
-
return "insufficient-data"
|
|
143
|
-
return "ready-with-risks"
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
"""Validate internal Analysis → Diagnostics seam payloads.
|
|
2
|
-
|
|
3
|
-
Uses the existing dependency-free qa_analysis.contracts validator against
|
|
4
|
-
schemas under shared/diagnostics/schemas/internal/.
|
|
5
|
-
|
|
6
|
-
The schemas must be reachable in **both** layouts this package runs in:
|
|
7
|
-
|
|
8
|
-
- the repository, where they live at `shared/diagnostics/schemas/internal/`; and
|
|
9
|
-
- a bundled skill, where the installer materializes the package into
|
|
10
|
-
`<skill>/scripts/lib/qa_diagnostics/` and copies the schemas alongside it as
|
|
11
|
-
package data (`qa_diagnostics/schemas/internal/`).
|
|
12
|
-
|
|
13
|
-
Resolution therefore tries package-local data first and falls back to the
|
|
14
|
-
canonical repository path. Missing schemas raise `InternalContractError` naming
|
|
15
|
-
the locations tried, rather than a bare FileNotFoundError from deep inside a
|
|
16
|
-
diagnosis.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
from __future__ import annotations
|
|
20
|
-
|
|
21
|
-
import json
|
|
22
|
-
from pathlib import Path
|
|
23
|
-
|
|
24
|
-
from qa_analysis import contracts
|
|
25
|
-
|
|
26
|
-
_HERE = Path(__file__).resolve().parent
|
|
27
|
-
|
|
28
|
-
# Ordered candidates: bundled package data first, then the repository layout.
|
|
29
|
-
_SCHEMA_DIRS = (
|
|
30
|
-
_HERE / "schemas" / "internal",
|
|
31
|
-
_HERE.parents[1] / "schemas" / "internal",
|
|
32
|
-
)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
class InternalContractError(ValueError):
|
|
36
|
-
"""Raised when an internal seam payload fails its schema."""
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
def schema_dir() -> Path:
|
|
40
|
-
"""The directory holding the internal schemas, whichever layout applies."""
|
|
41
|
-
for candidate in _SCHEMA_DIRS:
|
|
42
|
-
if candidate.is_dir():
|
|
43
|
-
return candidate
|
|
44
|
-
tried = ", ".join(str(c) for c in _SCHEMA_DIRS)
|
|
45
|
-
raise InternalContractError(
|
|
46
|
-
f"internal schemas not found; tried: {tried}. "
|
|
47
|
-
"A bundled skill must carry qa_diagnostics/schemas/internal/ as package data."
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
def _load(name: str) -> dict:
|
|
52
|
-
path = schema_dir() / name
|
|
53
|
-
if not path.is_file():
|
|
54
|
-
raise InternalContractError(f"internal schema missing: {path}")
|
|
55
|
-
return json.loads(path.read_text(encoding="utf-8"))
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
def _require(payload: dict, schema_name: str) -> dict:
|
|
59
|
-
schema = _load(schema_name)
|
|
60
|
-
ok, errors = contracts.validate(payload, schema)
|
|
61
|
-
if not ok:
|
|
62
|
-
raise InternalContractError(f"{schema_name}: " + "; ".join(errors))
|
|
63
|
-
return payload
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
def validate_analysis_result(payload: dict) -> dict:
|
|
67
|
-
return _require(payload, "analysis-result.schema.json")
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def validate_execution_result_min(payload: dict) -> dict:
|
|
71
|
-
return _require(payload, "execution-result-min.schema.json")
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def validate_diagnosis(payload: dict) -> dict:
|
|
75
|
-
return _require(payload, "diagnosis.schema.json")
|