qa-engineer 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/COMPATIBILITY.md +50 -26
  2. package/README.md +40 -32
  3. package/package.json +12 -20
  4. package/packages/engine/bin/qa-engine.mjs +336 -0
  5. package/packages/engine/lib/analysis/branding.mjs +187 -0
  6. package/packages/engine/lib/analysis/context.mjs +294 -0
  7. package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
  8. package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
  9. package/packages/engine/lib/analysis/discovery.mjs +220 -0
  10. package/packages/engine/lib/analysis/evidence.mjs +116 -0
  11. package/packages/engine/lib/analysis/har.mjs +90 -0
  12. package/packages/engine/lib/analysis/junit.mjs +126 -0
  13. package/packages/engine/lib/analysis/redaction.mjs +127 -0
  14. package/packages/engine/lib/analysis/report-html.mjs +763 -0
  15. package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
  16. package/packages/engine/lib/analysis/xml.mjs +153 -0
  17. package/packages/engine/lib/analysis/zip.mjs +107 -0
  18. package/packages/engine/lib/diagnostics/engine.mjs +165 -0
  19. package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
  20. package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
  21. package/packages/engine/lib/diagnostics/repair.mjs +73 -0
  22. package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
  23. package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
  24. package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
  25. package/packages/engine/lib/frameworks/playwright.mjs +158 -0
  26. package/packages/engine/package.json +14 -0
  27. package/packages/installer/lib/agents/registry.mjs +46 -0
  28. package/packages/installer/lib/commands/doctor.mjs +7 -16
  29. package/packages/installer/lib/commands/install.mjs +59 -4
  30. package/packages/installer/lib/core/bundle.mjs +69 -92
  31. package/packages/installer/lib/core/config.mjs +1 -1
  32. package/packages/installer/lib/core/lockfile.mjs +1 -1
  33. package/packages/installer/lib/core/manifest.mjs +48 -54
  34. package/packages/installer/lib/core/validate-install.mjs +12 -19
  35. package/packages/installer/package.json +1 -1
  36. package/shared/tooling/qa-tool.mjs +125 -0
  37. package/skills/qa-api/SKILL.md +5 -5
  38. package/skills/qa-api/references/deterministic-tooling.md +34 -30
  39. package/skills/qa-api/references/evidence-and-reporting.md +29 -4
  40. package/skills/qa-api/scripts/qa-tool.mjs +126 -0
  41. package/skills/qa-audit/SKILL.md +5 -5
  42. package/skills/qa-audit/references/deterministic-tooling.md +34 -30
  43. package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
  44. package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
  45. package/skills/qa-debug/SKILL.md +6 -6
  46. package/skills/qa-debug/references/deterministic-tooling.md +34 -30
  47. package/skills/qa-debug/references/diagnostic-engine.md +3 -3
  48. package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
  49. package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
  50. package/skills/qa-explore/SKILL.md +22 -5
  51. package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
  52. package/skills/qa-explore/references/deterministic-tooling.md +132 -0
  53. package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
  54. package/skills/qa-explore/references/report-pipeline.md +89 -8
  55. package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
  56. package/skills/qa-fix/SKILL.md +4 -4
  57. package/skills/qa-fix/references/deterministic-tooling.md +34 -30
  58. package/skills/qa-fix/references/diagnostic-engine.md +3 -3
  59. package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
  60. package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
  61. package/skills/qa-flaky/SKILL.md +4 -4
  62. package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
  63. package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
  64. package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
  65. package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
  66. package/skills/qa-init/SKILL.md +4 -4
  67. package/skills/qa-init/references/deterministic-tooling.md +34 -30
  68. package/skills/qa-init/references/evidence-and-reporting.md +29 -4
  69. package/skills/qa-init/scripts/qa-tool.mjs +126 -0
  70. package/skills/qa-report/SKILL.md +8 -7
  71. package/skills/qa-report/references/deterministic-tooling.md +34 -30
  72. package/skills/qa-report/references/diagnostic-engine.md +3 -3
  73. package/skills/qa-report/references/evidence-and-reporting.md +29 -4
  74. package/skills/qa-report/scripts/qa-tool.mjs +126 -0
  75. package/skills/qa-review/references/evidence-and-reporting.md +29 -4
  76. package/skills/qa-run/SKILL.md +6 -6
  77. package/skills/qa-run/references/deterministic-tooling.md +34 -30
  78. package/skills/qa-run/references/evidence-and-reporting.md +29 -4
  79. package/skills/qa-run/scripts/qa-tool.mjs +126 -0
  80. package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
  81. package/shared/analysis/lib/qa_analysis/branding.py +0 -175
  82. package/shared/analysis/lib/qa_analysis/cli.py +0 -129
  83. package/shared/analysis/lib/qa_analysis/context.py +0 -233
  84. package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
  85. package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
  86. package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
  87. package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
  88. package/shared/analysis/lib/qa_analysis/har.py +0 -58
  89. package/shared/analysis/lib/qa_analysis/junit.py +0 -80
  90. package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
  91. package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
  92. package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
  93. package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
  94. package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
  95. package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
  96. package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
  97. package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
  98. package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
  99. package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
  100. package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
  101. package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
  102. package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
  103. package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
  104. package/shared/tooling/qa_tool.py +0 -127
  105. /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
  106. /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
  107. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
  108. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
  109. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
@@ -1,80 +0,0 @@
1
- """JUnit XML parser.
2
-
3
- Framework-agnostic: Playwright, Selenium, Cypress, WebdriverIO, and most unit
4
- runners emit JUnit XML, so this one parser normalizes them all into the pack's
5
- per-test result shape. This is the concrete proof that different frameworks
6
- share one contract — only where the file lives differs, not how it is read.
7
-
8
- Parses deterministically; a malformed document raises rather than guessing.
9
- """
10
-
11
- import xml.etree.ElementTree as ET
12
-
13
- from .redaction import redact_text
14
-
15
-
16
- class MalformedArtifact(ValueError):
17
- """Raised when an artifact cannot be parsed. Never swallowed into a guess."""
18
-
19
-
20
- def _int(value, default=0):
21
- try:
22
- return int(value)
23
- except (TypeError, ValueError):
24
- return default
25
-
26
-
27
- def parse_junit(path):
28
- """Parse a JUnit XML file into a normalized result.
29
-
30
- Returns a dict: {tests: {total, passed, failed, skipped}, executed: [...]}.
31
- Each executed entry: {title, file, status, durationMs, message?}.
32
- Raises MalformedArtifact if the XML is unreadable or not JUnit-shaped.
33
- """
34
- try:
35
- tree = ET.parse(path)
36
- except (ET.ParseError, OSError) as exc:
37
- raise MalformedArtifact(f"could not parse JUnit XML at {path}: {exc}") from exc
38
-
39
- root = tree.getroot()
40
- # Accept either a <testsuites> root or a single <testsuite> root.
41
- if root.tag == "testsuites":
42
- suites = root.findall("testsuite")
43
- elif root.tag == "testsuite":
44
- suites = [root]
45
- else:
46
- raise MalformedArtifact(f"not a JUnit document (root <{root.tag}>) at {path}")
47
-
48
- executed = []
49
- for suite in suites:
50
- for case in suite.findall("testcase"):
51
- failure = case.find("failure")
52
- error = case.find("error")
53
- skipped = case.find("skipped")
54
- if failure is not None or error is not None:
55
- status = "failed"
56
- node = failure if failure is not None else error
57
- message = redact_text((node.get("message") or node.text or "").strip())
58
- elif skipped is not None:
59
- status = "skipped"
60
- message = redact_text((skipped.get("message") or "").strip())
61
- else:
62
- status = "passed"
63
- message = ""
64
- entry = {
65
- "title": case.get("name", ""),
66
- "file": case.get("classname", ""),
67
- "status": status,
68
- "durationMs": int(round(float(case.get("time", "0") or "0") * 1000)),
69
- }
70
- if message:
71
- entry["message"] = message
72
- executed.append(entry)
73
-
74
- counts = {
75
- "total": len(executed),
76
- "passed": sum(1 for e in executed if e["status"] == "passed"),
77
- "failed": sum(1 for e in executed if e["status"] == "failed"),
78
- "skipped": sum(1 for e in executed if e["status"] == "skipped"),
79
- }
80
- return {"tests": counts, "executed": executed}
@@ -1,99 +0,0 @@
1
- """Credential, secret, token, and PII redaction.
2
-
3
- Every analyzer runs artifact text through redaction before it appears in a
4
- finding, a report, or stdout. Redaction happens as evidence is captured, not
5
- after — a secret must never reach a model's context or a log. Standard library
6
- only; deterministic.
7
- """
8
-
9
- import re
10
-
11
- # Ordered (name, pattern, replacement). Order matters: match structured,
12
- # high-signal secrets (JWTs, provider keys) before generic fallbacks.
13
- _RULES = [
14
- ("jwt", re.compile(r"eyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}")),
15
- ("aws-access-key", re.compile(r"AKIA[0-9A-Z]{16}")),
16
- ("github-token", re.compile(r"gh[pousr]_[A-Za-z0-9]{20,}")),
17
- ("slack-token", re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}")),
18
- ("openai-key", re.compile(r"sk-[A-Za-z0-9]{20,}")),
19
- ("bearer", re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/-]+=*")),
20
- # Sensitive header lines: keep the header name, mask the value.
21
- ("auth-header", re.compile(r"(?im)^(\s*(?:authorization|proxy-authorization)\s*[:=]\s*).+$")),
22
- ("cookie-header", re.compile(r"(?im)^(\s*(?:set-cookie|cookie)\s*[:=]\s*).+$")),
23
- # Secret-like assignments: key=value / "key": "value".
24
- ("assigned-secret", re.compile(
25
- r'(?i)(\b(?:password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|client[_-]?secret)\b\s*[:=]\s*["\']?)'
26
- r'([^\s"\'&]{4,})')),
27
- # Credentials in URLs and query strings.
28
- ("url-credential", re.compile(r"(?i)(://[^:/@\s]+:)([^@/\s]+)(@)")),
29
- ("query-secret", re.compile(
30
- r"(?i)([?&](?:password|passwd|pwd|token|secret|api[_-]?key|access[_-]?key)=)([^&\s#]+)")),
31
- # PII: email addresses.
32
- ("email", re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b")),
33
- ]
34
-
35
- # Rules whose replacement preserves a leading captured group (the label/prefix).
36
- _PREFIX_PRESERVING = {
37
- "auth-header": r"\1[REDACTED:auth-header]",
38
- "cookie-header": r"\1[REDACTED:cookie-header]",
39
- "assigned-secret": r"\1[REDACTED:secret]",
40
- "url-credential": r"\1[REDACTED:credential]\3",
41
- "query-secret": r"\1[REDACTED:secret]",
42
- }
43
-
44
-
45
- def redact_text(text):
46
- """Return text with every recognized secret or PII value masked."""
47
- if not text:
48
- return text
49
- result = text
50
- for name, pattern in _RULES:
51
- if name in _PREFIX_PRESERVING:
52
- result = pattern.sub(_PREFIX_PRESERVING[name], result)
53
- else:
54
- result = pattern.sub(f"[REDACTED:{name}]", result)
55
- return result
56
-
57
-
58
- def detect_secrets(text):
59
- """Return a list of {type, start, end} for secrets found — never the values.
60
-
61
- Used to decide whether an artifact is safe to expose, without surfacing the
62
- secret itself.
63
- """
64
- findings = []
65
- if not text:
66
- return findings
67
- for name, pattern in _RULES:
68
- for match in pattern.finditer(text):
69
- findings.append({"type": name, "start": match.start(), "end": match.end()})
70
- findings.sort(key=lambda f: f["start"])
71
- return findings
72
-
73
-
74
- _SENSITIVE_HEADERS = {
75
- "authorization", "proxy-authorization", "cookie", "set-cookie",
76
- "x-api-key", "x-auth-token", "api-key", "x-csrf-token",
77
- }
78
-
79
-
80
- def redact_headers(headers):
81
- """Mask the values of sensitive HTTP headers.
82
-
83
- Accepts a dict, or a list of {"name","value"} entries (HAR shape), and
84
- returns the same shape with sensitive values replaced.
85
- """
86
- def mask(name, value):
87
- return "[REDACTED:header]" if name.strip().lower() in _SENSITIVE_HEADERS else redact_text(value)
88
-
89
- if isinstance(headers, dict):
90
- return {k: mask(k, str(v)) for k, v in headers.items()}
91
- if isinstance(headers, list):
92
- out = []
93
- for entry in headers:
94
- if isinstance(entry, dict) and "name" in entry:
95
- out.append({**entry, "value": mask(str(entry["name"]), str(entry.get("value", "")))})
96
- else:
97
- out.append(entry)
98
- return out
99
- return headers
@@ -1,98 +0,0 @@
1
- """The canonical QA failure taxonomy and its deterministic classifier.
2
-
3
- Classification maps observed signals to one of a closed set of failure classes.
4
- It is rule-based and conservative: when no rule matches with sufficient signal,
5
- the result is UNKNOWN. Unknown is preferable to incorrect.
6
- """
7
-
8
- import re
9
-
10
- # The closed set of failure classifications. Every value implies a different
11
- # owner and next action; if two would lead to the same action, they are merged.
12
- ASSERTION = "assertion-failure"
13
- LOCATOR = "locator-failure"
14
- TIMEOUT = "timeout"
15
- NETWORK = "network"
16
- AUTH = "authentication"
17
- AUTHORIZATION = "authorization"
18
- ENVIRONMENT = "environment"
19
- CONFIGURATION = "configuration"
20
- INFRASTRUCTURE = "infrastructure"
21
- TEST_DATA = "test-data"
22
- APPLICATION_BUG = "application-bug"
23
- FRAMEWORK = "framework-failure"
24
- FLAKY = "flaky"
25
- UNKNOWN = "unknown"
26
-
27
- CLASSES = {
28
- ASSERTION, LOCATOR, TIMEOUT, NETWORK, AUTH, AUTHORIZATION, ENVIRONMENT,
29
- CONFIGURATION, INFRASTRUCTURE, TEST_DATA, APPLICATION_BUG, FRAMEWORK,
30
- FLAKY, UNKNOWN,
31
- }
32
-
33
- # Ordered rules: (classification, message-pattern, confidence, reason).
34
- #
35
- # Order encodes priority, and the first two rules exist because modern runners
36
- # print a timeout budget in *every* assertion failure. Playwright's message for a
37
- # plain text mismatch ends with "Timeout: 5000ms" even though nothing timed out,
38
- # so a naive timeout rule captures assertion and locator failures alike and sends
39
- # the reader to raise a timeout — the one action guaranteed not to help. The
40
- # patterns below are derived from real captured runner output; see
41
- # tests/test_analysis.py::RealRunnerMessageTests, which pins the exact strings.
42
- #
43
- # The discriminators, from that output:
44
- # element missing -> "Error: element(s) not found" (no Received: value)
45
- # value mismatch -> "Expected: X" + "Received: Y" (element was resolved)
46
- # real timeout -> a timeout with neither of the above
47
- _RULES = [
48
- (LOCATOR, re.compile(r"(?i)(no such element|element(?:\(s\))?\s+not\s+(?:found|visible|attached)|locator.*(?:resolved to 0|not found)|waiting for (?:locator|selector)|unable to locate element)"), 0.8,
49
- "Error indicates the target element could not be found or resolved."),
50
- # A concrete expected-vs-received comparison is an assertion result, not a
51
- # time budget: the runner resolved the target and compared values.
52
- (ASSERTION, re.compile(r"(?is)expected:.*received:"), 0.8,
53
- "Error shows a concrete expected-versus-received comparison, so the assertion did not hold."),
54
- (TIMEOUT, re.compile(r"(?i)(timeout|timed out|exceeded .*ms|deadline exceeded)"), 0.75,
55
- "Error indicates an operation exceeded its time budget."),
56
- (AUTH, re.compile(r"(?i)(401 unauthorized|authentication failed|invalid (?:credentials|token)|login failed|not authenticated)"), 0.85,
57
- "Error indicates an authentication failure — the identity could not be established."),
58
- (AUTHORIZATION, re.compile(r"(?i)(403 forbidden|permission denied|access denied|not authorized|forbidden\b|insufficient (?:permission|privilege))"), 0.85,
59
- "Error indicates an authorization failure — the identity lacked permission."),
60
- (NETWORK, re.compile(r"(?i)(ECONNREFUSED| ENOTFOUND|net::ERR|connection (?:refused|reset)|5\d\d (?:internal server error|bad gateway|service unavailable)|fetch failed)"), 0.8,
61
- "Error indicates a network or upstream service failure."),
62
- (ASSERTION, re.compile(r"(?i)(assertion|expect(?:ed)?\b|toBe|toEqual|toHaveText|toBeVisible|AssertionError|expected .* (?:to|but))"), 0.7,
63
- "Error indicates an assertion did not hold."),
64
- (TEST_DATA, re.compile(r"(?i)(duplicate key|constraint violation|no rows|seed data|fixture .*(?:missing|not found)|invalid test data)"), 0.7,
65
- "Error indicates a test-data problem."),
66
- (CONFIGURATION, re.compile(r"(?i)(cannot find module|config(?:uration)? (?:error|not found)|missing (?:config|environment variable)|unknown option)"), 0.7,
67
- "Error indicates a configuration problem."),
68
- (ENVIRONMENT, re.compile(r"(?i)(base ?url|env(?:ironment)? .*(?:not set|missing)|ECONNREFUSED .*localhost|dev server)"), 0.6,
69
- "Error indicates an environment problem such as a missing base URL or unreachable local service."),
70
- (INFRASTRUCTURE, re.compile(r"(?i)(out of memory|OOM|disk (?:full|space)|browser .*(?:crash|closed unexpectedly)|worker .*(?:died|crashed))"), 0.7,
71
- "Error indicates an infrastructure problem such as a crash or resource exhaustion."),
72
- (FRAMEWORK, re.compile(r"(?i)(internal error|unexpected error in (?:playwright|selenium|cypress|webdriver)|driver .*mismatch)"), 0.6,
73
- "Error indicates a fault in the test framework or driver itself."),
74
- ]
75
-
76
-
77
- def classify(message, http_status=None):
78
- """Classify a failure from its error message and optional HTTP status.
79
-
80
- Returns (classification, confidence, reason). Never guesses: an
81
- unrecognized signal yields UNKNOWN at low confidence with an honest reason.
82
- """
83
- text = message or ""
84
-
85
- # A concrete HTTP status is stronger evidence than message text.
86
- if http_status is not None:
87
- if http_status == 401:
88
- return (AUTH, 0.9, "HTTP 401 indicates an authentication failure.")
89
- if http_status == 403:
90
- return (AUTHORIZATION, 0.9, "HTTP 403 indicates an authorization failure.")
91
- if 500 <= http_status <= 599:
92
- return (NETWORK, 0.85, f"HTTP {http_status} indicates an upstream server failure.")
93
-
94
- for classification, pattern, confidence, reason in _RULES:
95
- if pattern.search(text):
96
- return (classification, confidence, reason)
97
-
98
- return (UNKNOWN, 0.2, "No classification rule matched the available signals; manual review needed.")
@@ -1,13 +0,0 @@
1
- """qa_diagnostics — the pack's shared diagnostic engine.
2
-
3
- One place where failure reasoning lives: root-cause analysis, timeline
4
- reconstruction, finding prioritization, and repair planning. The three
5
- diagnostic skills (qa-debug, qa-fix, qa-report) consume this engine and differ
6
- only in presentation — the reasoning is not duplicated across them.
7
-
8
- Standard library only. Builds on qa_analysis (taxonomy, evidence, diff guard);
9
- adds no framework-specific logic. Deterministic: the same inputs yield the same
10
- diagnosis every time.
11
- """
12
-
13
- __version__ = "0.1.0"
@@ -1,124 +0,0 @@
1
- """Unified CLI for the diagnostic engine.
2
-
3
- The engine's reasoning is deterministic; this is how a skill reaches it. Every
4
- subcommand reads JSON files, validates them against the internal seam contracts,
5
- and writes JSON to stdout — so an agent never has to invent glue code, and the
6
- input and output shapes are checked rather than assumed.
7
-
8
- Mirrors `qa_analysis.cli` exactly: JSON to stdout, exit 0 on success, exit 2 on
9
- bad input or usage. Standard library only.
10
-
11
- Usage:
12
- python -m qa_diagnostics.cli diagnose --execution-result <path> [--analysis-result <path>]
13
- python -m qa_diagnostics.cli plan-repairs --diagnosis <path>
14
- python -m qa_diagnostics.cli summarize --execution-result <path> --diagnosis <path>
15
- python -m qa_diagnostics.cli report --execution-result <path> [--analysis-result <path>]
16
-
17
- `report` is the one-shot path: it diagnoses, plans, and summarizes in a single
18
- invocation, so the common case is one command instead of three.
19
-
20
- Inputs
21
- --execution-result a qa-run execution result (or the minimal subset:
22
- `tests` counts plus `executed[]` with `status`)
23
- --analysis-result an analysis result with `findings[]`; preferred over
24
- `executed[]` when present
25
- --diagnosis the object emitted by `diagnose`
26
-
27
- Exit codes
28
- 0 success
29
- 2 unreadable file, malformed JSON, or a payload that fails its seam contract
30
- """
31
-
32
- import argparse
33
- import json
34
- import sys
35
-
36
- from . import engine
37
- from . import internal_contracts
38
- from .internal_contracts import InternalContractError
39
-
40
-
41
- def _emit(obj):
42
- json.dump(obj, sys.stdout, indent=2, sort_keys=True)
43
- sys.stdout.write("\n")
44
-
45
-
46
- def _read(path):
47
- with open(path, "r", encoding="utf-8") as handle:
48
- return json.load(handle)
49
-
50
-
51
- def _read_execution(path):
52
- """Read an execution result and hold it to the seam contract."""
53
- return internal_contracts.validate_execution_result_min(_read(path))
54
-
55
-
56
- def _read_analysis(path):
57
- """Read an analysis result and hold it to the seam contract."""
58
- if path is None:
59
- return None
60
- return internal_contracts.validate_analysis_result(_read(path))
61
-
62
-
63
- def _read_diagnosis(path):
64
- """Read a diagnosis and hold it to the seam contract."""
65
- return internal_contracts.validate_diagnosis(_read(path))
66
-
67
-
68
- def main(argv=None):
69
- parser = argparse.ArgumentParser(
70
- prog="qa-diagnostics",
71
- description="Deterministic diagnostic engine: root cause, timeline, priority, repairs",
72
- )
73
- sub = parser.add_subparsers(dest="command", required=True)
74
-
75
- p = sub.add_parser("diagnose", help="root causes, timeline, prioritization, recommendations")
76
- p.add_argument("--execution-result", required=True)
77
- p.add_argument("--analysis-result")
78
-
79
- p = sub.add_parser("plan-repairs", help="repair plans for a diagnosis (never code)")
80
- p.add_argument("--diagnosis", required=True)
81
-
82
- p = sub.add_parser("summarize", help="totals, breakdown, top priorities, release readiness")
83
- p.add_argument("--execution-result", required=True)
84
- p.add_argument("--diagnosis", required=True)
85
-
86
- p = sub.add_parser("report", help="diagnose + plan-repairs + summarize in one call")
87
- p.add_argument("--execution-result", required=True)
88
- p.add_argument("--analysis-result")
89
-
90
- args = parser.parse_args(argv)
91
-
92
- try:
93
- if args.command == "diagnose":
94
- execution = _read_execution(args.execution_result)
95
- analysis = _read_analysis(args.analysis_result)
96
- _emit(engine.diagnose(execution, analysis))
97
- elif args.command == "plan-repairs":
98
- _emit({"plans": engine.plan_repairs(_read_diagnosis(args.diagnosis))})
99
- elif args.command == "summarize":
100
- _emit(engine.summarize(_read_execution(args.execution_result),
101
- _read_diagnosis(args.diagnosis)))
102
- elif args.command == "report":
103
- execution = _read_execution(args.execution_result)
104
- analysis = _read_analysis(args.analysis_result)
105
- diagnosis = engine.diagnose(execution, analysis)
106
- _emit({
107
- "diagnosis": diagnosis,
108
- "plans": engine.plan_repairs(diagnosis),
109
- "summary": engine.summarize(execution, diagnosis),
110
- })
111
- except InternalContractError as exc:
112
- _emit({"error": "invalid-payload", "detail": str(exc)})
113
- return 2
114
- except (OSError, json.JSONDecodeError) as exc:
115
- _emit({"error": "io-error", "detail": str(exc)})
116
- return 2
117
- except (KeyError, TypeError) as exc:
118
- _emit({"error": "unexpected-shape", "detail": f"{type(exc).__name__}: {exc}"})
119
- return 2
120
- return 0
121
-
122
-
123
- if __name__ == "__main__":
124
- sys.exit(main())
@@ -1,143 +0,0 @@
1
- """The diagnostic engine — the one place failure reasoning happens.
2
-
3
- Orchestrates the analysis platform and the diagnostics modules into a single
4
- diagnosis: per-failure root cause, prioritization, a reconstructed timeline, and
5
- ranked recommendations. qa-debug presents the diagnosis; qa-fix turns it into
6
- repair plans; qa-report aggregates diagnoses. None of them re-implements this.
7
- """
8
-
9
- from qa_analysis import taxonomy
10
-
11
- from . import root_cause, prioritization, repair, timeline
12
- from . import internal_contracts
13
-
14
- _PRIORITY_RANK = {"P1": 3, "P2": 2, "P3": 1}
15
-
16
- # Classifications that block a release when they are the cause of a failure.
17
- _RELEASE_BLOCKING = {
18
- taxonomy.APPLICATION_BUG, taxonomy.NETWORK, taxonomy.INFRASTRUCTURE,
19
- taxonomy.AUTH, taxonomy.AUTHORIZATION,
20
- }
21
-
22
-
23
- def diagnose(execution_result, analysis_result=None, generation_result=None):
24
- """Produce a full diagnosis from the available results.
25
-
26
- Returns {entries: [...], timeline: [...], recommendations: [...]}, where each
27
- entry combines a root cause with its prioritization and affected tests.
28
- """
29
- signals = _signals(execution_result, analysis_result)
30
- findings = (analysis_result or {}).get("findings", [])
31
-
32
- entries = []
33
- for signal in signals:
34
- rc = root_cause.analyze(signal)
35
- blocking = rc["classification"] in _RELEASE_BLOCKING
36
- prio = prioritization.prioritize(rc, blocking=blocking)
37
- entries.append({
38
- "rootCause": rc,
39
- "priority": prio,
40
- "affectedTests": signal.get("affectedTests", []),
41
- })
42
-
43
- entries.sort(key=lambda e: (_PRIORITY_RANK.get(e["priority"]["priority"], 0),
44
- e["rootCause"]["confidence"]), reverse=True)
45
-
46
- diagnosis = {
47
- "entries": entries,
48
- "timeline": timeline.build_timeline(execution_result, findings),
49
- "recommendations": _recommendations(entries),
50
- }
51
- # Mechanical seam enforcement: diagnosis must match the internal contract.
52
- return internal_contracts.validate_diagnosis(diagnosis)
53
-
54
-
55
- def plan_repairs(diagnosis):
56
- """Turn a diagnosis into repair plans (for qa-fix). One plan per entry;
57
- non-repairable causes yield an escalation plan. Never produces code."""
58
- plans = []
59
- for entry in diagnosis["entries"]:
60
- plan = repair.plan_repair(entry["rootCause"], affected_files=entry.get("affectedTests"))
61
- plan["priority"] = entry["priority"]["priority"]
62
- plans.append(plan)
63
- return plans
64
-
65
-
66
- def summarize(execution_result, diagnosis):
67
- """Aggregate a diagnosis for qa-report: totals, breakdown by classification,
68
- the top-priority findings, and a deterministic release-readiness call."""
69
- tests = (execution_result or {}).get("tests", {})
70
- by_class = {}
71
- for entry in diagnosis["entries"]:
72
- cls = entry["rootCause"]["classification"]
73
- by_class[cls] = by_class.get(cls, 0) + 1
74
-
75
- return {
76
- "totals": tests,
77
- "byClassification": by_class,
78
- "topPriority": [e for e in diagnosis["entries"] if e["priority"]["priority"] == "P1"],
79
- "releaseReadiness": _release_readiness(execution_result, diagnosis),
80
- }
81
-
82
-
83
- def _signals(execution_result, analysis_result):
84
- """Derive failure signals, preferring the analysis platform's findings."""
85
- signals = []
86
- findings = (analysis_result or {}).get("findings", [])
87
- if findings:
88
- for f in findings:
89
- signals.append({
90
- "message": f.get("reason", ""),
91
- "classification": f.get("classification"),
92
- "confidence": f.get("confidence"),
93
- "reason": f.get("reason"),
94
- "httpStatus": f.get("httpStatus"),
95
- "retries": f.get("retries", 0),
96
- "finalStatus": f.get("finalStatus"),
97
- "evidence": f.get("evidence", []),
98
- "affectedTests": f.get("affectedTests", []),
99
- })
100
- return signals
101
-
102
- for test in (execution_result or {}).get("executed", []):
103
- if test.get("status") in ("failed", "flaky"):
104
- signals.append({
105
- "message": test.get("message", ""),
106
- "retries": test.get("retries", 0),
107
- "finalStatus": test.get("status"),
108
- "affectedTests": [test.get("title", "")],
109
- "evidence": [{
110
- "type": "junit",
111
- "description": f"{test.get('status')}: {test.get('title', '')}",
112
- "source": test.get("file", "execution-result"),
113
- }],
114
- })
115
- return signals
116
-
117
-
118
- def _recommendations(entries):
119
- """Ranked, de-duplicated recommendations — highest priority first."""
120
- seen, ranked = set(), []
121
- for entry in entries:
122
- rec = entry["rootCause"]["recommendation"]
123
- if rec not in seen:
124
- seen.add(rec)
125
- ranked.append({
126
- "action": rec,
127
- "priority": entry["priority"]["priority"],
128
- "owner": entry["rootCause"]["ownership"],
129
- "classification": entry["rootCause"]["classification"],
130
- })
131
- return ranked
132
-
133
-
134
- def _release_readiness(execution_result, diagnosis):
135
- failed = (execution_result or {}).get("tests", {}).get("failed", 0)
136
- if failed == 0 and not diagnosis["entries"]:
137
- return "ready"
138
- classes = {e["rootCause"]["classification"] for e in diagnosis["entries"]}
139
- if classes & _RELEASE_BLOCKING:
140
- return "not-ready"
141
- if classes <= {taxonomy.UNKNOWN}:
142
- return "insufficient-data"
143
- return "ready-with-risks"
@@ -1,75 +0,0 @@
1
- """Validate internal Analysis → Diagnostics seam payloads.
2
-
3
- Uses the existing dependency-free qa_analysis.contracts validator against
4
- schemas under shared/diagnostics/schemas/internal/.
5
-
6
- The schemas must be reachable in **both** layouts this package runs in:
7
-
8
- - the repository, where they live at `shared/diagnostics/schemas/internal/`; and
9
- - a bundled skill, where the installer materializes the package into
10
- `<skill>/scripts/lib/qa_diagnostics/` and copies the schemas alongside it as
11
- package data (`qa_diagnostics/schemas/internal/`).
12
-
13
- Resolution therefore tries package-local data first and falls back to the
14
- canonical repository path. Missing schemas raise `InternalContractError` naming
15
- the locations tried, rather than a bare FileNotFoundError from deep inside a
16
- diagnosis.
17
- """
18
-
19
- from __future__ import annotations
20
-
21
- import json
22
- from pathlib import Path
23
-
24
- from qa_analysis import contracts
25
-
26
- _HERE = Path(__file__).resolve().parent
27
-
28
- # Ordered candidates: bundled package data first, then the repository layout.
29
- _SCHEMA_DIRS = (
30
- _HERE / "schemas" / "internal",
31
- _HERE.parents[1] / "schemas" / "internal",
32
- )
33
-
34
-
35
- class InternalContractError(ValueError):
36
- """Raised when an internal seam payload fails its schema."""
37
-
38
-
39
- def schema_dir() -> Path:
40
- """The directory holding the internal schemas, whichever layout applies."""
41
- for candidate in _SCHEMA_DIRS:
42
- if candidate.is_dir():
43
- return candidate
44
- tried = ", ".join(str(c) for c in _SCHEMA_DIRS)
45
- raise InternalContractError(
46
- f"internal schemas not found; tried: {tried}. "
47
- "A bundled skill must carry qa_diagnostics/schemas/internal/ as package data."
48
- )
49
-
50
-
51
- def _load(name: str) -> dict:
52
- path = schema_dir() / name
53
- if not path.is_file():
54
- raise InternalContractError(f"internal schema missing: {path}")
55
- return json.loads(path.read_text(encoding="utf-8"))
56
-
57
-
58
- def _require(payload: dict, schema_name: str) -> dict:
59
- schema = _load(schema_name)
60
- ok, errors = contracts.validate(payload, schema)
61
- if not ok:
62
- raise InternalContractError(f"{schema_name}: " + "; ".join(errors))
63
- return payload
64
-
65
-
66
- def validate_analysis_result(payload: dict) -> dict:
67
- return _require(payload, "analysis-result.schema.json")
68
-
69
-
70
- def validate_execution_result_min(payload: dict) -> dict:
71
- return _require(payload, "execution-result-min.schema.json")
72
-
73
-
74
- def validate_diagnosis(payload: dict) -> dict:
75
- return _require(payload, "diagnosis.schema.json")