qa-engineer 0.9.1 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/COMPATIBILITY.md +50 -26
  2. package/README.md +40 -32
  3. package/package.json +12 -20
  4. package/packages/engine/bin/qa-engine.mjs +336 -0
  5. package/packages/engine/lib/analysis/branding.mjs +187 -0
  6. package/packages/engine/lib/analysis/context.mjs +294 -0
  7. package/packages/{installer/lib/core/schema-validate.mjs → engine/lib/analysis/contracts.mjs} +11 -4
  8. package/packages/engine/lib/analysis/diff-guard.mjs +425 -0
  9. package/packages/engine/lib/analysis/discovery.mjs +220 -0
  10. package/packages/engine/lib/analysis/evidence.mjs +116 -0
  11. package/packages/engine/lib/analysis/har.mjs +90 -0
  12. package/packages/engine/lib/analysis/junit.mjs +126 -0
  13. package/packages/engine/lib/analysis/redaction.mjs +127 -0
  14. package/packages/engine/lib/analysis/report-html.mjs +763 -0
  15. package/packages/engine/lib/analysis/taxonomy.mjs +109 -0
  16. package/packages/engine/lib/analysis/xml.mjs +153 -0
  17. package/packages/engine/lib/analysis/zip.mjs +107 -0
  18. package/packages/engine/lib/diagnostics/engine.mjs +165 -0
  19. package/packages/engine/lib/diagnostics/internal-contracts.mjs +50 -0
  20. package/packages/engine/lib/diagnostics/prioritization.mjs +99 -0
  21. package/packages/engine/lib/diagnostics/repair.mjs +73 -0
  22. package/packages/engine/lib/diagnostics/root-cause.mjs +92 -0
  23. package/packages/engine/lib/diagnostics/timeline.mjs +101 -0
  24. package/packages/engine/lib/frameworks/junit-frameworks.mjs +48 -0
  25. package/packages/engine/lib/frameworks/playwright.mjs +158 -0
  26. package/packages/engine/package.json +14 -0
  27. package/packages/installer/lib/agents/registry.mjs +46 -0
  28. package/packages/installer/lib/commands/doctor.mjs +7 -16
  29. package/packages/installer/lib/commands/install.mjs +59 -4
  30. package/packages/installer/lib/core/bundle.mjs +69 -92
  31. package/packages/installer/lib/core/config.mjs +1 -1
  32. package/packages/installer/lib/core/lockfile.mjs +1 -1
  33. package/packages/installer/lib/core/manifest.mjs +48 -54
  34. package/packages/installer/lib/core/validate-install.mjs +12 -19
  35. package/packages/installer/package.json +1 -1
  36. package/shared/tooling/qa-tool.mjs +125 -0
  37. package/skills/qa-api/SKILL.md +5 -5
  38. package/skills/qa-api/references/deterministic-tooling.md +34 -30
  39. package/skills/qa-api/references/evidence-and-reporting.md +29 -4
  40. package/skills/qa-api/scripts/qa-tool.mjs +126 -0
  41. package/skills/qa-audit/SKILL.md +5 -5
  42. package/skills/qa-audit/references/deterministic-tooling.md +34 -30
  43. package/skills/qa-audit/references/evidence-and-reporting.md +29 -4
  44. package/skills/qa-audit/scripts/qa-tool.mjs +126 -0
  45. package/skills/qa-debug/SKILL.md +6 -6
  46. package/skills/qa-debug/references/deterministic-tooling.md +34 -30
  47. package/skills/qa-debug/references/diagnostic-engine.md +3 -3
  48. package/skills/qa-debug/references/evidence-and-reporting.md +29 -4
  49. package/skills/qa-debug/scripts/qa-tool.mjs +126 -0
  50. package/skills/qa-explore/SKILL.md +22 -5
  51. package/skills/qa-explore/contracts/explore-result.schema.json +26 -2
  52. package/skills/qa-explore/references/deterministic-tooling.md +132 -0
  53. package/skills/qa-explore/references/evidence-and-reporting.md +29 -4
  54. package/skills/qa-explore/references/report-pipeline.md +89 -8
  55. package/skills/qa-explore/scripts/qa-tool.mjs +126 -0
  56. package/skills/qa-fix/SKILL.md +4 -4
  57. package/skills/qa-fix/references/deterministic-tooling.md +34 -30
  58. package/skills/qa-fix/references/diagnostic-engine.md +3 -3
  59. package/skills/qa-fix/references/evidence-and-reporting.md +29 -4
  60. package/skills/qa-fix/scripts/qa-tool.mjs +126 -0
  61. package/skills/qa-flaky/SKILL.md +4 -4
  62. package/skills/qa-flaky/references/deterministic-tooling.md +34 -30
  63. package/skills/qa-flaky/references/evidence-and-reporting.md +29 -4
  64. package/skills/qa-flaky/scripts/qa-tool.mjs +126 -0
  65. package/skills/qa-generate/references/evidence-and-reporting.md +29 -4
  66. package/skills/qa-init/SKILL.md +4 -4
  67. package/skills/qa-init/references/deterministic-tooling.md +34 -30
  68. package/skills/qa-init/references/evidence-and-reporting.md +29 -4
  69. package/skills/qa-init/scripts/qa-tool.mjs +126 -0
  70. package/skills/qa-report/SKILL.md +8 -7
  71. package/skills/qa-report/references/deterministic-tooling.md +34 -30
  72. package/skills/qa-report/references/diagnostic-engine.md +3 -3
  73. package/skills/qa-report/references/evidence-and-reporting.md +29 -4
  74. package/skills/qa-report/scripts/qa-tool.mjs +126 -0
  75. package/skills/qa-review/references/evidence-and-reporting.md +29 -4
  76. package/skills/qa-run/SKILL.md +6 -6
  77. package/skills/qa-run/references/deterministic-tooling.md +34 -30
  78. package/skills/qa-run/references/evidence-and-reporting.md +29 -4
  79. package/skills/qa-run/scripts/qa-tool.mjs +126 -0
  80. package/shared/analysis/lib/qa_analysis/__init__.py +0 -11
  81. package/shared/analysis/lib/qa_analysis/branding.py +0 -175
  82. package/shared/analysis/lib/qa_analysis/cli.py +0 -129
  83. package/shared/analysis/lib/qa_analysis/context.py +0 -233
  84. package/shared/analysis/lib/qa_analysis/contracts.py +0 -158
  85. package/shared/analysis/lib/qa_analysis/diff_guard.py +0 -327
  86. package/shared/analysis/lib/qa_analysis/discovery.py +0 -113
  87. package/shared/analysis/lib/qa_analysis/evidence.py +0 -128
  88. package/shared/analysis/lib/qa_analysis/har.py +0 -58
  89. package/shared/analysis/lib/qa_analysis/junit.py +0 -80
  90. package/shared/analysis/lib/qa_analysis/redaction.py +0 -99
  91. package/shared/analysis/lib/qa_analysis/taxonomy.py +0 -98
  92. package/shared/diagnostics/lib/qa_diagnostics/__init__.py +0 -13
  93. package/shared/diagnostics/lib/qa_diagnostics/cli.py +0 -124
  94. package/shared/diagnostics/lib/qa_diagnostics/engine.py +0 -143
  95. package/shared/diagnostics/lib/qa_diagnostics/internal_contracts.py +0 -75
  96. package/shared/diagnostics/lib/qa_diagnostics/prioritization.py +0 -101
  97. package/shared/diagnostics/lib/qa_diagnostics/repair.py +0 -72
  98. package/shared/diagnostics/lib/qa_diagnostics/root_cause.py +0 -89
  99. package/shared/diagnostics/lib/qa_diagnostics/timeline.py +0 -71
  100. package/shared/frameworks/cypress/lib/cypress_analysis.py +0 -27
  101. package/shared/frameworks/playwright/lib/playwright_analysis.py +0 -167
  102. package/shared/frameworks/selenium/lib/selenium_analysis.py +0 -28
  103. package/shared/frameworks/webdriverio/lib/webdriverio_analysis.py +0 -24
  104. package/shared/tooling/qa_tool.py +0 -127
  105. /package/{shared/analysis/lib/qa_analysis → packages/engine/lib/analysis}/branding.json +0 -0
  106. /package/{shared → packages/engine/lib}/analysis/schemas/context.schema.json +0 -0
  107. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/analysis-result.schema.json +0 -0
  108. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/diagnosis.schema.json +0 -0
  109. /package/{shared → packages/engine/lib}/diagnostics/schemas/internal/execution-result-min.schema.json +0 -0
@@ -1,233 +0,0 @@
1
- """Deterministic parsing of `.qa/context.md`.
2
-
3
- `qa-init` writes the project profile as a Markdown file whose frontmatter holds
4
- the machine-readable facts, and every other skill reads it. The contract for
5
- those facts is a JSON Schema (`shared/analysis/schemas/context.schema.json`) —
6
- but nothing could check a *real* `.qa/context.md` against it, because the
7
- toolkit is standard-library-only and the frontmatter is YAML. Validation ran
8
- against a hand-written JSON fixture instead, so the contract was unenforced
9
- exactly where it mattered.
10
-
11
- This module closes that gap without adding a dependency or changing the file
12
- format: it parses the **explicit subset** of YAML the context contract uses,
13
- then hands the result to the existing contract validator.
14
-
15
- ## The supported subset
16
-
17
- Deliberately small, and everything outside it is an error rather than a guess:
18
-
19
- - `key: value` mappings, nested by two-space indentation
20
- - block sequences (`- item`), including nested under a key
21
- - flow collections only when empty: `[]` and `{}`
22
- - scalars: double- or single-quoted strings, bare strings, integers, floats,
23
- `true`/`false`, `null`/`~`/empty
24
- - `#` comments on their own line or after a value
25
- - the leading/trailing `---` fences
26
-
27
- Not supported, and rejected loudly: anchors/aliases, multi-line block scalars
28
- (`|`, `>`), non-empty flow collections, multi-document streams, and tabs for
29
- indentation. A generator that needs one of those has outgrown the contract, and
30
- the right response is to change the contract deliberately — not to have a parser
31
- quietly misread it.
32
- """
33
-
34
- import re
35
-
36
- from . import contracts
37
-
38
- _FENCE = "---"
39
- _TRUE = ("true", "True", "TRUE")
40
- _FALSE = ("false", "False", "FALSE")
41
- _NULL = ("null", "Null", "NULL", "~", "")
42
- _UNSUPPORTED_SCALARS = ("|", ">")
43
- _INT = re.compile(r"^-?\d+$")
44
- _FLOAT = re.compile(r"^-?\d+\.\d+$")
45
-
46
-
47
- class MalformedContext(ValueError):
48
- """Raised when `.qa/context.md` cannot be parsed or fails its contract."""
49
-
50
-
51
- def split_frontmatter(text):
52
- """Return (frontmatter_text, body_text). Raises if the fences are missing."""
53
- lines = text.split("\n")
54
- start = None
55
- for index, line in enumerate(lines):
56
- if line.strip() == "":
57
- continue
58
- if line.rstrip() == _FENCE:
59
- start = index
60
- break
61
- if start is None:
62
- raise MalformedContext(
63
- "no frontmatter: the file must open with a '---' fence "
64
- "(see the qa-init context template)"
65
- )
66
- for index in range(start + 1, len(lines)):
67
- if lines[index].rstrip() == _FENCE:
68
- return "\n".join(lines[start + 1:index]), "\n".join(lines[index + 1:])
69
- raise MalformedContext("unterminated frontmatter: no closing '---' fence")
70
-
71
-
72
- def _strip_comment(value):
73
- """Remove a trailing `#` comment that is not inside quotes."""
74
- quote = None
75
- for index, char in enumerate(value):
76
- if quote:
77
- if char == quote:
78
- quote = None
79
- elif char in ("'", '"'):
80
- quote = char
81
- elif char == "#" and (index == 0 or value[index - 1] in " \t"):
82
- return value[:index]
83
- return value
84
-
85
-
86
- def _scalar(raw, line_number):
87
- text = _strip_comment(raw).strip()
88
- if text[:1] in _UNSUPPORTED_SCALARS:
89
- raise MalformedContext(
90
- f"line {line_number}: block scalars ('|', '>') are outside the supported subset"
91
- )
92
- if text[:1] in ("&", "*"):
93
- raise MalformedContext(
94
- f"line {line_number}: anchors and aliases are outside the supported subset"
95
- )
96
- if len(text) >= 2 and text[0] == text[-1] and text[0] in ("'", '"'):
97
- return text[1:-1]
98
- if text == "[]":
99
- return []
100
- if text == "{}":
101
- return {}
102
- if text.startswith("[") or text.startswith("{"):
103
- raise MalformedContext(
104
- f"line {line_number}: only empty flow collections ('[]', '{{}}') are supported; "
105
- "use a block sequence or mapping"
106
- )
107
- if text in _TRUE:
108
- return True
109
- if text in _FALSE:
110
- return False
111
- if text in _NULL:
112
- return None
113
- if _INT.match(text):
114
- return int(text)
115
- if _FLOAT.match(text):
116
- return float(text)
117
- return text
118
-
119
-
120
- def parse_frontmatter(text):
121
- """Parse the supported YAML subset into a dict. Raises MalformedContext.
122
-
123
- The parser keeps a stack of `(key_indent, container)` frames. `key_indent` is
124
- the column at which that container's own entries start, so a line's indent
125
- alone decides which container it belongs to: pop every frame deeper than the
126
- line, and the top of the stack is the target.
127
- """
128
- root = {}
129
- stack = [(0, root)]
130
- pending = None # (key, indent) — a key whose value is a block that follows
131
-
132
- def resolve_pending_as_null():
133
- """A key with no value and no nested block is a null field."""
134
- nonlocal pending
135
- if pending is not None:
136
- stack[-1][1][pending[0]] = None
137
- pending = None
138
-
139
- for number, raw_line in enumerate(text.split("\n"), start=1):
140
- leading = raw_line[: len(raw_line) - len(raw_line.lstrip())]
141
- if "\t" in leading:
142
- raise MalformedContext(f"line {number}: tab indentation is not supported")
143
- stripped = raw_line.strip()
144
- if not stripped or stripped.startswith("#"):
145
- continue
146
- if stripped == _FENCE:
147
- raise MalformedContext(f"line {number}: unexpected '---' inside frontmatter")
148
-
149
- indent = len(raw_line) - len(raw_line.lstrip(" "))
150
-
151
- # --- sequence item ---
152
- if stripped.startswith("- ") or stripped == "-":
153
- item = _scalar(stripped[2:] if len(stripped) > 1 else "", number)
154
- if pending is not None:
155
- # A sequence may sit at the key's indent or deeper.
156
- parent = stack[-1][1]
157
- parent[pending[0]] = []
158
- stack.append((indent, parent[pending[0]]))
159
- pending = None
160
- while len(stack) > 1 and stack[-1][0] > indent:
161
- stack.pop()
162
- container = stack[-1][1]
163
- if not isinstance(container, list):
164
- raise MalformedContext(f"line {number}: sequence item outside a sequence")
165
- container.append(item)
166
- continue
167
-
168
- # --- mapping entry ---
169
- if ":" not in stripped:
170
- raise MalformedContext(
171
- f"line {number}: expected 'key: value' or '- item', got {stripped!r}"
172
- )
173
-
174
- if pending is not None:
175
- if indent > pending[1]:
176
- # The pending key opens a nested mapping at this indent.
177
- parent = stack[-1][1]
178
- parent[pending[0]] = {}
179
- stack.append((indent, parent[pending[0]]))
180
- pending = None
181
- else:
182
- resolve_pending_as_null()
183
-
184
- # Leave the frames this line does not belong to. A sequence frame at the
185
- # same indent as a key also ends here (`packages:` / `- a` / `ci:`).
186
- while len(stack) > 1 and (
187
- stack[-1][0] > indent
188
- or (isinstance(stack[-1][1], list) and stack[-1][0] >= indent)
189
- ):
190
- stack.pop()
191
-
192
- key, _, value = stripped.partition(":")
193
- key = key.strip()
194
- if not key:
195
- raise MalformedContext(f"line {number}: empty key")
196
- container = stack[-1][1]
197
- if not isinstance(container, dict):
198
- raise MalformedContext(f"line {number}: mapping key inside a sequence")
199
-
200
- if _strip_comment(value).strip() == "":
201
- pending = (key, indent) # a block or a null follows
202
- else:
203
- container[key] = _scalar(value, number)
204
-
205
- resolve_pending_as_null()
206
- return root
207
-
208
-
209
- def parse(text, schema=None):
210
- """Parse `.qa/context.md` text and, when a schema is given, validate it.
211
-
212
- Returns {"context": <dict>, "body": <str>, "valid": bool, "errors": [...]}.
213
- Raises MalformedContext when the text cannot be parsed at all — a parse
214
- failure is not a validation result.
215
- """
216
- frontmatter, body = split_frontmatter(text)
217
- context = parse_frontmatter(frontmatter)
218
- result = {"context": context, "body": body, "valid": True, "errors": []}
219
- if schema is not None:
220
- ok, errors = contracts.validate(context, schema)
221
- result["valid"] = ok
222
- result["errors"] = errors
223
- return result
224
-
225
-
226
- def parse_file(path, schema=None):
227
- """Read and parse a `.qa/context.md` file."""
228
- try:
229
- with open(path, "r", encoding="utf-8") as handle:
230
- text = handle.read()
231
- except OSError as exc:
232
- raise MalformedContext(f"could not read {path}: {exc}") from exc
233
- return parse(text, schema=schema)
@@ -1,158 +0,0 @@
1
- """Deterministic contract validation.
2
-
3
- A dependency-free validator for the subset of JSON Schema the pack's contracts
4
- use. It exists so no skill ever emits a result that violates its own contract,
5
- and so CI can gate on it without a third-party dependency.
6
-
7
- The supported subset is declared once, in `SUPPORTED_KEYWORDS`, and is the
8
- **same** subset the JavaScript twin implements
9
- (`packages/installer/lib/core/schema-validate.mjs`). Two rules keep that promise
10
- honest:
11
-
12
- 1. A keyword outside the subset is an **error**, never a silent no-op — a
13
- contract author cannot accidentally write a constraint that looks enforced
14
- and isn't.
15
- 2. `tests/parity/validator-cases.json` runs through both validators and both
16
- must agree on every case.
17
-
18
- The subset includes `allOf` / `if` / `then` / `else` because the pack's safety
19
- invariants are cross-field implications ("classification `passed` implies exit
20
- code 0"). Those must be enforced at runtime by the contract itself, not only by
21
- evaluation fixtures.
22
-
23
- This validates the pack's own schemas; it is not a general-purpose JSON Schema
24
- implementation, and it says so rather than pretending to be one.
25
- """
26
-
27
- import json
28
- import re
29
-
30
- _TYPES = {
31
- "object": dict,
32
- "array": list,
33
- "string": str,
34
- "number": (int, float),
35
- "integer": int,
36
- "boolean": bool,
37
- "null": type(None),
38
- }
39
-
40
- # The complete supported subset. Anything else is reported as a schema error.
41
- # Keep in sync with SUPPORTED in packages/installer/lib/core/schema-validate.mjs
42
- # and the table in docs/skills/output-contracts.md (checked by
43
- # scripts/check-spec-code-sync.mjs).
44
- SUPPORTED_KEYWORDS = frozenset({
45
- "$schema", "$id", "title", "description", "type", "properties", "required",
46
- "additionalProperties", "items", "enum", "const", "pattern", "minimum",
47
- "maximum", "minItems", "maxItems", "minLength", "maxLength", "default",
48
- "examples", "format", "allOf", "if", "then", "else",
49
- })
50
-
51
- # RFC 3339 date-time. Defined here (rather than delegating to
52
- # datetime.fromisoformat, which also accepts date-only strings and varies by
53
- # interpreter version) so both validators accept exactly the same set.
54
- _DATE_TIME = re.compile(
55
- r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$"
56
- )
57
-
58
-
59
- def load_schema(path):
60
- with open(path, "r", encoding="utf-8") as handle:
61
- return json.load(handle)
62
-
63
-
64
- def validate(instance, schema, path="$"):
65
- """Validate instance against schema. Returns (ok, errors) where errors is a
66
- list of human-readable "path: problem" strings."""
67
- errors = []
68
- _validate(instance, schema, path, errors)
69
- return (not errors, errors)
70
-
71
-
72
- def _type_ok(value, expected):
73
- types = expected if isinstance(expected, list) else [expected]
74
- for name in types:
75
- py = _TYPES.get(name)
76
- if py is None:
77
- continue
78
- # bool is a subclass of int; keep them distinct.
79
- if name == "integer" and isinstance(value, bool):
80
- continue
81
- if name == "number" and isinstance(value, bool):
82
- continue
83
- if isinstance(value, py):
84
- return True
85
- return False
86
-
87
-
88
- def _matches(value, schema):
89
- """True when value satisfies schema. Used for if/then branch selection, so
90
- it must not leak errors into the caller's list."""
91
- probe = []
92
- _validate(value, schema, "$", probe)
93
- return not probe
94
-
95
-
96
- def _validate(value, schema, path, errors):
97
- for keyword in schema:
98
- if keyword not in SUPPORTED_KEYWORDS:
99
- errors.append(f"{path}: schema uses unsupported keyword \"{keyword}\"")
100
-
101
- if "type" in schema and not _type_ok(value, schema["type"]):
102
- errors.append(f"{path}: expected type {schema['type']}, got {type(value).__name__}")
103
- return # further checks assume the type held
104
-
105
- if "const" in schema and value != schema["const"]:
106
- errors.append(f"{path}: expected const {schema['const']!r}, got {value!r}")
107
-
108
- if "enum" in schema and value not in schema["enum"]:
109
- errors.append(f"{path}: {value!r} is not one of {schema['enum']}")
110
-
111
- if isinstance(value, str):
112
- if "pattern" in schema and not re.search(schema["pattern"], value):
113
- errors.append(f"{path}: {value!r} does not match pattern {schema['pattern']}")
114
- if schema.get("format") == "date-time" and not _DATE_TIME.match(value):
115
- errors.append(f"{path}: {value!r} is not a valid date-time")
116
- if "minLength" in schema and len(value) < schema["minLength"]:
117
- errors.append(f"{path}: shorter than minLength {schema['minLength']}")
118
- if "maxLength" in schema and len(value) > schema["maxLength"]:
119
- errors.append(f"{path}: longer than maxLength {schema['maxLength']}")
120
-
121
- if isinstance(value, (int, float)) and not isinstance(value, bool):
122
- if "minimum" in schema and value < schema["minimum"]:
123
- errors.append(f"{path}: {value} < minimum {schema['minimum']}")
124
- if "maximum" in schema and value > schema["maximum"]:
125
- errors.append(f"{path}: {value} > maximum {schema['maximum']}")
126
-
127
- if isinstance(value, list):
128
- if "minItems" in schema and len(value) < schema["minItems"]:
129
- errors.append(f"{path}: fewer than minItems {schema['minItems']}")
130
- if "maxItems" in schema and len(value) > schema["maxItems"]:
131
- errors.append(f"{path}: more than maxItems {schema['maxItems']}")
132
- if "items" in schema:
133
- for i, item in enumerate(value):
134
- _validate(item, schema["items"], f"{path}[{i}]", errors)
135
-
136
- if isinstance(value, dict):
137
- for key in schema.get("required", []):
138
- if key not in value:
139
- errors.append(f"{path}: missing required property '{key}'")
140
- properties = schema.get("properties", {})
141
- for key, sub in properties.items():
142
- if key in value:
143
- _validate(value[key], sub, f"{path}.{key}", errors)
144
- if schema.get("additionalProperties") is False:
145
- for key in value:
146
- if key not in properties:
147
- errors.append(f"{path}: additional property '{key}' is not allowed")
148
-
149
- # Applicators last: the invariant layer. `additionalProperties` deliberately
150
- # does not see properties introduced by these subschemas (same rule as
151
- # JSON Schema 2020-12, and the same as the JavaScript twin).
152
- for sub in schema.get("allOf", []):
153
- _validate(value, sub, path, errors)
154
-
155
- if "if" in schema:
156
- branch = schema.get("then") if _matches(value, schema["if"]) else schema.get("else")
157
- if branch is not None:
158
- _validate(value, branch, path, errors)