qa-engineer 0.9.0

package/shared/analysis/lib/qa_analysis/contracts.py

@@ -0,0 +1,158 @@
+"""Deterministic contract validation.
+
+A dependency-free validator for the subset of JSON Schema the pack's contracts
+use. It exists so no skill ever emits a result that violates its own contract,
+and so CI can gate on it without a third-party dependency.
+
+The supported subset is declared once, in `SUPPORTED_KEYWORDS`, and is the
+**same** subset the JavaScript twin implements
+(`packages/installer/lib/core/schema-validate.mjs`). Two rules keep that promise
+honest:
+
+1. A keyword outside the subset is an **error**, never a silent no-op — a
+   contract author cannot accidentally write a constraint that looks enforced
+   and isn't.
+2. `tests/parity/validator-cases.json` runs through both validators and both
+   must agree on every case.
+
+The subset includes `allOf` / `if` / `then` / `else` because the pack's safety
+invariants are cross-field implications ("classification `passed` implies exit
+code 0"). Those must be enforced at runtime by the contract itself, not only by
+evaluation fixtures.
+
+This validates the pack's own schemas; it is not a general-purpose JSON Schema
+implementation, and it says so rather than pretending to be one.
+"""
+
+import json
+import re
+
+_TYPES = {
+    "object": dict,
+    "array": list,
+    "string": str,
+    "number": (int, float),
+    "integer": int,
+    "boolean": bool,
+    "null": type(None),
+}
+
+# The complete supported subset. Anything else is reported as a schema error.
+# Keep in sync with SUPPORTED in packages/installer/lib/core/schema-validate.mjs
+# and the table in docs/skills/output-contracts.md (checked by
+# scripts/check-spec-code-sync.mjs).
+SUPPORTED_KEYWORDS = frozenset({
+    "$schema", "$id", "title", "description", "type", "properties", "required",
+    "additionalProperties", "items", "enum", "const", "pattern", "minimum",
+    "maximum", "minItems", "maxItems", "minLength", "maxLength", "default",
+    "examples", "format", "allOf", "if", "then", "else",
+})
+
+# RFC 3339 date-time. Defined here (rather than delegating to
+# datetime.fromisoformat, which also accepts date-only strings and varies by
+# interpreter version) so both validators accept exactly the same set.
+_DATE_TIME = re.compile(
+    r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(\.\d+)?([Zz]|[+-]\d{2}:\d{2})$"
+)
+
+
+def load_schema(path):
+    with open(path, "r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def validate(instance, schema, path="$"):
+    """Validate instance against schema. Returns (ok, errors) where errors is a
+    list of human-readable "path: problem" strings."""
+    errors = []
+    _validate(instance, schema, path, errors)
+    return (not errors, errors)
+
+
+def _type_ok(value, expected):
+    types = expected if isinstance(expected, list) else [expected]
+    for name in types:
+        py = _TYPES.get(name)
+        if py is None:
+            continue
+        # bool is a subclass of int; keep them distinct.
+        if name == "integer" and isinstance(value, bool):
+            continue
+        if name == "number" and isinstance(value, bool):
+            continue
+        if isinstance(value, py):
+            return True
+    return False
+
+
+def _matches(value, schema):
+    """True when value satisfies schema. Used for if/then branch selection, so
+    it must not leak errors into the caller's list."""
+    probe = []
+    _validate(value, schema, "$", probe)
+    return not probe
+
+
+def _validate(value, schema, path, errors):
+    for keyword in schema:
+        if keyword not in SUPPORTED_KEYWORDS:
+            errors.append(f"{path}: schema uses unsupported keyword \"{keyword}\"")
+
+    if "type" in schema and not _type_ok(value, schema["type"]):
+        errors.append(f"{path}: expected type {schema['type']}, got {type(value).__name__}")
+        return  # further checks assume the type held
+
+    if "const" in schema and value != schema["const"]:
+        errors.append(f"{path}: expected const {schema['const']!r}, got {value!r}")
+
+    if "enum" in schema and value not in schema["enum"]:
+        errors.append(f"{path}: {value!r} is not one of {schema['enum']}")
+
+    if isinstance(value, str):
+        if "pattern" in schema and not re.search(schema["pattern"], value):
+            errors.append(f"{path}: {value!r} does not match pattern {schema['pattern']}")
+        if schema.get("format") == "date-time" and not _DATE_TIME.match(value):
+            errors.append(f"{path}: {value!r} is not a valid date-time")
+        if "minLength" in schema and len(value) < schema["minLength"]:
+            errors.append(f"{path}: shorter than minLength {schema['minLength']}")
+        if "maxLength" in schema and len(value) > schema["maxLength"]:
+            errors.append(f"{path}: longer than maxLength {schema['maxLength']}")
+
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        if "minimum" in schema and value < schema["minimum"]:
+            errors.append(f"{path}: {value} < minimum {schema['minimum']}")
+        if "maximum" in schema and value > schema["maximum"]:
+            errors.append(f"{path}: {value} > maximum {schema['maximum']}")
+
+    if isinstance(value, list):
+        if "minItems" in schema and len(value) < schema["minItems"]:
+            errors.append(f"{path}: fewer than minItems {schema['minItems']}")
+        if "maxItems" in schema and len(value) > schema["maxItems"]:
+            errors.append(f"{path}: more than maxItems {schema['maxItems']}")
+        if "items" in schema:
+            for i, item in enumerate(value):
+                _validate(item, schema["items"], f"{path}[{i}]", errors)
+
+    if isinstance(value, dict):
+        for key in schema.get("required", []):
+            if key not in value:
+                errors.append(f"{path}: missing required property '{key}'")
+        properties = schema.get("properties", {})
+        for key, sub in properties.items():
+            if key in value:
+                _validate(value[key], sub, f"{path}.{key}", errors)
+        if schema.get("additionalProperties") is False:
+            for key in value:
+                if key not in properties:
+                    errors.append(f"{path}: additional property '{key}' is not allowed")
+
+    # Applicators last: the invariant layer. `additionalProperties` deliberately
+    # does not see properties introduced by these subschemas (same rule as
+    # JSON Schema 2020-12, and the same as the JavaScript twin).
+    for sub in schema.get("allOf", []):
+        _validate(value, sub, path, errors)
+
+    if "if" in schema:
+        branch = schema.get("then") if _matches(value, schema["if"]) else schema.get("else")
+        if branch is not None:
+            _validate(value, branch, path, errors)

package/shared/analysis/lib/qa_analysis/diff_guard.py

@@ -0,0 +1,327 @@
+"""The diff guard: a deterministic protection layer against unsafe "fixes".
+
+Given a unified diff of test code, it flags changes that make a suite pass
+without proving the software works. Each flag explains why it is unsafe. It
+never edits; it only judges.
+
+The guard has two jobs, and they pull against each other:
+
+1. **Catch every way a suite is made to lie.** Deleting or weakening
+   assertions, skipping, forcing a pass, excluding specs from the run, making the
+   test command always exit zero, swallowing failures, inflating timeouts,
+   deleting test files.
+2. **Not cry wolf on real repairs.** Healing a stale locator *is* the job of
+   `/qa-fix`, and it necessarily rewrites an assertion line. A guard that flags
+   every legitimate repair as `high` trains the agent — and the human — to
+   override it, which is worse than having no guard.
+
+Job 2 is why assertions are compared by **strength** rather than by presence. An
+assertion replaced by one that is at least as strong, keeping the same expected
+values, is a modification worth confirming (`low`) — not a removal (`high`). An
+assertion replaced by a weaker one, or one that drops the expected value, is
+`weakened-assertion` (`high`), because that is how a test quietly stops checking
+anything.
+"""
+
+import re
+
+# --- assertion recognition ---------------------------------------------------
+
+_ASSERTION = re.compile(r"(?i)\b(expect|assert|should|toBe|toEqual|toHaveText|toBeVisible|toContain|assertThat)\b")
+
+# Matchers that pin a specific value or state. Replacing one of these with a
+# weaker matcher means the test no longer checks what it used to.
+_STRONG_MATCHERS = re.compile(
+    r"(?i)\b(toBe|toEqual|toStrictEqual|toHaveText|toContainText|toHaveValue|toHaveURL|"
+    r"toHaveTitle|toHaveCount|toHaveLength|toHaveAttribute|toHaveClass|toHaveScreenshot|"
+    r"toMatch|toMatchObject|toMatchSnapshot|toBeVisible|toBeChecked|toBeEnabled|toBeDisabled|"
+    r"toBeEmpty|toBeGreaterThan|toBeLessThan|toBeCloseTo|"
+    r"assertEqual|assertEquals|assertIn|assertTrue|assertFalse|isEqualTo|containsExactly)\b"
+)
+
+# Matchers that only prove something exists or is loosely truthy.
+_WEAK_MATCHERS = re.compile(
+    r"(?i)\b(toBeDefined|toBeUndefined|toBeTruthy|toBeFalsy|toBeNull|toBeNaN|toBeAttached|"
+    r"toBeOk|assertIsNotNone|assertIsNone|isNotNull|isNotEmpty|notNull|toBeInstanceOf)\b"
+)
+
+# A soft assertion does not stop the test at the point of failure; swapping a
+# hard assertion for a soft one weakens it.
+_SOFT_ASSERTION = re.compile(r"(?i)\b(expect\.soft|softAssert|assertSoftly)\b")
+
+# String and numeric literals — the expected values an assertion pins down.
+_STRING_LITERAL = re.compile(r"""(['"])((?:(?!\1).){2,})\1""")
+_NUMBER_LITERAL = re.compile(r"(?<![\w.])\d+(?:\.\d+)?(?![\w.])")
+
+_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+# --- unsafe-change patterns --------------------------------------------------
+
+_SKIP_ADDED = re.compile(
+    r"(?i)(\.skip\b|\.fixme\b|\bxit\b|\bxdescribe\b|test\.skip|it\.skip|describe\.skip|"
+    r"@pytest\.mark\.skip|@Ignore\b|@Disabled\b|\.only\b|this\.skip\(|t\.Skip\()"
+)
+_FORCED_PASS = re.compile(
+    r"(?i)(assert\s+True\b|expect\(true\)\.toBe\(true\)|expect\(1\)\.toBe\(1\)|"
+    r"return\s*;?\s*//\s*pass|assert\s+1\s*==\s*1|pass\s*#\s*(?:todo|skip|pass))"
+)
+
+# A bare early return inside a test body ends the test before it verifies
+# anything — the quietest way to fake a pass. Only value-free returns count, so
+# `return page.click(...)` and `return await expect(...)` stay clean.
+_EARLY_RETURN = re.compile(r"^\s*(?:if\s*\(.*?\)\s*\{?\s*)?return\s*;?\s*\}?\s*$")
+
+# Excluding specs from the run drops coverage without touching a test file.
+_SUITE_EXCLUSION = re.compile(
+    r"(?i)(testIgnore|testPathIgnorePatterns|excludeSpecPattern|"
+    r"exclude\s*[:=]|ignorePatterns|--grep-invert|--ignore-pattern|"
+    r"specs?\s*:\s*\[\s*\]|testMatch\s*[:=]\s*\[\s*\])"
+)
+
+# Making the test command exit zero regardless of the result.
+_FORCED_PASS_COMMAND = re.compile(
+    r"(?i)(\|\|\s*true\b|\|\|\s*exit\s+0\b|;\s*exit\s+0\b|--passWithNoTests\b|"
+    r"--pass-with-no-tests\b|continue-on-error\s*:\s*true|set\s+\+e\b|"
+    r"-DskipTests\b|--exit-zero\b|\|\|\s*:)"
+)
+
+# A catch block that discards the failure.
+_SWALLOWED_FAILURE = re.compile(r"(?i)(catch\s*(?:\([^)]*\))?\s*\{\s*\}|except\s*[\w.]*\s*:\s*pass\b)")
+_CATCH_ADDED = re.compile(r"(?i)(\bcatch\s*[({]|^\s*except\b)")
+
+_TIMEOUT = re.compile(r"(?i)(timeout|setTimeout|wait_?for|implicitly_wait|setDefaultTimeout)\D{0,20}?(\d{3,})")
+# Deliberately excludes `expect(`: an expectation is an assertion, not a wait,
+# and including it made this rule unreachable.
+_WAIT = re.compile(r"(?i)(waitFor\w*|wait_?for\w*|\.wait\(|awaitVisible|implicitly_wait)")
+_LOCATOR = re.compile(r"""(?i)(getBy\w+|locator|find_element|\$\(|css=|xpath=|By\.\w+)""")
+_EMPTY_BODY = re.compile(r"(?i)(async\s+)?(\([^)]*\)\s*=>\s*\{\s*\}|function[^{]*\{\s*\}|def\s+test\w*\([^)]*\):\s*pass)")
+_RETRY = re.compile(r"(?i)(retries?\s*[:=]\s*(\d+)|\.retry\((\d+)\))")
+
+_TEST_FILE = re.compile(r"(?i)(\.spec\.|\.test\.|_test\.|test_|/tests?/|/e2e/|\.feature$|Test\.java$|Tests\.cs$)")
+
+MASS_DELETION_THRESHOLD = 15
+
+# How much token overlap makes an added assertion the counterpart of a removed
+# one. Tuned so a locator rewrite on the same expectation pairs up, while two
+# unrelated assertions in the same file do not.
+_COUNTERPART_THRESHOLD = 0.34
+
+
+def _parse_diff(diff_text):
+    """Yield (hunk_file, sign, text) for each changed line. sign is '+' or '-'."""
+    current = "unknown"
+    for line in diff_text.splitlines():
+        if line.startswith("+++ ") or line.startswith("--- "):
+            path = line[4:].strip()
+            if path not in ("/dev/null", ""):
+                current = re.sub(r"^[ab]/", "", path)
+            continue
+        if line.startswith("@@"):
+            continue
+        if line.startswith("+") and not line.startswith("+++"):
+            yield (current, "+", line[1:])
+        elif line.startswith("-") and not line.startswith("---"):
+            yield (current, "-", line[1:])
+
+
+def _deleted_files(diff_text):
+    """Files whose new side is /dev/null — deleted outright."""
+    deleted, pending = set(), None
+    for line in diff_text.splitlines():
+        if line.startswith("--- "):
+            path = line[4:].strip()
+            pending = re.sub(r"^[ab]/", "", path) if path != "/dev/null" else None
+        elif line.startswith("+++ "):
+            if line[4:].strip() == "/dev/null" and pending:
+                deleted.add(pending)
+            pending = None
+    return deleted
+
+
+def _tokens(text):
+    return {t.lower() for t in _IDENTIFIER.findall(text)}
+
+
+def _literals(text):
+    strings = {m.group(2) for m in _STRING_LITERAL.finditer(text)}
+    numbers = set(_NUMBER_LITERAL.findall(text))
+    return strings, numbers
+
+
+def _strength(text):
+    """3 = pins a value/state, 2 = unclassified assertion, 1 = existence only."""
+    if _SOFT_ASSERTION.search(text):
+        return 1
+    if _WEAK_MATCHERS.search(text) and not _STRONG_MATCHERS.search(text):
+        return 1
+    if _STRONG_MATCHERS.search(text):
+        return 3
+    return 2
+
+
+def _counterpart(removed_text, candidates):
+    """The added assertion line most plausibly replacing this removed one."""
+    removed_tokens = _tokens(removed_text)
+    if not removed_tokens:
+        return None
+    best, best_score = None, 0.0
+    for text in candidates:
+        overlap = removed_tokens & _tokens(text)
+        score = len(overlap) / max(len(removed_tokens), 1)
+        if score > best_score:
+            best, best_score = text, score
+    return best if best_score >= _COUNTERPART_THRESHOLD else None
+
+
+def check_diff(diff_text):
+    """Analyze a unified diff. Returns a list of issue dicts:
+    {rule, severity, file, why, sample}. An empty list means no unsafe change
+    was detected (which is not a guarantee of correctness, only of safety)."""
+    issues = []
+    removed = [(f, t) for f, s, t in _parse_diff(diff_text) if s == "-"]
+    added = [(f, t) for f, s, t in _parse_diff(diff_text) if s == "+"]
+    deleted_files = _deleted_files(diff_text)
+
+    def flag(rule, severity, file, why, sample):
+        issues.append({"rule": rule, "severity": severity, "file": file,
+                       "why": why, "sample": str(sample).strip()[:200]})
+
+    added_by_file = {}
+    for file, text in added:
+        added_by_file.setdefault(file, []).append(text)
+
+    # --- removed or weakened assertions --------------------------------------
+    for file, text in removed:
+        if _ASSERTION.search(text):
+            same_file_added = added_by_file.get(file, [])
+            # An identical line re-added is a move, not a change.
+            if any(a.strip() == text.strip() for a in same_file_added):
+                continue
+
+            assertion_candidates = [a for a in same_file_added if _ASSERTION.search(a)]
+            replacement = _counterpart(text, assertion_candidates)
+
+            if replacement is None:
+                if file in deleted_files:
+                    continue  # reported once as a deleted test file, below
+                flag("removed-assertion", "high", file,
+                     "An assertion or expectation was removed with nothing replacing it; "
+                     "a test that no longer asserts proves nothing.", text)
+                continue
+
+            old_strength, new_strength = _strength(text), _strength(replacement)
+            old_strings, old_numbers = _literals(text)
+            new_strings, new_numbers = _literals(replacement)
+            dropped_strings = old_strings - new_strings
+            dropped_numbers = old_numbers - new_numbers
+
+            if new_strength < old_strength:
+                flag("weakened-assertion", "high", file,
+                     "An assertion was replaced by a weaker one (existence or truthiness "
+                     "instead of a specific value or state); the test can now pass without "
+                     "the behaviour being correct.", replacement)
+            elif dropped_strings or dropped_numbers:
+                dropped = sorted(dropped_strings | dropped_numbers)
+                flag("weakened-assertion", "high", file,
+                     "An assertion kept its matcher but dropped the expected value(s) "
+                     f"{dropped}; it no longer pins down what it used to.", replacement)
+            else:
+                flag("assertion-modified", "low", file,
+                     "An assertion was rewritten with equal or greater strength and the same "
+                     "expected values — consistent with a legitimate repair. Confirm it still "
+                     "targets the same behaviour.", replacement)
+
+        if _WAIT.search(text) and not _ASSERTION.search(text):
+            flag("removed-wait", "medium", file,
+                 "A wait/synchronization was removed, which can mask a real failure or "
+                 "introduce flakiness.", text)
+
+    # --- added skips, forced passes, early returns, empty bodies -------------
+    for file, text in added:
+        if _SKIP_ADDED.search(text):
+            flag("added-skip-or-only", "high", file,
+                 "A skip/ignore/only marker was added; skipping or narrowing hides failures "
+                 "instead of fixing them.", text)
+        if _FORCED_PASS.search(text):
+            flag("forced-pass", "high", file,
+                 "A tautological or forced-pass assertion was added; it makes the test pass "
+                 "without checking behavior.", text)
+        if _EMPTY_BODY.search(text):
+            flag("empty-test-body", "high", file,
+                 "A test body was emptied; an empty test passes vacuously.", text)
+        if _EARLY_RETURN.match(text) and _TEST_FILE.search(file):
+            flag("conditional-skip", "high", file,
+                 "An unconditional or condition-guarded early return was added to a test; "
+                 "the test exits before verifying anything, which is a skip in disguise.", text)
+        if _SUITE_EXCLUSION.search(text):
+            flag("suite-exclusion", "high", file,
+                 "Specs were excluded from the run (ignore pattern, exclusion list, or empty "
+                 "spec list); coverage is dropped without any test file changing.", text)
+        if _FORCED_PASS_COMMAND.search(text):
+            flag("forced-pass-command", "high", file,
+                 "The test command was made to succeed regardless of the result "
+                 "(`|| true`, `exit 0`, `--passWithNoTests`, `continue-on-error`); the suite "
+                 "can no longer fail a pipeline.", text)
+        if _SWALLOWED_FAILURE.search(text):
+            flag("swallowed-failure", "high", file,
+                 "A failure is caught and discarded; an assertion inside a swallowed catch "
+                 "cannot fail the test.", text)
+        elif _CATCH_ADDED.search(text) and _TEST_FILE.search(file):
+            flag("added-error-handling", "medium", file,
+                 "Error handling was added inside a test; confirm the failure still surfaces "
+                 "rather than being caught and logged.", text)
+
+    # --- deleted test files --------------------------------------------------
+    for file in sorted(deleted_files):
+        if _TEST_FILE.search(file):
+            flag("test-file-deleted", "high", file,
+                 "A test file was deleted outright; every case it held stops running, "
+                 "regardless of the file's size.", file)
+
+    # --- timeout inflation and unsafe retry increases -------------------------
+    _flag_numeric_inflation(_TIMEOUT, removed, added, "timeout-inflation", "medium",
+                            "A timeout was increased substantially, which can paper over a real slowness or hang.", flag)
+    _flag_numeric_inflation(_RETRY, removed, added, "unsafe-retry-increase", "medium",
+                            "Retry count was increased, which can convert a real failure into an intermittent pass.", flag)
+
+    # --- suspicious locator changes ------------------------------------------
+    removed_locators = {f: t for f, t in removed if _LOCATOR.search(t)}
+    for file, text in added:
+        if _LOCATOR.search(text) and file in removed_locators and text.strip() != removed_locators[file].strip():
+            flag("suspicious-locator-change", "low", file,
+                 "A locator was changed; confirm it targets the same element and was not "
+                 "loosened to pass.", text)
+
+    # --- mass deletion per file ----------------------------------------------
+    per_file = {}
+    for file, _ in removed:
+        per_file[file] = per_file.get(file, 0) + 1
+    for file, count in per_file.items():
+        if count >= MASS_DELETION_THRESHOLD and file not in deleted_files:
+            flag("mass-deletion", "high", file,
+                 f"{count} lines were deleted from a test file; large deletions can silently drop coverage.",
+                 f"{count} lines removed")
+
+    return issues
+
+
+def _flag_numeric_inflation(pattern, removed, added, rule, severity, why, flag):
+    def max_number(text):
+        nums = [int(n) for n in re.findall(r"\d{2,}", text)]
+        return max(nums) if nums else None
+
+    removed_by_file = {}
+    for file, text in removed:
+        if pattern.search(text):
+            value = max_number(text)
+            if value is not None:
+                removed_by_file.setdefault(file, []).append(value)
+    for file, text in added:
+        if pattern.search(text):
+            new_value = max_number(text)
+            if new_value is None:
+                continue
+            for old_value in removed_by_file.get(file, []):
+                if new_value >= old_value * 2 and new_value > old_value:
+                    flag(rule, severity, file, why, text)
+                    break

package/shared/analysis/lib/qa_analysis/discovery.py

@@ -0,0 +1,113 @@
+"""Deterministic artifact discovery.
+
+Locates the artifacts a run produced — automatically by convention or from
+explicit paths — and classifies each as present, partial, or corrupted. It
+handles multiple runs, parallel workers, and sharded output (many result files),
+and it reports what is missing rather than inventing it.
+"""
+
+import json
+import os
+import zipfile
+from glob import glob
+
+from .evidence import Artifact
+
+# Convention globs for known artifact types, relative to a run root.
+_PATTERNS = [
+    ("junit", ["**/results.xml", "**/junit*.xml", "**/*junit*.xml"]),
+    ("report", ["**/results.json", "**/report.json"]),
+    ("html-report", ["**/playwright-report/index.html", "**/*-report/index.html"]),
+    ("trace", ["**/trace.zip", "**/*-trace.zip"]),
+    ("har", ["**/*.har"]),
+    ("video", ["**/*.webm", "**/*.mp4"]),
+    ("screenshot", ["**/*.png", "**/*-actual.png"]),
+]
+
+
+def _integrity(artifact_type, path):
+    """Classify an artifact as present, partial (empty), or corrupted."""
+    try:
+        size = os.path.getsize(path)
+    except OSError:
+        return "missing"
+    if size == 0:
+        return "partial"
+    # Structural spot-checks for formats we can cheaply verify.
+    if artifact_type == "trace":
+        if not zipfile.is_zipfile(path):
+            return "corrupted"
+    elif artifact_type in ("report", "har"):
+        try:
+            with open(path, "r", encoding="utf-8") as handle:
+                json.load(handle)
+        except (json.JSONDecodeError, OSError, UnicodeDecodeError):
+            return "corrupted"
+    return "present"
+
+
+def discover(root=".", explicit=None, framework="unknown"):
+    """Discover artifacts under root, or validate explicit paths.
+
+    Returns {present: [Artifact...], partial: [...], corrupted: [...],
+    missing: [paths]}. Explicit paths that do not exist are reported as missing;
+    convention discovery never reports missing (absence is simply not found).
+    """
+    present, partial, corrupted, missing = [], [], [], []
+
+    def record(artifact_type, path):
+        state = _integrity(artifact_type, path)
+        artifact = Artifact(type=artifact_type, location=os.path.relpath(path, root)
+                            if os.path.isabs(path) else path, framework=framework,
+                            present=(state == "present"))
+        if state == "present":
+            present.append(artifact)
+        elif state == "partial":
+            partial.append(artifact)
+        elif state == "corrupted":
+            corrupted.append(artifact)
+        else:
+            missing.append(path)
+
+    if explicit:
+        for path in explicit:
+            if not os.path.exists(path):
+                missing.append(path)
+                continue
+            matched = next((t for t, _ in _PATTERNS if _matches_type(t, path)), "attachment")
+            record(matched, path)
+    else:
+        seen = set()
+        for artifact_type, patterns in _PATTERNS:
+            for pattern in patterns:
+                for path in sorted(glob(os.path.join(root, pattern), recursive=True)):
+                    if path in seen:
+                        continue
+                    seen.add(path)
+                    record(artifact_type, path)
+
+    return {
+        "present": present,
+        "partial": partial,
+        "corrupted": corrupted,
+        "missing": missing,
+    }
+
+
+def _matches_type(artifact_type, path):
+    lower = path.lower()
+    if artifact_type == "junit":
+        return lower.endswith(".xml")
+    if artifact_type == "report":
+        return lower.endswith(".json")
+    if artifact_type == "trace":
+        return lower.endswith(".zip")
+    if artifact_type == "har":
+        return lower.endswith(".har")
+    if artifact_type == "video":
+        return lower.endswith((".webm", ".mp4"))
+    if artifact_type == "screenshot":
+        return lower.endswith(".png")
+    if artifact_type == "html-report":
+        return lower.endswith(".html")
+    return False