ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/validators/__init__.py

@@ -0,0 +1,419 @@
+"""Role-output validator registry and `validate` dispatcher.
+
+Each per-role validator lives in a sibling module (e.g.,
+`ref_agents.tools.validators.product_manager`) and registers itself via
+the `register_validator` decorator. Foundation only — STORY-021 ships
+the registry and dispatcher; per-role validators arrive in FEAT-004..007.
+
+Contract:
+    validate(role, artifact_id, artifact_path) -> ValidationReport
+
+Unknown role / missing module / unexpected exception → FAIL_AUTO with a
+machine-readable `Gap.code`. Dispatcher never raises.
+"""
+
+import hashlib
+from collections.abc import Callable
+from pathlib import Path
+
+import structlog
+
+from ref_agents.core.validation_models import (
+    Gap,
+    ValidationReport,
+    Verdict,
+)
+
+logger = structlog.get_logger(__name__)
+
+ValidatorFn = Callable[[str, str], ValidationReport]
+VALIDATOR_REGISTRY: dict[str, ValidatorFn] = {}
+
+# v3.13.1: roles allowed to phase-close without a story id. Narrower than
+# constants.STORY_EXEMPT_AGENTS — discovery + platform_engineer have
+# project-root-scoped validators that must always run.
+GATE_EXEMPT_WITHOUT_STORY: frozenset[str] = frozenset({"product_manager", "strategist"})
+
+# v3.13.2: surface validator-module load failures so silent fail-open does
+# not mask a broken validator. Populated by `_autoload_validators` at
+# server startup; readable via `validate(role="__health__")`.
+VALIDATOR_LOAD_FAILURES: list[tuple[str, str]] = []
+HEALTH_ROLE = "__health__"
+
+# Gap 9: gate-firing metrics (verdict counts per role).
+GATE_METRICS: dict[str, dict[str, int]] = {}
+
+# Gap 10: per-(role, artifact_hash) report cache (LRU-bounded).
+_REPORT_CACHE: dict[tuple[str, str], ValidationReport] = {}
+_CACHE_MAX = 256
+
+
+def _cache_get(role: str, ahash: str) -> ValidationReport | None:
+    if not ahash:
+        return None
+    return _REPORT_CACHE.get((role, ahash))
+
+
+def _cache_put(role: str, ahash: str, report: ValidationReport) -> None:
+    if not ahash:
+        return
+    if len(_REPORT_CACHE) >= _CACHE_MAX:
+        # Drop oldest entry (FIFO)
+        _REPORT_CACHE.pop(next(iter(_REPORT_CACHE)))
+    _REPORT_CACHE[(role, ahash)] = report
+
+
+def _record_metric(role: str, verdict: str) -> None:
+    bucket = GATE_METRICS.setdefault(role, {})
+    bucket[verdict] = bucket.get(verdict, 0) + 1
+
+
+def register_validator(role: str) -> Callable[[ValidatorFn], ValidatorFn]:
+    """Register a per-role validator function.
+
+    Usage:
+        @register_validator("product_manager")
+        def validate_story_quality(artifact_id: str, artifact_path: str) -> ValidationReport:
+            ...
+
+    Args:
+        role: Role name (must match `AVAILABLE_AGENTS` key).
+
+    Returns:
+        Decorator that registers the function in `VALIDATOR_REGISTRY`.
+    """
+
+    def decorator(fn: ValidatorFn) -> ValidatorFn:
+        VALIDATOR_REGISTRY[role] = fn
+        return fn
+
+    return decorator
+
+
+def _artifact_hash(artifact_path: str) -> str:
+    """Compute sha256 of artifact bytes. Empty path → ''.
+
+    Args:
+        artifact_path: Path string; may be empty.
+
+    Returns:
+        Hex digest, or "" when path is empty or unreadable.
+    """
+    if not artifact_path:
+        return ""
+    try:
+        return hashlib.sha256(Path(artifact_path).read_bytes()).hexdigest()
+    except OSError:
+        return ""
+
+
+def validate(
+    role: str, artifact_id: str = "", artifact_path: str = ""
+) -> ValidationReport:
+    """Dispatch a role-output validation pass.
+
+    Args:
+        role: Role name; routes to `VALIDATOR_REGISTRY[role]`.
+        artifact_id: Story / artifact identifier (e.g., "STORY-021").
+        artifact_path: Optional explicit path to artifact on disk.
+
+    Returns:
+        ValidationReport. Unknown role / load failure / unexpected
+        exception → FAIL_AUTO with a structured Gap. Never raises.
+    """
+    log = logger.bind(role=role, artifact_id=artifact_id)
+
+    # v3.13.2: surface registry health for operators / CI.
+    if role == HEALTH_ROLE:
+        return _health_report()
+
+    fn = VALIDATOR_REGISTRY.get(role)
+    if fn is None:
+        log.info("validate_unknown_role")
+        report = ValidationReport(
+            verdict=Verdict.FAIL_AUTO,
+            gaps=[
+                Gap(
+                    code="unknown_role",
+                    detail=f"No validator registered for role '{role}'.",
+                )
+            ],
+        )
+        _record_metric(role, report.verdict.value)
+        return report
+
+    ahash = _artifact_hash(artifact_path)
+    cached = _cache_get(role, ahash)
+    if cached is not None:
+        log.info("validate_cache_hit", verdict=cached.verdict.value)
+        _record_metric(role, cached.verdict.value)
+        return cached
+
+    try:
+        report = fn(artifact_id, artifact_path)
+    except Exception as exc:  # noqa: BLE001 - dispatcher must never raise
+        log.exception("validator_exception", error=str(exc))
+        report = ValidationReport(
+            verdict=Verdict.FAIL_AUTO,
+            gaps=[
+                Gap(
+                    code="validator_exception",
+                    detail=f"{type(exc).__name__}: {exc}",
+                )
+            ],
+        )
+        _record_metric(role, report.verdict.value)
+        return report
+
+    _cache_put(role, ahash, report)
+    _record_metric(role, report.verdict.value)
+    log.info(
+        "validate_complete",
+        verdict=report.verdict.value,
+        gap_count=len(report.gaps),
+    )
+    return report
+
+
+def get_gate_metrics() -> dict[str, dict[str, int]]:
+    """Return per-role verdict counts since process start (Gap 9)."""
+    return {role: dict(counts) for role, counts in GATE_METRICS.items()}
+
+
+def clear_validator_cache() -> None:
+    """Drop the per-(role, hash) cache. Used by tests and live re-runs."""
+    _REPORT_CACHE.clear()
+
+
+def phase_gate(
+    role: str, story_id: str, artifact_path: str = ""
+) -> tuple[bool, str, ValidationReport | None]:
+    """Centralised G4 phase-close gate used by log() and state_machine.
+
+    Encapsulates: story-exempt skip (Gap 5), per-story lock (Gap 7), retry
+    counter + escalation (Gap 2), structured NEEDS_USER payload (Gap 3),
+    and audit-log emission on block (Gap 6).
+
+    Args:
+        role: Phase / role name.
+        story_id: Active story (may be empty for story-exempt agents).
+        artifact_path: Optional explicit artifact path; resolves through
+            validator if blank.
+
+    Returns:
+        (allowed, message, report) where:
+          allowed=True  → caller proceeds (message empty)
+          allowed=False → caller returns message verbatim to user.
+        `report` is the underlying ValidationReport when one was produced.
+    """
+    from ref_agents.session import SessionManager
+    from ref_agents.utils import handoff_logger as _hl
+
+    # Gap 5 (narrowed in v3.13.1): only PM + strategist legitimately need
+    # to close phases without a story id (they may be creating epics/features
+    # or producing strategy docs that have no story-scoped artifact yet).
+    # Discovery + platform_engineer are story-exempt at the agent layer but
+    # their validators check project-root-scoped artifacts (codemap/,
+    # health_report.json) and MUST run on every phase close.
+    if role in GATE_EXEMPT_WITHOUT_STORY and not story_id:
+        return True, "", None
+
+    # Skip silently when role has no registered validator.
+    if role not in VALIDATOR_REGISTRY:
+        return True, "", None
+
+    session = SessionManager.get()
+
+    # Gap 7: per-story lock prevents concurrent G4 runs and audit dupes.
+    lock_key = story_id or f"__role:{role}"
+    with session.story_lock(lock_key):
+        report = validate(role=role, artifact_id=story_id, artifact_path=artifact_path)
+
+        if report.verdict == Verdict.PASS:
+            return True, "", report
+
+        # Pass-through when validator cannot resolve its input artifact.
+        # This avoids breaking flows where the artifact lives outside the
+        # validator's default search path (e.g., unit-test fixtures, partial
+        # rollouts). Callers can still see the gap in `report` if they want
+        # to enforce stricter behavior themselves.
+        if (
+            report.verdict == Verdict.NEEDS_USER
+            and report.gaps
+            and all(
+                g.code.endswith("_unresolved") or g.code == "no_project_root"
+                for g in report.gaps
+            )
+        ):
+            return True, "", report
+
+        # Gap 11: validator_version is on the report; surface in audit log.
+        # Gap 6: write handoff_log escalation entry before blocking.
+        try:
+            _hl.log_escalation(
+                level="L1",
+                from_agent=role,
+                to_agents=["user"],
+                reason=(
+                    f"validator_blocked verdict={report.verdict.value} "
+                    f"gaps={[g.code for g in report.gaps][:5]}"
+                ),
+                story_id=story_id or None,
+            )
+        except Exception as exc:  # noqa: BLE001 — audit best-effort
+            logger.warning("gate_audit_log_failed", error=str(exc))
+
+        if report.verdict == Verdict.HALT:
+            session.set_halt(story_id or role)
+            return (
+                False,
+                (
+                    f"🛑 Phase close blocked (HALT) for role={role}: "
+                    f"{report.halt_reason or 'validator HALT'}\n"
+                    f"Clear via session.clear_halt({story_id!r} or role)."
+                ),
+                report,
+            )
+
+        # v3.13.2: no supervision escape valve. FAIL_AUTO/NEEDS_USER block
+        # unconditionally; standards hold.
+
+        # Gap 2: retry counter + auto-escalation after N=2 failures.
+        ahash = _artifact_hash(artifact_path)
+        count = session.increment_retry_count(story_id or role, role, ahash)
+        escalate = count >= 2  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+
+        gap_lines = "\n".join(f"  - [{g.code}] {g.detail}" for g in report.gaps[:10])
+        q_lines = "\n".join(
+            f"  ?[{q.text}]" + (f" options={q.options}" if q.options else "")
+            for q in report.user_questions[:5]
+        )
+
+        # Gap 3: structured marker so caller agents can route NEEDS_USER
+        # to AskUserQuestion. The "USER_QUESTIONS:" sentinel is documented
+        # in the LOGGING_PROTOCOL prompt.
+        marker = (
+            "\nUSER_QUESTIONS_REQUIRED: agent MUST surface questions via "
+            "AskUserQuestion before retry.\n"
+            if report.verdict == Verdict.NEEDS_USER and report.user_questions
+            else ""
+        )
+        escalation_note = (
+            f"\n↗️ Retry threshold ({count}/2) reached — escalate to user."
+            if escalate
+            else ""
+        )
+
+        return (
+            False,
+            (
+                f"❌ Phase close blocked by {role} validator "
+                f"(verdict={report.verdict.value}, version={report.validator_version}).\n"
+                f"{gap_lines}\n{q_lines}{marker}{escalation_note}\n\n"
+                f"Fix gaps then re-call."
+            ),
+            report,
+        )
+
+
+def artifact_hash(artifact_path: str) -> str:
+    """Public alias for `_artifact_hash` used by state-machine wiring."""
+    return _artifact_hash(artifact_path)
+
+
+def _autoload_validators() -> None:
+    """Import all role validator modules so VALIDATOR_REGISTRY is populated.
+
+    Per-validator modules self-register via `@register_validator(role)` at
+    import time. Import failures are logged and ignored so server startup
+    survives a single broken validator.
+    """
+    import importlib
+
+    role_modules = (
+        "product_manager",
+        "architect",
+        "qa_lead",
+        "scrum_master",
+        "pr_reviewer",
+        "developer",
+        "impact_architect",
+        "specifier",
+        "tester",
+        "security_owner",
+        "discovery",
+        "platform_engineer",
+        "forensic_engineer",
+        "strategist",
+        "migration_planner",
+        "parity_tester",
+        "cutover_engineer",
+    )
+    for mod in role_modules:
+        try:
+            importlib.import_module(f"ref_agents.tools.validators.{mod}")
+        except Exception as exc:  # noqa: BLE001 - one broken validator must not kill server
+            VALIDATOR_LOAD_FAILURES.append((mod, f"{type(exc).__name__}: {exc}"))
+            logger.warning("validator_module_load_failed", module=mod, error=str(exc))
+
+
+_EXPECTED_ROLES: tuple[str, ...] = (
+    "product_manager",
+    "architect",
+    "qa_lead",
+    "scrum_master",
+    "pr_reviewer",
+    "developer",
+    "impact_architect",
+    "specifier",
+    "tester",
+    "security_owner",
+    "discovery",
+    "platform_engineer",
+    "forensic_engineer",
+    "strategist",
+    "migration_planner",
+    "parity_tester",
+    "cutover_engineer",
+)
+
+
+def _health_report() -> ValidationReport:
+    """Return a ValidationReport describing validator-registry health.
+
+    Used by `validate(role="__health__")`. Verdict is PASS when every
+    expected validator loaded; FAIL_AUTO otherwise with one Gap per
+    failed or missing role.
+    """
+    registered = set(VALIDATOR_REGISTRY.keys())
+    expected = set(_EXPECTED_ROLES)
+    missing = sorted(expected - registered)
+
+    gaps: list[Gap] = []
+    for role, error in VALIDATOR_LOAD_FAILURES:
+        gaps.append(
+            Gap(
+                code=f"validator_load_failed:{role}",
+                detail=error,
+                location=role,
+            )
+        )
+    for role in missing:
+        if any(role in entry[0] for entry in VALIDATOR_LOAD_FAILURES):
+            continue
+        gaps.append(
+            Gap(
+                code=f"validator_missing:{role}",
+                detail=f"Expected validator '{role}' not registered.",
+                location=role,
+            )
+        )
+
+    verdict = Verdict.PASS if not gaps else Verdict.FAIL_AUTO
+    return ValidationReport(
+        verdict=verdict,
+        gaps=gaps,
+    )
+
+
+_autoload_validators()

ref_agents/tools/validators/architect.py

@@ -0,0 +1,268 @@
+"""Architect output validator — validate_design_artifact.
+
+Reads DESIGN-{id}.md, checks mermaid presence, decision log completeness,
+component-codemap bijection, regression risk, and NFR coverage.
+Registered under role "architect".
+
+See SPEC-STORY-023 / DESIGN-STORY-023.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import structlog
+
+from ref_agents.core.validation_models import (
+    Gap,
+    UserQuestion,
+    ValidationReport,
+    Verdict,
+)
+from ref_agents.session import SessionManager
+from ref_agents.tools.validators import register_validator
+
+logger = structlog.get_logger(__name__)
+
+NFR_KEYWORDS = (
+    "latency",
+    "scale",
+    "throughput",
+    "security",
+    "observability",
+    "performance",
+)
+RISK_LEVELS = ("LOW", "MEDIUM", "HIGH", "CRITICAL")
+DECISION_REQUIRED_FIELDS = ("decision", "options", "chosen", "rationale")
+
+
+def _resolve_design_path(artifact_id: str, artifact_path: str) -> Path | None:
+    """Resolve design file path under project root with traversal guard."""
+    root = SessionManager.get().project_root
+    if artifact_path:
+        candidate = Path(artifact_path).resolve()
+    elif artifact_id and root is not None:
+        candidate = (root / "docs" / "design" / f"DESIGN-{artifact_id}.md").resolve()
+    else:
+        return None
+
+    if root is not None:
+        try:
+            candidate.relative_to(root.resolve())
+        except ValueError:
+            return None
+
+    if not candidate.exists():
+        return None
+    return candidate
+
+
+def _has_mermaid(text: str) -> bool:
+    return bool(re.search(r"^```mermaid\b", text, flags=re.MULTILINE))
+
+
+def _decision_log_gaps(text: str) -> list[str]:
+    """Return list of decision entries missing required fields."""
+    block_match = re.search(
+        r"##\s+Decision Log\s*\n+(.*?)(?=\n##\s|\Z)", text, flags=re.DOTALL
+    )
+    if not block_match:
+        return ["__missing_section__"]
+
+    block = "\n" + block_match.group(1)
+    entries = re.split(r"\n###\s+", block)
+    issues: list[str] = []
+    for raw in entries[1:]:  # skip preamble before first ###
+        name = raw.splitlines()[0].strip() if raw.strip() else "?"
+        body_lower = raw.lower()
+        for field in DECISION_REQUIRED_FIELDS:
+            if field not in body_lower:
+                issues.append(f"{name}:{field}")
+                break
+    return issues
+
+
+def _component_names_from_design(text: str) -> list[str]:
+    """Extract component identifiers from Component Registry table (col 1)."""
+    block_match = re.search(
+        r"##\s+Component(?:\s+Registry)?\s*\n+(.*?)(?=\n##\s|\Z)",
+        text,
+        flags=re.DOTALL,
+    )
+    if not block_match:
+        return []
+
+    block = block_match.group(1)
+    names: list[str] = []
+    for line in block.splitlines():
+        if not line.startswith("|"):
+            continue
+        if re.match(r"\|\s*-+", line):  # separator row
+            continue
+        cells = [c.strip() for c in line.strip("|").split("|")]
+        if not cells:
+            continue
+        first = cells[0]
+        if first.lower() in ("component", ""):
+            continue
+        first = re.sub(r"^[`*_]+|[`*_]+$", "", first)
+        if first:
+            names.append(first)
+    return names
+
+
+def _codemap_corpus(project_root: Path) -> str:
+    """Read all codemap files into a single search corpus."""
+    codemap_root = project_root / "codemap"
+    if not codemap_root.exists():
+        return ""
+    parts: list[str] = []
+    for md in codemap_root.rglob("*.md"):
+        try:
+            parts.append(md.read_text(encoding="utf-8"))
+        except OSError:
+            continue
+    return "\n".join(parts)
+
+
+def _component_unmapped(names: list[str], corpus: str) -> list[str]:
+    return [n for n in names if n and n not in corpus]
+
+
+def _risk_level(text: str) -> str | None:
+    block_match = re.search(
+        r"##\s+Regression Risk(?:\s+Assessment)?\s*\n+(.*?)(?=\n##\s|\Z)",
+        text,
+        flags=re.DOTALL,
+    )
+    if not block_match:
+        return None
+    body = block_match.group(1).upper()
+    for level in ("CRITICAL", "HIGH", "MEDIUM", "LOW"):
+        if re.search(rf"\bRISK:?\s*[*_]*{level}\b", body) or re.search(
+            rf"\b{level}\b", body
+        ):
+            return level
+    return None
+
+
+def _nfr_mentioned(text: str) -> bool:
+    lower = text.lower()
+    return any(kw in lower for kw in NFR_KEYWORDS)
+
+
+@register_validator("architect")
+def validate_design_artifact(
+    artifact_id: str = "", artifact_path: str = ""
+) -> ValidationReport:
+    """Validate an Architect design artifact.
+
+    Args:
+        artifact_id: Story identifier; resolves to
+            `docs/design/DESIGN-<artifact_id>.md` when path not supplied.
+        artifact_path: Explicit design file path.
+
+    Returns:
+        ValidationReport per SPEC-STORY-023.
+    """
+    log = logger.bind(role="architect", artifact_id=artifact_id)
+
+    path = _resolve_design_path(artifact_id, artifact_path)
+    if path is None:
+        return ValidationReport(
+            verdict=Verdict.NEEDS_USER,
+            gaps=[
+                Gap(code="design_path_unresolved", detail=artifact_id or artifact_path)
+            ],
+            user_questions=[
+                UserQuestion(
+                    text=f"Provide a path to the design file for {artifact_id or '?'}."
+                )
+            ],
+        )
+
+    text = path.read_text(encoding="utf-8")
+    gaps: list[Gap] = []
+    user_questions: list[UserQuestion] = []
+
+    # FR-1: Mermaid
+    if not _has_mermaid(text):
+        gaps.append(Gap(code="no_diagram", detail="No ```mermaid block found."))
+
+    # FR-2: Decision Log
+    decision_issues = _decision_log_gaps(text)
+    if decision_issues == ["__missing_section__"]:
+        gaps.append(
+            Gap(code="decision_log_missing", detail="No '## Decision Log' section.")
+        )
+    else:
+        for issue in decision_issues:
+            gaps.append(
+                Gap(
+                    code=f"decision_log_incomplete:{issue}",
+                    detail="Required field missing from decision entry.",
+                )
+            )
+
+    # FR-3: Component Registry bijection vs codemap
+    root = SessionManager.get().project_root
+    component_names = _component_names_from_design(text)
+    if root is not None and component_names:
+        corpus = _codemap_corpus(root)
+        unmapped = _component_unmapped(component_names, corpus)
+        for name in unmapped:
+            gaps.append(
+                Gap(
+                    code=f"component_unmapped:{name}",
+                    detail="Component not found in any codemap file.",
+                    location=name,
+                )
+            )
+
+    # FR-4: Regression Risk
+    risk = _risk_level(text)
+    if risk is None:
+        gaps.append(
+            Gap(code="risk_missing", detail="No Regression Risk section/level.")
+        )
+    elif risk in ("HIGH", "CRITICAL"):
+        user_questions.append(
+            UserQuestion(
+                text=(
+                    f"Regression risk = {risk}. Approve proceeding, request "
+                    "feature-flag wrapper, or send back for redesign?"
+                ),
+                options=["proceed", "feature_flag", "redesign"],
+            )
+        )
+
+    # FR-5: NFR coverage
+    if not _nfr_mentioned(text):
+        user_questions.append(
+            UserQuestion(
+                text=(
+                    "Design does not mention any NFR (latency / scale / throughput / "
+                    "security / observability / performance). Confirm which apply."
+                )
+            )
+        )
+
+    if gaps:
+        verdict = Verdict.FAIL_AUTO
+    elif user_questions:
+        verdict = Verdict.NEEDS_USER
+    else:
+        verdict = Verdict.PASS
+
+    log.info(
+        "architect_validate_complete",
+        verdict=verdict.value,
+        gap_count=len(gaps),
+        risk=risk,
+    )
+    return ValidationReport(
+        verdict=verdict,
+        gaps=gaps,
+        user_questions=user_questions,
+    )