ref-agents 1.0.0__py3-none-any.whl

ref_agents/tools/validators/pr_reviewer.py

@@ -0,0 +1,129 @@
+"""PR Reviewer output validator — validate_review_output.
+
+Checks PR review doc structure (3 stages), verdict presence, finding
+citation discipline, and APPROVE/L3 consistency.
+
+See SPEC-STORY-026.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import structlog
+
+from ref_agents.core.validation_models import (
+    Gap,
+    ValidationReport,
+    Verdict,
+)
+from ref_agents.session import SessionManager
+from ref_agents.tools.validators import register_validator
+
+logger = structlog.get_logger(__name__)
+
+STAGE_HEADERS = ("STAGE 0", "STAGE 1", "STAGE 2")
+APPROVE = "APPROVE"
+REQUEST_CHANGES = "REQUEST_CHANGES"
+
+
+def _resolve_under_root(candidate: Path) -> Path | None:
+    root = SessionManager.get().project_root
+    candidate = candidate.resolve()
+    if root is not None:
+        try:
+            candidate.relative_to(root.resolve())
+        except ValueError:
+            return None
+    return candidate if candidate.exists() else None
+
+
+def _resolve_review(artifact_id: str, artifact_path: str) -> Path | None:
+    if artifact_path:
+        return _resolve_under_root(Path(artifact_path))
+    root = SessionManager.get().project_root
+    if artifact_id and root is not None:
+        return _resolve_under_root(
+            root / "docs" / "analysis" / f"PR-REVIEW-{artifact_id}.md"
+        )
+    return None
+
+
+def _extract_verdict(text: str) -> str | None:
+    match = re.search(
+        r"verdict[:*\s_]*\*{0,2}(APPROVE|REQUEST[_ ]CHANGES)\*{0,2}",
+        text,
+        flags=re.IGNORECASE,
+    )
+    if not match:
+        return None
+    raw = match.group(1).upper().replace(" ", "_")
+    return raw
+
+
+@register_validator("pr_reviewer")
+def validate_review_output(
+    artifact_id: str = "", artifact_path: str = ""
+) -> ValidationReport:
+    log = logger.bind(role="pr_reviewer", artifact_id=artifact_id)
+
+    path = _resolve_review(artifact_id, artifact_path)
+    if path is None:
+        return ValidationReport(
+            verdict=Verdict.NEEDS_USER,
+            gaps=[
+                Gap(code="pr_review_unresolved", detail=artifact_id or artifact_path)
+            ],
+        )
+
+    text = path.read_text(encoding="utf-8")
+    gaps: list[Gap] = []
+
+    # FR-1: stage headers
+    for stage in STAGE_HEADERS:
+        if not re.search(rf"^#+\s*.*{re.escape(stage)}", text, flags=re.MULTILINE):
+            gaps.append(
+                Gap(
+                    code=f"pr_stage_missing:{stage.replace(' ', '_')}",
+                    detail=f"Missing required header for {stage}.",
+                )
+            )
+
+    # FR-2: verdict
+    verdict = _extract_verdict(text)
+    if verdict is None:
+        gaps.append(Gap(code="pr_verdict_missing", detail="No verdict line found."))
+
+    # FR-3: REQUEST_CHANGES requires path:line citations
+    if verdict == REQUEST_CHANGES:
+        citations = re.findall(r"[\w./\\-]+\.\w+:\d+", text)
+        if not citations:
+            gaps.append(
+                Gap(
+                    code="pr_finding_no_citation",
+                    detail="Verdict REQUEST_CHANGES but no file:line citations found.",
+                )
+            )
+
+    # FR-4: APPROVE forbidden with L3/critical finding
+    if verdict == APPROVE:
+        if re.search(r"severity\s*[:\-]\s*critical", text, flags=re.IGNORECASE) or (
+            re.search(r"\bL3\b", text)
+            and re.search(r"\b(FAIL|BLOCK|UNRESOLVED)\b", text, flags=re.IGNORECASE)
+        ):
+            gaps.append(
+                Gap(
+                    code="pr_approve_with_l3",
+                    detail="Verdict APPROVE but L3 / Critical finding present.",
+                )
+            )
+
+    out_verdict = Verdict.PASS if not gaps else Verdict.FAIL_AUTO
+    log.info(
+        "pr_validate_complete",
+        verdict=out_verdict.value,
+        gap_count=len(gaps),
+        pr_verdict=verdict,
+    )
+    return ValidationReport(verdict=out_verdict, gaps=gaps)

ref_agents/tools/validators/product_manager.py

@@ -0,0 +1,291 @@
+"""Product Manager output validator — validate_story_quality.
+
+Reads STORY-{id}.md, extracts user story + acceptance criteria,
+scores via INVEST, applies AC count / format / measurability / edge-case /
+E2E-section checks, returns a ValidationReport. Registered under role
+"product_manager".
+
+See SPEC-STORY-022 / DESIGN-STORY-022.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import structlog
+
+from ref_agents.core.validation_models import (
+    Gap,
+    UserQuestion,
+    ValidationReport,
+    Verdict,
+)
+from ref_agents.session import SessionManager
+from ref_agents.tools.invest_scorer import score_invest_principles
+from ref_agents.tools.validators import register_validator
+
+logger = structlog.get_logger(__name__)
+
+VAGUE_PHRASES = (
+    "should work",
+    "works well",
+    "nice",
+    " fast ",
+    "easy",
+    "intuitive",
+    "user-friendly",
+)
+FRONTEND_KEYWORDS = (
+    "ui",
+    "route",
+    "component",
+    "screen",
+    "page",
+    "form",
+    "button",
+    "frontend",
+    "fullstack",
+)
+INVEST_FLOOR = 3.0
+INVEST_PASS = 4.0
+
+
+def _resolve_path(artifact_id: str, artifact_path: str) -> Path | None:
+    """Resolve story path safely under project root.
+
+    Args:
+        artifact_id: Story ID (e.g., "STORY-021").
+        artifact_path: Explicit path or empty.
+
+    Returns:
+        Resolved Path under project root, or None if traversal/missing.
+    """
+    root = SessionManager.get().project_root
+    if artifact_path:
+        candidate = Path(artifact_path).resolve()
+    elif artifact_id and root is not None:
+        candidate = (root / "docs" / "requirements" / f"{artifact_id}.md").resolve()
+    else:
+        return None
+
+    if root is not None:
+        try:
+            candidate.relative_to(root.resolve())
+        except ValueError:
+            return None
+
+    if not candidate.exists():
+        return None
+    return candidate
+
+
+def _parse_story(path: Path) -> dict[str, object]:
+    """Extract user_story, ACs, and presence flags from a story markdown file.
+
+    Args:
+        path: Story markdown file.
+
+    Returns:
+        Dict with keys: user_story (str), acs (list[str]),
+        has_edge_cases (bool), has_e2e (bool), full_text (str).
+    """
+    text = path.read_text(encoding="utf-8")
+
+    # User Story section: between '## User Story' and the next '## '
+    us_match = re.search(
+        r"##\s+User Story\s*\n+(.*?)(?=\n##\s|\Z)", text, flags=re.DOTALL
+    )
+    user_story = us_match.group(1).strip() if us_match else ""
+
+    # Acceptance Criteria block(s)
+    ac_blocks = re.findall(
+        r"##\s+Acceptance Criteria\s*\n+(.*?)(?=\n##\s|\Z)", text, flags=re.DOTALL
+    )
+    ac_body = "\n".join(ac_blocks)
+
+    # AC patterns: "**AC1:** ..." or "AC1: ..." or "- AC1: ..."
+    acs = re.findall(
+        r"(?:^|\n)\s*[-*]?\s*\*{0,2}AC\d+\*{0,2}\s*[:\.\-]?\s*(.+?)(?=\n\s*[-*]?\s*\*{0,2}AC\d+\*{0,2}\s*[:\.\-]|\n##\s|\Z)",
+        ac_body,
+        flags=re.DOTALL,
+    )
+    acs = [a.strip() for a in acs if a.strip()]
+
+    has_edge_cases = bool(
+        re.search(r"##\s+Edge Cases\s*\n+\s*[-*0-9]", text, flags=re.IGNORECASE)
+    )
+    has_e2e = bool(
+        re.search(r"##\s+E2E Testing Requirements", text, flags=re.IGNORECASE)
+    )
+
+    return {
+        "user_story": user_story,
+        "acs": acs,
+        "has_edge_cases": has_edge_cases,
+        "has_e2e": has_e2e,
+        "full_text": text,
+    }
+
+
+def _ac_has_given_when_then(ac: str) -> bool:
+    lower = ac.lower()
+    return "given" in lower and "when" in lower and "then" in lower
+
+
+def _detect_frontend(user_story: str, full_text: str) -> bool:
+    combined = (user_story + " " + full_text).lower()
+    return any(kw in combined for kw in FRONTEND_KEYWORDS)
+
+
+def _find_vague(ac: str) -> list[str]:
+    lower = ac.lower()
+    return [phrase.strip() for phrase in VAGUE_PHRASES if phrase in lower]
+
+
+@register_validator("product_manager")
+def validate_story_quality(
+    artifact_id: str = "", artifact_path: str = ""
+) -> ValidationReport:
+    """Validate a Product Manager story artifact.
+
+    Args:
+        artifact_id: Story identifier (e.g., "STORY-021"). Resolves to
+            `<project_root>/docs/requirements/<artifact_id>.md` when
+            `artifact_path` not supplied.
+        artifact_path: Explicit story file path.
+
+    Returns:
+        ValidationReport per SPEC-STORY-022.
+    """
+    log = logger.bind(role="product_manager", artifact_id=artifact_id)
+
+    path = _resolve_path(artifact_id, artifact_path)
+    if path is None:
+        return ValidationReport(
+            verdict=Verdict.NEEDS_USER,
+            gaps=[
+                Gap(code="story_path_unresolved", detail=artifact_id or artifact_path)
+            ],
+            user_questions=[
+                UserQuestion(
+                    text=f"Provide a path to the story file for {artifact_id or '?'}."
+                )
+            ],
+        )
+
+    parsed = _parse_story(path)
+    acs: list[str] = parsed["acs"]  # type: ignore[assignment]
+    user_story: str = parsed["user_story"]  # type: ignore[assignment]
+    has_edge_cases: bool = parsed["has_edge_cases"]  # type: ignore[assignment]
+    has_e2e: bool = parsed["has_e2e"]  # type: ignore[assignment]
+    full_text: str = parsed["full_text"]  # type: ignore[assignment]
+
+    gaps: list[Gap] = []
+    user_questions: list[UserQuestion] = []
+
+    # FR-2: AC count
+    if len(acs) < 3:  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+        gaps.append(
+            Gap(code="ac_count_lt_3", detail=f"Found {len(acs)} ACs; require ≥3.")
+        )
+
+    # FR-3: Given/When/Then
+    for idx, ac in enumerate(acs, start=1):
+        if not _ac_has_given_when_then(ac):
+            gaps.append(
+                Gap(
+                    code=f"ac_format_invalid:{idx}",
+                    detail="Missing Given/When/Then structure.",
+                    location=f"AC{idx}",
+                )
+            )
+
+    # FR-4: vague phrases → NEEDS_USER
+    for idx, ac in enumerate(acs, start=1):
+        vague = _find_vague(ac)
+        if vague:
+            user_questions.append(
+                UserQuestion(
+                    text=(
+                        f"AC{idx} uses vague phrasing ({', '.join(vague)}). "
+                        "Provide a measurable assertion."
+                    )
+                )
+            )
+
+    # FR-6: Edge Cases
+    if not has_edge_cases:
+        gaps.append(
+            Gap(
+                code="edge_cases_missing",
+                detail="No populated '## Edge Cases' section found.",
+            )
+        )
+
+    # FR-7: Frontend → E2E required
+    if _detect_frontend(user_story, full_text) and not has_e2e:
+        gaps.append(
+            Gap(
+                code="e2e_section_missing",
+                detail="Frontend tag detected but no '## E2E Testing Requirements' section.",
+            )
+        )
+
+    # FR-5: INVEST scoring
+    invest_avg: float | None = None
+    if user_story or acs:
+        try:
+            scores = score_invest_principles(
+                user_story=user_story, acceptance_criteria=acs
+            )
+            score_values = [
+                scores.independent,
+                scores.negotiable,
+                scores.valuable,
+                scores.estimable,
+                scores.small,
+                scores.testable,
+            ]
+            invest_avg = sum(score_values) / len(score_values)
+        except Exception as exc:  # noqa: BLE001 - scorer is best-effort
+            log.warning("invest_score_failed", error=str(exc))
+
+    # Verdict resolution
+    if gaps:
+        verdict = Verdict.FAIL_AUTO
+    elif user_questions or (invest_avg is not None and invest_avg < INVEST_PASS):
+        verdict = Verdict.NEEDS_USER
+        if invest_avg is not None and invest_avg < INVEST_FLOOR:
+            verdict = Verdict.FAIL_AUTO
+            gaps.append(
+                Gap(
+                    code="invest_below_floor",
+                    detail=f"INVEST avg {invest_avg:.2f} below floor {INVEST_FLOOR}.",
+                )
+            )
+        elif invest_avg is not None and invest_avg < INVEST_PASS:
+            user_questions.append(
+                UserQuestion(
+                    text=(
+                        f"INVEST average {invest_avg:.2f} (target ≥{INVEST_PASS}). "
+                        "Improve weakest dimensions before proceeding."
+                    )
+                )
+            )
+    else:
+        verdict = Verdict.PASS
+
+    log.info(
+        "pm_validate_complete",
+        verdict=verdict.value,
+        invest_avg=invest_avg,
+        ac_count=len(acs),
+        gap_count=len(gaps),
+    )
+    return ValidationReport(
+        verdict=verdict,
+        score=invest_avg,
+        gaps=gaps,
+        user_questions=user_questions,
+    )

ref_agents/tools/validators/qa_lead.py

@@ -0,0 +1,172 @@
+"""QA Lead output validator — validate_ac_review.
+
+Reads QA Lead AC validation report markdown; cross-checks FR coverage vs
+SPEC, INVEST score completeness, verdict enum, and verdict-INVEST
+consistency. Registered under role "qa_lead".
+
+See SPEC-STORY-024 / DESIGN-STORY-024.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import structlog
+
+from ref_agents.core.validation_models import (
+    Gap,
+    ValidationReport,
+    Verdict,
+)
+from ref_agents.session import SessionManager
+from ref_agents.tools.validators import register_validator
+
+logger = structlog.get_logger(__name__)
+
+INVEST_DIMS = (
+    "independent",
+    "negotiable",
+    "valuable",
+    "estimable",
+    "small",
+    "testable",
+)
+VERDICT_LITERALS = ("APPROVED", "NEEDS_REVISION", "BLOCKED")
+
+
+def _resolve_under_root(candidate: Path) -> Path | None:
+    root = SessionManager.get().project_root
+    candidate = candidate.resolve()
+    if root is not None:
+        try:
+            candidate.relative_to(root.resolve())
+        except ValueError:
+            return None
+    return candidate if candidate.exists() else None
+
+
+def _resolve_report_path(artifact_id: str, artifact_path: str) -> Path | None:
+    root = SessionManager.get().project_root
+    if artifact_path:
+        return _resolve_under_root(Path(artifact_path))
+    if artifact_id and root is not None:
+        return _resolve_under_root(root / "docs" / "analysis" / f"QA-{artifact_id}.md")
+    return None
+
+
+def _resolve_spec_path(artifact_id: str) -> Path | None:
+    root = SessionManager.get().project_root
+    if artifact_id and root is not None:
+        return _resolve_under_root(root / "docs" / "specs" / f"SPEC-{artifact_id}.md")
+    return None
+
+
+def _extract_fr_ids(text: str) -> set[str]:
+    return set(re.findall(r"FR-\d+", text))
+
+
+def _extract_invest_scores(text: str) -> dict[str, int | None]:
+    scores: dict[str, int | None] = {dim: None for dim in INVEST_DIMS}
+    lower = text.lower()
+    for dim in INVEST_DIMS:
+        match = re.search(rf"\|\s*{dim}\b\s*\|\s*(\d)", lower) or re.search(
+            rf"{dim}\s*[:=]\s*(\d)", lower
+        )
+        if match:
+            scores[dim] = int(match.group(1))
+    return scores
+
+
+def _extract_verdict(text: str) -> str | None:
+    match = re.search(
+        r"verdict\s*[:*\s_]*\s*\*{0,2}(APPROVED|NEEDS_REVISION|BLOCKED)\*{0,2}",
+        text,
+        flags=re.IGNORECASE,
+    )
+    if match:
+        return match.group(1).upper()
+    return None
+
+
+@register_validator("qa_lead")
+def validate_ac_review(
+    artifact_id: str = "", artifact_path: str = ""
+) -> ValidationReport:
+    """Validate a QA Lead AC validation report."""
+    log = logger.bind(role="qa_lead", artifact_id=artifact_id)
+
+    report_path = _resolve_report_path(artifact_id, artifact_path)
+    if report_path is None:
+        return ValidationReport(
+            verdict=Verdict.NEEDS_USER,
+            gaps=[
+                Gap(code="qa_report_unresolved", detail=artifact_id or artifact_path)
+            ],
+        )
+
+    text = report_path.read_text(encoding="utf-8")
+    gaps: list[Gap] = []
+
+    # FR-2: FR coverage vs SPEC
+    spec_path = _resolve_spec_path(artifact_id)
+    if spec_path is not None:
+        spec_text = spec_path.read_text(encoding="utf-8")
+        spec_frs = _extract_fr_ids(spec_text)
+        report_frs = _extract_fr_ids(text)
+        missing = sorted(spec_frs - report_frs)
+        for fr in missing:
+            gaps.append(
+                Gap(
+                    code=f"fr_not_reviewed:{fr}",
+                    detail=f"{fr} present in SPEC but not cited in QA report.",
+                    location=fr,
+                )
+            )
+
+    # FR-3: INVEST scores present + numeric 1-5
+    scores = _extract_invest_scores(text)
+    for dim, score in scores.items():
+        if score is None:
+            gaps.append(
+                Gap(
+                    code=f"invest_missing:{dim}",
+                    detail=f"INVEST dimension '{dim}' missing or non-numeric.",
+                )
+            )
+        elif not 1 <= score <= 5:  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+            gaps.append(
+                Gap(
+                    code=f"invest_out_of_range:{dim}",
+                    detail=f"INVEST '{dim}' = {score} (must be 1-5).",
+                )
+            )
+
+    # FR-4: Verdict
+    verdict = _extract_verdict(text)
+    if verdict is None:
+        gaps.append(Gap(code="verdict_missing", detail="No verdict found in report."))
+
+    # FR-5: Consistency
+    if verdict == "APPROVED":
+        low_dims = [
+            dim
+            for dim, score in scores.items()
+            if score is not None and score < 3  # noqa: PLR2004  # TECH_DEBT: extract to named constant — G25
+        ]
+        if low_dims:
+            gaps.append(
+                Gap(
+                    code="verdict_inconsistent",
+                    detail=(f"Verdict APPROVED but INVEST dim(s) {low_dims} below 3."),
+                )
+            )
+
+    out_verdict = Verdict.PASS if not gaps else Verdict.FAIL_AUTO
+    log.info(
+        "qa_validate_complete",
+        verdict=out_verdict.value,
+        gap_count=len(gaps),
+        qa_verdict=verdict,
+    )
+    return ValidationReport(verdict=out_verdict, gaps=gaps)