devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/review/dag.py

@@ -0,0 +1,588 @@
+"""Review DAG implementation.
+
+DEPRECATED: This module is non-canonical. The `devflow review` CLI surface
+and run_review_dag() are retained for backward compatibility and debugging
+only. Canonical automated execution runs through the queue-driven worker
+pipeline (devflow.worker / `devflow worker start`).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import time
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from pydantic import BaseModel
+
+from ..implementation.green_gate import run_green_gate
+from ..review.review_story import review_story_contract
+from ..stores.execution_store import ExecutionStore
+from ..vendor.datalumina_genai.core.nodes.base import Node
+from ..vendor.datalumina_genai.core.schema import NodeConfig, WorkflowSchema
+from ..vendor.datalumina_genai.core.task import TaskContext
+from ..vendor.datalumina_genai.core.workflow import Workflow
+
+SEVERITY_ORDER = {"info": 0, "low": 1, "medium": 2, "high": 3, "critical": 4}
+SECRET_PATTERNS = [
+    ("private_key", re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----")),
+    ("token_like", re.compile(r"(?i)(api[_-]?key|secret|token|password)\s*[:=]\s*([A-Za-z0-9._-]{6,})")),
+]
+
+
+@dataclass(frozen=True)
+class ReviewFinding:
+    finding_id: str
+    category: str
+    severity: str
+    title: str
+    description: str
+    recommendation: str
+    contract_refs: list[str]
+    evidence_refs: list[str]
+    fingerprint: str
+
+
+class ReviewEvent(BaseModel):
+    repo_root: str
+    story: dict[str, Any]
+
+
+_CURRENT_STORE: ExecutionStore | None = None
+_CURRENT_RUN_ID: str | None = None
+_CURRENT_REVIEW_RUN_ID: str | None = None
+
+
+def _runtime() -> tuple[ExecutionStore, str, str]:
+    if _CURRENT_STORE is None or _CURRENT_RUN_ID is None or _CURRENT_REVIEW_RUN_ID is None:
+        raise RuntimeError("review workflow missing runtime bindings")
+    return _CURRENT_STORE, _CURRENT_RUN_ID, _CURRENT_REVIEW_RUN_ID
+
+
+def _stable_hash(payload: dict[str, Any]) -> str:
+    return hashlib.sha256(json.dumps(payload, sort_keys=True).encode("utf-8")).hexdigest()
+
+
+def _make_finding(
+    *,
+    category: str,
+    severity: str,
+    title: str,
+    description: str,
+    recommendation: str,
+    contract_refs: list[str] | None = None,
+    evidence_refs: list[str] | None = None,
+) -> ReviewFinding:
+    contract_refs = sorted(contract_refs or [])
+    evidence_refs = sorted(evidence_refs or [])
+    key = {
+        "category": category,
+        "severity": severity,
+        "title": title,
+        "description": description,
+        "recommendation": recommendation,
+        "contract_refs": contract_refs,
+        "evidence_refs": evidence_refs,
+    }
+    digest = _stable_hash(key)
+    return ReviewFinding(
+        finding_id=digest[:16],
+        category=category,
+        severity=severity,
+        title=title,
+        description=description,
+        recommendation=recommendation,
+        contract_refs=contract_refs,
+        evidence_refs=evidence_refs,
+        fingerprint=digest,
+    )
+
+
+def _redact_text(text: str) -> tuple[str, list[dict[str, str]]]:
+    redactions: list[dict[str, str]] = []
+    redacted = text
+    for kind, pattern in SECRET_PATTERNS:
+        for match in list(pattern.finditer(redacted)):
+            secret_text = match.group(0)
+            redactions.append({"kind": kind, "match": secret_text[:32]})
+            redacted = redacted.replace(secret_text, f"[REDACTED:{kind}]")
+    return redacted, redactions
+
+
+def _load_story_text(story: dict[str, Any], repo_root: Path) -> str:
+    contract = story.get("contract") or {}
+    raw_text = str(contract.get("raw_text") or "").strip()
+    if raw_text:
+        return raw_text + ("\n" if not raw_text.endswith("\n") else "")
+
+    path_value = story.get("path") or contract.get("source_path")
+    if path_value:
+        path = Path(str(path_value))
+        if not path.is_absolute():
+            path = repo_root / path
+        if path.exists():
+            return path.read_text(encoding="utf-8")
+    return ""
+
+
+def _extract_requirement_refs(story_id: str, story_text: str) -> list[str]:
+    refs: list[str] = []
+    lines = [line.strip() for line in story_text.splitlines()]
+    current_section = "contract"
+    index = 0
+    for line in lines:
+        if not line:
+            continue
+        if line.startswith("## "):
+            current_section = line[3:].strip().lower().replace(" ", "_")
+            index = 0
+            continue
+        if line.startswith("### "):
+            current_section = line[4:].strip().lower().replace(" ", "_")
+            index = 0
+            continue
+        if line[:1].isdigit() and ". " in line or line.startswith("- "):
+            index += 1
+            refs.append(f"{story_id}#{current_section}.{index}")
+    return refs
+
+
+def _persist_node(
+    *,
+    node_id: str,
+    node_name: str,
+    output: dict[str, Any] | None = None,
+    status: str = "succeeded",
+    error: dict[str, Any] | None = None,
+) -> str:
+    store, run_id, _review_run_id = _runtime()
+    node_exec_id = store.create_node_attempt(run_id=run_id, node_id=node_id, node_name=node_name, attempt=1)
+    store.mark_node_finished(node_exec_id=node_exec_id, status=status, output=output, error=error)
+    return node_exec_id
+
+
+class IngestNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        store, run_id, review_run_id = _runtime()
+        repo_root = Path(task_context.event.repo_root)
+        story = dict(task_context.event.story)
+        contract = dict(story.get("contract") or {})
+        story_id = str(contract.get("story_id") or story.get("story_id") or "").strip() or None
+        story_text = _load_story_text(story, repo_root)
+        packet = {
+            "story_id": story_id,
+            "story_uuid": contract.get("story_uuid"),
+            "title": contract.get("title") or story.get("title"),
+            "required_planes": contract.get("required_planes") or [],
+            "plane_oracles": contract.get("plane_oracles") or [],
+            "story_path": story.get("path") or contract.get("source_path"),
+            "repo_root": str(repo_root),
+        }
+        packet_hash = _stable_hash(packet)
+        packet_id = store.add_review_packet(
+            run_id=run_id,
+            story_id=story_id,
+            summary="review packet",
+            findings={"packet_hash": packet_hash, "review_run_id": review_run_id},
+        )
+        node_exec_id = _persist_node(
+            node_id="ingest",
+            node_name="Ingest",
+            output={"packet_id": packet_id, "packet_hash": packet_hash},
+        )
+        task_context.metadata["story_id"] = story_id
+        task_context.metadata["story_text"] = story_text
+        secret_scan_text = story_text + "\n" + json.dumps(contract, sort_keys=True)
+        redacted_text, redactions = _redact_text(secret_scan_text)
+        task_context.metadata["story_text_redacted"] = redacted_text
+        task_context.metadata["redactions"] = redactions
+        task_context.metadata["packet"] = packet
+        task_context.metadata["packet_hash"] = packet_hash
+        task_context.metadata["packet_id"] = packet_id
+        task_context.metadata["findings"] = []
+        if redactions:
+            task_context.metadata["findings"].append(
+                _make_finding(
+                    category="security",
+                    severity="high",
+                    title="Secrets detected in review inputs",
+                    description="Review inputs contained token-like or private-key material and were redacted.",
+                    recommendation="Remove secrets from story contracts and evidence before review.",
+                    contract_refs=[f"{story_id or 'story'}#functional_requirements.1"],
+                    evidence_refs=[f"redaction:{item['kind']}" for item in redactions],
+                )
+            )
+        task_context.metadata["node_exec_ids"] = {"ingest": node_exec_id}
+        return task_context
+
+
+class ValidationGateNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        results = run_green_gate(Path(task_context.event.repo_root))
+        failing = [name for name, result in results.items() if not result.ok]
+        output = {
+            "checks": {
+                name: {
+                    "ok": result.ok,
+                    "returncode": result.returncode,
+                    "stderr": result.stderr,
+                    "stdout": result.stdout,
+                }
+                for name, result in results.items()
+            }
+        }
+        status = "succeeded"
+        error = None
+        if failing:
+            status = "failed"
+            story_id = str(task_context.metadata.get("story_id") or "story")
+            finding = _make_finding(
+                category="compliance",
+                severity="high",
+                title="Validation gate failed",
+                description=f"Validation checks failed: {', '.join(sorted(failing))}.",
+                recommendation="Fix build/type/test/lint failures before accepting the review.",
+                contract_refs=[f"{story_id}#functional_requirements.1"],
+                evidence_refs=[f"gate:{name}" for name in sorted(failing)],
+            )
+            task_context.metadata["findings"].append(finding)
+            task_context.metadata["halted"] = True
+            error = {"message": "validation gate failed", "checks": failing}
+        node_exec_id = _persist_node(
+            node_id="validation_gate",
+            node_name="ValidationGate",
+            output=output,
+            status=status,
+            error=error,
+        )
+        task_context.metadata["validation_results"] = output["checks"]
+        task_context.metadata["node_exec_ids"]["validation_gate"] = node_exec_id
+        return task_context
+
+
+class SecurityReviewNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        if task_context.metadata.get("halted"):
+            node_exec_id = _persist_node(
+                node_id="security_review",
+                node_name="SecurityReview",
+                output={"reason": "validation gate failed"},
+                status="skipped",
+            )
+            task_context.metadata["node_exec_ids"]["security_review"] = node_exec_id
+            return task_context
+
+        node_exec_id = _persist_node(
+            node_id="security_review",
+            node_name="SecurityReview",
+            output={"redaction_count": len(task_context.metadata.get("redactions") or [])},
+        )
+        task_context.metadata["node_exec_ids"]["security_review"] = node_exec_id
+        return task_context
+
+
+class ContractReviewNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        if task_context.metadata.get("halted"):
+            node_exec_id = _persist_node(
+                node_id="contract_review",
+                node_name="ContractReview",
+                output={"reason": "validation gate failed"},
+                status="skipped",
+            )
+            task_context.metadata["node_exec_ids"]["contract_review"] = node_exec_id
+            return task_context
+
+        repo_root = Path(task_context.event.repo_root)
+        story = dict(task_context.event.story)
+        contract = dict(story.get("contract") or {})
+        story_id = str(task_context.metadata.get("story_id") or "story")
+        requirement_refs = _extract_requirement_refs(story_id, str(task_context.metadata.get("story_text") or ""))
+
+        for finding in review_story_contract(repo_root, contract):
+            task_context.metadata["findings"].append(
+                _make_finding(
+                    category="compliance" if finding.kind == "missing_anchor" else "soundness",
+                    severity=finding.severity,
+                    title=finding.kind.replace("_", " ").title(),
+                    description=finding.message,
+                    recommendation="Add or repair the missing contract evidence/anchor.",
+                    contract_refs=requirement_refs[:1] or [f"{story_id}#contract.1"],
+                    evidence_refs=[str(v) for v in finding.evidence.values() if v],
+                )
+            )
+
+        if not requirement_refs:
+            task_context.metadata["findings"].append(
+                _make_finding(
+                    category="soundness",
+                    severity="medium",
+                    title="Requirement extraction incomplete",
+                    description="No contract requirement anchors could be extracted from the story markdown.",
+                    recommendation=(
+                        "Use canonical story sections so coverage and compliance checks "
+                        "can map to stable anchors."
+                    ),
+                    contract_refs=[f"{story_id}#contract.1"],
+                    evidence_refs=[],
+                )
+            )
+
+        node_exec_id = _persist_node(
+            node_id="contract_review",
+            node_name="ContractReview",
+            output={"requirement_refs": requirement_refs},
+        )
+        task_context.metadata["requirement_refs"] = requirement_refs
+        task_context.metadata["node_exec_ids"]["contract_review"] = node_exec_id
+        return task_context
+
+
+class RegressionReviewNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        if task_context.metadata.get("halted"):
+            node_exec_id = _persist_node(
+                node_id="regression_review",
+                node_name="RegressionReview",
+                output={"reason": "validation gate failed"},
+                status="skipped",
+            )
+            task_context.metadata["node_exec_ids"]["regression_review"] = node_exec_id
+            return task_context
+
+        validation_results = dict(task_context.metadata.get("validation_results") or {})
+        failing = sorted(name for name, result in validation_results.items() if not result.get("ok"))
+        if failing:
+            story_id = str(task_context.metadata.get("story_id") or "story")
+            task_context.metadata["findings"].append(
+                _make_finding(
+                    category="soundness",
+                    severity="high",
+                    title="Regression suite failed",
+                    description=f"Full-suite validation reported regressions in: {', '.join(failing)}.",
+                    recommendation="Stabilize the failing regression suite before approval.",
+                    contract_refs=[f"{story_id}#test_plan.1"],
+                    evidence_refs=[f"gate:{name}" for name in failing],
+                )
+            )
+
+        node_exec_id = _persist_node(
+            node_id="regression_review",
+            node_name="RegressionReview",
+            output={"failing_checks": failing},
+        )
+        task_context.metadata["node_exec_ids"]["regression_review"] = node_exec_id
+        return task_context
+
+
+class AggregateOutputNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        story_id = str(task_context.metadata.get("story_id") or "story")
+        packet = dict(task_context.metadata.get("packet") or {})
+        redacted_story_text = str(
+            task_context.metadata.get("story_text_redacted")
+            or task_context.metadata.get("story_text")
+            or ""
+        )
+        findings = sorted(
+            task_context.metadata.get("findings") or [],
+            key=lambda item: (-SEVERITY_ORDER.get(item.severity, 0), item.category, item.title, item.finding_id),
+        )
+        report = {
+            "review_id": _CURRENT_REVIEW_RUN_ID,
+            "story_id": story_id,
+            "packet_hash": task_context.metadata.get("packet_hash"),
+            "story_hash": _stable_hash({"story_id": story_id, "story_text": redacted_story_text}),
+            "review_status": "fail"
+            if any(SEVERITY_ORDER.get(item.severity, 0) >= SEVERITY_ORDER["high"] for item in findings)
+            else "pass",
+            "counts_by_severity": {
+                severity: sum(1 for item in findings if item.severity == severity)
+                for severity in sorted(SEVERITY_ORDER, key=SEVERITY_ORDER.get)
+            },
+            "counts_by_category": {
+                category: sum(1 for item in findings if item.category == category)
+                for category in sorted({item.category for item in findings})
+            },
+            "findings": [
+                {
+                    "finding_id": item.finding_id,
+                    "category": item.category,
+                    "severity": item.severity,
+                    "title": item.title,
+                    "description": item.description,
+                    "recommendation": item.recommendation,
+                    "contract_refs": item.contract_refs,
+                    "evidence_refs": item.evidence_refs,
+                    "fingerprint": item.fingerprint,
+                }
+                for item in findings
+            ],
+            "review_packet": packet,
+        }
+        node_exec_id = _persist_node(
+            node_id="aggregate_output",
+            node_name="AggregateOutput",
+            output={"review_status": report["review_status"], "finding_count": len(findings)},
+        )
+        task_context.metadata["report"] = report
+        task_context.metadata["node_exec_ids"]["aggregate_output"] = node_exec_id
+        return task_context
+
+
+class EmitArtifactsNode(Node):
+    async def process(self, task_context: TaskContext) -> TaskContext:
+        store, run_id, review_run_id = _runtime()
+        repo_root = Path(task_context.event.repo_root)
+        report = dict(task_context.metadata.get("report") or {})
+        packet = dict(task_context.metadata.get("packet") or {})
+        output_dir = repo_root / ".devflow" / "reviews" / review_run_id
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+        report_path = output_dir / "review_report.json"
+        summary_path = output_dir / "review_summary.md"
+        packet_path = output_dir / "review_packet.canonical.json"
+
+        report_text = json.dumps(report, sort_keys=True, indent=2) + "\n"
+        packet_text = json.dumps(packet, sort_keys=True, indent=2) + "\n"
+        summary_text = "\n".join(
+            [
+                "# Review Summary",
+                "",
+                f"Review status: {report.get('review_status', 'pass')}",
+                f"Story: {task_context.metadata.get('story_id') or ''}",
+                f"Findings: {len(report.get('findings') or [])}",
+            ]
+        ) + "\n"
+
+        report_path.write_text(report_text, encoding="utf-8")
+        summary_path.write_text(summary_text, encoding="utf-8")
+        packet_path.write_text(packet_text, encoding="utf-8")
+
+        node_exec_id = _persist_node(
+            node_id="emit_artifacts",
+            node_name="EmitArtifacts",
+            output={"output_dir": str(output_dir)},
+        )
+        for kind, path, content in [
+            ("review_report", report_path, report_text),
+            ("review_summary", summary_path, summary_text),
+            ("review_packet", packet_path, packet_text),
+        ]:
+            store.add_artifact(
+                run_id=run_id,
+                node_exec_id=node_exec_id,
+                kind=kind,
+                uri=str(path),
+                content_type="application/json" if path.suffix == ".json" else "text/markdown",
+                byte_size=len(content.encode("utf-8")),
+                sha256=hashlib.sha256(content.encode("utf-8")).hexdigest(),
+                metadata={"review_run_id": review_run_id},
+            )
+
+        task_context.metadata["output_dir"] = str(output_dir)
+        task_context.metadata["node_exec_ids"]["emit_artifacts"] = node_exec_id
+        return task_context
+
+
+class ReviewWorkflow(Workflow):
+    workflow_schema = WorkflowSchema(
+        description="DevFlow review DAG (GenAI workflow framework)",
+        event_schema=ReviewEvent,
+        start=IngestNode,
+        nodes=[
+            NodeConfig(node=IngestNode, connections=[ValidationGateNode]),
+            NodeConfig(node=ValidationGateNode, connections=[SecurityReviewNode]),
+            NodeConfig(node=SecurityReviewNode, connections=[ContractReviewNode]),
+            NodeConfig(node=ContractReviewNode, connections=[RegressionReviewNode]),
+            NodeConfig(node=RegressionReviewNode, connections=[AggregateOutputNode]),
+            NodeConfig(node=AggregateOutputNode, connections=[EmitArtifactsNode]),
+            NodeConfig(node=EmitArtifactsNode, connections=[]),
+        ],
+    )
+
+
+def run_review_dag(*, repo_root: Path, store: ExecutionStore, story: dict[str, Any]) -> tuple[int, str]:
+    contract = dict(story.get("contract") or {})
+    story_id = str(contract.get("story_id") or story.get("story_id") or "").strip() or None
+    story_text = _load_story_text(story, repo_root)
+    packet = {
+        "story_id": story_id,
+        "story_uuid": contract.get("story_uuid"),
+        "repo_root": str(repo_root),
+        "required_planes": contract.get("required_planes") or [],
+    }
+    packet_hash = _stable_hash(packet)
+    story_hash = _stable_hash({"story_id": story_id, "story_text": story_text})
+    config_hash = _stable_hash(
+        {
+            "checks": [
+                "validation_gate",
+                "security_review",
+                "contract_review",
+                "regression_review",
+            ]
+        }
+    )
+
+    run_id = store.create_run(
+        dag_id="review_dag",
+        dag_version="v1",
+        root_correlation_id=f"corr_{uuid.uuid4()}",
+        config={"story_id": story_id, "story_hash": story_hash},
+    )
+    store.mark_run_started(run_id=run_id)
+    review_run_id = store.create_review_run(
+        run_id=run_id,
+        story_id=story_id,
+        packet_hash=packet_hash,
+        story_hash=story_hash,
+        config_hash=config_hash,
+        output_dir=str(repo_root / ".devflow" / "reviews" / "pending"),
+    )
+
+    wf = ReviewWorkflow()
+    global _CURRENT_STORE, _CURRENT_RUN_ID, _CURRENT_REVIEW_RUN_ID
+    _CURRENT_STORE = store
+    _CURRENT_RUN_ID = run_id
+    _CURRENT_REVIEW_RUN_ID = review_run_id
+    try:
+        ctx = wf.run({"repo_root": str(repo_root), "story": story})
+    finally:
+        _CURRENT_STORE = None
+        _CURRENT_RUN_ID = None
+        _CURRENT_REVIEW_RUN_ID = None
+
+    report = dict(ctx.metadata.get("report") or {})
+    findings = list(report.get("findings") or [])
+    for item in findings:
+        store.add_review_finding(
+            review_run_id=review_run_id,
+            finding_id=str(item.get("finding_id") or ""),
+            category=str(item.get("category") or "compliance"),
+            severity=str(item.get("severity") or "medium"),
+            title=str(item.get("title") or ""),
+            description=str(item.get("description") or ""),
+            recommendation=str(item.get("recommendation") or ""),
+            contract_refs=[str(v) for v in item.get("contract_refs") or []],
+            evidence_refs=[str(v) for v in item.get("evidence_refs") or []],
+            fingerprint=str(item.get("fingerprint") or "") or None,
+        )
+
+    output_dir = str(ctx.metadata.get("output_dir") or (repo_root / ".devflow" / "reviews" / review_run_id))
+    with store._connect() as conn:
+        conn.execute(
+            "UPDATE review_runs SET output_dir=?, updated_at=? WHERE review_run_id=?",
+            (output_dir, int(time.time()), review_run_id),
+        )
+
+    review_status = str(report.get("review_status") or "pass")
+    store.mark_review_run_finished(review_run_id=review_run_id, status="succeeded", review_status=review_status)
+    store.mark_run_finished(run_id=run_id, status="succeeded")
+
+    exit_code = 2 if review_status == "fail" else 0
+    msg = f"review {'failed' if exit_code else 'passed'}: review_run_id={review_run_id} output_dir={output_dir}\n"
+    return exit_code, msg

devflow_engine/review/review_story.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Finding:
+    kind: str
+    severity: str
+    message: str
+    evidence: dict[str, Any]
+
+
+def _anchor_exists(repo_root: Path, anchor: str) -> bool:
+    # For now: treat anchor that looks like a path as filesystem check.
+    p = (repo_root / anchor).resolve()
+    try:
+        p.relative_to(repo_root.resolve())
+    except Exception:
+        return False
+    return p.exists()
+
+
+def review_story_contract(repo_root: Path, contract: dict[str, Any]) -> list[Finding]:
+    findings: list[Finding] = []
+
+    for oracle in contract.get("plane_oracles", []) or []:
+        anchor = oracle.get("anchor")
+        plane = oracle.get("plane")
+        if not isinstance(anchor, str) or not anchor:
+            continue
+        if _anchor_exists(repo_root, anchor):
+            continue
+        findings.append(
+            Finding(
+                kind="missing_anchor",
+                severity="medium",
+                message=f"Plane oracle anchor not found: {anchor}",
+                evidence={"plane": plane, "anchor": anchor},
+            )
+        )
+
+    # Lightweight security/compliance heuristics.
+    sec = (contract.get("planes") or {}).get("security", "")
+    comp = (contract.get("planes") or {}).get("compliance", "")
+    if isinstance(sec, str) and "TODO" in sec:
+        findings.append(
+            Finding(
+                kind="security_todo",
+                severity="low",
+                message="Security plane still contains TODO",
+                evidence={},
+            )
+        )
+    if isinstance(comp, str) and "TODO" in comp:
+        findings.append(
+            Finding(
+                kind="compliance_todo",
+                severity="low",
+                message="Compliance plane still contains TODO",
+                evidence={},
+            )
+        )
+
+    return findings

devflow_engine/scope_idea/__init__.py

@@ -0,0 +1,3 @@
+from .dag import DAG_ID, ScopeToIdeaDagResult, run_scope_to_idea_dag
+
+__all__ = ["DAG_ID", "ScopeToIdeaDagResult", "run_scope_to_idea_dag"]