devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/story/reconciliation.py

@@ -0,0 +1,408 @@
+from __future__ import annotations
+
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+from ..stores.execution_store import ExecutionStore
+from .discovery import canonical_story_reconciliation_doc_path, get_story_source_paths
+from .markdown_contracts import parse_story_contracts_from_markdown, render_canonical_story_markdown
+
+
+ACTIVE_STORY_QUEUE_STATUSES = ("queued", "claimed", "in_progress", "blocked")
+GENERATED_STORY_ARTIFACT_KIND = "generated_devflow_story_json"
+
+
+def _iter_live_story_json_paths(repo_root: Path) -> list[Path]:
+    ideas_root = repo_root / ".devflow" / "ideas"
+    if not ideas_root.exists():
+        return []
+    return sorted(ideas_root.glob("*/devflow_story_sets/*/story_*.json"))
+
+
+def _read_json(path: Path) -> dict[str, Any] | None:
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    return payload if isinstance(payload, dict) else None
+
+
+def _resolve_artifact_uri(repo_root: Path, artifact_uri: str) -> Path:
+    path = Path(artifact_uri).expanduser()
+    if not path.is_absolute():
+        path = repo_root / path
+    return path.resolve()
+
+
+def _load_live_story_index(repo_root: Path) -> dict[str, list[dict[str, Any]]]:
+    grouped: dict[str, list[dict[str, Any]]] = {}
+    for path in _iter_live_story_json_paths(repo_root):
+        payload = _read_json(path)
+        if payload is None:
+            continue
+        story_id = str(payload.get("story_id") or "").strip()
+        if not story_id:
+            continue
+        grouped.setdefault(story_id, []).append(
+            {
+                "path": str(path.resolve()),
+                "story_uuid": str(payload.get("story_uuid") or "").strip(),
+                "title": str(payload.get("title") or "").strip(),
+            }
+        )
+    for story_id in grouped:
+        grouped[story_id] = sorted(grouped[story_id], key=lambda item: str(item["path"]))
+    return grouped
+
+
+def _load_live_story_payloads(repo_root: Path) -> dict[str, list[dict[str, Any]]]:
+    grouped: dict[str, list[dict[str, Any]]] = {}
+    for path in _iter_live_story_json_paths(repo_root):
+        payload = _read_json(path)
+        if payload is None:
+            continue
+        story_id = str(payload.get("story_id") or "").strip()
+        if not story_id:
+            continue
+        grouped.setdefault(story_id, []).append({"path": str(path.resolve()), "payload": payload})
+    for story_id in grouped:
+        grouped[story_id] = sorted(grouped[story_id], key=lambda item: str(item["path"]))
+    return grouped
+
+
+def _load_canonical_doc_story_ids(repo_root: Path) -> dict[str, Any]:
+    try:
+        source_paths = get_story_source_paths(repo_root)
+    except FileNotFoundError:
+        return {"available": False, "story_ids": [], "source_paths": []}
+
+    story_ids: set[str] = set()
+    for path in source_paths:
+        try:
+            text = path.read_text(encoding="utf-8")
+        except Exception:
+            continue
+        for block in parse_story_contracts_from_markdown(text, source_path=str(path)):
+            story_id = str(block.contract.story_id or "").strip()
+            if story_id:
+                story_ids.add(story_id)
+    return {
+        "available": True,
+        "story_ids": sorted(story_ids),
+        "source_paths": [str(path.resolve()) for path in source_paths],
+    }
+
+
+def _load_story_queue_rows(store: ExecutionStore, *, project_id: str | None) -> list[dict[str, Any]]:
+    query = (
+        "SELECT sq.story_queue_id, sq.project_id, sq.enqueue_run_id, sq.story_artifact_id, sq.story_id, sq.title, "
+        "sq.status, sq.claimed_by_worker_id, sq.claimed_at, sq.started_run_id, sq.finished_run_id, "
+        "sq.failure_message, sq.failure_context_json, sq.created_at, sq.updated_at, "
+        "a.kind AS artifact_kind, a.uri AS artifact_uri, a.metadata_json AS artifact_metadata_json "
+        "FROM story_queue sq "
+        "LEFT JOIN artifacts a ON a.artifact_id = sq.story_artifact_id "
+    )
+    params: tuple[Any, ...] = ()
+    if project_id is not None:
+        query += "WHERE sq.project_id=? "
+        params = (project_id,)
+    query += "ORDER BY sq.story_id ASC, sq.created_at ASC, sq.story_queue_id ASC"
+
+    with store._connect() as conn:
+        rows = conn.execute(query, params).fetchall()
+
+    payloads: list[dict[str, Any]] = []
+    for row in rows:
+        payload = dict(row)
+        try:
+            payload["artifact_metadata"] = json.loads(str(payload.get("artifact_metadata_json") or "")) if payload.get("artifact_metadata_json") else {}
+        except Exception:
+            payload["artifact_metadata"] = {}
+        try:
+            payload["failure_context"] = json.loads(str(payload.get("failure_context_json") or "")) if payload.get("failure_context_json") else None
+        except Exception:
+            payload["failure_context"] = None
+        payloads.append(payload)
+    return payloads
+
+
+def _choose_canonical_live_path(*, story_id: str, live_candidates: list[dict[str, Any]], referenced_uris: set[str]) -> str | None:
+    if not live_candidates:
+        return None
+    ranked = sorted(
+        live_candidates,
+        key=lambda item: (0 if str(item["path"]) in referenced_uris else 1, str(item["path"])),
+    )
+    return str(ranked[0]["path"])
+
+
+def _queue_status_priority(status: str) -> int:
+    return {"in_progress": 0, "claimed": 1, "queued": 2, "blocked": 3}.get(status, 9)
+
+
+def _choose_canonical_queue_row(repo_root: Path, rows: list[dict[str, Any]], *, canonical_path: str | None) -> dict[str, Any] | None:
+    active_rows = [row for row in rows if str(row.get("status") or "") in ACTIVE_STORY_QUEUE_STATUSES]
+    if not active_rows:
+        return None
+    ranked = sorted(
+        active_rows,
+        key=lambda row: (
+            0
+            if canonical_path
+            and str(row.get("artifact_uri") or "").strip()
+            and str(_resolve_artifact_uri(repo_root, str(row.get("artifact_uri") or "").strip())) == canonical_path
+            else 1,
+            _queue_status_priority(str(row.get("status") or "")),
+            int(row.get("created_at") or 0),
+            str(row.get("story_queue_id") or ""),
+        ),
+    )
+    return ranked[0]
+
+
+def _inventory_story_path_issue(repo_root: Path, row: dict[str, Any]) -> dict[str, Any] | None:
+    artifact_uri = str(row.get("artifact_uri") or "").strip()
+    if not artifact_uri:
+        return {
+            "story_queue_id": row["story_queue_id"],
+            "artifact_id": row["story_artifact_id"],
+            "issue": "missing_artifact_uri",
+        }
+    path = _resolve_artifact_uri(repo_root, artifact_uri)
+    if not path.exists():
+        return {
+            "story_queue_id": row["story_queue_id"],
+            "artifact_id": row["story_artifact_id"],
+            "artifact_uri": artifact_uri,
+            "issue": "missing_path",
+        }
+    payload = _read_json(path)
+    if payload is None:
+        return {
+            "story_queue_id": row["story_queue_id"],
+            "artifact_id": row["story_artifact_id"],
+            "artifact_uri": artifact_uri,
+            "issue": "invalid_json",
+        }
+    payload_story_id = str(payload.get("story_id") or "").strip()
+    queue_story_id = str(row.get("story_id") or "").strip()
+    if payload_story_id and queue_story_id and payload_story_id != queue_story_id:
+        return {
+            "story_queue_id": row["story_queue_id"],
+            "artifact_id": row["story_artifact_id"],
+            "artifact_uri": artifact_uri,
+            "issue": "story_id_mismatch",
+            "payload_story_id": payload_story_id,
+            "queue_story_id": queue_story_id,
+        }
+    return None
+
+
+def _rebuild_canonical_story_doc(
+    *,
+    repo_root: Path,
+    live_story_payloads: dict[str, list[dict[str, Any]]],
+    referenced_uris_by_story_id: dict[str, set[str]],
+) -> dict[str, Any]:
+    canonical_doc_path = canonical_story_reconciliation_doc_path(repo_root)
+    if not live_story_payloads:
+        return {
+            "changed": False,
+            "path": str(canonical_doc_path.resolve()),
+            "story_count": 0,
+            "story_ids": [],
+        }
+
+    story_blocks: list[str] = []
+    story_ids: list[str] = []
+    for story_id in sorted(live_story_payloads):
+        candidates = live_story_payloads[story_id]
+        canonical_path = _choose_canonical_live_path(
+            story_id=story_id,
+            live_candidates=[{"path": str(item["path"])} for item in candidates],
+            referenced_uris=referenced_uris_by_story_id.get(story_id, set()),
+        )
+        selected = next((item for item in candidates if str(item["path"]) == canonical_path), candidates[0])
+        story_blocks.append(render_canonical_story_markdown(dict(selected["payload"])))
+        story_ids.append(story_id)
+
+    canonical_doc_text = "\n".join(block.rstrip() for block in story_blocks).rstrip() + "\n"
+    previous_text = canonical_doc_path.read_text(encoding="utf-8") if canonical_doc_path.exists() else None
+    changed = previous_text != canonical_doc_text
+    if changed:
+        canonical_doc_path.parent.mkdir(parents=True, exist_ok=True)
+        canonical_doc_path.write_text(canonical_doc_text, encoding="utf-8")
+
+    return {
+        "changed": changed,
+        "path": str(canonical_doc_path.resolve()),
+        "story_count": len(story_ids),
+        "story_ids": story_ids,
+    }
+
+
+def reconcile_story_queue(
+    *,
+    repo_root: Path,
+    project_id: str | None = None,
+    apply: bool = False,
+) -> dict[str, Any]:
+    repo_root = repo_root.expanduser().resolve()
+    store = ExecutionStore(repo_root / ".devflow" / "execution.sqlite")
+    live_story_index = _load_live_story_index(repo_root)
+    live_story_payloads = _load_live_story_payloads(repo_root)
+    doc_index = _load_canonical_doc_story_ids(repo_root)
+    queue_rows = _load_story_queue_rows(store, project_id=project_id)
+
+    grouped_rows: dict[str, list[dict[str, Any]]] = {}
+    for row in queue_rows:
+        story_id = str(row.get("story_id") or "").strip()
+        if not story_id:
+            continue
+        grouped_rows.setdefault(story_id, []).append(row)
+
+    report_stories: list[dict[str, Any]] = []
+    artifact_updates: list[dict[str, Any]] = []
+    queue_inertions: list[dict[str, Any]] = []
+    canonical_doc_repair = {
+        "changed": False,
+        "path": str(canonical_story_reconciliation_doc_path(repo_root).resolve()),
+        "story_count": 0,
+        "story_ids": [],
+    }
+    now = int(time.time())
+    referenced_uris_by_story_id: dict[str, set[str]] = {}
+
+    with store._connect() as conn:
+        for story_id in sorted(grouped_rows):
+            rows = grouped_rows[story_id]
+            live_candidates = live_story_index.get(story_id, [])
+            referenced_uris = {
+                str(_resolve_artifact_uri(repo_root, str(row.get("artifact_uri") or "").strip()))
+                for row in rows
+                if str(row.get("artifact_uri") or "").strip()
+            }
+            referenced_uris_by_story_id[story_id] = referenced_uris
+            canonical_path = _choose_canonical_live_path(
+                story_id=story_id,
+                live_candidates=live_candidates,
+                referenced_uris=referenced_uris,
+            )
+            canonical_queue_row = _choose_canonical_queue_row(repo_root, rows, canonical_path=canonical_path)
+            active_rows = [row for row in rows if str(row.get("status") or "") in ACTIVE_STORY_QUEUE_STATUSES]
+            path_issues = [issue for issue in (_inventory_story_path_issue(repo_root, row) for row in rows) if issue is not None]
+            unknown_story_id_risk = canonical_path is None
+            doc_mismatch = bool(doc_index["available"] and canonical_path and story_id not in set(doc_index["story_ids"]))
+
+            story_report = {
+                "story_id": story_id,
+                "project_ids": sorted({str(row.get("project_id") or "") for row in rows if str(row.get("project_id") or "")}),
+                "story_queue_ids": [str(row["story_queue_id"]) for row in rows],
+                "duplicate_queue_rows": len(rows) > 1,
+                "duplicate_active_queue_rows": len(active_rows) > 1,
+                "distinct_artifact_uris": sorted(referenced_uris),
+                "broken_story_json_paths": path_issues,
+                "canonical_story_json_path": canonical_path,
+                "canonical_queue_row_id": None if canonical_queue_row is None else str(canonical_queue_row["story_queue_id"]),
+                "unknown_story_id_risk": unknown_story_id_risk,
+                "canonical_doc_mismatch": doc_mismatch,
+                "live_story_json_candidates": [str(item["path"]) for item in live_candidates],
+            }
+
+            if apply and canonical_path:
+                seen_artifact_ids: set[str] = set()
+                for row in rows:
+                    artifact_id = str(row.get("story_artifact_id") or "").strip()
+                    if not artifact_id or artifact_id in seen_artifact_ids:
+                        continue
+                    seen_artifact_ids.add(artifact_id)
+                    artifact_uri = str(_resolve_artifact_uri(repo_root, str(row.get("artifact_uri") or "").strip())) if str(row.get("artifact_uri") or "").strip() else ""
+                    if artifact_uri == canonical_path:
+                        continue
+                    conn.execute(
+                        "UPDATE artifacts SET uri=?, updated_at=? WHERE artifact_id=?",
+                        (canonical_path, now, artifact_id),
+                    )
+                    artifact_updates.append(
+                        {
+                            "story_id": story_id,
+                            "artifact_id": artifact_id,
+                            "from_uri": artifact_uri,
+                            "to_uri": canonical_path,
+                        }
+                    )
+
+                if canonical_queue_row is not None:
+                    for row in active_rows:
+                        story_queue_id = str(row["story_queue_id"])
+                        if story_queue_id == str(canonical_queue_row["story_queue_id"]):
+                            continue
+                        failure_context = row.get("failure_context") if isinstance(row.get("failure_context"), dict) else {}
+                        failure_context["reconciled_duplicate"] = True
+                        failure_context["canonical_story_queue_id"] = str(canonical_queue_row["story_queue_id"])
+                        failure_context["canonical_story_json_path"] = canonical_path
+                        message = (
+                            "story queue reconciled: duplicate active row inerted; "
+                            f"canonical_story_queue_id={canonical_queue_row['story_queue_id']}"
+                        )
+                        conn.execute(
+                            (
+                                "UPDATE story_queue SET status='cancelled', claimed_by_worker_id=NULL, claimed_at=NULL, "
+                                "failure_message=?, failure_context_json=?, updated_at=? WHERE story_queue_id=?"
+                            ),
+                            (
+                                message,
+                                json.dumps(failure_context, sort_keys=True),
+                                now,
+                                story_queue_id,
+                            ),
+                        )
+                        queue_inertions.append(
+                            {
+                                "story_id": story_id,
+                                "story_queue_id": story_queue_id,
+                                "canonical_story_queue_id": str(canonical_queue_row["story_queue_id"]),
+                            }
+                        )
+
+            report_stories.append(story_report)
+
+    if apply:
+        canonical_doc_repair = _rebuild_canonical_story_doc(
+            repo_root=repo_root,
+            live_story_payloads=live_story_payloads,
+            referenced_uris_by_story_id=referenced_uris_by_story_id,
+        )
+        doc_index = _load_canonical_doc_story_ids(repo_root)
+        doc_story_ids = set(doc_index["story_ids"])
+        for story_report in report_stories:
+            story_id = str(story_report.get("story_id") or "").strip()
+            story_report["canonical_doc_mismatch"] = bool(story_report["canonical_story_json_path"] and story_id not in doc_story_ids)
+            story_report["canonical_doc_repaired"] = bool(canonical_doc_repair["changed"] and not story_report["canonical_doc_mismatch"])
+
+    summary = {
+        "story_count": len(report_stories),
+        "story_ids_with_duplicate_queue_rows": sum(1 for item in report_stories if item["duplicate_queue_rows"]),
+        "story_ids_with_duplicate_active_queue_rows": sum(1 for item in report_stories if item["duplicate_active_queue_rows"]),
+        "story_ids_with_broken_story_json_paths": sum(1 for item in report_stories if item["broken_story_json_paths"]),
+        "story_ids_with_unknown_story_id_risk": sum(1 for item in report_stories if item["unknown_story_id_risk"]),
+        "story_ids_with_canonical_doc_mismatch": sum(1 for item in report_stories if item["canonical_doc_mismatch"]),
+        "artifact_uri_updates": len(artifact_updates),
+        "queue_rows_inerted": len(queue_inertions),
+        "canonical_doc_repairs": 1 if canonical_doc_repair["changed"] else 0,
+    }
+
+    return {
+        "repo_root": str(repo_root),
+        "db_path": str((repo_root / ".devflow" / "execution.sqlite").resolve()),
+        "project_id": project_id,
+        "apply": apply,
+        "canonical_doc_index": doc_index,
+        "summary": summary,
+        "stories": report_stories,
+        "artifact_updates": artifact_updates,
+        "queue_inertions": queue_inertions,
+        "canonical_doc_repair": canonical_doc_repair,
+    }

devflow_engine/story/validate_stories.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Deterministic story contract validator for devflow_engine.
+
+Hard gates:
+- No silent skips: every canonical story file must contain ≥1 parseable story block.
+- Contract formation mode line present (verbatim)
+- story_uuid uuid-v4, required_planes valid
+- plane_oracles present with 1 entry per required plane
+- each plane_oracles item has oracle + anchor
+- anchor must match a stable-address allowlist (prefix-based)
+
+Output:
+- human-readable diagnostics to stdout
+- optional JSON report via --json <path>
+
+Exit codes:
+- 0 pass
+- 2 validation failed
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from dataclasses import asdict, dataclass
+from pathlib import Path
+
+from devflow_engine.story.contracts import validate_parsed_story_block
+from devflow_engine.story.markdown_contracts import iter_canonical_story_paths, parse_story_contracts_from_markdown
+
+
+ANCHOR_PREFIX_ALLOWLIST = (
+    # UI / routing
+    "route:",
+    # API contracts
+    "api:",
+    # Integrations boundary oracles
+    "outbox:",
+    "audit:",
+    # CLI and operational commands (common in devflow_engine stories)
+    "cli:",
+    "command:",
+    # Artifacts and files
+    "artifact:",
+    "file:",
+    # Environment + docs + scripts are legitimate stable anchors in story specs
+    "env:",
+    "docs:",
+    "script:",
+)
+
+# Also allow short-form HTTP anchors like: "GET /health" or "POST /api/v1/foo"
+HTTP_ANCHOR_RE = re.compile(r"^(GET|POST|PUT|PATCH|DELETE)\s+/\S+")
+SCHEME_ANCHOR_RE = re.compile(r"^[a-z][a-z0-9_-]{1,32}:")
+
+
+@dataclass(frozen=True)
+class Issue:
+    code: str
+    message: str
+    path: str
+
+
+def _anchor_is_allowed(anchor: str) -> bool:
+    a = (anchor or "").strip()
+    if not a:
+        return False
+    # Prefer scheme-like anchors (stable, parseable addresses), e.g.
+    # cli:..., route:..., api:..., outbox:..., docs:..., tests:...
+    if SCHEME_ANCHOR_RE.match(a):
+        return True
+    # Also allow short-form HTTP anchors like "GET /health".
+    if HTTP_ANCHOR_RE.match(a):
+        return True
+    return False
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--repo-root", default=".")
+    ap.add_argument("--json", default=None, help="Write JSON report to this path")
+    args = ap.parse_args()
+
+    repo_root = Path(args.repo_root).resolve()
+
+    issues: list[Issue] = []
+    story_count = 0
+
+    for p in iter_canonical_story_paths(repo_root):
+        text = p.read_text(encoding="utf-8")
+        blocks = parse_story_contracts_from_markdown(text, source_path=str(p))
+
+        if not blocks:
+            issues.append(
+                Issue(
+                    code="no_story_blocks",
+                    message="No parseable story blocks found (expected a story_uuid: header). This file would otherwise be silently skipped.",
+                    path=str(p),
+                )
+            )
+            continue
+
+        for b in blocks:
+            story_count += 1
+            contract_issues = validate_parsed_story_block(b)
+            for ci in contract_issues:
+                issues.append(Issue(code=ci.code, message=ci.message, path=ci.path))
+
+            # Additional deterministic semantic gate: anchor must be stable-address formatted.
+            c = b.contract
+            for idx, po in enumerate(c.plane_oracles):
+                if po.anchor and not _anchor_is_allowed(po.anchor):
+                    issues.append(
+                        Issue(
+                            code="invalid_plane_oracle_anchor_format",
+                            message=(
+                                f"plane_oracles[{idx}].anchor must be a stable address; "
+                                f"allowed prefixes={list(ANCHOR_PREFIX_ALLOWLIST)} or HTTP like 'GET /health'"
+                            ),
+                            path=f"{c.source_path}:{b.start_line}:plane_oracles[{idx}].anchor",
+                        )
+                    )
+
+    report = {
+        "ok": len(issues) == 0,
+        "story_count": story_count,
+        "issue_count": len(issues),
+        "issues": [asdict(i) for i in issues],
+    }
+
+    if args.json:
+        Path(args.json).write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+
+    if issues:
+        print(f"STORY VALIDATION FAILED: {len(issues)} issue(s) across {story_count} story block(s)")
+        # Print up to 200 issues to avoid drowning CI logs.
+        for i, iss in enumerate(issues[:200]):
+            print(f"- {iss.code}: {iss.path}: {iss.message}")
+        if len(issues) > 200:
+            print(f"... ({len(issues) - 200} more)")
+        return 2
+
+    print(f"STORY VALIDATION OK: {story_count} story block(s)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())