devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/recovery/models.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+QueueItemKind = Literal["scope", "idea_creation", "idea", "story", "integration", "recovery"]
+RecoveryDisposition = Literal["replayable", "blocked"]
+RecoveryAction = Literal[
+    "requeue_without_fix",
+    "repair_artifact_then_requeue",
+    "adjust_story_shaping_policy_then_requeue",
+    "manual_review_required",
+    "dead_letter",
+    "none",
+]
+FixScope = Literal["item_local", "systemic"]
+ReplayScope = Literal["single_item", "similar_failed_items"]
+FailureType = Literal["code_error", "process_error"]
+RecoveryStrategy = Literal[
+    "handoff_to_code_error",
+    "replay_boundary_recovery",
+    "state_reconciliation_recovery",
+    "artifact_regeneration_recovery",
+    "preflight_health_repair_recovery",
+    "escalate_process_bug",
+]
+
+
+class FailedQueueItemArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    project_id: str | None = None
+    dfs_project_id: str | None = None
+    enqueue_run_id: str
+    status: str
+    title: str
+    payload_ref: str | None = None
+    payload_exists: bool = False
+    story_id: str | None = None
+    failure_message: str | None = None
+    failure_context: dict = Field(default_factory=dict)
+    raw_row: dict = Field(default_factory=dict)
+    discovered_artifacts: dict[str, Any] = Field(default_factory=dict)
+
+
+class RecoverySuccessCriterion(BaseModel):
+    criterion: str
+    oracle: str
+    evidence_ref: str | None = None
+
+
+class RecoveryNonConvergenceArtifact(BaseModel):
+    summary: str
+    reason: str
+    attempts_reviewed: int = 0
+    same_failure_surface: bool = False
+    unchanged_test_surface: bool = False
+    wrong_seam: bool = False
+    downstream_blocker: bool = False
+    downstream_blocker_stage: str | None = None
+    schema_or_runtime_drift: bool = False
+    measurable_progress: bool = True
+    evidence: list[str] = Field(default_factory=list)
+
+
+class RecoveryInvestigationArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    summary: str
+    failure_nature: str
+    evidence: list[str] = Field(default_factory=list)
+    primary_evidence_source: str | None = None
+    primary_evidence_refs: list[str] = Field(default_factory=list)
+    primary_log_insight: str | None = None
+    affected_boundary: str | None = None
+    likely_failed_stage: str | None = None
+    confidence: str = "medium"
+    recovery_goal: str
+    success_criteria: list[RecoverySuccessCriterion] = Field(default_factory=list)
+    verification_evidence: list[str] = Field(default_factory=list)
+    replay_path: str | None = None
+    escalation_conditions: list[str] = Field(default_factory=list)
+    non_convergence: RecoveryNonConvergenceArtifact | None = None
+
+
+class RecoveryDiagnosisArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    strategy: RecoveryStrategy
+    summary: str
+    rationale: str
+    verification_targets: list[RecoverySuccessCriterion] = Field(default_factory=list)
+    replay_path: str | None = None
+    suggested_action: RecoveryAction = "none"
+    evidence: list[str] = Field(default_factory=list)
+
+
+class RemediationPlanArtifact(BaseModel):
+    queue_type: QueueItemKind
+    action: RecoveryAction
+    summary: str
+    steps: list[str] = Field(default_factory=list)
+    bounded: bool = True
+    preserve_failure_context: bool = True
+    fix_scope: FixScope = "item_local"
+    replay_scope: ReplayScope = "single_item"
+    replay_path: str | None = None
+    remediation_artifact: str | None = None
+
+
+class FixApplicationArtifact(BaseModel):
+    queue_type: QueueItemKind
+    action_taken: RecoveryAction
+    changed: bool
+    notes: list[str] = Field(default_factory=list)
+    remediation_artifact: str | None = None
+
+
+class RecoveryExecutionArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    outcome: Literal["reenqueued", "delegated", "blocked"]
+    execution_summary: str
+    preserve_failure_context: bool = True
+    delegation_summary: str | None = None
+    attempts_used: int = 1
+    success_criteria: list[RecoverySuccessCriterion] = Field(default_factory=list)
+    verification_summary: str | None = None
+    primary_evidence_source: str | None = None
+    primary_evidence_refs: list[str] = Field(default_factory=list)
+    primary_log_insight: str | None = None
+    recovery_handoff_artifact_path: str | None = None
+    recovery_handoff_summary: str | None = None
+
+
+class PreReplayCheckArtifact(BaseModel):
+    queue_type: QueueItemKind
+    ready: bool
+    checks: list[str] = Field(default_factory=list)
+    blocking_reasons: list[str] = Field(default_factory=list)
+
+
+class ReenqueueArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    status: str
+    failure_context: dict | None = None
+    replay_metadata: dict[str, Any] | None = None
+
+
+class RecoveryHandoffArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    story_id: str | None = None
+    implementation_run_id: str | None = None
+    failed_stage: str | None = None
+    primary_evidence_source: str | None = None
+    primary_evidence_refs: list[str] = Field(default_factory=list)
+    key_log_insight: str | None = None
+    failing_surface_summary: str
+    likely_seam: str | None = None
+    disproven_dead_ends: list[str] = Field(default_factory=list)
+    verification_blockers: list[str] = Field(default_factory=list)
+    non_convergence_insight: str | None = None
+    produced_by_recovery_run_id: str | None = None
+    consumer_hint: str = "Attach this artifact to the next Green attempt for the same story when present."
+
+
+class RecoveryOutcomeArtifact(BaseModel):
+    queue_type: QueueItemKind
+    item_id: str
+    project_id: str | None = None
+    outcome: Literal["reenqueued", "blocked", "delegated", "recovered"]
+    summary: str
+    investigation: RecoveryInvestigationArtifact | None = None
+    plan: RemediationPlanArtifact | None = None
+    reenqueue: ReenqueueArtifact | None = None
+    recovery_handoff_artifact_path: str | None = None
+
+
+class SystemicPatternArtifact(BaseModel):
+    failure_signature: str
+    is_systemic: bool
+    affected_queue_type: str
+    affected_item_ids: list[str]
+    sample_failure_messages: list[str]
+    failed_stages: list[str]
+    total_affected: int
+    pattern_summary: str
+
+
+class FilePatch(BaseModel):
+    path: str
+    content: str
+
+
+class CodeRootCauseArtifact(BaseModel):
+    failure_signature: str
+    root_cause_location: str
+    root_cause_description: str
+    fix_type: Literal["code_fix", "config_fix", "environment_fix", "dead_letter", "item_local_only"]
+    specific_fix_plan: str
+    files_inspected: list[str]
+    confidence: Literal["high", "medium", "low"]
+    affects_all_items: bool
+    residual_items_needing_attention: list[str]
+
+
+class RemediationResultArtifact(BaseModel):
+    fix_type: str
+    fix_applied: bool
+    files_changed: list[str]
+    fix_summary: str
+    verification_notes: str
+    ready_to_requeue: bool
+    items_to_requeue: list[str]
+    items_to_dead_letter: list[str]
+    file_patches: list[FilePatch] = Field(default_factory=list)

devflow_engine/refactor.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+
+"""Unified refactor runner.
+
+Consumes a `devflow enforce` report (missing + directives) and asks an LLM to
+produce a unified diff to fix the issues.
+
+This intentionally does not attempt to be clever per-kind; it is one refactor
+loop for packages + modules.
+"""
+
+import json
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from .llm.cli_stream import run_streaming
+from .llm.cli_one_shot import run_one_shot
+
+
+def _extract_diff(text: str) -> str | None:
+    # Prefer fenced ```diff blocks.
+    if "```" in text:
+        parts = text.split("```")
+        for i in range(len(parts) - 1):
+            body = parts[i + 1]
+            lines = body.splitlines()
+            if lines and lines[0].strip().lower() in {"diff", "patch"}:
+                body = "\n".join(lines[1:])
+            body = body.strip("\n")
+            if body.startswith("diff --git"):
+                return body + "\n"
+    # Fallback: find first diff header.
+    idx = text.find("diff --git")
+    if idx != -1:
+        return text[idx:].strip("\n") + "\n"
+    return None
+
+
+def _run(cmd: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(cmd, cwd=str(cwd), text=True, capture_output=True, check=False)
+
+
+def refactor_from_enforce_report(
+    *,
+    repo_root: Path,
+    report: dict[str, object],
+    base_cmd: str,
+    delivery: str,
+    stream: bool,
+) -> dict[str, object]:
+    prompt = {
+        "task": "refactor_to_satisfy_devflow_enforce",
+        "repo_root": str(repo_root),
+        "enforce_report": report,
+        "instructions": [
+            "You must output ONLY a unified git diff (no commentary).",
+            "Fix the enforce_report issues by refactoring code:",
+            "- For refactor_internal_module: rewrite imports/specifiers to use the canonical module path.",
+            "- For refactor_external_package: remove usage/imports of the forbidden package and migrate to the canonical one.",
+            "- For missing imports: either remove the dependency or replace it with already-registered canonical pathways.",
+            "Prefer small, safe edits.",
+        ],
+    }
+
+    prompt_text = json.dumps(prompt, indent=2, sort_keys=True)
+
+    if stream:
+        sres = run_streaming(provider="cli", base_cmd=base_cmd, delivery=delivery, prompt=prompt_text, cwd=repo_root)
+        if not sres.ok:
+            return {"ok": False, "stderr": sres.stderr or sres.stdout, "stdout": sres.stdout, "log_path": sres.log_path}
+        out_text = sres.stdout
+    else:
+        r = run_one_shot(base_cmd=base_cmd, delivery=delivery, prompt=prompt_text, cwd=repo_root)
+        if not r.ok:
+            return {"ok": False, "stderr": r.stderr or r.stdout, "stdout": r.stdout}
+        out_text = r.stdout
+
+    diff = _extract_diff(out_text)
+    if not diff:
+        return {"ok": False, "stderr": "LLM did not return a diff", "stdout": out_text}
+
+    # Apply diff.
+    patch_path = repo_root / ".devflow" / "refactor" / "last.patch"
+    patch_path.parent.mkdir(parents=True, exist_ok=True)
+    patch_path.write_text(diff, encoding="utf-8")
+
+    cp = _run(["git", "apply", "--whitespace=nowarn", str(patch_path)], cwd=repo_root)
+    if cp.returncode != 0:
+        return {"ok": False, "stderr": cp.stderr, "stdout": cp.stdout, "patch": str(patch_path)}
+
+    return {"ok": True, "patch": str(patch_path)}

devflow_engine/registry/__init__.py

@@ -0,0 +1 @@
+"""Registry helpers (module cards + package policy gate)."""

devflow_engine/registry/cards.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+
+_TS_EXPORT_RE = re.compile(
+    r"^\s*export\s+(?:default\s+)?(?:(?:async\s+)?function\s+(?P<fn>[A-Za-z_\$][\w\$]*)|class\s+(?P<class>[A-Za-z_\$][\w\$]*)|const\s+(?P<const>[A-Za-z_\$][\w\$]*)|let\s+(?P<let>[A-Za-z_\$][\w\$]*)|var\s+(?P<var>[A-Za-z_\$][\w\$]*)|type\s+(?P<type>[A-Za-z_\$][\w\$]*)|interface\s+(?P<iface>[A-Za-z_\$][\w\$]*))\b"
+)
+
+_TS_EXPORT_LIST_RE = re.compile(r"^\s*export\s*\{(?P<body>[^}]+)\}\s*;?\s*$")
+
+_PY_DEF_RE = re.compile(r"^def\s+(?P<fn>[A-Za-z_][\w]*)\s*\(")
+_PY_CLASS_RE = re.compile(r"^class\s+(?P<class>[A-Za-z_][\w]*)\s*(\(|:)")
+
+
+@dataclass(frozen=True)
+class ExportSymbol:
+    name: str
+    kind: str  # function|class|const|type|interface|reexport
+
+
+def _iter_project_files(repo_root: Path, roots: list[Path]) -> list[Path]:
+    exts = {".ts", ".tsx", ".js", ".jsx", ".py"}
+
+    ignore_dirnames = {
+        "node_modules",
+        ".next",
+        "dist",
+        "build",
+        "coverage",
+        ".turbo",
+        ".git",
+        ".venv",
+        "__pycache__",
+    }
+
+    import os
+
+    out: list[Path] = []
+    for r in roots:
+        if not r.exists():
+            continue
+        for dirpath, dirnames, filenames in os.walk(r):
+            dirnames[:] = [d for d in dirnames if d not in ignore_dirnames and not d.startswith(".")]
+            for fn in filenames:
+                p = Path(dirpath) / fn
+                if p.suffix in exts:
+                    out.append(p)
+    return sorted(out)
+
+
+def extract_exports(path: Path) -> list[ExportSymbol]:
+    """Best-effort export extraction.
+
+    Deterministic + cheap; not a full parser.
+    """
+
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except Exception:
+        return []
+
+    out: list[ExportSymbol] = []
+
+    if path.suffix == ".py":
+        for ln in lines:
+            if ln.startswith(" ") or ln.startswith("\t"):
+                continue
+            if m := _PY_DEF_RE.match(ln):
+                out.append(ExportSymbol(name=m.group("fn"), kind="function"))
+            elif m := _PY_CLASS_RE.match(ln):
+                out.append(ExportSymbol(name=m.group("class"), kind="class"))
+        return out
+
+    # TS/JS
+    for ln in lines:
+        if m := _TS_EXPORT_RE.match(ln):
+            for kind, key in [
+                ("function", "fn"),
+                ("class", "class"),
+                ("const", "const"),
+                ("const", "let"),
+                ("const", "var"),
+                ("type", "type"),
+                ("interface", "iface"),
+            ]:
+                name = m.groupdict().get(key)
+                if name:
+                    out.append(ExportSymbol(name=name, kind=kind))
+                    break
+            continue
+
+        if m := _TS_EXPORT_LIST_RE.match(ln):
+            body = m.group("body")
+            parts = [p.strip() for p in body.split(",") if p.strip()]
+            for p in parts:
+                # handle `a as b`
+                name = p.split(" as ")[-1].strip()
+                if name:
+                    out.append(ExportSymbol(name=name, kind="reexport"))
+
+    # stable
+    out.sort(key=lambda s: (s.kind, s.name))
+    return out
+
+
+def make_module_card(
+    *,
+    repo_root: Path,
+    importer_rel: str,
+    specifier: str,
+    export_symbols: list[ExportSymbol],
+    used_by: list[str],
+    snippets: list[str],
+    external_packages: list[str],
+    internal_deps: list[str],
+) -> dict[str, object]:
+    return {
+        "item_id": f"module:{importer_rel}",
+        "path": importer_rel,
+        "exports": [s.__dict__ for s in export_symbols],
+        "usage": {
+            "importer_count": len(used_by),
+            "top_importers": used_by,
+            "snippets": snippets,
+        },
+        "imports": {
+            "external_packages": external_packages,
+            "internal_modules": internal_deps,
+        },
+        "heuristic_summary": {
+            "export_count": len(export_symbols),
+            "external_pkg_count": len(external_packages),
+            "internal_dep_count": len(internal_deps),
+            "used_by_count": len(used_by),
+        },
+    }
+
+
+def extract_import_snippets(*, text: str, needle: str, context: int = 2, limit: int = 3) -> list[str]:
+    lines = text.splitlines()
+    out: list[str] = []
+    for i, ln in enumerate(lines):
+        if needle not in ln:
+            continue
+        start = max(0, i - context)
+        end = min(len(lines), i + context + 1)
+        snippet = "\n".join(lines[start:end])
+        out.append(snippet)
+        if len(out) >= limit:
+            break
+    return out
+
+
+def build_module_cards(
+    *,
+    repo_root: Path,
+    roots: list[Path],
+    imports_rows: list[tuple[str, str, str, str]],
+    usage_snippet_limit: int = 3,
+) -> list[dict[str, object]]:
+    """Build module cards for internal modules and their usage.
+
+    imports_rows matches schema: (importer, specifier, kind, resolved_id).
+    """
+
+    # Build reverse index: resolved_id -> list[(importer, specifier)]
+    used_by: dict[str, list[tuple[str, str]]] = defaultdict(list)
+    for importer, spec, kind, resolved in imports_rows:
+        if kind != "internal":
+            continue
+        used_by[str(resolved)].append((str(importer), str(spec)))
+
+    # Build per-file imports summary
+    per_importer_external: dict[str, Counter[str]] = defaultdict(Counter)
+    per_importer_internal: dict[str, Counter[str]] = defaultdict(Counter)
+    for importer, spec, kind, resolved in imports_rows:
+        if kind == "external":
+            pkg = str(resolved).split(":", 1)[1]
+            per_importer_external[str(importer)][pkg] += 1
+        elif kind == "internal":
+            per_importer_internal[str(importer)][str(resolved)] += 1
+
+    cards: list[dict[str, object]] = []
+    for p in _iter_project_files(repo_root, roots):
+        rel = p.relative_to(repo_root).as_posix()
+        rel_no_ext = re.sub(r"\.[a-zA-Z0-9]+$", "", rel)
+        item_id = f"module:{rel_no_ext}"
+
+        exports = extract_exports(p)
+
+        # usage (where used)
+        importers = sorted({imp for (imp, _spec) in used_by.get(item_id, [])})
+        importers_top = importers[:10]
+
+        snippets: list[str] = []
+        for imp, spec in used_by.get(item_id, [])[:50]:
+            imp_path = repo_root / imp
+            if not imp_path.exists():
+                continue
+            try:
+                text = imp_path.read_text(encoding="utf-8")
+            except Exception:
+                continue
+            snippets.extend(extract_import_snippets(text=text, needle=spec, limit=usage_snippet_limit))
+            if len(snippets) >= usage_snippet_limit:
+                snippets = snippets[:usage_snippet_limit]
+                break
+
+        external_pkgs = sorted(per_importer_external.get(rel, Counter()).keys())
+        internal_mods = sorted({rid for rid in per_importer_internal.get(rel, Counter()).keys()})
+
+        cards.append(
+            make_module_card(
+                repo_root=repo_root,
+                importer_rel=rel_no_ext,
+                specifier="",
+                export_symbols=exports,
+                used_by=importers_top,
+                snippets=snippets,
+                external_packages=external_pkgs,
+                internal_deps=internal_mods,
+            )
+        )
+
+    cards.sort(key=lambda c: str(c.get("path") or ""))
+    return cards
+
+
+def write_cards_jsonl(cards: list[dict[str, object]], out_path: Path) -> None:
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with out_path.open("w", encoding="utf-8") as f:
+        for c in cards:
+            f.write(json.dumps(c, sort_keys=True) + "\n")

devflow_engine/registry/domain_normalize.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+"""Deterministic domain name normalization.
+
+Goal: reduce taxonomy entropy without requiring a human in the loop.
+
+Rules:
+- slugify (lowercase, hyphens)
+- apply small synonym map (config->configuration, etc.)
+- collapse trivial plurals/singulars (contract/contracts)
+
+This is intentionally conservative.
+"""
+
+import re
+
+
+_SYNONYMS: dict[str, str] = {
+    "config": "configuration",
+    "configs": "configuration",
+    "configuration": "configuration",
+    "planning": "planning",
+    "plan": "planning",
+    "plans": "planning",
+    "validate": "validation",
+    "validations": "validation",
+    "validation": "validation",
+    "contract": "contracts",
+    "contracts": "contracts",
+    "testing": "testing",
+    "test": "testing",
+    "tests": "testing",
+    "workflow": "workflow",
+}
+
+
+def slugify_domain(name: str) -> str:
+    s = (name or "").strip().lower()
+    s = s.replace("_", "-")
+    s = re.sub(r"[^a-z0-9-]+", "-", s)
+    s = re.sub(r"-+", "-", s).strip("-")
+    return s
+
+
+def normalize_domain(name: str) -> str:
+    s = slugify_domain(name)
+    if not s:
+        return "unknown"
+
+    # Direct synonym
+    if s in _SYNONYMS:
+        return _SYNONYMS[s]
+
+    # Singular/plural fold for simple cases.
+    if s.endswith("s") and s[:-1] in _SYNONYMS:
+        return _SYNONYMS[s[:-1]]
+    if (s + "s") in _SYNONYMS:
+        return _SYNONYMS[s + "s"]
+
+    return s

devflow_engine/registry/effects.py

@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+"""Cheap static effect-signature extraction.
+
+Goal: provide a general-purpose signal for "what this module does" without
+building a full parser or hardcoding repo-specific detectors.
+
+This intentionally uses conservative heuristics (regex-ish token matches).
+"""
+
+import re
+from pathlib import Path
+
+
+_EFFECT_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
+    ("subprocess.run", re.compile(r"\bsubprocess\.run\b")),
+    ("filesystem.write", re.compile(r"\bwrite_text\b|\bwrite_bytes\b|\bopen\([^)]*['\"]w")),
+    ("filesystem.read", re.compile(r"\bread_text\b|\bread_bytes\b|\bopen\([^)]*['\"]r")),
+    ("execution_store", re.compile(r"\bExecutionStore\b|\bstores\.execution_store\b")),
+    ("timeline.events", re.compile(r"\badd_event\b|\badd_error\b")),
+    ("story.contract", re.compile(r"\bget_story\b|\bupsert_story\b|\bstory\b.*\bcontract\b")),
+    ("devflow.artifacts", re.compile(r"\.devflow/")),
+    ("cli.typer", re.compile(r"\btyper\.")),
+    ("network.http", re.compile(r"\brequests\b|\bhttpx\b|\burllib\b")),
+]
+
+
+def extract_effect_tokens_from_text(text: str) -> list[str]:
+    toks: list[str] = []
+    for name, pat in _EFFECT_PATTERNS:
+        if pat.search(text):
+            toks.append(name)
+    return sorted(set(toks))
+
+
+def read_module_text(repo_root: Path, module_id: str) -> str | None:
+    if not module_id.startswith("module:"):
+        return None
+    rel = module_id.split(":", 1)[1]
+
+    # Try py first, then ts/tsx/js/jsx.
+    for ext in [".py", ".ts", ".tsx", ".js", ".jsx"]:
+        p = repo_root / (rel + ext)
+        if p.exists():
+            try:
+                return p.read_text(encoding="utf-8")
+            except Exception:
+                return None
+
+    # Also allow package __init__.py for python packages.
+    init = repo_root / rel / "__init__.py"
+    if init.exists():
+        try:
+            return init.read_text(encoding="utf-8")
+        except Exception:
+            return None
+
+    return None
+
+
+def extract_effect_tokens(repo_root: Path, module_id: str) -> list[str]:
+    text = read_module_text(repo_root, module_id)
+    if not text:
+        return []
+    return extract_effect_tokens_from_text(text)