devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/registry/packages.py

@@ -0,0 +1,347 @@
+from __future__ import annotations
+
+import json
+from collections import Counter, defaultdict
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Any
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
+
+
+@dataclass(frozen=True)
+class DomainState:
+    domain: str
+    packages: list[str]
+    canonical: str | None
+    allow_multiple: bool
+
+
+def infer_domain_heuristic(pkg: str) -> tuple[str | None, float]:
+    """Heuristic-only domain inference.
+
+    Returns (domain, confidence).
+
+    Important: must avoid substring false-positives (e.g. "jsesc" contains "ses").
+    """
+
+    p = pkg.lower().strip()
+
+    email_markers = {
+        "mailjet",
+        "nodemailer",
+        "postmark",
+        "mailgun",
+        "@sendgrid/mail",
+        "sendgrid",
+        "@aws-sdk/client-ses",
+        "aws-sdk",
+    }
+
+    testing_markers = {
+        "pytest",
+        "vitest",
+        "jest",
+        "mocha",
+        "ava",
+        "cypress",
+        "playwright",
+        "playwright-core",
+        "@playwright/test",
+    }
+
+    llm_markers = {
+        "openai",
+        "anthropic",
+        "@anthropic-ai/sdk",
+        "@google/generative-ai",
+        "vertexai",
+        "gemini",
+        "mistral",
+        "cohere",
+        "ollama",
+    }
+
+    if p in email_markers or any(p.startswith(m + "/") for m in email_markers if m.startswith("@")):
+        return ("email", 0.9)
+    if p in testing_markers:
+        return ("testing", 0.9)
+    if p in llm_markers or p.startswith("@google/"):
+        return ("llm", 0.8)
+
+    return (None, 0.3)
+
+
+def refresh_policy(conn) -> dict[str, Any]:
+    """Populate/update package_domains/domain_policy/package_status based on imports + allowed_packages.
+
+    Deterministic rule:
+    - domain is inferred via package_domains.manual else heuristic.
+    - canonical per domain is package with highest import count in imports(kind='external'), tie-break lexicographically.
+    - if allow_multiple=0 and multiple packages exist in a domain => non-canonical packages become heretical.
+
+    Safety:
+    - ignore invalid "package" strings that can arise from scanning vendor/compiled code.
+    """
+
+    now = utc_now_iso()
+
+    def _valid_pkg(s: str) -> bool:
+        # npm-style package name (very approximate) with optional scope.
+        # Reject templates, paths, and weird punctuation.
+        import re
+
+        return bool(re.match(r"^(@[a-z0-9][\w.-]*/)?[a-z0-9][\w.-]*$", s))
+
+    # Ensure schema compatibility (older DBs).
+    # - add lane column if missing
+    # - add strict-domain tables if missing
+    try:
+        cols = [r[1] for r in conn.execute("pragma table_info(package_status)").fetchall()]
+        if "lane" not in cols:
+            conn.execute("alter table package_status add column lane text")
+    except Exception:
+        pass
+
+    # Create strict-domain tables if missing (older DBs).
+    conn.execute(
+        "create table if not exists known_domains(domain text primary key, is_controlled integer not null default 1, updated_at text not null)"
+    )
+    conn.execute(
+        "create table if not exists package_purpose(package text primary key, domain text not null, purpose text not null, decided_by text not null, confidence real, updated_at text not null)"
+    )
+    conn.execute(
+        "create table if not exists package_status_overrides(domain text not null, package text not null, status text not null, updated_at text not null, primary key(domain, package))"
+    )
+
+    # Clear derived tables (we recompute deterministically each refresh).
+    conn.execute("delete from package_status")
+
+    # Import counts per external package
+    pkg_counts: Counter[str] = Counter()
+    for (resolved_id,) in conn.execute("select resolved_id from imports where kind='external'").fetchall():
+        rid = str(resolved_id)
+        if rid.startswith("pkg:"):
+            pkg = rid.split(":", 1)[1]
+            if not _valid_pkg(pkg):
+                continue
+            pkg_counts[pkg] += 1
+
+    # Ensure allowed_packages includes anything in imports (belt+braces)
+    for pkg in pkg_counts.keys():
+        conn.execute("insert or ignore into allowed_packages(name) values (?)", (pkg,))
+
+    # Load manual domain overrides
+    manual_domains: dict[str, str] = {
+        str(pkg): str(dom)
+        for pkg, dom in conn.execute("select package, domain from package_domain_overrides").fetchall()
+    }
+
+    # Also respect any previously stored manual entries in package_domains.
+    for pkg, dom, inferred_by in conn.execute(
+        "select package, domain, inferred_by from package_domains where domain is not null"
+    ).fetchall():
+        if str(inferred_by) == "manual":
+            manual_domains.setdefault(str(pkg), str(dom))
+
+    # Upsert package_domains
+    domains_to_pkgs: dict[str, list[str]] = defaultdict(list)
+    unknown: list[str] = []
+
+    # Ensure inferred domains are tracked (controlled-by-default).
+    # We'll upsert discovered domains during the loop below.
+
+    for pkg in sorted({*pkg_counts.keys()}):
+        if pkg in manual_domains:
+            dom = manual_domains[pkg]
+            conf = 1.0
+            inferred_by = "manual"
+        else:
+            dom, conf = infer_domain_heuristic(pkg)
+            inferred_by = "heuristic"
+
+        conn.execute(
+            "insert or replace into package_domains(package, domain, inferred_by, confidence, updated_at) values (?,?,?,?,?)",
+            (pkg, dom, inferred_by, float(conf), now),
+        )
+
+        if dom is None:
+            unknown.append(pkg)
+        else:
+            dom_s = str(dom)
+            domains_to_pkgs[dom_s].append(pkg)
+            # Upsert known domain as controlled-by-default.
+            conn.execute(
+                "insert or ignore into known_domains(domain, is_controlled, updated_at) values (?,?,?)",
+                (dom_s, 1, now),
+            )
+            # Touch timestamp (keep deterministic but current).
+            conn.execute("update known_domains set updated_at=? where domain=?", (now, dom_s))
+
+    # Drop stale domain_policy rows for domains that no longer exist in this repo.
+    current_domains = set(domains_to_pkgs.keys())
+    for (dom_existing,) in conn.execute("select domain from domain_policy").fetchall():
+        if str(dom_existing) not in current_domains:
+            conn.execute("delete from domain_policy where domain=?", (str(dom_existing),))
+
+    # Determine canonical per domain by import counts
+    for dom, pkgs in domains_to_pkgs.items():
+        # ensure a policy row exists
+        row = conn.execute("select canonical_package, allow_multiple from domain_policy where domain=?", (dom,)).fetchone()
+        if row:
+            canonical_existing, allow_multiple = row
+            allow_multiple = bool(int(allow_multiple))
+        else:
+            canonical_existing, allow_multiple = None, False
+
+        # If canonical is unset, compute deterministically
+        canonical = str(canonical_existing) if canonical_existing else None
+        if canonical is None:
+            ranked = sorted(pkgs, key=lambda p: (-pkg_counts.get(p, 0), p))
+            canonical = ranked[0] if ranked else None
+
+        conn.execute(
+            "insert or replace into domain_policy(domain, canonical_package, allow_multiple, updated_at) values (?,?,?,?)",
+            (dom, canonical, 1 if allow_multiple else 0, now),
+        )
+
+        # Update package_status for packages in the domain
+        # Load any lane assignments
+        lanes: dict[str, str] = {
+            str(pkg): str(lane)
+            for pkg, lane in conn.execute(
+                "select package, lane from package_lanes where domain=?",
+                (dom,),
+            ).fetchall()
+        }
+
+        # Load any status overrides
+        overrides: dict[str, str] = {
+            str(pkg): str(st)
+            for pkg, st in conn.execute(
+                "select package, status from package_status_overrides where domain=?",
+                (dom,),
+            ).fetchall()
+        }
+
+        for p in pkgs:
+            if p in overrides:
+                status = overrides[p]
+            else:
+                status = "canonical" if canonical and p == canonical else "allowed" if allow_multiple else "heretical"
+
+            reason = None
+            if status == "heretical":
+                reason = f"heretical (non-canonical) in domain {dom}"
+
+            lane = lanes.get(p)
+            conn.execute(
+                "insert or replace into package_status(package, domain, status, lane, reason, updated_at) values (?,?,?,?,?,?)",
+                (p, dom, status, lane, reason, now),
+            )
+
+    # Unknown packages
+    for pkg in unknown:
+        conn.execute(
+            "insert or replace into package_status(package, domain, status, lane, reason, updated_at) values (?,?,?,?,?,?)",
+            (pkg, None, "unknown", None, "domain_unknown", now),
+        )
+
+    conn.commit()
+
+    return {
+        "unknown": unknown,
+        "domains": {d: sorted(ps) for d, ps in domains_to_pkgs.items()},
+        "import_counts": dict(pkg_counts),
+    }
+
+
+def gate_from_db(conn) -> dict[str, Any]:
+    """Compute gate report from DB policy tables (refreshing first if needed)."""
+
+    # Always refresh; this is cheap relative to registry init, and keeps autonomy.
+    refresh_meta = refresh_policy(conn)
+
+    violations: list[dict[str, Any]] = []
+
+    # domain -> packages
+    domain_rows = conn.execute(
+        "select domain, canonical_package, allow_multiple from domain_policy order by domain"
+    ).fetchall()
+
+    controlled_map = {
+        str(d): bool(int(c))
+        for d, c in conn.execute("select domain, is_controlled from known_domains").fetchall()
+    }
+
+    for dom, canonical, allow_multiple in domain_rows:
+        dom = str(dom)
+        canonical = str(canonical) if canonical else None
+        allow_multiple = bool(int(allow_multiple))
+        is_controlled = controlled_map.get(dom, True)
+
+        # Pull statuses+lanes for this domain
+        rows = [
+            (str(pkg), str(st), (str(lane) if lane is not None else ""))
+            for pkg, st, lane in conn.execute(
+                "select package, status, lane from package_status where domain=? and status in ('canonical','allowed','heretical') order by package",
+                (dom,),
+            ).fetchall()
+        ]
+        pkgs = [r[0] for r in rows]
+
+        if not is_controlled:
+            continue
+
+        if len(pkgs) > 1 and not allow_multiple:
+            # Strict: must have exactly one non-heretical package.
+            non_heresy = [pkg for (pkg, st, _lane) in rows if st in {"canonical", "allowed"}]
+            if len(non_heresy) != 1:
+                violations.append({"domain": dom, "kind": "domain_not_converged", "packages": rows})
+            continue
+
+        if allow_multiple and pkgs:
+            # Lane required for all packages
+            missing_lane = [pkg for (pkg, _st, lane) in rows if not lane]
+            if missing_lane:
+                violations.append({"domain": dom, "kind": "missing_lane", "packages": sorted(missing_lane)})
+                continue
+
+            # Per-lane: exactly one non-heretical.
+            by_lane: dict[str, list[tuple[str, str]]] = {}
+            for pkg, st, lane in rows:
+                by_lane.setdefault(lane, []).append((pkg, st))
+
+            for lane, items in sorted(by_lane.items()):
+                non_heresy = [pkg for (pkg, st) in items if st in {"canonical", "allowed"}]
+                if len(non_heresy) != 1:
+                    violations.append({"domain": dom, "kind": "lane_not_converged", "lane": lane, "packages": items})
+
+    unknown = [
+        str(r[0])
+        for r in conn.execute("select package from package_status where status='unknown' order by package").fetchall()
+    ]
+
+    # Strict mode: unknown packages are not allowed.
+    if unknown:
+        violations.append({"kind": "unknown_packages", "packages": unknown})
+
+    return {
+        "domains": {
+            r[0]: {
+                "canonical": r[1],
+                "allow_multiple": bool(int(r[2])),
+                "controlled": controlled_map.get(str(r[0]), True),
+            }
+            for r in domain_rows
+        },
+        "unknown": unknown,
+        "violations": violations,
+        "refresh": refresh_meta,
+    }
+
+
+def dump_report(report: dict[str, Any]) -> str:
+    return json.dumps(report, indent=2, sort_keys=True) + "\n"

devflow_engine/registry/pathways.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+"""Pathway ("module card") detector.
+
+This is a lightweight, deterministic analyzer over the internal import graph.
+
+Design goals:
+- general signals only (graph structure + optional signatures)
+- stable output ordering
+- easy to test against synthetic graphs
+
+It is *not* a policy enforcer.
+"""
+
+from dataclasses import dataclass
+from typing import Iterable
+
+
+@dataclass(frozen=True)
+class ModuleSignals:
+    module_id: str
+    in_degree: int
+    out_degree: int
+    importers: tuple[str, ...]
+    internal_deps: tuple[str, ...]
+    exports: tuple[str, ...]
+    external_pkgs: tuple[str, ...]
+
+
+def _jaccard(a: set[str], b: set[str]) -> float:
+    if not a and not b:
+        return 1.0
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
+def _stable_sorted(xs: Iterable[str]) -> list[str]:
+    return sorted({str(x) for x in xs if str(x)})
+
+
+def build_module_graph(
+    *,
+    imports_rows: list[tuple[str, str, str, str]],
+    module_exports: dict[str, list[str]] | None = None,
+    module_external_pkgs: dict[str, list[str]] | None = None,
+) -> dict[str, ModuleSignals]:
+    """Build per-module signals from imports rows.
+
+    imports_rows schema: (importer, specifier, kind, resolved_id)
+    - importer is a file path (repo-relative)
+    - resolved_id for internal modules is the module item_id (e.g. module:src/x)
+
+    module_exports/module_external_pkgs are optional signature maps keyed by module_id.
+    """
+
+    module_exports = module_exports or {}
+    module_external_pkgs = module_external_pkgs or {}
+
+    importers: dict[str, set[str]] = {}
+    deps: dict[str, set[str]] = {}
+
+    # Initialize keys for any module we see.
+    for importer, _spec, kind, resolved_id in imports_rows:
+        if kind != "internal":
+            continue
+        resolved = str(resolved_id)
+        importers.setdefault(resolved, set()).add(str(importer))
+        # Track deps per importing module if importer itself is a module id.
+
+    # Build deps by mapping importer file path -> module id form if present.
+    # For synthetic tests we allow importer to already be a module id.
+    for importer, _spec, kind, resolved_id in imports_rows:
+        if kind != "internal":
+            continue
+        importer_s = str(importer)
+        if importer_s.startswith("module:"):
+            importer_id = importer_s
+        else:
+            # best-effort normalization: treat as module:<path-without-ext>
+            importer_id = "module:" + importer_s.rsplit(".", 1)[0]
+        deps.setdefault(importer_id, set()).add(str(resolved_id))
+        importers.setdefault(importer_id, importers.get(importer_id, set()))
+
+    out: dict[str, ModuleSignals] = {}
+    for mid in sorted(set(importers.keys()) | set(deps.keys())):
+        imp = importers.get(mid, set())
+        d = deps.get(mid, set())
+        exp = _stable_sorted(module_exports.get(mid, []))
+        pkgs = _stable_sorted(module_external_pkgs.get(mid, []))
+        out[mid] = ModuleSignals(
+            module_id=mid,
+            in_degree=len(imp),
+            out_degree=len(d),
+            importers=tuple(sorted(imp)),
+            internal_deps=tuple(sorted(d)),
+            exports=tuple(exp),
+            external_pkgs=tuple(pkgs),
+        )
+
+    return out
+
+
+def _reachable(
+    g: dict[str, ModuleSignals], start: str, target: str, *, max_depth: int = 6
+) -> bool:
+    if start == target:
+        return True
+    seen: set[str] = set()
+    stack: list[tuple[str, int]] = [(start, 0)]
+    while stack:
+        cur, depth = stack.pop()
+        if cur == target:
+            return True
+        if cur in seen:
+            continue
+        seen.add(cur)
+        if depth >= max_depth:
+            continue
+        for nxt in g.get(cur).internal_deps if g.get(cur) else []:
+            if nxt not in seen:
+                stack.append((nxt, depth + 1))
+    return False
+
+
+def detect_pathways(
+    *,
+    imports_rows: list[tuple[str, str, str, str]],
+    module_exports: dict[str, list[str]] | None = None,
+    module_external_pkgs: dict[str, list[str]] | None = None,
+    module_effect_tokens: dict[str, list[str]] | None = None,
+    min_hub_in_degree: int = 2,
+    max_hubs: int = 5000,
+    competing_importer_jaccard: float = 0.65,
+    competing_export_jaccard: float = 0.40,
+    overlap_effect_jaccard: float = 0.6,
+) -> dict[str, object]:
+    """Detect seam candidates + competing seams.
+
+    Output is stable + deterministic.
+
+    Heuristics:
+    - hub seam: in_degree >= min_hub_in_degree
+    - orchestrator seam: imports multiple internal deps AND depends on at least one hub
+    - competing seams: hubs that share importer neighborhoods + (optionally) export signature
+    """
+
+    g = build_module_graph(
+        imports_rows=imports_rows,
+        module_exports=module_exports,
+        module_external_pkgs=module_external_pkgs,
+    )
+
+    module_effect_tokens = module_effect_tokens or {}
+
+    # Seam candidates
+    hubs = [s for s in g.values() if s.in_degree >= int(min_hub_in_degree)]
+    hubs.sort(key=lambda s: (-s.in_degree, s.module_id))
+    hubs = hubs[: max(0, int(max_hubs))]
+    hub_ids = {h.module_id for h in hubs}
+
+    seams: list[dict[str, object]] = []
+
+    for h in hubs:
+        seams.append(
+            {
+                "kind": "hub",
+                "module_id": h.module_id,
+                "score": float(h.in_degree),
+                "evidence": {
+                    "in_degree": h.in_degree,
+                    "out_degree": h.out_degree,
+                    "top_importers": list(h.importers)[:10],
+                },
+            }
+        )
+
+    # Orchestrators: modules that fan out to multiple internal modules (esp hubs)
+    # and are themselves used (in_degree>0).
+    for s in g.values():
+        if s.out_degree < 2 or s.in_degree < 1:
+            continue
+        dep_hubs = [d for d in s.internal_deps if d in hub_ids]
+        if not dep_hubs:
+            continue
+        # Score: fanout weighted by hub-ness.
+        score = 0.0
+        for d in dep_hubs:
+            score += float(g.get(d).in_degree if g.get(d) else 0)
+        score = score + 0.1 * float(s.out_degree)
+        seams.append(
+            {
+                "kind": "orchestrator",
+                "module_id": s.module_id,
+                "score": float(score),
+                "evidence": {
+                    "in_degree": s.in_degree,
+                    "out_degree": s.out_degree,
+                    "hub_deps": sorted(dep_hubs),
+                },
+            }
+        )
+
+    seams.sort(key=lambda r: (-float(r.get("score") or 0.0), str(r.get("kind")), str(r.get("module_id"))))
+
+    # Competing seams among hubs
+    competing: list[dict[str, object]] = []
+    for i in range(len(hubs)):
+        a = hubs[i]
+        a_imps = set(a.importers)
+        a_exp = set(a.exports)
+        for j in range(i + 1, len(hubs)):
+            b = hubs[j]
+            b_imps = set(b.importers)
+            sim_imp = _jaccard(a_imps, b_imps)
+            if sim_imp < float(competing_importer_jaccard):
+                continue
+
+            # Export similarity is optional; if neither has exports, treat as neutral.
+            b_exp = set(b.exports)
+            sim_exp = _jaccard(a_exp, b_exp) if (a_exp or b_exp) else 1.0
+            if sim_exp < float(competing_export_jaccard):
+                continue
+
+            competing.append(
+                {
+                    "a": a.module_id,
+                    "b": b.module_id,
+                    "similarity": {
+                        "importers_jaccard": round(sim_imp, 4),
+                        "exports_jaccard": round(sim_exp, 4),
+                    },
+                    "evidence": {
+                        "a_in_degree": a.in_degree,
+                        "b_in_degree": b.in_degree,
+                        "shared_importers": sorted(a_imps & b_imps)[:12],
+                    },
+                }
+            )
+
+    competing.sort(key=lambda r: (-float(r["similarity"]["importers_jaccard"]), str(r["a"]), str(r["b"])))
+
+    # Effect-signature overlaps.
+    # We consider any module that has effect tokens (not just detected seams),
+    # to avoid "missing" layered overlaps when the modules aren't hubs/orchestrators.
+    seam_ids = sorted({
+        *[str(s.get("module_id")) for s in seams if isinstance(s, dict) and s.get("module_id")],
+        *[str(k) for k, v in module_effect_tokens.items() if v],
+    })
+
+    overlaps: list[dict[str, object]] = []
+    for i in range(len(seam_ids)):
+        a = seam_ids[i]
+        a_tok = set(module_effect_tokens.get(a, []))
+        if not a_tok:
+            continue
+        for j in range(i + 1, len(seam_ids)):
+            b = seam_ids[j]
+            b_tok = set(module_effect_tokens.get(b, []))
+            if not b_tok:
+                continue
+            sim = _jaccard(a_tok, b_tok)
+            if sim < float(overlap_effect_jaccard):
+                continue
+
+            layered = _reachable(g, a, b) or _reachable(g, b, a)
+            overlaps.append(
+                {
+                    "a": a,
+                    "b": b,
+                    "similarity": {"effects_jaccard": round(sim, 4)},
+                    "kind": "layered" if layered else "competing",
+                    "evidence": {
+                        "shared_effects": sorted(a_tok & b_tok)[:20],
+                        "a_effect_count": len(a_tok),
+                        "b_effect_count": len(b_tok),
+                    },
+                }
+            )
+
+    overlaps.sort(key=lambda r: (-float(r["similarity"]["effects_jaccard"]), str(r["kind"]), str(r["a"]), str(r["b"])))
+
+    # Union-find clusters of effect-overlap (competing or layered; cluster is informational)
+    parent: dict[str, str] = {}
+
+    def find(x: str) -> str:
+        parent.setdefault(x, x)
+        while parent[x] != x:
+            parent[x] = parent[parent[x]]
+            x = parent[x]
+        return x
+
+    def union(a: str, b: str) -> None:
+        ra, rb = find(a), find(b)
+        if ra != rb:
+            parent[rb] = ra
+
+    for p in overlaps:
+        union(str(p["a"]), str(p["b"]))
+
+    clusters: dict[str, set[str]] = {}
+    for mid in parent.keys():
+        clusters.setdefault(find(mid), set()).add(mid)
+
+    overlap_clusters = [sorted(v) for v in clusters.values() if len(v) >= 2]
+    overlap_clusters.sort(key=lambda grp: (len(grp), grp))
+
+    return {
+        "stats": {
+            "modules": len(g),
+            "hubs": len(hubs),
+            "seams": len(seams),
+            "competing_pairs": len(competing),
+            "effect_overlap_pairs": len(overlaps),
+            "effect_overlap_clusters": len(overlap_clusters),
+        },
+        "seams": seams,
+        "competing": competing,
+        "effect_overlap": {
+            "pairs": overlaps,
+            "clusters": overlap_clusters,
+        },
+    }

devflow_engine/review/__init__.py

@@ -0,0 +1,11 @@
+"""Review workflow support.
+
+DEPRECATED: The `review` CLI surface and this module are non-canonical.
+Automated story execution should use the queue-driven worker pipeline
+(`devflow worker start`). This module is retained for backward compatibility
+and direct debugging only.
+"""
+
+from .dag import run_review_dag
+
+__all__ = ["run_review_dag"]