devflow-engine 1.0.0__py3-none-any.whl

devflow_engine/llm/repo_knowledge.py

@@ -0,0 +1,588 @@
+from __future__ import annotations
+
+import json
+import re
+import sqlite3
+import subprocess
+from collections import Counter, defaultdict
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from hashlib import sha256
+from pathlib import Path
+from typing import Any
+
+from ..registry.enforce_report import build_enforce_report
+from ..story.discovery import get_story_source_paths
+from ..story.markdown_contracts import parse_story_contracts_from_markdown
+
+SCHEMA_VERSION = 1
+DEFAULT_MAX_FILES = 2_000
+DEFAULT_MAX_BYTES_PER_FILE = 180_000
+OUTPUT_DIR = Path(".devflow/llm")
+JSON_INDEX_NAME = "repo_knowledge.json"
+MARKDOWN_INDEX_NAME = "repo_knowledge.md"
+
+_EXCLUDED_DIR_PARTS = {
+    ".git",
+    ".hg",
+    ".svn",
+    ".devflow",
+    ".mypy_cache",
+    ".pytest_cache",
+    ".ruff_cache",
+    ".venv",
+    "venv",
+    "env",
+    "node_modules",
+    "dist",
+    "build",
+    "coverage",
+    "__pycache__",
+}
+
+_IMPORTANT_FILENAMES = {
+    "README.md",
+    "pyproject.toml",
+    "package.json",
+    "go.mod",
+    "Cargo.toml",
+    "Makefile",
+    "Dockerfile",
+    "docker-compose.yml",
+    "docker-compose.yaml",
+    "requirements.txt",
+    "tsconfig.json",
+    "vite.config.ts",
+    "next.config.js",
+    "next.config.mjs",
+}
+
+_LANG_BY_SUFFIX = {
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript-react",
+    ".js": "javascript",
+    ".jsx": "javascript-react",
+    ".md": "markdown",
+    ".json": "json",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".toml": "toml",
+    ".sql": "sql",
+    ".sh": "shell",
+    ".css": "css",
+    ".html": "html",
+    ".go": "go",
+    ".rs": "rust",
+}
+
+_PLANE_HINTS = {
+    "auth": ("auth", "login", "session", "permission", "jwt", "oauth"),
+    "api": ("api", "route", "endpoint", "controller", "graphql", "rest"),
+    "ui": ("ui", "component", "page", "view", "screen", "playwright", "css", "tsx", "jsx"),
+    "integrations": ("integration", "webhook", "queue", "supabase", "stripe", "external"),
+    "ops": ("cli", "worker", "deploy", "config", "logging", "registry", "migration"),
+}
+
+_SYMBOL_PATTERNS = [
+    re.compile(r"^\s*(?:async\s+)?def\s+([A-Za-z_][A-Za-z0-9_]*)\s*\("),
+    re.compile(r"^\s*class\s+([A-Za-z_][A-Za-z0-9_]*)\s*[:\(]"),
+    re.compile(r"^\s*(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*\("),
+    re.compile(r"^\s*(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][A-Za-z0-9_$]*)\s*="),
+]
+
+
+@dataclass(frozen=True)
+class RepoFileSummary:
+    path: str
+    kind: str
+    language: str
+    size_bytes: int
+    line_count: int
+    content_hash: str
+    symbols: list[str] = field(default_factory=list)
+    story_ids: list[str] = field(default_factory=list)
+    story_uuids: list[str] = field(default_factory=list)
+    planes: list[str] = field(default_factory=list)
+
+
+@dataclass(frozen=True)
+class RepoDirectorySummary:
+    path: str
+    file_count: int
+    languages: list[str]
+    planes: list[str]
+    key_files: list[str]
+    child_dirs: list[str]
+
+
+@dataclass(frozen=True)
+class StoryPlaneSummary:
+    story_id: str
+    story_uuid: str
+    title: str
+    source_path: str
+    required_planes: list[str]
+    plane_oracles: list[dict[str, str]]
+    evidence_files: list[str]
+
+
+@dataclass(frozen=True)
+class RegistryKnowledgeSummary:
+    present: bool
+    meta: dict[str, str] = field(default_factory=dict)
+    import_counts: dict[str, int] = field(default_factory=dict)
+    allowed_packages: list[str] = field(default_factory=list)
+    registered_modules: list[dict[str, str]] = field(default_factory=list)
+    package_policy: list[dict[str, str]] = field(default_factory=list)
+    canonical_module_routes: list[dict[str, str]] = field(default_factory=list)
+    heretical_module_routes: list[dict[str, str]] = field(default_factory=list)
+    enforce_ok: bool | None = None
+    enforce_directives: list[dict[str, Any]] = field(default_factory=list)
+    missing_registrations: list[dict[str, str]] = field(default_factory=list)
+    error: str | None = None
+
+
+@dataclass(frozen=True)
+class RepoKnowledgeIndex:
+    schema_version: int
+    generated_at: str
+    repo_root: str
+    git_head: str | None
+    file_count: int
+    indexed_file_count: int
+    skipped_file_count: int
+    directories: list[RepoDirectorySummary]
+    files: list[RepoFileSummary]
+    story_planes: list[StoryPlaneSummary]
+    registry: RegistryKnowledgeSummary
+    search_hints: dict[str, list[str]]
+
+    def to_payload(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+def _git_output(repo_root: Path, args: list[str], *, timeout: int = 15) -> str | None:
+    try:
+        proc = subprocess.run(
+            ["git", "-C", str(repo_root), *args],
+            capture_output=True,
+            text=True,
+            check=False,
+            timeout=timeout,
+        )
+    except Exception:
+        return None
+    if proc.returncode != 0:
+        return None
+    return proc.stdout.strip()
+
+
+def _git_head(repo_root: Path) -> str | None:
+    value = _git_output(repo_root, ["rev-parse", "--short", "HEAD"])
+    return value or None
+
+
+def _is_indexable_path(path: Path) -> bool:
+    parts = set(path.parts)
+    if parts & _EXCLUDED_DIR_PARTS:
+        return False
+    binary_suffixes = (".pyc", ".pyo", ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf", ".zip", ".sqlite", ".db")
+    return not path.name.endswith(binary_suffixes)
+
+
+def _list_repo_files(repo_root: Path, *, max_files: int) -> tuple[list[Path], int]:
+    raw = _git_output(repo_root, ["ls-files", "-co", "--exclude-standard"], timeout=30)
+    paths: list[Path] = []
+    if raw is not None:
+        for line in raw.splitlines():
+            rel = Path(line)
+            if _is_indexable_path(rel):
+                paths.append(rel)
+    else:
+        for path in repo_root.rglob("*"):
+            if not path.is_file():
+                continue
+            try:
+                rel = path.relative_to(repo_root)
+            except ValueError:
+                continue
+            if _is_indexable_path(rel):
+                paths.append(rel)
+    paths = sorted(set(paths), key=lambda p: p.as_posix())
+    return paths[:max_files], max(0, len(paths) - max_files)
+
+
+def _file_kind(rel: Path) -> str:
+    path = rel.as_posix()
+    name = rel.name
+    if name in _IMPORTANT_FILENAMES:
+        return "key_file"
+    if path.startswith("tests/") or "/test" in path or name.startswith("test_") or name.endswith(".spec.ts"):
+        return "test"
+    if path.startswith("docs/") or path.startswith("ai_docs/") or rel.suffix == ".md":
+        return "docs"
+    if "prompt" in path:
+        return "prompt"
+    if "src/" in path or rel.suffix in {".py", ".ts", ".tsx", ".js", ".jsx", ".go", ".rs"}:
+        return "source"
+    return "support"
+
+
+def _language(rel: Path) -> str:
+    if rel.name == "Dockerfile":
+        return "dockerfile"
+    if rel.name == "Makefile":
+        return "make"
+    return _LANG_BY_SUFFIX.get(rel.suffix.lower(), "text")
+
+
+def _extract_symbols(text: str, *, limit: int = 24) -> list[str]:
+    symbols: list[str] = []
+    for line in text.splitlines():
+        for pattern in _SYMBOL_PATTERNS:
+            match = pattern.match(line)
+            if match:
+                symbols.append(match.group(1))
+                break
+        if len(symbols) >= limit:
+            break
+    return symbols
+
+
+def _extract_marker_values(text: str, marker: str) -> list[str]:
+    values: list[str] = []
+    patterns = [
+        re.compile(rf"{re.escape(marker)}\s*[:=]\s*[\"']?([A-Za-z0-9_.:-]+)"),
+        re.compile(rf"pytest\.mark\.{re.escape(marker)}\([\"']([^\"']+)[\"']\)"),
+    ]
+    for pattern in patterns:
+        for match in pattern.finditer(text):
+            value = match.group(1).strip()
+            if value and value not in values:
+                values.append(value)
+    return values
+
+
+def _infer_planes(rel: Path, text: str) -> list[str]:
+    path_text = rel.as_posix().lower()
+    haystack = f"{path_text}\n{text[:12000].lower()}"
+    planes: list[str] = []
+    marked = _extract_marker_values(text, "plane")
+    for plane in marked:
+        if plane in _PLANE_HINTS and plane not in planes:
+            planes.append(plane)
+    for plane, hints in _PLANE_HINTS.items():
+        if any(hint in haystack for hint in hints) and plane not in planes:
+            planes.append(plane)
+    return planes
+
+
+def _read_text(path: Path, *, max_bytes: int) -> tuple[str, bool]:
+    raw = path.read_bytes()
+    truncated = len(raw) > max_bytes
+    raw = raw[:max_bytes]
+    try:
+        return raw.decode("utf-8", errors="replace"), truncated
+    except Exception:
+        return "", True
+
+
+def _summarize_file(repo_root: Path, rel: Path, *, max_bytes_per_file: int) -> RepoFileSummary | None:
+    path = repo_root / rel
+    try:
+        stat = path.stat()
+        text, _ = _read_text(path, max_bytes=max_bytes_per_file)
+    except OSError:
+        return None
+    if "\x00" in text:
+        return None
+    return RepoFileSummary(
+        path=rel.as_posix(),
+        kind=_file_kind(rel),
+        language=_language(rel),
+        size_bytes=stat.st_size,
+        line_count=0 if not text else text.count("\n") + (0 if text.endswith("\n") else 1),
+        content_hash=sha256(text.encode("utf-8", errors="replace")).hexdigest()[:16],
+        symbols=_extract_symbols(text),
+        story_ids=_extract_marker_values(text, "story_id"),
+        story_uuids=_extract_marker_values(text, "story_uuid"),
+        planes=_infer_planes(rel, text),
+    )
+
+
+def _summarize_directories(files: list[RepoFileSummary]) -> list[RepoDirectorySummary]:
+    by_dir: dict[str, list[RepoFileSummary]] = defaultdict(list)
+    child_dirs: dict[str, set[str]] = defaultdict(set)
+    for item in files:
+        rel = Path(item.path)
+        parent = rel.parent.as_posix()
+        parent = "." if parent == "." else parent
+        by_dir[parent].append(item)
+        parts = rel.parts
+        for idx in range(len(parts) - 1):
+            d = "." if idx == 0 else Path(*parts[:idx]).as_posix()
+            child_dirs[d].add(Path(*parts[: idx + 1]).as_posix())
+    summaries: list[RepoDirectorySummary] = []
+    for dirname, items in sorted(by_dir.items()):
+        langs = [name for name, _ in Counter(item.language for item in items).most_common(8)]
+        planes = sorted({plane for item in items for plane in item.planes})
+        key_files = [item.path for item in items if item.kind == "key_file"][:12]
+        summaries.append(
+            RepoDirectorySummary(
+                path=dirname,
+                file_count=len(items),
+                languages=langs,
+                planes=planes,
+                key_files=key_files,
+                child_dirs=sorted(child_dirs.get(dirname, set()))[:32],
+            )
+        )
+    return summaries
+
+
+def _story_plane_summaries(repo_root: Path, files: list[RepoFileSummary]) -> list[StoryPlaneSummary]:
+    evidence_by_story: dict[str, set[str]] = defaultdict(set)
+    for item in files:
+        for story_id in item.story_ids:
+            evidence_by_story[story_id].add(item.path)
+        for story_uuid in item.story_uuids:
+            evidence_by_story[story_uuid].add(item.path)
+
+    try:
+        story_paths = get_story_source_paths(repo_root)
+    except FileNotFoundError:
+        return []
+
+    summaries: list[StoryPlaneSummary] = []
+    for path in story_paths:
+        try:
+            text = path.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        rel = path.relative_to(repo_root).as_posix() if path.is_relative_to(repo_root) else str(path)
+        for block in parse_story_contracts_from_markdown(text, source_path=rel):
+            contract = block.contract
+            evidence = set(evidence_by_story.get(contract.story_id, set()))
+            evidence.update(evidence_by_story.get(contract.story_uuid, set()))
+            evidence.add(rel)
+            summaries.append(
+                StoryPlaneSummary(
+                    story_id=contract.story_id,
+                    story_uuid=contract.story_uuid,
+                    title=contract.title,
+                    source_path=rel,
+                    required_planes=list(contract.required_planes),
+                    plane_oracles=[
+                        {"plane": item.plane, "oracle": item.oracle, "anchor": item.anchor}
+                        for item in contract.plane_oracles
+                    ],
+                    evidence_files=sorted(evidence)[:32],
+                )
+            )
+    return sorted(summaries, key=lambda item: (item.story_id, item.story_uuid, item.source_path))
+
+
+def _build_search_hints(files: list[RepoFileSummary], stories: list[StoryPlaneSummary]) -> dict[str, list[str]]:
+    by_plane: dict[str, list[str]] = defaultdict(list)
+    for item in files:
+        for plane in item.planes:
+            if item.path not in by_plane[plane]:
+                by_plane[plane].append(item.path)
+    hints = {f"plane:{plane}": paths[:24] for plane, paths in sorted(by_plane.items())}
+    hints["key_files"] = [item.path for item in files if item.kind == "key_file"][:32]
+    hints["story_sources"] = [item.source_path for item in stories[:64]]
+    return hints
+
+
+def _registry_knowledge(repo_root: Path) -> RegistryKnowledgeSummary:
+    db_path = repo_root / ".devflow" / "registry.sqlite"
+    if not db_path.exists():
+        return RegistryKnowledgeSummary(present=False)
+    try:
+        with sqlite3.connect(str(db_path)) as conn:
+            meta = {
+                str(key): str(value)
+                for key, value in conn.execute(
+                    "select key, value from registry_meta order by key"
+                ).fetchall()
+            }
+            import_counts = {
+                str(kind): int(count)
+                for kind, count in conn.execute(
+                    "select kind, count(*) from imports group by kind order by kind"
+                ).fetchall()
+            }
+            allowed_packages = [
+                str(row[0])
+                for row in conn.execute("select name from allowed_packages order by name limit 200").fetchall()
+            ]
+            registered_modules = [
+                {"module_id": str(module_id), "path": str(key)}
+                for module_id, key in conn.execute(
+                    "select item_id, key from registered_items where kind='internal_module' order by item_id limit 300"
+                ).fetchall()
+            ]
+            package_policy = [
+                {
+                    "package": str(package),
+                    "domain": str(domain or ""),
+                    "status": str(status),
+                    "lane": str(lane or ""),
+                    "reason": str(reason or ""),
+                }
+                for package, domain, status, lane, reason in conn.execute(
+                    """
+                    select package, domain, status, lane, reason
+                    from package_status
+                    order by coalesce(domain,''), status, package
+                    limit 300
+                    """
+                ).fetchall()
+            ]
+            module_routes = [
+                {
+                    "domain": str(domain),
+                    "card_id": str(card_id),
+                    "lane": str(lane),
+                    "module_id": str(module_id),
+                    "status": str(status),
+                    "reason": str(reason or ""),
+                }
+                for domain, card_id, lane, module_id, status, reason in conn.execute(
+                    """
+                    select domain, card_id, lane, module_id, status, reason
+                    from module_card_membership
+                    order by domain, card_id, lane, status, module_id
+                    limit 500
+                    """
+                ).fetchall()
+            ]
+            enforce = build_enforce_report(conn)
+    except Exception as exc:
+        return RegistryKnowledgeSummary(present=True, error=str(exc))
+
+    return RegistryKnowledgeSummary(
+        present=True,
+        meta=meta,
+        import_counts=import_counts,
+        allowed_packages=allowed_packages,
+        registered_modules=registered_modules,
+        package_policy=package_policy,
+        canonical_module_routes=[item for item in module_routes if item["status"] == "canonical"],
+        heretical_module_routes=[item for item in module_routes if item["status"] == "heretical"],
+        enforce_ok=bool(enforce.get("ok")),
+        enforce_directives=list(enforce.get("directives") or [])[:100],
+        missing_registrations=list(enforce.get("missing") or [])[:100],
+    )
+
+
+def build_repo_knowledge_index(
+    repo_root: Path,
+    *,
+    max_files: int = DEFAULT_MAX_FILES,
+    max_bytes_per_file: int = DEFAULT_MAX_BYTES_PER_FILE,
+) -> RepoKnowledgeIndex:
+    repo_root = Path(repo_root).expanduser().resolve()
+    rel_paths, skipped_for_limit = _list_repo_files(repo_root, max_files=max_files)
+    files: list[RepoFileSummary] = []
+    skipped = skipped_for_limit
+    for rel in rel_paths:
+        summary = _summarize_file(repo_root, rel, max_bytes_per_file=max_bytes_per_file)
+        if summary is None:
+            skipped += 1
+            continue
+        files.append(summary)
+    stories = _story_plane_summaries(repo_root, files)
+    registry = _registry_knowledge(repo_root)
+    return RepoKnowledgeIndex(
+        schema_version=SCHEMA_VERSION,
+        generated_at=datetime.now(UTC).replace(microsecond=0).isoformat(),
+        repo_root=str(repo_root),
+        git_head=_git_head(repo_root),
+        file_count=len(rel_paths) + skipped_for_limit,
+        indexed_file_count=len(files),
+        skipped_file_count=skipped,
+        directories=_summarize_directories(files),
+        files=files,
+        story_planes=stories,
+        registry=registry,
+        search_hints=_build_search_hints(files, stories),
+    )
+
+
+def write_repo_knowledge_index(repo_root: Path, *, max_files: int = DEFAULT_MAX_FILES) -> dict[str, Any]:
+    repo_root = Path(repo_root).expanduser().resolve()
+    index = build_repo_knowledge_index(repo_root, max_files=max_files)
+    out_dir = repo_root / OUTPUT_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
+    json_path = out_dir / JSON_INDEX_NAME
+    md_path = out_dir / MARKDOWN_INDEX_NAME
+    payload = index.to_payload()
+    json_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    md_path.write_text(render_repo_knowledge_markdown(index), encoding="utf-8")
+    return {"json_path": str(json_path), "markdown_path": str(md_path), **payload}
+
+
+def load_repo_knowledge_index(repo_root: Path) -> dict[str, Any] | None:
+    path = Path(repo_root).expanduser().resolve() / OUTPUT_DIR / JSON_INDEX_NAME
+    if not path.exists():
+        return None
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return None
+    return payload if isinstance(payload, dict) else None
+
+
+def render_repo_knowledge_markdown(index: RepoKnowledgeIndex) -> str:
+    lines = [
+        "# Repo Knowledge Index",
+        "",
+        f"- generated_at: {index.generated_at}",
+        f"- git_head: {index.git_head or 'unknown'}",
+        f"- indexed_files: {index.indexed_file_count}",
+        f"- skipped_files: {index.skipped_file_count}",
+        "",
+        "## Key Files",
+    ]
+    for path in index.search_hints.get("key_files", [])[:32]:
+        lines.append(f"- `{path}`")
+    lines.extend(["", "## Story Planes"])
+    if not index.story_planes:
+        lines.append("- No canonical story contracts found.")
+    for story in index.story_planes[:80]:
+        planes = ", ".join(story.required_planes) if story.required_planes else "none"
+        lines.append(f"- `{story.story_id}` {story.title} [{planes}]")
+        for evidence in story.evidence_files[:8]:
+            lines.append(f"  - `{evidence}`")
+    lines.extend(["", "## Directory Map"])
+    for directory in index.directories[:160]:
+        planes = f" planes={','.join(directory.planes)}" if directory.planes else ""
+        langs = ",".join(directory.languages)
+        lines.append(f"- `{directory.path}` files={directory.file_count} languages={langs}{planes}")
+    lines.extend(["", "## Registry Canonicals"])
+    if not index.registry.present:
+        lines.append("- Registry DB not present.")
+    elif index.registry.error:
+        lines.append(f"- Registry read failed: {index.registry.error}")
+    else:
+        lines.append(f"- enforce_ok: {index.registry.enforce_ok}")
+        phase = index.registry.meta.get("module_registry_phase") or index.registry.meta.get(
+            "project_registration_state"
+        )
+        if phase:
+            lines.append(f"- phase: {phase}")
+        for item in index.registry.canonical_module_routes[:40]:
+            lines.append(
+                f"- canonical `{item['module_id']}` for `{item['domain']}/{item['card_id']}` lane `{item['lane']}`"
+            )
+        for item in index.registry.package_policy[:40]:
+            if item["status"] == "canonical":
+                lines.append(
+                    f"- canonical package `{item['package']}` for domain `{item['domain']}` lane `{item['lane']}`"
+                )
+        for directive in index.registry.enforce_directives[:20]:
+            lines.append(f"- refactor directive: `{directive.get('from')}` -> `{directive.get('to')}`")
+    lines.append("")
+    return "\n".join(lines)