osscodeiq 0.0.0__py3-none-any.whl

osscodeiq/discovery/change_detector.py

@@ -0,0 +1,97 @@
+"""Git diff-based incremental change detection."""
+
+from __future__ import annotations
+
+import re
+import subprocess
+from pathlib import Path
+
+from osscodeiq.discovery.file_discovery import (
+    ChangeType,
+    DiscoveredFile,
+    _compute_sha256,
+    _map_extension_to_language,
+)
+
+
+# Mapping from git status letters to ChangeType.
+_GIT_STATUS_MAP: dict[str, ChangeType] = {
+    "A": ChangeType.ADDED,
+    "M": ChangeType.MODIFIED,
+    "D": ChangeType.DELETED,
+    "R": ChangeType.MODIFIED,  # Rename treated as modified
+}
+
+
+class ChangeDetector:
+    """Detects file changes between two git commits."""
+
+    def detect_changes(
+        self, repo_path: Path, last_commit: str
+    ) -> list[DiscoveredFile]:
+        """Return files that changed between *last_commit* and HEAD.
+
+        Uses ``git diff --name-status <last_commit>..HEAD``.
+        """
+        repo_path = repo_path.resolve()
+
+        if not re.fullmatch(r'[0-9a-fA-F]{4,40}', last_commit):
+            raise ValueError(f"Invalid commit SHA: {last_commit}")
+
+        result = subprocess.run(
+            ["git", "diff", "--name-status", f"{last_commit}..HEAD"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+
+        discovered: list[DiscoveredFile] = []
+        for line in result.stdout.splitlines():
+            if not line.strip():
+                continue
+            parts = line.split("\t")
+            if len(parts) < 2:
+                continue
+
+            status_letter = parts[0][0]  # First char handles R100 etc.
+            # For renames the *destination* path is the last element.
+            file_rel = parts[-1]
+            change_type = _GIT_STATUS_MAP.get(status_letter)
+            if change_type is None:
+                continue
+
+            rel_path = Path(file_rel)
+            lang = _map_extension_to_language(rel_path)
+            if lang is None:
+                continue
+
+            abs_path = repo_path / rel_path
+
+            if change_type is ChangeType.DELETED:
+                discovered.append(
+                    DiscoveredFile(
+                        path=rel_path,
+                        language=lang,
+                        content_hash="",
+                        size_bytes=0,
+                        change_type=change_type,
+                    )
+                )
+            else:
+                try:
+                    size = abs_path.stat().st_size
+                    content_hash = _compute_sha256(abs_path)
+                except OSError:
+                    continue
+                discovered.append(
+                    DiscoveredFile(
+                        path=rel_path,
+                        language=lang,
+                        content_hash=content_hash,
+                        size_bytes=size,
+                        change_type=change_type,
+                    )
+                )
+
+        return discovered

osscodeiq/discovery/file_discovery.py

@@ -0,0 +1,342 @@
+"""Git-aware file discovery for OSSCodeIQ."""
+
+from __future__ import annotations
+
+import fnmatch
+import hashlib
+import logging
+import os
+import re
+import subprocess
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+
+import pathspec
+
+logger = logging.getLogger(__name__)
+
+from osscodeiq.config import Config
+
+
+class ChangeType(Enum):
+    """Type of file change detected by git."""
+
+    ADDED = "added"
+    MODIFIED = "modified"
+    DELETED = "deleted"
+
+
+# Map file extensions to language identifiers.
+_EXTENSION_MAP: dict[str, str] = {
+    ".java": "java",
+    ".py": "python",
+    ".ts": "typescript",
+    ".tsx": "typescript",
+    ".js": "javascript",
+    ".jsx": "javascript",
+    ".xml": "xml",
+    ".yaml": "yaml",
+    ".yml": "yaml",
+    ".json": "json",
+    ".properties": "properties",
+    ".gradle": "gradle",
+    ".gradle.kts": "gradle",
+    ".sql": "sql",
+    ".graphql": "graphql",
+    ".gql": "graphql",
+    ".proto": "proto",
+    ".md": "markdown",
+    ".markdown": "markdown",
+    ".bicep": "bicep",
+    ".tf": "terraform",
+    ".tfvars": "terraform",
+    ".cs": "csharp",
+    ".go": "go",
+    ".cpp": "cpp",
+    ".cc": "cpp",
+    ".cxx": "cpp",
+    ".hpp": "cpp",
+    ".c": "c",
+    ".h": "c",
+    ".sh": "bash",
+    ".bash": "bash",
+    ".zsh": "bash",
+    ".ps1": "powershell",
+    ".psm1": "powershell",
+    ".psd1": "powershell",
+    ".bat": "batch",
+    ".cmd": "batch",
+    ".rb": "ruby",
+    ".rs": "rust",
+    ".kt": "kotlin",
+    ".kts": "kotlin",
+    ".scala": "scala",
+    ".swift": "swift",
+    ".r": "r",
+    ".R": "r",
+    ".pl": "perl",
+    ".pm": "perl",
+    ".lua": "lua",
+    ".dart": "dart",
+    ".hcl": "terraform",
+    ".dockerfile": "dockerfile",
+    ".toml": "toml",
+    ".ini": "ini",
+    ".cfg": "ini",
+    ".conf": "ini",
+    ".env": "dotenv",
+    ".csv": "csv",
+    ".vue": "vue",
+    ".svelte": "svelte",
+    ".html": "html",
+    ".htm": "html",
+    ".css": "css",
+    ".scss": "scss",
+    ".less": "less",
+    ".mjs": "javascript",
+    ".cjs": "javascript",
+    ".mts": "typescript",
+    ".cts": "typescript",
+    ".jsonc": "json",
+    ".groovy": "groovy",
+    ".pyi": "python",
+    ".razor": "razor",
+    ".cshtml": "cshtml",
+    ".adoc": "asciidoc",
+}
+
+
+_FILENAME_MAP: dict[str, str] = {
+    "Dockerfile": "dockerfile",
+    "Makefile": "makefile",
+    "GNUmakefile": "makefile",
+    "Jenkinsfile": "groovy",
+    "Vagrantfile": "ruby",
+    "Gemfile": "ruby",
+    "Rakefile": "ruby",
+    "go.mod": "gomod",
+    "go.sum": "gosum",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class DiscoveredFile:
+    """A file discovered during repository scanning."""
+
+    path: Path
+    language: str
+    content_hash: str
+    size_bytes: int
+    change_type: ChangeType | None = None
+
+
+def _map_extension_to_language(file_path: Path) -> str | None:
+    """Map a file's extension to a language string.
+
+    Falls back to :data:`_FILENAME_MAP` when no extension matches, using the
+    basename of *file_path* (e.g. ``"Dockerfile"`` from ``"app/Dockerfile"``).
+    """
+    name = file_path.name
+    # Check compound extensions first (e.g. .gradle.kts)
+    for ext, lang in _EXTENSION_MAP.items():
+        if name.endswith(ext):
+            return lang
+    # Fallback: match the full filename (extensionless files like Dockerfile)
+    return _FILENAME_MAP.get(name)
+
+
+def _matches_any_pattern(path_str: str, patterns: list[str]) -> bool:
+    """Check if a path matches any of the given glob patterns."""
+    for pattern in patterns:
+        if fnmatch.fnmatch(path_str, pattern):
+            return True
+    return False
+
+
+def _compile_exclude_patterns(patterns: list[str]) -> re.Pattern[str] | None:
+    """Compile a list of glob patterns into a single regex for fast matching."""
+    if not patterns:
+        return None
+    return re.compile("|".join(fnmatch.translate(p) for p in patterns))
+
+
+def _build_ignore_spec(repo_path: Path, config_patterns: list[str]) -> pathspec.PathSpec:
+    """Build a combined ignore spec from config patterns + ignore files.
+
+    Reads .codeignore and .gitignore files from the repo root and any
+    subdirectory, combining them with the config exclude_patterns.
+    Uses gitignore-style matching (handles node_modules at any depth).
+    """
+    all_patterns: list[str] = []
+
+    # 1. Config exclude patterns (convert ** glob to gitignore style)
+    for p in config_patterns:
+        # Strip leading **/ — gitignore patterns match at any depth by default
+        cleaned = p.replace("**/", "").rstrip("/**")
+        all_patterns.append(cleaned)
+        # Also keep original for explicit **/ matching
+        all_patterns.append(p)
+
+    # 2. Read .codeignore from repo root
+    codeignore = repo_path / ".codeignore"
+    if codeignore.is_file():
+        try:
+            lines = codeignore.read_text().splitlines()
+            for line in lines:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    all_patterns.append(line)
+            logger.debug("Loaded %d patterns from .codeignore", len(lines))
+        except OSError:
+            pass
+
+    # 3. Read .gitignore from repo root (supplementary)
+    gitignore = repo_path / ".gitignore"
+    if gitignore.is_file():
+        try:
+            lines = gitignore.read_text().splitlines()
+            for line in lines:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    all_patterns.append(line)
+            logger.debug("Loaded %d patterns from .gitignore", len(lines))
+        except OSError:
+            pass
+
+    return pathspec.PathSpec.from_lines("gitwildmatch", all_patterns)
+
+
+def _compute_sha256(file_path: Path) -> str:
+    """Compute SHA-256 hex digest for a file.
+
+    Delegates to :func:`osscodeiq.cache.hasher.hash_file` for
+    consistency, falling back to a local implementation if the import fails.
+    """
+    from osscodeiq.cache.hasher import hash_file
+
+    return hash_file(file_path)
+    return h.hexdigest()
+
+
+class FileDiscovery:
+    """Discovers files in a repository using git or filesystem walk."""
+
+    def __init__(self, config: Config | None = None) -> None:
+        self._config = config or Config()
+        self._current_commit: str | None = None
+
+    @property
+    def current_commit(self) -> str | None:
+        """The HEAD commit hash from the last discovery run."""
+        return self._current_commit
+
+    def discover(
+        self, repo_path: Path, incremental: bool = True
+    ) -> list[DiscoveredFile]:
+        """Discover tracked files in a repository.
+
+        Uses ``git ls-files`` for git repos (fast, ~50ms for large repos).
+        Falls back to ``os.walk`` for non-git directories.
+        """
+        repo_path = repo_path.resolve()
+        discovery_cfg = self._config.discovery
+
+        if self._is_git_repo(repo_path):
+            self._current_commit = self._git_head(repo_path)
+            relative_paths = self._git_ls_files(repo_path)
+        else:
+            self._current_commit = None
+            relative_paths = self._walk_files(repo_path)
+
+        ignore_spec = _build_ignore_spec(repo_path, discovery_cfg.exclude_patterns)
+
+        result: list[DiscoveredFile] = []
+        for rel in relative_paths:
+            abs_path = repo_path / rel
+            rel_path = Path(rel)
+
+            # Check ignore patterns first (fastest rejection)
+            if ignore_spec.match_file(str(rel_path)):
+                continue
+
+            # Extension filter
+            lang = _map_extension_to_language(rel_path)
+            if lang is None:
+                continue
+
+            # Check include extensions (skip for extensionless filename matches)
+            is_filename_match = rel_path.name in _FILENAME_MAP
+            if not is_filename_match and not any(
+                rel.endswith(ext) for ext in discovery_cfg.include_extensions
+            ):
+                continue
+
+            # Size guard
+            try:
+                size = abs_path.stat().st_size
+            except OSError:
+                continue
+            if size > discovery_cfg.max_file_size_bytes:
+                continue
+
+            content_hash = _compute_sha256(abs_path)
+            result.append(
+                DiscoveredFile(
+                    path=rel_path,
+                    language=lang,
+                    content_hash=content_hash,
+                    size_bytes=size,
+                )
+            )
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _is_git_repo(path: Path) -> bool:
+        try:
+            subprocess.run(
+                ["git", "rev-parse", "--git-dir"],
+                cwd=path,
+                capture_output=True,
+                check=True,
+            )
+            return True
+        except (subprocess.CalledProcessError, FileNotFoundError):
+            return False
+
+    @staticmethod
+    def _git_head(repo_path: Path) -> str:
+        result = subprocess.run(
+            ["git", "rev-parse", "HEAD"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return result.stdout.strip()
+
+    @staticmethod
+    def _git_ls_files(repo_path: Path) -> list[str]:
+        result = subprocess.run(
+            ["git", "ls-files"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return [line for line in result.stdout.splitlines() if line]
+
+    @staticmethod
+    def _walk_files(root: Path) -> list[str]:
+        paths: list[str] = []
+        for dirpath, _dirnames, filenames in os.walk(root):
+            for fname in filenames:
+                abs_p = Path(dirpath) / fname
+                rel = str(abs_p.relative_to(root))
+                paths.append(rel)
+        return paths

osscodeiq/flow/engine.py

@@ -0,0 +1,78 @@
+"""FlowEngine — core library for generating architecture flow diagrams.
+
+All consumers (CLI, HTTP API, MCP tool, HTML UI) call the same methods.
+FlowDiagram is the single source of truth — renderers only change format, never data.
+"""
+
+from __future__ import annotations
+
+from osscodeiq.flow.models import FlowDiagram
+from osscodeiq.flow.renderer import render_html, render_json, render_mermaid
+from osscodeiq.flow.views import (
+    build_auth_view,
+    build_ci_view,
+    build_deploy_view,
+    build_overview,
+    build_runtime_view,
+)
+from osscodeiq.graph.store import GraphStore
+
+_VIEWS = {
+    "overview": build_overview,
+    "ci": build_ci_view,
+    "deploy": build_deploy_view,
+    "runtime": build_runtime_view,
+    "auth": build_auth_view,
+}
+
+AVAILABLE_VIEWS = tuple(_VIEWS.keys())
+
+
+class FlowEngine:
+    """Generate and render architecture flow diagrams from a OSSCodeIQ graph.
+
+    Usage::
+
+        engine = FlowEngine(store)
+        diagram = engine.generate("overview")
+        print(engine.render(diagram, "mermaid"))
+        # Or generate interactive HTML with all views:
+        html = engine.render_interactive()
+    """
+
+    def __init__(self, store: GraphStore) -> None:
+        self._store = store
+
+    def generate(self, view: str = "overview") -> FlowDiagram:
+        """Generate a single flow view diagram."""
+        builder = _VIEWS.get(view)
+        if builder is None:
+            raise ValueError(f"Unknown view: {view}. Available: {', '.join(AVAILABLE_VIEWS)}")
+        return builder(self._store)
+
+    def generate_all(self) -> dict[str, FlowDiagram]:
+        """Generate all views. Used for HTML interactive output."""
+        return {name: self.generate(name) for name in AVAILABLE_VIEWS}
+
+    def render(self, diagram: FlowDiagram, format: str = "mermaid") -> str:
+        """Render a diagram to string.
+
+        Args:
+            diagram: The FlowDiagram to render.
+            format: Output format — "mermaid" or "json".
+        """
+        if format == "mermaid":
+            return render_mermaid(diagram)
+        elif format == "json":
+            return render_json(diagram)
+        else:
+            raise ValueError(f"Unknown format: {format}. Available: mermaid, json")
+
+    def render_interactive(self, project_name: str = "Project") -> str:
+        """Generate all views and bake into a self-contained interactive HTML file."""
+        all_views = self.generate_all()
+        stats = {
+            "total_nodes": self._store.node_count,
+            "total_edges": self._store.edge_count,
+        }
+        return render_html(all_views, stats, project_name=project_name)

osscodeiq/flow/models.py

@@ -0,0 +1,72 @@
+"""Data models for flow diagrams — the single source of truth for all renderers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass
+class FlowNode:
+    """A node in a flow diagram (collapsed/summarized from graph nodes)."""
+    id: str
+    label: str
+    kind: str               # "trigger", "job", "service", "endpoint", "database", "guard", etc.
+    properties: dict = field(default_factory=dict)
+    style: str = "default"  # "default", "success", "warning", "danger"
+
+
+@dataclass
+class FlowSubgraph:
+    """A labeled group of nodes in a flow diagram."""
+    id: str
+    label: str
+    nodes: list[FlowNode] = field(default_factory=list)
+    drill_down_view: str | None = None  # "ci", "deploy", "runtime", "auth"
+
+
+@dataclass
+class FlowEdge:
+    """An edge in a flow diagram."""
+    source: str
+    target: str
+    label: str | None = None
+    style: str = "solid"    # "solid", "dotted", "thick"
+
+
+@dataclass
+class FlowDiagram:
+    """A complete flow diagram — the single source of truth for all renderers."""
+    title: str
+    view: str               # "overview", "ci", "deploy", "runtime", "auth"
+    direction: str = "LR"
+    subgraphs: list[FlowSubgraph] = field(default_factory=list)
+    loose_nodes: list[FlowNode] = field(default_factory=list)
+    edges: list[FlowEdge] = field(default_factory=list)
+    stats: dict = field(default_factory=dict)
+
+    def all_nodes(self) -> list[FlowNode]:
+        """Return all nodes across subgraphs and loose nodes."""
+        result = list(self.loose_nodes)
+        for sg in self.subgraphs:
+            result.extend(sg.nodes)
+        return result
+
+    def to_dict(self) -> dict:
+        """Serialize to a plain dict (for JSON renderer and API responses)."""
+        return {
+            "title": self.title,
+            "view": self.view,
+            "direction": self.direction,
+            "subgraphs": [
+                {
+                    "id": sg.id,
+                    "label": sg.label,
+                    "drill_down_view": sg.drill_down_view,
+                    "nodes": [{"id": n.id, "label": n.label, "kind": n.kind, "properties": n.properties, "style": n.style} for n in sg.nodes],
+                }
+                for sg in self.subgraphs
+            ],
+            "loose_nodes": [{"id": n.id, "label": n.label, "kind": n.kind, "properties": n.properties, "style": n.style} for n in self.loose_nodes],
+            "edges": [{"source": e.source, "target": e.target, "label": e.label, "style": e.style} for e in self.edges],
+            "stats": self.stats,
+        }