modwire-extraction 1.0.0__py3-none-any.whl

modwire_extraction/__init__.py

@@ -0,0 +1,3 @@
+from .extraction import ModwireExtraction
+
+__all__ = ["ModwireExtraction"]

modwire_extraction/_version.py

@@ -0,0 +1,6 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("modwire-extraction")
+except PackageNotFoundError:
+    __version__ = "0.0.0"

modwire_extraction/code/__init__.py

@@ -0,0 +1,20 @@
+from .code_map import CodeMap
+from .query import (
+    DependencyEdgeResult,
+    DependencyNodeResult,
+    QueryableCodeMap,
+    QueryBuilder,
+    SourceFileResult,
+    SourceItemResult,
+)
+
+
+__all__ = [
+    "CodeMap",
+    "DependencyEdgeResult",
+    "DependencyNodeResult",
+    "QueryableCodeMap",
+    "QueryBuilder",
+    "SourceFileResult",
+    "SourceItemResult",
+]

modwire_extraction/code/code_map.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict
+
+from ..dependency.graph import DependencyGraph
+from ..extractors.languages.base import SourceExtraction
+
+
+class CodeMap(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    language: str
+    extraction: SourceExtraction
+    dependency_graph: DependencyGraph
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.model_dump(mode="python")
+
+    @classmethod
+    def from_dict(cls, payload: object) -> CodeMap:
+        return cls.model_validate(payload)
+
+    def to_json(self) -> str:
+        return self.model_dump_json()
+
+    @classmethod
+    def from_json(cls, payload: str | bytes) -> CodeMap:
+        return cls.model_validate_json(payload)

modwire_extraction/code/query.py

@@ -0,0 +1,223 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable
+from dataclasses import dataclass
+from typing import Generic, TypeVar
+
+from ..dependency.graph import Edge, Node
+from ..extractors.source import (
+    SourceAbstractClass,
+    SourceCall,
+    SourceCallable,
+    SourceClass,
+    SourceExport,
+    SourceFile,
+    SourceFunction,
+    SourceImport,
+    SourceInterface,
+    SourceType,
+    SourceValue,
+)
+from .code_map import CodeMap
+
+
+T = TypeVar("T")
+SourceItem = TypeVar("SourceItem")
+
+
+class QueryBuilder(Generic[T]):
+    def __init__(
+        self,
+        items: Iterable[T],
+        predicates: tuple[Callable[[T], bool], ...] = (),
+    ):
+        self._items = tuple(items)
+        self._predicates = predicates
+
+    def where(self, predicate: Callable[[T], bool]) -> QueryBuilder[T]:
+        return QueryBuilder(self._items, (*self._predicates, predicate))
+
+    def where_equal(
+        self,
+        selector: Callable[[T], object],
+        expected: object,
+    ) -> QueryBuilder[T]:
+        return self.where(lambda item: selector(item) == expected)
+
+    def where_contains(
+        self,
+        selector: Callable[[T], str],
+        expected: str,
+        *,
+        case_sensitive: bool = True,
+    ) -> QueryBuilder[T]:
+        if case_sensitive:
+            return self.where(lambda item: expected in selector(item))
+
+        lowered = expected.casefold()
+        return self.where(lambda item: lowered in selector(item).casefold())
+
+    def all(self) -> tuple[T, ...]:
+        return tuple(
+            item
+            for item in self._items
+            if all(predicate(item) for predicate in self._predicates)
+        )
+
+    def first(self) -> T | None:
+        for item in self.all():
+            return item
+        return None
+
+    def count(self) -> int:
+        return len(self.all())
+
+
+@dataclass(frozen=True)
+class SourceFileResult:
+    source_id: str
+    file: SourceFile
+
+
+@dataclass(frozen=True)
+class SourceItemResult(Generic[SourceItem]):
+    source_id: str
+    file: SourceFile
+    item: SourceItem
+
+
+@dataclass(frozen=True)
+class DependencyNodeResult:
+    node_id: str
+    node: Node
+    file: SourceFile | None
+
+
+@dataclass(frozen=True)
+class DependencyEdgeResult:
+    edge: Edge
+    source_file: SourceFile | None
+    target_file: SourceFile | None
+
+
+class QueryableCodeMap:
+    def __init__(self, code_map: CodeMap):
+        self.code_map = code_map
+        self.cm = code_map
+
+    def query(self, items: Iterable[T]) -> QueryBuilder[T]:
+        return QueryBuilder(items)
+
+    def source_ids(self) -> tuple[str, ...]:
+        return tuple(self.code_map.extraction.files)
+
+    def has_source_file(self, source_id: str) -> bool:
+        return source_id in self.code_map.extraction.files
+
+    def source_file(self, source_id: str) -> SourceFileResult | None:
+        source_file = self.code_map.extraction.files.get(source_id)
+        if source_file is None:
+            return None
+        return SourceFileResult(source_id=source_id, file=source_file)
+
+    def files(self) -> QueryBuilder[SourceFileResult]:
+        return self.source_files()
+
+    def source_files(self) -> QueryBuilder[SourceFileResult]:
+        return QueryBuilder(
+            SourceFileResult(source_id=source_id, file=source_file)
+            for source_id, source_file in self.code_map.extraction.files.items()
+        )
+
+    def imports(self) -> QueryBuilder[SourceItemResult[SourceImport]]:
+        return self._source_items(lambda source_file: source_file.imports)
+
+    def exports(self) -> QueryBuilder[SourceItemResult[SourceExport]]:
+        return self._source_items(lambda source_file: source_file.exports)
+
+    def classes(self) -> QueryBuilder[SourceItemResult[SourceClass]]:
+        return self._source_items(lambda source_file: source_file.classes)
+
+    def interfaces(self) -> QueryBuilder[SourceItemResult[SourceInterface]]:
+        return self._source_items(lambda source_file: source_file.interfaces)
+
+    def types(self) -> QueryBuilder[SourceItemResult[SourceType]]:
+        return self._source_items(lambda source_file: source_file.types)
+
+    def abstract_classes(self) -> QueryBuilder[SourceItemResult[SourceAbstractClass]]:
+        return self._source_items(lambda source_file: source_file.abstract_classes)
+
+    def functions(self) -> QueryBuilder[SourceItemResult[SourceFunction]]:
+        return self._source_items(lambda source_file: source_file.functions)
+
+    def values(self) -> QueryBuilder[SourceItemResult[SourceValue]]:
+        return self._source_items(lambda source_file: source_file.values)
+
+    def callables(self) -> QueryBuilder[SourceItemResult[SourceCallable]]:
+        return self._source_items(lambda source_file: source_file.callables)
+
+    def calls(self) -> QueryBuilder[SourceItemResult[SourceCall]]:
+        return self._source_items(lambda source_file: source_file.calls)
+
+    def dependency_nodes(self) -> QueryBuilder[DependencyNodeResult]:
+        files = self.code_map.extraction.files
+        return QueryBuilder(
+            DependencyNodeResult(
+                node_id=node_id,
+                node=node,
+                file=files.get(node_id),
+            )
+            for node_id, node in self.code_map.dependency_graph.nodes.items()
+        )
+
+    def dependency_edges(self) -> QueryBuilder[DependencyEdgeResult]:
+        return self._dependency_edges(self.code_map.dependency_graph.edges)
+
+    def outgoing_dependencies(self, source_id: str) -> QueryBuilder[DependencyEdgeResult]:
+        return self._dependency_edges(self.code_map.dependency_graph.outgoing(source_id))
+
+    def incoming_dependencies(self, source_id: str) -> QueryBuilder[DependencyEdgeResult]:
+        return self._dependency_edges(self.code_map.dependency_graph.incoming(source_id))
+
+    def dependencies_between(
+        self,
+        source_id: str,
+        target_id: str,
+    ) -> QueryBuilder[DependencyEdgeResult]:
+        return self._dependency_edges(
+            self.code_map.dependency_graph.edges_between(source_id, target_id)
+        )
+
+    def tracked_dependency_edges(self) -> QueryBuilder[DependencyEdgeResult]:
+        return self._dependency_edges(
+            self.code_map.dependency_graph.tracked_edges(self.source_ids())
+        )
+
+    def external_dependency_edges(self) -> QueryBuilder[DependencyEdgeResult]:
+        return self._dependency_edges(
+            self.code_map.dependency_graph.external_edges(self.source_ids())
+        )
+
+    def _source_items(
+        self,
+        selector: Callable[[SourceFile], Iterable[SourceItem]],
+    ) -> QueryBuilder[SourceItemResult[SourceItem]]:
+        return QueryBuilder(
+            SourceItemResult(source_id=source_id, file=source_file, item=item)
+            for source_id, source_file in self.code_map.extraction.files.items()
+            for item in selector(source_file)
+        )
+
+    def _dependency_edges(
+        self,
+        edges: Iterable[Edge],
+    ) -> QueryBuilder[DependencyEdgeResult]:
+        files = self.code_map.extraction.files
+        return QueryBuilder(
+            DependencyEdgeResult(
+                edge=edge,
+                source_file=files.get(edge.from_id),
+                target_file=files.get(edge.to_id),
+            )
+            for edge in edges
+        )

modwire_extraction/dependency/__init__.py

@@ -0,0 +1,6 @@
+from .graph import build_dependency_graph
+
+
+__all__ = [
+    "build_dependency_graph",
+]

modwire_extraction/dependency/graph.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
+
+from ..extractors.source import SourceFile
+
+
+class Node(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str
+    kind: str = "file"
+
+
+class Edge(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    from_id: str
+    to_id: str
+    kind: str = "import"
+
+
+class DependencyGraph(BaseModel):
+    nodes: dict[str, Node] = Field(default_factory=dict)
+    edges: list[Edge] = Field(default_factory=list)
+    _outgoing_by_node: dict[str, list[Edge]] = PrivateAttr(default_factory=dict)
+    _incoming_by_node: dict[str, list[Edge]] = PrivateAttr(default_factory=dict)
+
+    def model_post_init(self, __context: Any) -> None:
+        self._rebuild_indexes()
+
+    def _rebuild_indexes(self) -> None:
+        self._outgoing_by_node = {node_id: [] for node_id in self.nodes}
+        self._incoming_by_node = {node_id: [] for node_id in self.nodes}
+        for edge in self.edges:
+            self._outgoing_by_node.setdefault(edge.from_id, []).append(edge)
+            self._incoming_by_node.setdefault(edge.to_id, []).append(edge)
+
+    def add_node(self, node_id: str, *, kind: str = "file") -> None:
+        self.nodes.setdefault(node_id, Node(id=node_id, kind=kind))
+        self._outgoing_by_node.setdefault(node_id, [])
+        self._incoming_by_node.setdefault(node_id, [])
+
+    def add_edge(self, from_id: str, to_id: str, *, kind: str = "import") -> None:
+        self.add_node(from_id)
+        self.add_node(to_id)
+        edge = Edge(from_id=from_id, to_id=to_id, kind=kind)
+        self.edges.append(edge)
+        self._outgoing_by_node[from_id].append(edge)
+        self._incoming_by_node[to_id].append(edge)
+
+    def outgoing(self, node_id: str) -> tuple[Edge, ...]:
+        return tuple(self._outgoing_by_node.get(node_id, ()))
+
+    def incoming(self, node_id: str) -> tuple[Edge, ...]:
+        return tuple(self._incoming_by_node.get(node_id, ()))
+
+    def edges_between(self, source: str, target: str) -> tuple[Edge, ...]:
+        return tuple(
+            edge
+            for edge in self._outgoing_by_node.get(source, ())
+            if edge.to_id == target
+        )
+
+    def has_node(self, node_id: str) -> bool:
+        return node_id in self.nodes
+
+    def node_ids(self) -> tuple[str, ...]:
+        return tuple(self.nodes.keys())
+
+    def sorted_nodes(self) -> tuple[Node, ...]:
+        return tuple(self.nodes[node_id] for node_id in sorted(self.nodes))
+
+    def sorted_edges(self) -> tuple[Edge, ...]:
+        return tuple(
+            sorted(
+                self.edges,
+                key=lambda edge: (edge.from_id, edge.to_id, edge.kind),
+            )
+        )
+
+    def subgraph(self, node_ids: set[str] | tuple[str, ...]) -> DependencyGraph:
+        selected = set(node_ids)
+        graph = DependencyGraph()
+        for node_id in sorted(selected):
+            if node_id in self.nodes:
+                graph.add_node(node_id, kind=self.nodes[node_id].kind)
+        for edge in self.edges:
+            if edge.from_id in selected and edge.to_id in selected:
+                graph.add_edge(edge.from_id, edge.to_id, kind=edge.kind)
+        return graph
+
+    def without_external_nodes(
+        self,
+        tracked_ids: set[str] | tuple[str, ...],
+    ) -> DependencyGraph:
+        return self.subgraph(set(tracked_ids))
+
+    def tracked_edges(self, tracked_ids: set[str] | tuple[str, ...]) -> tuple[Edge, ...]:
+        tracked = set(tracked_ids)
+        return tuple(
+            edge
+            for edge in self.edges
+            if edge.from_id in tracked and edge.to_id in tracked
+        )
+
+    def external_edges(self, tracked_ids: set[str] | tuple[str, ...]) -> tuple[Edge, ...]:
+        tracked = set(tracked_ids)
+        return tuple(
+            edge
+            for edge in self.edges
+            if edge.from_id not in tracked or edge.to_id not in tracked
+        )
+
+
+def build_dependency_graph(extracted_files: dict[str, SourceFile]) -> DependencyGraph:
+    graph = DependencyGraph()
+
+    for file_path, extracted_file in extracted_files.items():
+        graph.add_node(file_path)
+        for imported_reference in extracted_file.imports:
+            graph.add_edge(file_path, imported_reference.normalized_path)
+
+    return graph

modwire_extraction/extraction.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+
+from .dependency import build_dependency_graph
+from .extractors import languages
+from .code import CodeMap, QueryableCodeMap
+
+
+class ModwireExtraction:
+    def __init__(self, root: Path):
+        self._root = root.resolve()
+
+    def discover(self) -> tuple[str, ...]:
+        discovered: list[str] = []
+        for language in languages.get_supported_languages():
+            extractor = languages.load_extractor(language)
+            if extractor.has_source_files(self._root):
+                discovered.append(language)
+        return tuple(discovered)
+
+    def generate_map(self, language: str) -> CodeMap:
+        available = languages.get_supported_languages()
+        if language not in available:
+            raise ValueError(f"Language is not supported: {language}")
+
+        extraction = languages.load_extractor(language).extract_source(self._root)
+        dependency_graph = build_dependency_graph(extraction.files)
+        return CodeMap(
+            language=language,
+            extraction=extraction,
+            dependency_graph=dependency_graph,
+        )
+
+    def generate_queryable_map(self, language: str) -> QueryableCodeMap:
+        code_map = self.generate_map(language)
+        return QueryableCodeMap(code_map=code_map)

modwire_extraction/extractors/__init__.py

@@ -0,0 +1,2 @@
+from .languages import load_extractor
+

modwire_extraction/extractors/languages/__init__.py

@@ -0,0 +1,7 @@
+from .loader import load_extractor, get_supported_languages
+
+
+__all__ = [
+    "load_extractor",
+    "get_supported_languages",
+]

modwire_extraction/extractors/languages/base.py

@@ -0,0 +1,219 @@
+import abc
+import json
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from typing import Any, Literal, cast
+
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict
+
+from ..source import SourceFile
+
+
+class SourceExtraction(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    files: dict[str, SourceFile]
+    files_found: int
+    files_excluded: int
+
+    def files_dict(self) -> dict[str, SourceFile]:
+        return dict(self.files)
+
+
+@dataclass(frozen=True)
+class BatchConfig:
+    size: int = 500
+    parallel_threshold: int = 0
+    parallel_size: int = 0
+    max_workers: int = 1
+    output_format: Literal["json", "jsonl"] = "json"
+
+
+@dataclass(frozen=True)
+class ExtractorRuntime:
+    language: str
+    file_extensions: tuple[str, ...]
+    command: tuple[str, ...]
+    script_path: Path
+
+
+class SourceExtractor(abc.ABC):
+    excluded_dir_names = frozenset(
+        {
+            ".git",
+            ".hg",
+            ".mypy_cache",
+            ".pytest_cache",
+            ".ruff_cache",
+            ".svn",
+            ".venv",
+            "__pycache__",
+            "build",
+            "coverage",
+            "dist",
+            "ignored",
+            "node_modules",
+            "vendor",
+        }
+    )
+
+    @property
+    @abc.abstractmethod
+    def runtime(self) -> ExtractorRuntime:
+        raise NotImplementedError
+
+    @property
+    @abc.abstractmethod
+    def batch_config(self) -> BatchConfig:
+        raise NotImplementedError
+
+    def has_source_files(self, root: Path) -> bool:
+        resolved_root = root.resolve()
+        if not resolved_root.is_dir():
+            raise ValueError(f"Source root is not a directory: {root}")
+
+        source_paths, _ = self._discover_source_files(resolved_root)
+        return bool(source_paths)
+
+    def extract_source(self, root: Path) -> SourceExtraction:
+        resolved_root = root.resolve()
+        if not resolved_root.is_dir():
+            raise ValueError(f"Source root is not a directory: {root}")
+
+        source_paths, files_excluded = self._discover_source_files(resolved_root)
+        files: dict[str, SourceFile] = {}
+        batch_size = max(1, self.batch_config.size)
+
+        for start in range(0, len(source_paths), batch_size):
+            batch_paths = source_paths[start : start + batch_size]
+            files.update(self._extract_batch(resolved_root, batch_paths))
+
+        return SourceExtraction(
+            files=files,
+            files_found=len(source_paths),
+            files_excluded=files_excluded,
+        )
+
+    def _discover_source_files(self, root: Path) -> tuple[list[Path], int]:
+        source_paths: list[Path] = []
+        files_excluded = 0
+        extensions = self.runtime.file_extensions
+
+        for current_root, dir_names, file_names in os.walk(root):
+            current_path = Path(current_root)
+            excluded_dirs = [
+                dir_name for dir_name in dir_names if self._is_excluded_dir(dir_name)
+            ]
+            files_excluded += sum(
+                self._count_source_files(current_path / dir_name)
+                for dir_name in excluded_dirs
+            )
+            dir_names[:] = [
+                dir_name for dir_name in dir_names if dir_name not in excluded_dirs
+            ]
+
+            for file_name in file_names:
+                file_path = current_path / file_name
+                if file_path.suffix.lower() in extensions:
+                    source_paths.append(file_path.resolve())
+
+        return sorted(source_paths), files_excluded
+
+    def _count_source_files(self, root: Path) -> int:
+        count = 0
+        extensions = self.runtime.file_extensions
+        for current_root, dir_names, file_names in os.walk(root):
+            dir_names[:] = [
+                dir_name
+                for dir_name in dir_names
+                if not self._is_excluded_dir(dir_name)
+            ]
+            count += sum(
+                1
+                for file_name in file_names
+                if (Path(current_root) / file_name).suffix.lower() in extensions
+            )
+        return count
+
+    def _is_excluded_dir(self, name: str) -> bool:
+        return name in self.excluded_dir_names or name.startswith(".")
+
+    def _extract_batch(self, root: Path, source_paths: list[Path]) -> dict[str, SourceFile]:
+        if not source_paths:
+            return {}
+
+        runtime = self.runtime
+        if not runtime.script_path.is_file():
+            raise RuntimeError(
+                f"{runtime.language} extractor script is missing: {runtime.script_path}"
+            )
+        executable = runtime.command[0]
+        if shutil.which(executable) is None:
+            raise RuntimeError(
+                f"{runtime.language} extractor runtime is not available on PATH: "
+                f"{executable}"
+            )
+
+        paths_by_source_id = {
+            self._source_id_for_path(root, source_path): str(source_path)
+            for source_path in source_paths
+        }
+        command = [
+            *runtime.command,
+            str(runtime.script_path),
+            "--batch",
+            str(root),
+        ]
+        if self.batch_config.output_format == "jsonl":
+            command.append("--jsonl")
+
+        result = subprocess.run(
+            command,
+            input=json.dumps(paths_by_source_id),
+            text=True,
+            capture_output=True,
+            check=False,
+        )
+        if result.returncode != 0:
+            message = result.stderr.strip() or result.stdout.strip()
+            raise RuntimeError(
+                f"{runtime.language} extractor failed with exit code "
+                f"{result.returncode}: {message}"
+            )
+
+        extracted = self._parse_batch_output(result.stdout)
+        return {
+            source_id: SourceFile.model_validate(source_file)
+            for source_id, source_file in extracted.items()
+        }
+
+    def _parse_batch_output(self, output: str) -> dict[str, Any]:
+        if self.batch_config.output_format == "jsonl":
+            result: dict[str, Any] = {}
+            for line in output.splitlines():
+                if not line.strip():
+                    continue
+                item: Any = json.loads(line)
+                if not isinstance(item, list):
+                    raise RuntimeError("Extractor returned invalid JSONL batch output.")
+                item_list = cast(list[Any], item)
+                if len(item_list) != 2:
+                    raise RuntimeError("Extractor returned invalid JSONL batch output.")
+                source_id, source_file = item_list
+                if not isinstance(source_id, str):
+                    raise RuntimeError("Extractor returned a non-string source id.")
+                result[source_id] = source_file
+            return result
+
+        parsed: Any = json.loads(output)
+        if not isinstance(parsed, dict):
+            raise RuntimeError("Extractor returned invalid JSON batch output.")
+        return cast(dict[str, Any], parsed)
+
+    def _source_id_for_path(self, root: Path, path: Path) -> str:
+        relative_path = path.relative_to(root)
+        return relative_path.with_suffix("").as_posix().strip("/")