modwire 1.0.0__py3-none-any.whl

modwire/extractors/base.py

@@ -0,0 +1,177 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from fnmatch import fnmatch
+from pathlib import Path
+from subprocess import run
+from typing import Protocol
+
+from ..definitions import SourceFile, SourceImport
+
+
+@dataclass(frozen=True)
+class SourceExtraction:
+    files: dict[str, SourceFile]
+    files_found: int
+    files_excluded: int
+
+
+@dataclass(frozen=True)
+class ExtractionTarget:
+    source_id: str
+    path: Path
+
+
+class SourceExtractor(Protocol):
+    language: str
+    file_extensions: tuple[str, ...]
+    command: str
+    extractor_file: str
+
+    def normalize_source_id(self, value: str) -> str:
+        source_id = value.strip().strip("/")
+        for file_extension in self.file_extensions:
+            if source_id.endswith(file_extension):
+                return source_id[: -len(file_extension)]
+        return source_id
+
+    def normalize_import(
+        self,
+        source_id: str,
+        source_import: SourceImport,
+        known_source_ids: set[str],
+    ) -> SourceImport:
+        if source_import.normalized_path in known_source_ids:
+            return source_import
+        return SourceImport(
+            path=source_import.path,
+            is_relative=source_import.is_relative,
+            normalized_path=source_import.normalized_path.strip().strip("/"),
+            imported_name=source_import.imported_name,
+            is_aliased=source_import.is_aliased,
+            crossing_type=source_import.crossing_type,
+            file_barrier_crossed=False,
+            statement_id=source_import.statement_id,
+            join_key=source_import.join_key,
+            uses_joined_import=source_import.uses_joined_import,
+        )
+
+    def extract_files(
+        self,
+        sources_root: Path,
+        exclusions: tuple[str, ...],
+    ) -> SourceExtraction:
+        script = Path(__file__).parent / "scripts" / self.extractor_file
+        assert script.is_file(), f"Extractor script {script} not found"
+
+        targets, files_found, files_excluded = _collect_extraction_targets(
+            sources_root,
+            self.file_extensions,
+            exclusions,
+        )
+        result = {}
+        for target in targets:
+            cmd = [
+                self.command,
+                str(script),
+                str(target.path.resolve()),
+                str(sources_root.resolve()),
+            ]
+            result[self.normalize_source_id(target.source_id)] = (
+                SourceFile.model_validate(_json_from_output(cmd))
+            )
+
+        known_source_ids = set(result)
+        result = {
+            source_id: self.normalize_source_file(
+                source_id,
+                source_file,
+                known_source_ids,
+            )
+            for source_id, source_file in result.items()
+        }
+
+        return SourceExtraction(
+            files=result,
+            files_found=files_found,
+            files_excluded=files_excluded,
+        )
+
+    def normalize_source_file(
+        self,
+        source_id: str,
+        source_file: SourceFile,
+        known_source_ids: set[str],
+    ) -> SourceFile:
+        return SourceFile(
+            imports=[
+                self.normalize_import(source_id, source_import, known_source_ids)
+                for source_import in source_file.imports
+            ],
+            classes=source_file.classes,
+            interfaces=source_file.interfaces,
+            types=source_file.types,
+            abstract_classes=source_file.abstract_classes,
+            functions=source_file.functions,
+            line_count=source_file.line_count,
+            code_line_count=source_file.code_line_count,
+            public_symbol_count=source_file.public_symbol_count,
+        )
+
+
+def _collect_extraction_targets(
+    sources_root: Path,
+    file_extensions: tuple[str, ...],
+    exclusions: tuple[str, ...],
+) -> tuple[tuple[ExtractionTarget, ...], int, int]:
+    targets = []
+    files_found = 0
+    files_excluded = 0
+    for path in sorted(sources_root.rglob("*")):
+        if path.suffix not in file_extensions:
+            continue
+
+        files_found += 1
+        source_id = path.relative_to(sources_root).as_posix()
+        if any(_matches_exclusion(source_id, exclusion) for exclusion in exclusions):
+            files_excluded += 1
+            continue
+
+        targets.append(ExtractionTarget(source_id, path))
+
+    return tuple(targets), files_found, files_excluded
+
+
+def _json_from_output(cmd: list[str], input_json: str | None = None) -> dict:
+    output_json = run(
+        cmd,
+        capture_output=True,
+        text=True,
+        input=input_json,
+        check=True,
+    ).stdout
+
+    try:
+        return json.loads(output_json)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to parse JSON from output: {output_json}") from e
+
+
+def _matches_exclusion(source_id: str, exclusion: str) -> bool:
+    if fnmatch(source_id, exclusion):
+        return True
+
+    normalized = exclusion.strip("/")
+    has_glob = any(char in normalized for char in "*?[")
+    if not normalized or has_glob:
+        return False
+
+    return source_id.startswith(f"{normalized}/")
+
+
+__all__ = [
+    "ExtractionTarget",
+    "SourceExtraction",
+    "SourceExtractor",
+]

modwire/extractors/loader.py

@@ -0,0 +1,31 @@
+from .base import SourceExtractor
+
+from .php import PhpExtractor
+from .python import PythonExtractor
+from .typescript import TypeScriptExtractor
+
+
+_map: dict[str, type[SourceExtractor]] = {
+    "python": PythonExtractor,
+    "typescript": TypeScriptExtractor,
+    "php": PhpExtractor,
+}
+
+_instances: dict[str, SourceExtractor] = {}
+
+
+def supported_languages() -> tuple[str, ...]:
+    return tuple(_map)
+
+
+def load_extractor(language: str) -> SourceExtractor:
+    assert language in _map, f"Unsupported language: {language}"
+
+    if language not in _instances:
+        _instances[language] = _map[language]()
+    return _instances[language]
+
+
+def normalize_source_id(language: str, value: str) -> str:
+    normalized = value.replace("\\", "/").strip().strip("/")
+    return load_extractor(language).normalize_source_id(normalized)

modwire/extractors/php.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+
+from ..definitions import SourceFile, SourceImport
+from .base import (
+    SourceExtraction,
+    SourceExtractor,
+    _collect_extraction_targets,
+    _json_from_output,
+)
+
+
+@dataclass(frozen=True)
+class PhpExtractor(SourceExtractor):
+    language = "php"
+    file_extensions = (".php",)
+    command = "php"
+    extractor_file = "php_extractor.php"
+
+    def extract_files(
+        self,
+        sources_root: Path,
+        exclusions: tuple[str, ...],
+    ) -> SourceExtraction:
+        script = Path(__file__).parent / "scripts" / self.extractor_file
+        assert script.is_file(), f"Extractor script {script} not found"
+
+        targets, files_found, files_excluded = _collect_extraction_targets(
+            sources_root,
+            self.file_extensions,
+            exclusions,
+        )
+        if not targets:
+            return SourceExtraction(
+                files={},
+                files_found=files_found,
+                files_excluded=files_excluded,
+            )
+
+        input_data = {
+            self.normalize_source_id(target.source_id): str(target.path.resolve())
+            for target in targets
+        }
+        cmd = [self.command, str(script), "--batch", str(sources_root.resolve())]
+        raw_files = _json_from_output(cmd, json.dumps(input_data))
+        result = {
+            source_id: SourceFile.model_validate(source_file)
+            for source_id, source_file in raw_files.items()
+        }
+
+        known_source_ids = set(result)
+        result = {
+            source_id: self.normalize_source_file(
+                source_id,
+                source_file,
+                known_source_ids,
+            )
+            for source_id, source_file in result.items()
+        }
+
+        return SourceExtraction(
+            files=result,
+            files_found=files_found,
+            files_excluded=files_excluded,
+        )
+
+    def normalize_import(
+        self,
+        source_id: str,
+        source_import: SourceImport,
+        known_source_ids: set[str],
+    ) -> SourceImport:
+        normalized_path = self._known_source_id(
+            source_import.normalized_path,
+            known_source_ids,
+        )
+        return SourceImport(
+            path=source_import.path,
+            is_relative=source_import.is_relative,
+            normalized_path=normalized_path,
+            imported_name=source_import.imported_name,
+            is_aliased=source_import.is_aliased,
+            crossing_type=source_import.crossing_type,
+            file_barrier_crossed=(
+                source_import.file_barrier_crossed
+                and normalized_path in known_source_ids
+            ),
+            statement_id=source_import.statement_id,
+            join_key=self._normalized_join_key(normalized_path, source_import),
+            uses_joined_import=source_import.uses_joined_import,
+        )
+
+    def _normalized_join_key(
+        self,
+        normalized_path: str,
+        source_import: SourceImport,
+    ) -> str:
+        if not source_import.join_key or not normalized_path:
+            return ""
+        parts = normalized_path.split("/")
+        return "/".join(parts[:-1])
+
+    def _known_source_id(
+        self,
+        normalized_path: str,
+        known_source_ids: set[str],
+    ) -> str:
+        candidates = (normalized_path, *self._php_source_id_candidates(normalized_path))
+        for candidate in candidates:
+            if candidate in known_source_ids:
+                return candidate
+            match = self._unique_suffix_match(candidate, known_source_ids)
+            if match is not None:
+                return match
+
+        namespace_match = self._namespace_source_id(normalized_path, known_source_ids)
+        if namespace_match is not None:
+            return namespace_match
+        return normalized_path
+
+    def _php_source_id_candidates(self, normalized_path: str) -> tuple[str, ...]:
+        parts = [self._source_part(part) for part in normalized_path.split("/") if part]
+        return tuple("/".join(parts[index:]) for index in range(len(parts)))
+
+    def _namespace_source_id(
+        self,
+        normalized_path: str,
+        known_source_ids: set[str],
+    ) -> str | None:
+        namespace_matches: list[str] = []
+        for candidate in self._php_source_id_candidates(normalized_path):
+            namespace_path = PurePosixPath(candidate).parent.as_posix()
+            if not namespace_path or namespace_path == ".":
+                continue
+            namespace_suffix = f"/{namespace_path}"
+            namespace_matches.extend(
+                source_id
+                for source_id in known_source_ids
+                if self._source_parent(source_id) == namespace_path
+                or self._source_parent(source_id).endswith(namespace_suffix)
+            )
+
+        matches = sorted(set(namespace_matches))
+        return matches[0] if len(matches) == 1 else None
+
+    def _unique_suffix_match(
+        self,
+        candidate: str,
+        known_source_ids: set[str],
+    ) -> str | None:
+        suffix = f"/{candidate}"
+        matches = sorted(
+            source_id
+            for source_id in known_source_ids
+            if source_id == candidate or source_id.endswith(suffix)
+        )
+        return matches[0] if len(matches) == 1 else None
+
+    def _source_part(self, value: str) -> str:
+        return re.sub(r"(?<!^)(?=[A-Z])", "_", value).lower()
+
+    def _source_parent(self, source_id: str) -> str:
+        return PurePosixPath(source_id).parent.as_posix()
+
+
+__all__ = ["PhpExtractor"]

modwire/extractors/python.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import sys
+
+from dataclasses import dataclass
+from posixpath import normpath
+from pathlib import PurePosixPath
+
+from ..definitions import SourceImport
+from .base import SourceExtractor
+
+
+@dataclass(frozen=True)
+class PythonExtractor(SourceExtractor):
+    language = "python"
+    file_extensions = (".py",)
+    command = sys.executable
+    extractor_file = "python_extractor.py"
+
+    def normalize_import(
+        self,
+        source_id: str,
+        source_import: SourceImport,
+        known_source_ids: set[str],
+    ) -> SourceImport:
+        normalized_path = self._normalize_relative_import(source_id, source_import)
+        crossing_type = source_import.crossing_type
+        if source_import.imported_name and source_import.imported_name != "*":
+            module_import_path = "/".join(
+                part for part in (normalized_path, source_import.imported_name) if part
+            )
+            module_source_id = self._known_source_id(module_import_path, known_source_ids)
+            if module_source_id in known_source_ids:
+                normalized_path = module_source_id
+                crossing_type = "module"
+            else:
+                normalized_path = self._known_source_id(normalized_path, known_source_ids)
+        else:
+            normalized_path = self._known_source_id(normalized_path, known_source_ids)
+
+        return SourceImport(
+            path=source_import.path,
+            is_relative=source_import.is_relative,
+            normalized_path=normalized_path,
+            imported_name=source_import.imported_name,
+            is_aliased=source_import.is_aliased,
+            crossing_type=crossing_type,
+            file_barrier_crossed=(
+                source_import.file_barrier_crossed
+                and normalized_path in known_source_ids
+            ),
+            statement_id=source_import.statement_id,
+            join_key=self._normalized_join_key(normalized_path, source_import),
+            uses_joined_import=source_import.uses_joined_import,
+        )
+
+    def _normalize_relative_import(
+        self,
+        source_id: str,
+        source_import: SourceImport,
+    ) -> str:
+        if not source_import.is_relative:
+            return source_import.normalized_path
+
+        level = len(source_import.path) - len(source_import.path.lstrip("."))
+        module_path = source_import.path[level:].replace(".", "/").strip("/")
+        package_path = PurePosixPath(source_id).parent
+        for _ in range(max(level - 1, 0)):
+            package_path = package_path.parent
+
+        return normpath(
+            "/".join(part for part in (package_path.as_posix(), module_path) if part)
+        )
+
+    def _known_source_id(
+        self,
+        normalized_path: str,
+        known_source_ids: set[str],
+    ) -> str:
+        candidates = (normalized_path, f"{normalized_path}/__init__")
+        for candidate in candidates:
+            if candidate in known_source_ids:
+                return candidate
+            match = self._unique_suffix_match(candidate, known_source_ids)
+            if match is not None:
+                return match
+        return normalized_path
+
+    def _normalized_join_key(
+        self,
+        normalized_path: str,
+        source_import: SourceImport,
+    ) -> str:
+        if not source_import.join_key or not normalized_path:
+            return ""
+        parent = PurePosixPath(normalized_path).parent.as_posix()
+        return "" if parent == "." else parent
+
+    def _unique_suffix_match(
+        self,
+        candidate: str,
+        known_source_ids: set[str],
+    ) -> str | None:
+        suffix = f"/{candidate}"
+        matches = sorted(
+            source_id
+            for source_id in known_source_ids
+            if source_id == candidate or source_id.endswith(suffix)
+        )
+        return matches[0] if len(matches) == 1 else None
+
+
+__all__ = ["PythonExtractor"]