modwire 1.0.0__tar.gz → 1.1.0__tar.gz

{modwire-1.0.0 → modwire-1.1.0}/.github/workflows/ci.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 jobs:
   test:

{modwire-1.0.0 → modwire-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modwire
-Version: 1.0.0
+Version: 1.1.0
 Summary: Extract source-code dependencies and build dependency graphs.
 Author: Tomasz Szpak
 License-Expression: MIT
@@ -70,6 +70,7 @@ TypeScript, and PHP projects can be compared through the same graph shape.
 
 ## Supported Languages
 
+
 ```python
 from modwire import supported_languages
 

{modwire-1.0.0 → modwire-1.1.0}/README.md

@@ -37,6 +37,7 @@ TypeScript, and PHP projects can be compared through the same graph shape.
 
 ## Supported Languages
 
+
 ```python
 from modwire import supported_languages
 

{modwire-1.0.0 → modwire-1.1.0}/src/modwire/__init__.py

@@ -1,5 +1,6 @@
 from .extraction import CodeMap, extract_code
 from .extractors.loader import normalize_source_id, supported_languages
+from .exports import UnusedExport, find_unused_exports
 from .graph import DependencyGraph, Edge, Node, build_dependency_graph
 
 
@@ -8,8 +9,10 @@ __all__ = [
     "DependencyGraph",
     "Edge",
     "Node",
+    "UnusedExport",
     "build_dependency_graph",
     "extract_code",
+    "find_unused_exports",
     "normalize_source_id",
     "supported_languages",
 ]

{modwire-1.0.0 → modwire-1.1.0}/src/modwire/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '1.0.0'
-__version_tuple__ = version_tuple = (1, 0, 0)
+__version__ = version = '1.1.0'
+__version_tuple__ = version_tuple = (1, 1, 0)
 
-__commit_id__ = commit_id = 'gb2874aa61'
+__commit_id__ = commit_id = 'g371c8ff6e'

{modwire-1.0.0 → modwire-1.1.0}/src/modwire/definitions.py

@@ -6,6 +6,25 @@ from pydantic import BaseModel, Field
 ImportCrossingType = Literal["module", "symbol"]
 SourceVisibility = Literal["public", "protected", "private"]
 SourceSignatureKind = Literal["call", "construct", "index"]
+SourceExportKind = Literal[
+    "module",
+    "class",
+    "interface",
+    "type",
+    "abstract_class",
+    "function",
+    "value",
+    "unknown",
+]
+
+
+class SourceImportedSymbol(BaseModel):
+    name: str
+    alias: str
+    is_aliased: bool
+    is_default: bool
+    is_namespace: bool
+    is_star: bool
 
 
 class SourceImport(BaseModel):
@@ -19,6 +38,21 @@ class SourceImport(BaseModel):
     statement_id: int
     join_key: str
     uses_joined_import: bool
+    imported_symbols: list[SourceImportedSymbol] = Field(default_factory=list)
+
+
+class SourceExport(BaseModel):
+    name: str
+    local_name: str
+    kind: SourceExportKind
+    crossing_type: ImportCrossingType
+    path: str
+    is_relative: bool
+    normalized_path: str
+    is_reexport: bool
+    is_default: bool
+    is_aliased: bool
+    statement_id: int
 
 
 class SourceFunction(BaseModel):
@@ -91,6 +125,7 @@ class SourceAbstractClass(BaseModel):
 
 class SourceFile(BaseModel):
     imports: list[SourceImport]
+    exports: list[SourceExport] = Field(default_factory=list)
     classes: list[SourceClass]
     interfaces: list[SourceInterface] = Field(default_factory=list)
     types: list[SourceType] = Field(default_factory=list)

modwire-1.1.0/src/modwire/exports.py

@@ -0,0 +1,188 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from .definitions import SourceExport, SourceFile, SourceImportedSymbol, SourceImport
+from .extraction import ExtractionResult
+
+
+@dataclass(frozen=True)
+class UnusedExport:
+    source_id: str
+    name: str
+    kind: str
+    crossing_type: str
+    reason: str
+
+
+def find_unused_exports(
+    extraction_result: ExtractionResult | dict[str, SourceFile],
+) -> tuple[UnusedExport, ...]:
+    files = _source_files(extraction_result)
+    exports_by_source = {
+        source_id: tuple(source_file.exports)
+        for source_id, source_file in files.items()
+    }
+    export_index: dict[tuple[str, str, str], list[SourceExport]] = {}
+    module_exports: dict[str, list[SourceExport]] = {}
+    star_reexports: dict[str, list[SourceExport]] = {}
+
+    for source_id, source_exports in exports_by_source.items():
+        for source_export in source_exports:
+            export_index.setdefault(
+                (source_id, source_export.name, source_export.crossing_type),
+                [],
+            ).append(source_export)
+            if source_export.kind == "module" and source_export.crossing_type == "module":
+                module_exports.setdefault(source_id, []).append(source_export)
+            if (
+                source_export.is_reexport
+                and source_export.crossing_type == "module"
+                and source_export.name == "*"
+            ):
+                star_reexports.setdefault(source_id, []).append(source_export)
+
+    used: set[tuple[str, str, str, str, str, bool]] = set()
+
+    def mark_export(source_id: str, source_export: SourceExport) -> None:
+        key = _export_key(source_id, source_export)
+        if key in used:
+            return
+        used.add(key)
+        if not source_export.is_reexport:
+            return
+        if source_export.normalized_path not in files:
+            return
+        mark_module(source_export.normalized_path)
+        if source_export.crossing_type == "symbol":
+            mark_symbol(
+                source_export.normalized_path,
+                source_export.local_name,
+            )
+
+    def mark_module(source_id: str) -> None:
+        for source_export in module_exports.get(source_id, ()):
+            mark_export(source_id, source_export)
+
+    def mark_symbol(source_id: str, name: str) -> None:
+        matched_exports = export_index.get((source_id, name, "symbol"), [])
+        if matched_exports:
+            for source_export in matched_exports:
+                mark_export(source_id, source_export)
+            return
+
+        for source_export in star_reexports.get(source_id, ()):
+            mark_export(source_id, source_export)
+            if source_export.normalized_path in files:
+                mark_module(source_export.normalized_path)
+                mark_symbol(source_export.normalized_path, name)
+
+    def mark_all_symbols(source_id: str) -> None:
+        for source_export in exports_by_source.get(source_id, ()):
+            if source_export.crossing_type == "symbol":
+                mark_export(source_id, source_export)
+
+    for source_id, source_file in files.items():
+        for source_import in source_file.imports:
+            target_id = source_import.normalized_path
+            if target_id not in files or target_id == source_id:
+                continue
+
+            symbols = _imported_symbols(source_import)
+            if source_import.crossing_type == "symbol" and symbols:
+                usable_symbols = [
+                    symbol
+                    for symbol in symbols
+                    if not _is_reexport_plumbing(
+                        exports_by_source.get(source_id, ()),
+                        target_id,
+                        symbol,
+                    )
+                ]
+                if not usable_symbols:
+                    continue
+                mark_module(target_id)
+                for symbol in usable_symbols:
+                    if symbol.is_namespace:
+                        continue
+                    if symbol.is_star or symbol.name == "*":
+                        mark_all_symbols(target_id)
+                    else:
+                        mark_symbol(target_id, symbol.name)
+                continue
+
+            mark_module(target_id)
+
+    unused = []
+    for source_id, source_exports in exports_by_source.items():
+        for source_export in source_exports:
+            if _export_key(source_id, source_export) in used:
+                continue
+            unused.append(
+                UnusedExport(
+                    source_id=source_id,
+                    name=source_export.name,
+                    kind=source_export.kind,
+                    crossing_type=source_export.crossing_type,
+                    reason="export is not imported by another source file",
+                )
+            )
+
+    return tuple(unused)
+
+
+def _source_files(
+    extraction_result: ExtractionResult | dict[str, SourceFile],
+) -> dict[str, SourceFile]:
+    if isinstance(extraction_result, dict):
+        return extraction_result
+    return extraction_result.files
+
+
+def _imported_symbols(source_import: SourceImport) -> tuple[SourceImportedSymbol, ...]:
+    if source_import.imported_symbols:
+        return tuple(source_import.imported_symbols)
+    if not source_import.imported_name:
+        return ()
+    return (
+        SourceImportedSymbol(
+            name=source_import.imported_name,
+            alias="",
+            is_aliased=False,
+            is_default=False,
+            is_namespace=False,
+            is_star=source_import.imported_name == "*",
+        ),
+    )
+
+
+def _is_reexport_plumbing(
+    source_exports: tuple[SourceExport, ...],
+    target_id: str,
+    symbol: SourceImportedSymbol,
+) -> bool:
+    for source_export in source_exports:
+        if not source_export.is_reexport or source_export.normalized_path != target_id:
+            continue
+        if source_export.name == "*":
+            return True
+        if source_export.local_name == symbol.name or source_export.name == symbol.name:
+            return True
+    return False
+
+
+def _export_key(
+    source_id: str,
+    source_export: SourceExport,
+) -> tuple[str, str, str, str, str, bool]:
+    return (
+        source_id,
+        source_export.name,
+        source_export.crossing_type,
+        source_export.normalized_path,
+        source_export.local_name,
+        source_export.is_reexport,
+    )
+
+
+__all__ = ["UnusedExport", "find_unused_exports"]

modwire-1.1.0/src/modwire/extractors/base.py

@@ -0,0 +1,340 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from fnmatch import fnmatch
+from pathlib import Path
+from subprocess import run
+from typing import Protocol
+
+from ..definitions import SourceExport, SourceFile, SourceImport
+
+
+@dataclass(frozen=True)
+class SourceExtraction:
+    files: dict[str, SourceFile]
+    files_found: int
+    files_excluded: int
+
+
+@dataclass(frozen=True)
+class ExtractionTarget:
+    source_id: str
+    path: Path
+
+
+class SourceExtractor(Protocol):
+    language: str
+    file_extensions: tuple[str, ...]
+    command: str
+    extractor_file: str
+
+    def normalize_source_id(self, value: str) -> str:
+        source_id = value.strip().strip("/")
+        for file_extension in self.file_extensions:
+            if source_id.endswith(file_extension):
+                return source_id[: -len(file_extension)]
+        return source_id
+
+    def normalize_import(
+        self,
+        source_id: str,
+        source_import: SourceImport,
+        known_source_ids: set[str],
+    ) -> SourceImport:
+        if source_import.normalized_path in known_source_ids:
+            return source_import
+        return SourceImport(
+            path=source_import.path,
+            is_relative=source_import.is_relative,
+            normalized_path=source_import.normalized_path.strip().strip("/"),
+            imported_name=source_import.imported_name,
+            is_aliased=source_import.is_aliased,
+            crossing_type=source_import.crossing_type,
+            file_barrier_crossed=False,
+            statement_id=source_import.statement_id,
+            join_key=source_import.join_key,
+            uses_joined_import=source_import.uses_joined_import,
+            imported_symbols=source_import.imported_symbols,
+        )
+
+    def normalize_export(
+        self,
+        source_id: str,
+        source_export: SourceExport,
+        known_source_ids: set[str],
+    ) -> SourceExport:
+        return SourceExport(
+            name=source_export.name,
+            local_name=source_export.local_name,
+            kind=source_export.kind,
+            crossing_type=source_export.crossing_type,
+            path=source_export.path,
+            is_relative=source_export.is_relative,
+            normalized_path=source_export.normalized_path.strip().strip("/"),
+            is_reexport=source_export.is_reexport,
+            is_default=source_export.is_default,
+            is_aliased=source_export.is_aliased,
+            statement_id=source_export.statement_id,
+        )
+
+    def extract_files(
+        self,
+        sources_root: Path,
+        exclusions: tuple[str, ...],
+    ) -> SourceExtraction:
+        script = Path(__file__).parent / "scripts" / self.extractor_file
+        assert script.is_file(), f"Extractor script {script} not found"
+
+        targets, files_found, files_excluded = _collect_extraction_targets(
+            sources_root,
+            self.file_extensions,
+            exclusions,
+        )
+        if not targets:
+            return SourceExtraction(
+                files={},
+                files_found=files_found,
+                files_excluded=files_excluded,
+            )
+
+        input_data = {
+            self.normalize_source_id(target.source_id): str(target.path.resolve())
+            for target in targets
+        }
+        cmd = [self.command, str(script), "--batch", str(sources_root.resolve())]
+        raw_files = _json_from_output(cmd, json.dumps(input_data))
+        result = {
+            source_id: SourceFile.model_validate(source_file)
+            for source_id, source_file in raw_files.items()
+        }
+
+        known_source_ids = set(result)
+        result = {
+            source_id: self.normalize_source_file(
+                source_id,
+                source_file,
+                known_source_ids,
+            )
+            for source_id, source_file in result.items()
+        }
+
+        return SourceExtraction(
+            files=result,
+            files_found=files_found,
+            files_excluded=files_excluded,
+        )
+
+    def normalize_source_file(
+        self,
+        source_id: str,
+        source_file: SourceFile,
+        known_source_ids: set[str],
+    ) -> SourceFile:
+        exports = [
+            self.normalize_export(source_id, source_export, known_source_ids)
+            for source_export in source_file.exports
+        ]
+        exports = self._with_module_export(source_id, exports)
+
+        return SourceFile(
+            imports=[
+                self.normalize_import(source_id, source_import, known_source_ids)
+                for source_import in source_file.imports
+            ],
+            exports=exports,
+            classes=source_file.classes,
+            interfaces=source_file.interfaces,
+            types=source_file.types,
+            abstract_classes=source_file.abstract_classes,
+            functions=source_file.functions,
+            line_count=source_file.line_count,
+            code_line_count=source_file.code_line_count,
+            public_symbol_count=source_file.public_symbol_count,
+        )
+
+    def _with_module_export(
+        self,
+        source_id: str,
+        exports: list[SourceExport],
+    ) -> list[SourceExport]:
+        module_export = SourceExport(
+            name=source_id,
+            local_name=source_id,
+            kind="module",
+            crossing_type="module",
+            path=source_id,
+            is_relative=False,
+            normalized_path=source_id,
+            is_reexport=False,
+            is_default=False,
+            is_aliased=False,
+            statement_id=0,
+        )
+        seen = {
+            (
+                source_export.name,
+                source_export.kind,
+                source_export.crossing_type,
+                source_export.normalized_path,
+            )
+            for source_export in exports
+        }
+        module_key = (
+            module_export.name,
+            module_export.kind,
+            module_export.crossing_type,
+            module_export.normalized_path,
+        )
+        if module_key in seen:
+            return exports
+        return [module_export, *exports]
+
+
+def _collect_extraction_targets(
+    sources_root: Path,
+    file_extensions: tuple[str, ...],
+    exclusions: tuple[str, ...],
+) -> tuple[tuple[ExtractionTarget, ...], int, int]:
+    directory_exclusions, file_exclusions = _partition_exclusions(
+        exclusions,
+        file_extensions,
+    )
+    targets: list[ExtractionTarget] = []
+    files_found = 0
+    files_excluded = 0
+
+    def walk(directory: Path, relative_dir: str = "") -> None:
+        nonlocal files_found, files_excluded
+        directories: list[tuple[str, Path]] = []
+        files: list[tuple[str, Path]] = []
+
+        for entry in sorted(directory.iterdir(), key=lambda path: path.name):
+            source_id = (
+                f"{relative_dir}/{entry.name}" if relative_dir else entry.name
+            )
+            if entry.is_dir() and not entry.is_symlink():
+                if _matches_directory_exclusion(source_id, directory_exclusions):
+                    # Exact file counts would require descending into the pruned tree.
+                    files_found += 1
+                    files_excluded += 1
+                    continue
+                directories.append((source_id, entry))
+                continue
+
+            if entry.is_file() and entry.suffix in file_extensions:
+                files.append((source_id, entry))
+
+        for source_id, path in files:
+            files_found += 1
+            if any(
+                _matches_exclusion(source_id, exclusion)
+                for exclusion in file_exclusions
+            ):
+                files_excluded += 1
+                continue
+
+            targets.append(ExtractionTarget(source_id, path))
+
+        for source_id, path in directories:
+            walk(path, source_id)
+
+    walk(sources_root)
+
+    return tuple(targets), files_found, files_excluded
+
+
+def _json_from_output(cmd: list[str], input_json: str | None = None) -> dict:
+    output_json = run(
+        cmd,
+        capture_output=True,
+        text=True,
+        input=input_json,
+        check=True,
+    ).stdout
+
+    try:
+        return json.loads(output_json)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Failed to parse JSON from output: {output_json}") from e
+
+
+def _matches_exclusion(source_id: str, exclusion: str) -> bool:
+    normalized = exclusion.replace("\\", "/").strip().strip("/")
+    if not normalized:
+        return False
+
+    if _matches_path_pattern(source_id, normalized):
+        return True
+
+    has_glob = any(char in normalized for char in "*?[")
+    if not normalized or has_glob:
+        return False
+
+    return source_id.startswith(f"{normalized}/")
+
+
+def _partition_exclusions(
+    exclusions: tuple[str, ...],
+    file_extensions: tuple[str, ...],
+) -> tuple[tuple[str, ...], tuple[str, ...]]:
+    directory_exclusions = []
+    file_exclusions = []
+    for exclusion in exclusions:
+        normalized = exclusion.replace("\\", "/").strip()
+        if not normalized:
+            continue
+        if _is_recursive_directory_exclusion(normalized, file_extensions):
+            directory_exclusions.append(_directory_exclusion_pattern(normalized))
+        else:
+            file_exclusions.append(normalized.strip("/"))
+    return tuple(directory_exclusions), tuple(file_exclusions)
+
+
+def _is_recursive_directory_exclusion(
+    exclusion: str,
+    file_extensions: tuple[str, ...],
+) -> bool:
+    normalized = exclusion.strip("/")
+    if not normalized:
+        return False
+    if exclusion.endswith("/") or normalized.endswith("/**"):
+        return True
+    if any(char in normalized for char in "*?["):
+        return False
+    return not normalized.endswith(file_extensions)
+
+
+def _directory_exclusion_pattern(exclusion: str) -> str:
+    normalized = exclusion.strip("/")
+    if normalized.endswith("/**"):
+        return normalized[:-3].rstrip("/")
+    return normalized
+
+
+def _matches_directory_exclusion(
+    source_id: str,
+    directory_exclusions: tuple[str, ...],
+) -> bool:
+    for exclusion in directory_exclusions:
+        if _matches_path_pattern(source_id, exclusion):
+            return True
+        if "/" not in exclusion and not any(char in exclusion for char in "*?["):
+            if Path(source_id).name == exclusion:
+                return True
+    return False
+
+
+def _matches_path_pattern(source_id: str, pattern: str) -> bool:
+    if fnmatch(source_id, pattern):
+        return True
+    if pattern.startswith("**/") and fnmatch(source_id, pattern[3:]):
+        return True
+    return False
+
+
+__all__ = [
+    "ExtractionTarget",
+    "SourceExtraction",
+    "SourceExtractor",
+]

{modwire-1.0.0 → modwire-1.1.0}/src/modwire/extractors/php.py

@@ -5,7 +5,7 @@ import re
 from dataclasses import dataclass
 from pathlib import Path, PurePosixPath
 
-from ..definitions import SourceFile, SourceImport
+from ..definitions import SourceExport, SourceFile, SourceImport
 from .base import (
     SourceExtraction,
     SourceExtractor,
@@ -92,6 +92,31 @@ class PhpExtractor(SourceExtractor):
             statement_id=source_import.statement_id,
             join_key=self._normalized_join_key(normalized_path, source_import),
             uses_joined_import=source_import.uses_joined_import,
+            imported_symbols=source_import.imported_symbols,
+        )
+
+    def normalize_export(
+        self,
+        source_id: str,
+        source_export: SourceExport,
+        known_source_ids: set[str],
+    ) -> SourceExport:
+        normalized_path = source_export.normalized_path
+        if normalized_path:
+            normalized_path = self._known_source_id(normalized_path, known_source_ids)
+
+        return SourceExport(
+            name=source_export.name,
+            local_name=source_export.local_name,
+            kind=source_export.kind,
+            crossing_type=source_export.crossing_type,
+            path=source_export.path,
+            is_relative=source_export.is_relative,
+            normalized_path=normalized_path,
+            is_reexport=source_export.is_reexport,
+            is_default=source_export.is_default,
+            is_aliased=source_export.is_aliased,
+            statement_id=source_export.statement_id,
         )
 
     def _normalized_join_key(