grimp 3.2__cp312-none-win_amd64.whl

grimp/__init__.py

@@ -0,0 +1,17 @@
+__version__ = "3.2"
+
+from .application.ports.graph import DetailedImport, ImportGraph
+from .domain.analysis import PackageDependency, Route
+from .domain.valueobjects import DirectImport, Module, Layer
+from .main import build_graph
+
+__all__ = [
+    "Module",
+    "DetailedImport",
+    "DirectImport",
+    "ImportGraph",
+    "PackageDependency",
+    "Route",
+    "build_graph",
+    "Layer",
+]

grimp/adaptors/_layers.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, Iterator, Sequence, TypedDict
+
+from grimp import Route
+from grimp import _rustgrimp as rust  # type: ignore[attr-defined]
+
+if TYPE_CHECKING:
+    from grimp.adaptors.graph import ImportGraph
+
+from grimp.domain.analysis import PackageDependency
+from grimp.exceptions import NoSuchContainer
+from grimp.domain.valueobjects import Layer
+
+
+def parse_layers(layers: Sequence[Layer | str | set[str]]) -> tuple[Layer, ...]:
+    """
+    Convert the passed raw `layers` into `Layer`s.
+    """
+    out_layers = []
+    for layer in layers:
+        if isinstance(layer, Layer):
+            out_layers.append(layer)
+        elif isinstance(layer, str):
+            out_layers.append(Layer(layer, independent=True))
+        else:
+            out_layers.append(Layer(*tuple(layer), independent=True))
+    return tuple(out_layers)
+
+
+def find_illegal_dependencies(
+    graph: ImportGraph,
+    layers: Sequence[Layer],
+    containers: set[str],
+) -> set[PackageDependency]:
+    """
+    Find dependencies that don't conform to the supplied layered architecture.
+
+    See ImportGraph.find_illegal_dependencies_for_layers.
+
+    The only difference between this and the method is that the containers passed in
+    is already a (potentially empty) set.
+    """
+    try:
+        rust_package_dependency_tuple = rust.find_illegal_dependencies(
+            levels=tuple(
+                {"layers": layer.module_tails, "independent": layer.independent}
+                for layer in layers
+            ),
+            containers=set(containers),
+            importeds_by_importer=graph._importeds_by_importer,
+        )
+    except rust.NoSuchContainer as e:
+        raise NoSuchContainer(str(e))
+
+    rust_package_dependencies = _dependencies_from_tuple(rust_package_dependency_tuple)
+    return rust_package_dependencies
+
+
+class _RustRoute(TypedDict):
+    heads: frozenset[str]
+    middle: tuple[str, ...]
+    tails: frozenset[str]
+
+
+class _RustPackageDependency(TypedDict):
+    importer: str
+    imported: str
+    routes: tuple[_RustRoute, ...]
+
+
+def _dependencies_from_tuple(
+    rust_package_dependency_tuple: tuple[_RustPackageDependency, ...]
+) -> set[PackageDependency]:
+    return {
+        PackageDependency(
+            imported=dep_dict["imported"],
+            importer=dep_dict["importer"],
+            routes=frozenset(
+                {
+                    Route(
+                        heads=route_dict["heads"],
+                        middle=route_dict["middle"],
+                        tails=route_dict["tails"],
+                    )
+                    for route_dict in dep_dict["routes"]
+                }
+            ),
+        )
+        for dep_dict in rust_package_dependency_tuple
+    }
+
+
+class _Module:
+    """
+    A Python module.
+    """
+
+    def __init__(self, name: str) -> None:
+        """
+        Args:
+            name: The fully qualified name of a Python module, e.g. 'package.foo.bar'.
+        """
+        self.name = name
+
+    def __str__(self) -> str:
+        return self.name
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, self.__class__):
+            return hash(self) == hash(other)
+        else:
+            return False
+
+    def __hash__(self) -> int:
+        return hash(str(self))
+
+    def is_descendant_of(self, module: "_Module") -> bool:
+        return self.name.startswith(f"{module.name}.")
+
+
+@dataclass(frozen=True)
+class _Link:
+    importer: str
+    imported: str
+
+
+# A chain of modules, each of which imports the next.
+if TYPE_CHECKING:
+    # TODO: remove TYPE_CHECKING conditional once on Python 3.9.
+    _Chain = tuple[str, ...]
+
+
+def _generate_module_permutations(
+    graph: ImportGraph,
+    layers: Sequence[str],
+    containers: set[str],
+) -> Iterator[tuple[_Module, _Module, str | None]]:
+    """
+    Return all possible combinations of higher level and lower level modules, in pairs.
+
+    Each pair of modules consists of immediate children of two different layers. The first
+    module is in a layer higher than the layer of the second module. This means the first
+    module is allowed to import the second, but not the other way around.
+
+    Returns:
+        module_in_higher_layer, module_in_lower_layer, container
+    """
+    # If there are no containers, we still want to run the loop once.
+    quasi_containers = containers or [None]
+
+    for container in quasi_containers:
+        for index, higher_layer in enumerate(layers):
+            higher_layer_module = _module_from_layer(higher_layer, container)
+
+            if higher_layer_module.name not in graph.modules:
+                continue
+
+            for lower_layer in layers[index + 1 :]:
+                lower_layer_module = _module_from_layer(lower_layer, container)
+
+                if lower_layer_module.name not in graph.modules:
+                    continue
+
+                yield higher_layer_module, lower_layer_module, container
+
+
+def _module_from_layer(layer: str, container: str | None = None) -> _Module:
+    if container:
+        name = ".".join([container, layer])
+    else:
+        name = layer
+    return _Module(name)

grimp/adaptors/caching.py

@@ -0,0 +1,286 @@
+import hashlib
+import json
+import logging
+from typing import Dict, List, Optional, Set, Tuple, Type
+
+from grimp.application.ports.filesystem import AbstractFileSystem
+from grimp.application.ports.modulefinder import FoundPackage, ModuleFile
+from grimp.domain.valueobjects import DirectImport, Module
+
+from ..application.ports.caching import Cache as AbstractCache
+from ..application.ports.caching import CacheMiss
+
+logger = logging.getLogger(__name__)
+PrimitiveFormat = Dict[str, List[Tuple[str, Optional[int], str]]]
+
+
+class CacheFileNamer:
+    @classmethod
+    def make_meta_file_name(cls, found_package: FoundPackage) -> str:
+        return f"{found_package.name}.meta.json"
+
+    @classmethod
+    def make_data_file_name(
+        cls,
+        found_packages: Set[FoundPackage],
+        include_external_packages: bool,
+        exclude_type_checking_imports: bool,
+    ) -> str:
+        identifier = cls.make_data_file_unique_string(
+            found_packages, include_external_packages, exclude_type_checking_imports
+        )
+
+        bytes_identifier = identifier.encode()
+        # Use a hash algorithm with a limited size to avoid cache filenames that are too long
+        # the filesystem, which can happen if there are more than a few root packages
+        # being analyzed.
+        safe_unicode_identifier = hashlib.blake2b(bytes_identifier, digest_size=20).hexdigest()
+        return f"{safe_unicode_identifier}.data.json"
+
+    @classmethod
+    def make_data_file_unique_string(
+        cls,
+        found_packages: Set[FoundPackage],
+        include_external_packages: bool,
+        exclude_type_checking_imports: bool,
+    ) -> str:
+        """
+        Construct a unique string that identifies the analysis parameters.
+
+        Doesn't need to be safe to use for a filename.
+        """
+        package_names = (p.name for p in found_packages)
+        csv_packages = ",".join(sorted(package_names))
+        include_external_packages_option = ":external" if include_external_packages else ""
+        exclude_type_checking_imports_option = (
+            ":no_type_checking" if exclude_type_checking_imports else ""
+        )
+        return (
+            csv_packages + include_external_packages_option + exclude_type_checking_imports_option
+        )
+
+
+class Cache(AbstractCache):
+    DEFAULT_CACHE_DIR = ".grimp_cache"
+
+    def __init__(self, *args, namer: Type[CacheFileNamer], **kwargs) -> None:
+        """
+        Don't instantiate Cache directly; use Cache.setup().
+        """
+        super().__init__(*args, **kwargs)
+        self._mtime_map: Dict[str, float] = {}
+        self._data_map: Dict[Module, Set[DirectImport]] = {}
+        self._namer = namer
+
+    @classmethod
+    def setup(
+        cls,
+        file_system: AbstractFileSystem,
+        found_packages: Set[FoundPackage],
+        include_external_packages: bool,
+        exclude_type_checking_imports: bool = False,
+        cache_dir: Optional[str] = None,
+        namer: Type[CacheFileNamer] = CacheFileNamer,
+    ) -> "Cache":
+        cache = cls(
+            file_system=file_system,
+            found_packages=found_packages,
+            include_external_packages=include_external_packages,
+            exclude_type_checking_imports=exclude_type_checking_imports,
+            cache_dir=cls.cache_dir_or_default(cache_dir),
+            namer=namer,
+        )
+        cache._build_mtime_map()
+        cache._build_data_map()
+        assert cache.cache_dir
+        return cache
+
+    @classmethod
+    def cache_dir_or_default(cls, cache_dir: Optional[str]) -> str:
+        return cache_dir or cls.DEFAULT_CACHE_DIR
+
+    def read_imports(self, module_file: ModuleFile) -> Set[DirectImport]:
+        try:
+            cached_mtime = self._mtime_map[module_file.module.name]
+        except KeyError:
+            raise CacheMiss
+        if cached_mtime != module_file.mtime:
+            raise CacheMiss
+
+        try:
+            return self._data_map[module_file.module]
+        except KeyError:
+            # While we would expect the module to be in here,
+            # there's no point in crashing if, for some reason, it's not.
+            raise CacheMiss
+
+    def write(
+        self,
+        imports_by_module: Dict[Module, Set[DirectImport]],
+    ) -> None:
+        self._write_marker_files_if_not_already_there()
+        # Write data file.
+        primitives_map: PrimitiveFormat = {}
+        for found_package in self.found_packages:
+            primitives_map_for_found_package: PrimitiveFormat = {
+                module_file.module.name: [
+                    (
+                        direct_import.imported.name,
+                        direct_import.line_number,
+                        direct_import.line_contents,
+                    )
+                    for direct_import in imports_by_module[module_file.module]
+                ]
+                for module_file in found_package.module_files
+            }
+            primitives_map.update(primitives_map_for_found_package)
+
+        serialized = json.dumps(primitives_map)
+        data_cache_filename = self.file_system.join(
+            self.cache_dir,
+            self._namer.make_data_file_name(
+                found_packages=self.found_packages,
+                include_external_packages=self.include_external_packages,
+                exclude_type_checking_imports=self.exclude_type_checking_imports,
+            ),
+        )
+        self.file_system.write(data_cache_filename, serialized)
+        logger.info(f"Wrote data cache file {data_cache_filename}.")
+
+        # Write meta files.
+        for found_package in self.found_packages:
+            meta_filename = self.file_system.join(
+                self.cache_dir, self._namer.make_meta_file_name(found_package)
+            )
+            mtime_map = {
+                module_file.module.name: module_file.mtime
+                for module_file in found_package.module_files
+            }
+            serialized_meta = json.dumps(mtime_map)
+            self.file_system.write(meta_filename, serialized_meta)
+            logger.info(f"Wrote meta cache file {meta_filename}.")
+
+    def _build_mtime_map(self) -> None:
+        self._mtime_map = self._read_mtime_map_files()
+
+    def _read_mtime_map_files(self) -> Dict[str, float]:
+        all_mtimes: Dict[str, float] = {}
+        for found_package in self.found_packages:
+            all_mtimes.update(self._read_mtime_map_file(found_package))
+        return all_mtimes
+
+    def _read_mtime_map_file(self, found_package: FoundPackage) -> Dict[str, float]:
+        meta_cache_filename = self.file_system.join(
+            self.cache_dir, self._namer.make_meta_file_name(found_package)
+        )
+        try:
+            serialized = self.file_system.read(meta_cache_filename)
+        except FileNotFoundError:
+            logger.info(f"No cache file: {meta_cache_filename}.")
+            return {}
+        try:
+            deserialized = json.loads(serialized)
+            logger.info(f"Used cache meta file {meta_cache_filename}.")
+            return deserialized
+        except json.JSONDecodeError:
+            logger.warning(f"Could not use corrupt cache file {meta_cache_filename}.")
+            return {}
+
+    def _build_data_map(self) -> None:
+        self._data_map = self._read_data_map_file()
+
+    def _read_data_map_file(self) -> Dict[Module, Set[DirectImport]]:
+        data_cache_filename = self.file_system.join(
+            self.cache_dir,
+            self._namer.make_data_file_name(
+                found_packages=self.found_packages,
+                include_external_packages=self.include_external_packages,
+                exclude_type_checking_imports=self.exclude_type_checking_imports,
+            ),
+        )
+        try:
+            serialized = self.file_system.read(data_cache_filename)
+        except FileNotFoundError:
+            logger.info(f"No cache file: {data_cache_filename}.")
+            return {}
+
+        # Deserialize to primitives.
+        try:
+            deserialized_json = json.loads(serialized)
+            logger.info(f"Used cache data file {data_cache_filename}.")
+        except json.JSONDecodeError:
+            logger.warning(f"Could not use corrupt cache file {data_cache_filename}.")
+            return {}
+
+        primitives_map: PrimitiveFormat = self._to_primitives_data_map(deserialized_json)
+
+        return {
+            Module(name=name): {
+                DirectImport(
+                    importer=Module(name),
+                    imported=Module(import_data[0]),
+                    line_number=int(import_data[1]),  # type: ignore
+                    line_contents=import_data[2],
+                )
+                for import_data in imports_data
+            }
+            for name, imports_data in primitives_map.items()
+        }
+
+    def _build_data_cache_filename(self, found_package: FoundPackage) -> str:
+        return self.file_system.join(self.cache_dir, f"{found_package.name}.data.json")
+
+    def _to_primitives_data_map(self, deserialized_json: object) -> PrimitiveFormat:
+        """
+        Convert the deserialized json from a data file to a narrower schema.
+
+        Anything that doesn't fit the schema will be removed.
+        """
+        if not isinstance(deserialized_json, dict):
+            return {}
+
+        primitives_map: PrimitiveFormat = {}
+
+        for key, value in deserialized_json.items():
+            if not isinstance(key, str):
+                continue
+            if not isinstance(value, list):
+                continue
+            primitive_imports = []
+            for deserialized_import in value:
+                try:
+                    [imported, line_number, line_contents] = deserialized_import
+                except ValueError:
+                    continue
+                try:
+                    primitive_imports.append(
+                        (
+                            str(imported),
+                            int(line_number) if line_number else None,
+                            str(line_contents),
+                        )
+                    )
+                except TypeError:
+                    continue
+
+            primitives_map[key] = primitive_imports
+
+        return primitives_map
+
+    def _write_marker_files_if_not_already_there(self) -> None:
+        marker_files_info = (
+            (".gitignore", "# Automatically created by Grimp.\n*"),
+            (
+                "CACHEDIR.TAG",
+                (
+                    "Signature: 8a477f597d28d172789f06886806bc55\n"
+                    "# This file is a cache directory tag automatically created by Grimp.\n"
+                    "# For information about cache directory tags see https://bford.info/cachedir/"
+                ),
+            ),
+        )
+
+        for filename, contents in marker_files_info:
+            full_filename = self.file_system.join(self.cache_dir, filename)
+            if not self.file_system.exists(full_filename):
+                self.file_system.write(full_filename, contents)

grimp/adaptors/filesystem.py

@@ -0,0 +1,46 @@
+import os
+import tokenize
+from typing import Iterator, List, Tuple
+
+from grimp.application.ports.filesystem import AbstractFileSystem
+
+
+class FileSystem(AbstractFileSystem):
+    """
+    Abstraction around file system calls.
+    """
+
+    @property
+    def sep(self) -> str:
+        return os.sep
+
+    def dirname(self, filename: str) -> str:
+        return os.path.dirname(filename)
+
+    def walk(self, directory_name: str) -> Iterator[Tuple[str, List[str], List[str]]]:
+        yield from os.walk(directory_name)
+
+    def join(self, *components: str) -> str:
+        return os.path.join(*components)
+
+    def split(self, file_name: str) -> Tuple[str, str]:
+        return os.path.split(file_name)
+
+    def read(self, file_name: str) -> str:
+        # Use tokenize.open to give us a better chance of successfully decoding
+        # source code in a non-ascii compatible encoding.
+        with tokenize.open(file_name) as file:
+            return file.read()
+
+    def exists(self, file_name: str) -> bool:
+        return os.path.isfile(file_name)
+
+    def get_mtime(self, file_name: str) -> float:
+        return os.path.getmtime(file_name)
+
+    def write(self, file_name: str, contents: str) -> None:
+        dirname = os.path.dirname(file_name)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(file_name, "w") as file:
+            print(contents, file=file)