grimp 3.7__tar.gz → 3.8__tar.gz

{grimp-3.7 → grimp-3.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grimp
-Version: 3.7
+Version: 3.8
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License

{grimp-3.7 → grimp-3.8}/pyproject.toml

@@ -8,7 +8,7 @@ module-name = "grimp._rustgrimp"
 
 [project]
 name = "grimp"
-version = "3.7"
+version = "3.8"
 license = {text = "BSD 2-Clause License"}
 description = "Builds a queryable graph of the imports within one or more Python packages."
 authors = [

{grimp-3.7 → grimp-3.8}/rust/Cargo.lock

@@ -293,9 +293,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.23.4"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc"
+checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229"
 dependencies = [
  "cfg-if",
  "indoc",
@@ -311,9 +311,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.23.4"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7"
+checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1"
 dependencies = [
  "once_cell",
  "target-lexicon",
@@ -321,9 +321,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.23.4"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d"
+checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -331,9 +331,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.23.4"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7"
+checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -343,9 +343,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.23.4"
+version = "0.24.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4"
+checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -494,9 +494,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.16"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
 
 [[package]]
 name = "thiserror"

{grimp-3.7 → grimp-3.8}/rust/Cargo.toml

@@ -24,7 +24,7 @@ regex = "1.11.1"
 const_format = "0.2.34"
 
 [dependencies.pyo3]
-version = "0.23.4"
+version = "0.24.1"
 
 [features]
 extension-module = ["pyo3/extension-module"]

{grimp-3.7 → grimp-3.8}/src/grimp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.7"
+__version__ = "3.8"
 
 from .application.ports.graph import DetailedImport, ImportGraph, Import
 from .domain.analysis import PackageDependency, Route

{grimp-3.7 → grimp-3.8}/src/grimp/application/usecases.py

@@ -1,13 +1,16 @@
 """
 Use cases handle application logic.
 """
-from typing import Dict, Sequence, Set, Type, Union, cast
+
+from typing import Dict, Sequence, Set, Type, Union, cast, Iterable, Collection
+import multiprocessing
+import math
 
 from ..application.ports import caching
 from ..application.ports.filesystem import AbstractFileSystem
 from ..application.ports.graph import ImportGraph
 from ..application.ports.importscanner import AbstractImportScanner
-from ..application.ports.modulefinder import AbstractModuleFinder, FoundPackage
+from ..application.ports.modulefinder import AbstractModuleFinder, FoundPackage, ModuleFile
 from ..application.ports.packagefinder import AbstractPackageFinder
 from ..domain.valueobjects import DirectImport, Module
 from .config import settings
@@ -17,6 +20,10 @@ class NotSupplied:
     pass
 
 
+# This is an arbitrary number, but setting it too low slows down our functional tests considerably.
+MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING = 50
+
+
 def build_graph(
     package_name,
     *additional_package_names,
@@ -106,7 +113,6 @@ def _scan_packages(
     exclude_type_checking_imports: bool,
     cache_dir: Union[str, Type[NotSupplied], None],
 ) -> Dict[Module, Set[DirectImport]]:
-    imports_by_module: Dict[Module, Set[DirectImport]] = {}
     if cache_dir is not None:
         cache_dir_if_supplied = cache_dir if cache_dir != NotSupplied else None
         cache: caching.Cache = settings.CACHE_CLASS.setup(
@@ -116,24 +122,33 @@ def _scan_packages(
             exclude_type_checking_imports=exclude_type_checking_imports,
             cache_dir=cache_dir_if_supplied,
         )
-    import_scanner: AbstractImportScanner = settings.IMPORT_SCANNER_CLASS(
-        file_system=file_system,
-        found_packages=found_packages,
-        include_external_packages=include_external_packages,
-    )
 
-    for found_package in found_packages:
-        for module_file in found_package.module_files:
-            module = module_file.module
-            try:
-                if cache_dir is None:
-                    raise caching.CacheMiss
-                direct_imports = cache.read_imports(module_file)
-            except caching.CacheMiss:
-                direct_imports = import_scanner.scan_for_imports(
-                    module, exclude_type_checking_imports=exclude_type_checking_imports
-                )
-            imports_by_module[module] = direct_imports
+    module_files_to_scan = {
+        module_file
+        for found_package in found_packages
+        for module_file in found_package.module_files
+    }
+
+    imports_by_module_file: Dict[ModuleFile, Set[DirectImport]] = {}
+
+    if cache_dir is not None:
+        imports_by_module_file.update(_read_imports_from_cache(module_files_to_scan, cache=cache))
+
+    remaining_module_files_to_scan = module_files_to_scan.difference(imports_by_module_file)
+    if remaining_module_files_to_scan:
+        imports_by_module_file.update(
+            _scan_imports(
+                remaining_module_files_to_scan,
+                file_system=file_system,
+                found_packages=found_packages,
+                include_external_packages=include_external_packages,
+                exclude_type_checking_imports=exclude_type_checking_imports,
+            )
+        )
+
+    imports_by_module: Dict[Module, Set[DirectImport]] = {
+        k.module: v for k, v in imports_by_module_file.items()
+    }
 
     if cache_dir is not None:
         cache.write(imports_by_module)
@@ -172,3 +187,100 @@ def _is_external(module: Module, found_packages: Set[FoundPackage]) -> bool:
         module.is_descendant_of(package_module) or module == package_module
         for package_module in package_modules
     )
+
+
+def _read_imports_from_cache(
+    module_files: Iterable[ModuleFile], *, cache: caching.Cache
+) -> Dict[ModuleFile, Set[DirectImport]]:
+    imports_by_module_file: Dict[ModuleFile, Set[DirectImport]] = {}
+    for module_file in module_files:
+        try:
+            direct_imports = cache.read_imports(module_file)
+        except caching.CacheMiss:
+            continue
+        else:
+            imports_by_module_file[module_file] = direct_imports
+    return imports_by_module_file
+
+
+def _scan_imports(
+    module_files: Collection[ModuleFile],
+    *,
+    file_system: AbstractFileSystem,
+    found_packages: Set[FoundPackage],
+    include_external_packages: bool,
+    exclude_type_checking_imports: bool,
+) -> Dict[ModuleFile, Set[DirectImport]]:
+    chunks = _create_chunks(module_files)
+    return _scan_chunks(
+        chunks,
+        file_system,
+        found_packages,
+        include_external_packages,
+        exclude_type_checking_imports,
+    )
+
+
+def _create_chunks(module_files: Collection[ModuleFile]) -> tuple[tuple[ModuleFile, ...], ...]:
+    """
+    Split the module files into chunks, each to be worked on by a separate OS process.
+    """
+    module_files_tuple = tuple(module_files)
+
+    number_of_module_files = len(module_files_tuple)
+    n_chunks = _decide_number_of_of_processes(number_of_module_files)
+    chunk_size = math.ceil(number_of_module_files / n_chunks)
+
+    return tuple(
+        module_files_tuple[i * chunk_size : (i + 1) * chunk_size] for i in range(n_chunks)
+    )
+
+
+def _decide_number_of_of_processes(number_of_module_files: int) -> int:
+    if number_of_module_files < MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING:
+        # Don't incur the overhead of multiprocessing.
+        return 1
+    return min(multiprocessing.cpu_count(), number_of_module_files)
+
+
+def _scan_chunks(
+    chunks: Collection[Collection[ModuleFile]],
+    file_system: AbstractFileSystem,
+    found_packages: Set[FoundPackage],
+    include_external_packages: bool,
+    exclude_type_checking_imports: bool,
+) -> Dict[ModuleFile, Set[DirectImport]]:
+    import_scanner: AbstractImportScanner = settings.IMPORT_SCANNER_CLASS(
+        file_system=file_system,
+        found_packages=found_packages,
+        include_external_packages=include_external_packages,
+    )
+
+    number_of_processes = len(chunks)
+    if number_of_processes == 1:
+        # No need to spawn a process if there's only one chunk.
+        [chunk] = chunks
+        return _scan_chunk(import_scanner, exclude_type_checking_imports, chunk)
+    else:
+        with multiprocessing.Pool(number_of_processes) as pool:
+            imports_by_module_file: Dict[ModuleFile, Set[DirectImport]] = {}
+            import_scanning_jobs = pool.starmap(
+                _scan_chunk,
+                [(import_scanner, exclude_type_checking_imports, chunk) for chunk in chunks],
+            )
+            for chunk_imports_by_module_file in import_scanning_jobs:
+                imports_by_module_file.update(chunk_imports_by_module_file)
+        return imports_by_module_file
+
+
+def _scan_chunk(
+    import_scanner: AbstractImportScanner,
+    exclude_type_checking_imports: bool,
+    chunk: Iterable[ModuleFile],
+) -> Dict[ModuleFile, Set[DirectImport]]:
+    return {
+        module_file: import_scanner.scan_for_imports(
+            module_file.module, exclude_type_checking_imports=exclude_type_checking_imports
+        )
+        for module_file in chunk
+    }

{grimp-3.7 → grimp-3.8}/src/grimp/exceptions.py

@@ -62,6 +62,11 @@ class SourceSyntaxError(GrimpException):
             other.text,
         )
 
+    def __reduce__(self):
+        # Implement __reduce__ to make this exception pickleable,
+        # allowing it to be sent between processes.
+        return SourceSyntaxError, (self.filename, self.lineno, self.text)
+
 
 class InvalidModuleExpression(GrimpException):
     pass

{grimp-3.7 → grimp-3.8}/rust/src/graph/graph_manipulation.rs

@@ -178,11 +178,6 @@ impl Graph {
             })
             .collect();
 
-        // Remove any descendants.
-        for descendant in descendants {
-            self.remove_module(descendant);
-        }
-
         // Add descendants and imports to parent module.
         for imported in modules_imported_by_descendants {
             self.add_import(module, imported);
@@ -192,6 +187,11 @@ impl Graph {
             self.add_import(importer, module);
         }
 
+        // Remove any descendants.
+        for descendant in descendants {
+            self.remove_module(descendant);
+        }
+
         self.mark_module_squashed(module);
     }
 }