grimp 3.8__tar.gz → 3.8.2__tar.gz

{grimp-3.8 → grimp-3.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: grimp
-Version: 3.8
+Version: 3.8.2
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: BSD License
@@ -16,6 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Rust
 Classifier: Topic :: Utilities
+Requires-Dist: joblib>=1.3.0
 Requires-Dist: typing-extensions>=3.10.0.0
 License-File: LICENSE
 License-File: AUTHORS.rst

{grimp-3.8 → grimp-3.8.2}/pyproject.toml

@@ -8,7 +8,7 @@ module-name = "grimp._rustgrimp"
 
 [project]
 name = "grimp"
-version = "3.8"
+version = "3.8.2"
 license = {text = "BSD 2-Clause License"}
 description = "Builds a queryable graph of the imports within one or more Python packages."
 authors = [
@@ -16,6 +16,7 @@ authors = [
 ]
 requires-python = ">=3.9"
 dependencies = [
+    "joblib>=1.3.0",
     "typing-extensions>=3.10.0.0",
 ]
 classifiers = [

{grimp-3.8 → grimp-3.8.2}/src/grimp/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.8"
+__version__ = "3.8.2"
 
 from .application.ports.graph import DetailedImport, ImportGraph, Import
 from .domain.analysis import PackageDependency, Route

{grimp-3.8 → grimp-3.8.2}/src/grimp/application/usecases.py

@@ -3,9 +3,10 @@ Use cases handle application logic.
 """
 
 from typing import Dict, Sequence, Set, Type, Union, cast, Iterable, Collection
-import multiprocessing
 import math
 
+import joblib  # type: ignore
+
 from ..application.ports import caching
 from ..application.ports.filesystem import AbstractFileSystem
 from ..application.ports.graph import ImportGraph
@@ -14,14 +15,19 @@ from ..application.ports.modulefinder import AbstractModuleFinder, FoundPackage,
 from ..application.ports.packagefinder import AbstractPackageFinder
 from ..domain.valueobjects import DirectImport, Module
 from .config import settings
+import os
 
 
 class NotSupplied:
     pass
 
 
+# Calling code can set this environment variable if it wants to tune when to switch to
+# multiprocessing, or set it to a large number to disable it altogether.
+MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING_ENV_NAME = "GRIMP_MIN_MULTIPROCESSING_MODULES"
 # This is an arbitrary number, but setting it too low slows down our functional tests considerably.
-MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING = 50
+# If you change this, update docs/usage.rst too!
+DEFAULT_MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING = 50
 
 
 def build_graph(
@@ -228,7 +234,7 @@ def _create_chunks(module_files: Collection[ModuleFile]) -> tuple[tuple[ModuleFi
     module_files_tuple = tuple(module_files)
 
     number_of_module_files = len(module_files_tuple)
-    n_chunks = _decide_number_of_of_processes(number_of_module_files)
+    n_chunks = _decide_number_of_processes(number_of_module_files)
     chunk_size = math.ceil(number_of_module_files / n_chunks)
 
     return tuple(
@@ -236,11 +242,17 @@ def _create_chunks(module_files: Collection[ModuleFile]) -> tuple[tuple[ModuleFi
     )
 
 
-def _decide_number_of_of_processes(number_of_module_files: int) -> int:
-    if number_of_module_files < MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING:
-        # Don't incur the overhead of multiprocessing.
+def _decide_number_of_processes(number_of_module_files: int) -> int:
+    min_number_of_modules = int(
+        os.environ.get(
+            MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING_ENV_NAME,
+            DEFAULT_MIN_NUMBER_OF_MODULES_TO_SCAN_USING_MULTIPROCESSING,
+        )
+    )
+    if number_of_module_files < min_number_of_modules:
+        # Don't incur the overhead of multiple processes.
         return 1
-    return min(multiprocessing.cpu_count(), number_of_module_files)
+    return min(joblib.cpu_count(), number_of_module_files)
 
 
 def _scan_chunks(
@@ -257,20 +269,15 @@ def _scan_chunks(
     )
 
     number_of_processes = len(chunks)
-    if number_of_processes == 1:
-        # No need to spawn a process if there's only one chunk.
-        [chunk] = chunks
-        return _scan_chunk(import_scanner, exclude_type_checking_imports, chunk)
-    else:
-        with multiprocessing.Pool(number_of_processes) as pool:
-            imports_by_module_file: Dict[ModuleFile, Set[DirectImport]] = {}
-            import_scanning_jobs = pool.starmap(
-                _scan_chunk,
-                [(import_scanner, exclude_type_checking_imports, chunk) for chunk in chunks],
-            )
-            for chunk_imports_by_module_file in import_scanning_jobs:
-                imports_by_module_file.update(chunk_imports_by_module_file)
-        return imports_by_module_file
+    import_scanning_jobs = joblib.Parallel(n_jobs=number_of_processes)(
+        joblib.delayed(_scan_chunk)(import_scanner, exclude_type_checking_imports, chunk)
+        for chunk in chunks
+    )
+
+    imports_by_module_file = {}
+    for chunk_imports_by_module_file in import_scanning_jobs:
+        imports_by_module_file.update(chunk_imports_by_module_file)
+    return imports_by_module_file
 
 
 def _scan_chunk(