pivot 0.3.7__tar.gz → 0.3.9__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. {pivot-0.3.7 → pivot-0.3.9}/PKG-INFO +3 -1
  2. {pivot-0.3.7 → pivot-0.3.9}/pyproject.toml +3 -1
  3. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/discovery.py +162 -3
  4. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/engine.py +0 -17
  5. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/worker_pool.py +58 -11
  6. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/AGENTS.md +8 -2
  7. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/core.py +23 -4
  8. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/worker.py +193 -14
  9. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/fingerprint.py +80 -72
  10. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/state.py +9 -16
  11. {pivot-0.3.7 → pivot-0.3.9}/LICENSE +0 -0
  12. {pivot-0.3.7 → pivot-0.3.9}/README.md +0 -0
  13. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/__init__.py +0 -0
  14. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/ast_utils.py +0 -0
  15. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/AGENTS.md +0 -0
  16. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/CLAUDE.md +0 -0
  17. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/__init__.py +0 -0
  18. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/_run_common.py +0 -0
  19. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/check_ignore.py +0 -0
  20. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/checkout.py +0 -0
  21. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/commit.py +0 -0
  22. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/completion.py +0 -0
  23. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/config.py +0 -0
  24. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/console.py +0 -0
  25. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/dag.py +0 -0
  26. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/data.py +0 -0
  27. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/decorators.py +0 -0
  28. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/doctor.py +0 -0
  29. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/errors.py +0 -0
  30. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/fingerprint.py +0 -0
  31. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/helpers.py +0 -0
  32. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/history.py +0 -0
  33. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/import_cmd.py +0 -0
  34. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/init.py +0 -0
  35. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/list.py +0 -0
  36. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/metrics.py +0 -0
  37. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/params.py +0 -0
  38. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/plots.py +0 -0
  39. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/remote.py +0 -0
  40. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/repro.py +0 -0
  41. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/run.py +0 -0
  42. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/status.py +0 -0
  43. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/targets.py +0 -0
  44. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/track.py +0 -0
  45. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/update.py +0 -0
  46. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/verify.py +0 -0
  47. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/compose.py +0 -0
  48. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/config/__init__.py +0 -0
  49. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/config/io.py +0 -0
  50. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/config/models.py +0 -0
  51. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/dag/__init__.py +0 -0
  52. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/dag/render.py +0 -0
  53. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/decorators.py +0 -0
  54. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/__init__.py +0 -0
  55. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/agent_rpc.py +0 -0
  56. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/graph.py +0 -0
  57. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/scheduler.py +0 -0
  58. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/sinks.py +0 -0
  59. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/sources.py +0 -0
  60. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/types.py +0 -0
  61. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/watch.py +0 -0
  62. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/exceptions.py +0 -0
  63. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/CLAUDE.md +0 -0
  64. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/__init__.py +0 -0
  65. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/commit.py +0 -0
  66. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/explain.py +0 -0
  67. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/git.py +0 -0
  68. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/ignore.py +0 -0
  69. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/import_artifact.py +0 -0
  70. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/loaders.py +0 -0
  71. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/metrics.py +0 -0
  72. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/names.py +0 -0
  73. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/outputs.py +0 -0
  74. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/parameters.py +0 -0
  75. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/path_policy.py +0 -0
  76. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/project.py +0 -0
  77. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/py.typed +0 -0
  78. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/registry.py +0 -0
  79. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/AGENTS.md +0 -0
  80. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/__init__.py +0 -0
  81. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/config.py +0 -0
  82. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/git_archive.py +0 -0
  83. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/github.py +0 -0
  84. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/storage.py +0 -0
  85. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/sync.py +0 -0
  86. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/run_history.py +0 -0
  87. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/__init__.py +0 -0
  88. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/common.py +0 -0
  89. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/data.py +0 -0
  90. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/metrics.py +0 -0
  91. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/params.py +0 -0
  92. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/plots.py +0 -0
  93. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/skip.py +0 -0
  94. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/stage_def.py +0 -0
  95. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/status.py +0 -0
  96. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/AGENTS.md +0 -0
  97. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/CLAUDE.md +0 -0
  98. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/__init__.py +0 -0
  99. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/artifact_lock.py +0 -0
  100. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/cache.py +0 -0
  101. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/lock.py +0 -0
  102. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/presentation.py +0 -0
  103. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/restore.py +0 -0
  104. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/store.py +0 -0
  105. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/track.py +0 -0
  106. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/types.py +0 -0
  107. {pivot-0.3.7 → pivot-0.3.9}/src/pivot/yaml_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pivot
3
- Version: 0.3.7
3
+ Version: 0.3.9
4
4
  Summary: High-performance Python pipeline tool with automatic code change detection
5
5
  Author: Sami Jawhar
6
6
  Author-email: Sami Jawhar <sami@metr.org>
@@ -23,6 +23,7 @@ Requires-Dist: loky>=3.4
23
23
  Requires-Dist: networkx>=3.0
24
24
  Requires-Dist: pathspec>=0.12.0
25
25
  Requires-Dist: pandas>=2.0
26
+ Requires-Dist: psutil>=7.0
26
27
  Requires-Dist: pydantic>=2.13
27
28
  Requires-Dist: pygtrie>=2.5.0
28
29
  Requires-Dist: pyyaml>=6.0
@@ -55,6 +56,7 @@ Requires-Dist: pytest-xdist>=3.8.0 ; extra == 'dev'
55
56
  Requires-Dist: pytest>=7.0 ; extra == 'dev'
56
57
  Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
57
58
  Requires-Dist: types-aioboto3[all]>=13.0.0 ; extra == 'dev'
59
+ Requires-Dist: types-psutil>=7.0 ; extra == 'dev'
58
60
  Requires-Dist: mkdocs>=1.6.0 ; extra == 'docs'
59
61
  Requires-Dist: mkdocs-material>=9.5.0 ; extra == 'docs'
60
62
  Requires-Dist: mkdocstrings[python]>=0.26.0 ; extra == 'docs'
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pivot"
3
- version = "0.3.7"
3
+ version = "0.3.9"
4
4
  description = "High-performance Python pipeline tool with automatic code change detection"
5
5
  requires-python = ">=3.13,<3.14"
6
6
  readme = "README.md"
@@ -27,6 +27,7 @@ dependencies = [
27
27
  "networkx>=3.0",
28
28
  "pathspec>=0.12.0",
29
29
  "pandas>=2.0",
30
+ "psutil>=7.0",
30
31
  "pydantic>=2.13",
31
32
  "pygtrie>=2.5.0",
32
33
  "pyyaml>=6.0",
@@ -76,6 +77,7 @@ dev = [
76
77
  "pytest>=7.0",
77
78
  "ruff>=0.8.0",
78
79
  "types-aioboto3[all]>=13.0.0",
80
+ "types-psutil>=7.0",
79
81
  ]
80
82
 
81
83
  [project.scripts]
@@ -1,21 +1,28 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import _imp
3
4
  import contextlib
4
5
  import dataclasses
5
6
  import importlib
6
7
  import importlib.machinery
7
8
  import importlib.util
8
9
  import logging
10
+ import os
9
11
  import pathlib
12
+ import py_compile
10
13
  import sys
11
14
  import types as _stdlib_types
12
- from typing import Final
15
+ from typing import TYPE_CHECKING, Final, override
13
16
 
14
17
  import cloudpickle
15
18
 
16
19
  from pivot import exceptions, fingerprint, metrics, project, types
17
20
  from pivot.registry import PipelineLike
18
21
 
22
+ if TYPE_CHECKING:
23
+ from collections.abc import Callable
24
+ from importlib.abc import PathEntryFinder
25
+
19
26
  logger = logging.getLogger(__name__)
20
27
 
21
28
  PIPELINE_PY_NAME: Final = "pipeline.py"
@@ -30,6 +37,133 @@ class DiscoveryError(Exception):
30
37
  # displaced or inserted, while the active pipeline stays importable for fingerprinting/workers.
31
38
  _MISSING_MODULE = object()
32
39
 
40
+ # A cached bytecode file's flags word (PEP 552): bit 0 marks it hash-based, bit 1 has every
41
+ # import check that hash against the source.
42
+ _CHECKED_HASH_PYC_FLAGS: Final = 0b11
43
+
44
+
45
+ def _is_checked_hash_pyc(path: str) -> bool:
46
+ """Whether ``path`` holds bytecode for this Python that every import checks by source hash."""
47
+ try:
48
+ with open(path, "rb") as cached:
49
+ header = cached.read(8)
50
+ except OSError:
51
+ return False
52
+ return (
53
+ header[:4] == importlib.util.MAGIC_NUMBER
54
+ and int.from_bytes(header[4:8], "little") == _CHECKED_HASH_PYC_FLAGS
55
+ )
56
+
57
+
58
+ class _ProjectSourceLoader(importlib.machinery.SourceFileLoader):
59
+ """Load a user-code file from bytecode checked against its source's content, not its mtime.
60
+
61
+ CPython's default bytecode cache is checked only by the source's size and whole-second mtime,
62
+ so an edit that keeps the size, made in the second the previous version was written and
63
+ cached, imports the previous version's code. A checked-hash cache (PEP 552) is checked by the
64
+ source's content on every import, by this process and by any other that imports the file.
65
+ Other bytecode for the file is replaced with one before the import, or removed when Python
66
+ is told not to write bytecode (``sys.dont_write_bytecode``), so no other process reads it.
67
+
68
+ The file is compiled from source when the cache cannot be written (a read-only
69
+ ``__pycache__``), when the source does not compile (which reports its syntax error), under
70
+ ``sys.dont_write_bytecode``, and when Python is told never to check hash-based bytecode
71
+ (``--check-hash-based-pycs never``). Its bytecode is replaced in that last case too: pivot's
72
+ workers do not inherit the flag, and they read it.
73
+
74
+ A file whose resolved path is a library's or pivot's, as a library package symlinked into
75
+ the project root, loads as CPython loads it, and its bytecode is left alone.
76
+ """
77
+
78
+ @override
79
+ def get_code(self, fullname: str) -> _stdlib_types.CodeType | None:
80
+ source = self.get_filename(fullname)
81
+ if _is_library_or_pivot(pathlib.Path(source).resolve()):
82
+ return super().get_code(fullname)
83
+ cached = importlib.util.cache_from_source(source)
84
+ if not _is_checked_hash_pyc(cached):
85
+ if sys.dont_write_bytecode:
86
+ with contextlib.suppress(OSError):
87
+ os.unlink(cached)
88
+ return self.source_to_code(self.get_data(source), source)
89
+ try:
90
+ py_compile.compile(
91
+ source,
92
+ cfile=cached,
93
+ doraise=True,
94
+ invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
95
+ )
96
+ except (py_compile.PyCompileError, OSError):
97
+ return self.source_to_code(self.get_data(source), source)
98
+ if _imp.check_hash_based_pycs == "never":
99
+ return self.source_to_code(self.get_data(source), source)
100
+ return super().get_code(fullname)
101
+
102
+
103
+ _project_finder_for = importlib.machinery.FileFinder.path_hook(
104
+ (importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES),
105
+ (_ProjectSourceLoader, importlib.machinery.SOURCE_SUFFIXES),
106
+ (importlib.machinery.SourcelessFileLoader, importlib.machinery.BYTECODE_SUFFIXES),
107
+ )
108
+
109
+ _PIVOT_PACKAGE_DIR = pathlib.Path(__file__).resolve().parent
110
+
111
+
112
+ def _is_library_or_pivot(resolved: pathlib.Path) -> bool:
113
+ """Whether resolved file or directory ``resolved`` is a library's
114
+ (``fingerprint.is_library_path``) or pivot's own source, neither of which is user code."""
115
+ return fingerprint.is_library_path(resolved) or resolved.is_relative_to(_PIVOT_PACKAGE_DIR)
116
+
117
+
118
+ def _holds_user_code(path: str, root: pathlib.Path) -> bool:
119
+ """Whether import path entry ``path`` is a directory of code fingerprinting treats as the
120
+ user's, imported through ``_ProjectSourceLoader``.
121
+
122
+ That is every directory but a library's (``fingerprint.is_library_path``: the interpreter's
123
+ prefix, so the stdlib even when the interpreter lives under the root, and site-packages or
124
+ dist-packages) and pivot's own source, minus, under the project root, the library and tooling
125
+ directories discovery skips. A stage's module outside the root, as a package installed from
126
+ a source checkout, is user code.
127
+ """
128
+ try:
129
+ resolved = pathlib.Path(path or ".").resolve()
130
+ except OSError:
131
+ return False
132
+ if _is_library_or_pivot(resolved):
133
+ return False
134
+ if not resolved.is_relative_to(root):
135
+ return True
136
+ return _SCAN_EXCLUDE_DIRS.isdisjoint(resolved.relative_to(root).parts)
137
+
138
+
139
+ def _project_path_hook(root: pathlib.Path) -> Callable[[str], PathEntryFinder]:
140
+ """A ``sys.path_hooks`` entry that imports user code with ``_ProjectSourceLoader`` and
141
+ declines every other path entry."""
142
+
143
+ def hook(path: str) -> PathEntryFinder:
144
+ if not _holds_user_code(path, root):
145
+ raise ImportError(f"{path} holds no user code")
146
+ return _project_finder_for(path)
147
+
148
+ return hook
149
+
150
+
151
+ def _forget_user_code_finders(root: pathlib.Path) -> None:
152
+ """Drop the import finders cached for user-code directories, so the next import from one
153
+ asks ``sys.path_hooks`` again."""
154
+ for entry in list(sys.path_importer_cache):
155
+ if _holds_user_code(entry, root):
156
+ del sys.path_importer_cache[entry]
157
+
158
+
159
+ def _install_path_hook(root: pathlib.Path) -> Callable[[str], PathEntryFinder]:
160
+ """Import user code through ``_ProjectSourceLoader`` from now on; return the hook."""
161
+ path_hook = _project_path_hook(root)
162
+ sys.path_hooks.insert(0, path_hook)
163
+ _forget_user_code_finders(root)
164
+ importlib.invalidate_caches()
165
+ return path_hook
166
+
33
167
 
34
168
  @dataclasses.dataclass
35
169
  class _ImportSession:
@@ -37,6 +171,7 @@ class _ImportSession:
37
171
  path_entry: str
38
172
  caller_path_entries: list[tuple[int, str]]
39
173
  preexisting_root_modules: dict[str, _stdlib_types.ModuleType]
174
+ path_hook: Callable[[str], PathEntryFinder]
40
175
  # Whether each module file lies under root; every scan of sys.modules shares one resolve.
41
176
  file_under_root: dict[str, bool] = dataclasses.field(default_factory=dict)
42
177
  owned_modules: dict[str, _stdlib_types.ModuleType] = dataclasses.field(default_factory=dict)
@@ -69,6 +204,9 @@ def _end_import_session(session: _ImportSession) -> None:
69
204
  sys.path.remove(session.path_entry)
70
205
  for index, entry in session.caller_path_entries:
71
206
  sys.path.insert(min(index, len(sys.path)), entry)
207
+ if session.path_hook in sys.path_hooks:
208
+ sys.path_hooks.remove(session.path_hook)
209
+ _forget_user_code_finders(session.root)
72
210
  importlib.invalidate_caches()
73
211
 
74
212
 
@@ -98,13 +236,30 @@ def _start_import_session(root: pathlib.Path) -> _ImportSession:
98
236
  root_str,
99
237
  caller_path_entries,
100
238
  preexisting_root_modules,
239
+ # User code imports through _ProjectSourceLoader while the session lasts.
240
+ _install_path_hook(root),
101
241
  file_under_root,
102
242
  preexisting_pickle_by_value_modules=cloudpickle.list_registry_pickle_by_value(),
103
243
  )
104
- importlib.invalidate_caches()
105
244
  return _import_session
106
245
 
107
246
 
247
+ def import_session_root() -> pathlib.Path | None:
248
+ """The project root of the import session the last discovery started, if it lasts."""
249
+ return None if _import_session is None else _import_session.root
250
+
251
+
252
+ def install_user_code_loader(root: str) -> None:
253
+ """Import user code through ``_ProjectSourceLoader`` in this process from now on, as the
254
+ import session for ``root`` does: the initializer of each worker (``executor.core``).
255
+
256
+ A worker imports user code by name: a stage's module that discovery could not send by
257
+ value (one outside the root, or under it in a ``src`` layout) and any module a stage body
258
+ imports. The session's loader is installed only in the process that discovered.
259
+ """
260
+ _install_path_hook(pathlib.Path(root))
261
+
262
+
108
263
  def _displace_module(module_name: str, session: _ImportSession) -> None:
109
264
  """Save one host module before session code claims its name."""
110
265
  if module_name in session.displaced_modules:
@@ -384,7 +539,11 @@ def _load_pipeline_from_module(
384
539
  try:
385
540
  if module is None:
386
541
  _ensure_parent_modules(module_name, session)
387
- spec = importlib.util.spec_from_file_location(module_name, resolved_path)
542
+ spec = importlib.util.spec_from_file_location(
543
+ module_name,
544
+ resolved_path,
545
+ loader=_ProjectSourceLoader(module_name, str(resolved_path)),
546
+ )
388
547
  if spec is None or spec.loader is None:
389
548
  raise DiscoveryError(f"Unable to create import spec for {path}")
390
549
  module = importlib.util.module_from_spec(spec)
@@ -5,7 +5,6 @@ import concurrent.futures
5
5
  import contextlib
6
6
  import contextvars
7
7
  import importlib
8
- import importlib.util
9
8
  import itertools
10
9
  import linecache
11
10
  import logging
@@ -2480,22 +2479,6 @@ class Engine:
2480
2479
  # Invalidate manifest cache entries for changed source files
2481
2480
  fingerprint.invalidate_manifests_for_paths(event["paths"])
2482
2481
 
2483
- # CPython reuses cached bytecode while the source's size and whole-second mtime match,
2484
- # so a same-size edit within a second of the last write would reload the old code.
2485
- for path in event["paths"]:
2486
- if path.endswith(".py"):
2487
- cached = pathlib.Path(importlib.util.cache_from_source(path))
2488
- try:
2489
- cached.unlink(missing_ok=True)
2490
- except OSError as exc:
2491
- _logger.warning(
2492
- "Could not remove %s (%s); its stale bytecode may be reused if %s "
2493
- + "kept its size and mtime second",
2494
- cached,
2495
- exc.strerror,
2496
- path,
2497
- )
2498
-
2499
2482
  # Reload registry - returns the old stages on success, the error message on failure
2500
2483
  reload_result = await self._reload_registry()
2501
2484
 
@@ -1,10 +1,13 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import contextlib
3
4
  import errno
4
5
  import logging
5
6
  import os
6
7
  import pathlib
8
+ import signal
7
9
  import threading
10
+ import time
8
11
  import typing
9
12
  from dataclasses import dataclass, field
10
13
 
@@ -17,6 +20,9 @@ _T = typing.TypeVar("_T")
17
20
 
18
21
  _logger = logging.getLogger(__name__)
19
22
 
23
+ # How long hard_cancel waits for a SIGKILLed worker to be reaped; it never blocks pivot's exit.
24
+ _KILLED_WORKER_JOIN_S = 5.0
25
+
20
26
  _FileStamp = tuple[int, int, int, int, int]
21
27
 
22
28
 
@@ -194,16 +200,57 @@ class WorkerPool:
194
200
  self._shutdown_event = None
195
201
 
196
202
  def hard_cancel(self) -> None:
197
- """Forcefully shut down workers and signal drain thread to stop."""
203
+ """Stop workers mid-stage and signal the drain thread to stop.
204
+
205
+ Each worker is sent SIGTERM, which unwinds its running stage
206
+ (``worker.unwind_when_stopped``) so the stage's cleanup runs, and is given
207
+ ``worker.TERMINATION_GRACE_S`` to exit; a worker still alive then is killed with its
208
+ descendants.
209
+
210
+ The workers are first taken out of loky's executor. Its manager thread takes a worker
211
+ that exits without being asked to for a broken pool and SIGKILLs every worker the
212
+ executor still lists, so the first worker to exit would cut the others' cleanup off.
213
+ """
198
214
  self._accepting = False
199
215
  self._in_run = False
200
- if self._executor is not None:
201
- self._executor.shutdown(wait=False, kill_workers=True)
202
- self._executor = None
203
- if self._manager is not None:
204
- self._manager.shutdown()
205
- self._manager = None
206
- if self._shutdown_event is not None:
207
- self._shutdown_event.set()
208
- self._output_queue = None
209
- self._shutdown_event = None
216
+ try:
217
+ if self._executor is not None:
218
+ executor = self._executor
219
+ self._executor = None
220
+ try:
221
+ self._stop_workers(executor)
222
+ finally:
223
+ # Only shutdown wakes loky's manager thread once its workers are gone.
224
+ executor.shutdown(wait=False, kill_workers=True)
225
+ finally:
226
+ if self._manager is not None:
227
+ self._manager.shutdown()
228
+ self._manager = None
229
+ if self._shutdown_event is not None:
230
+ self._shutdown_event.set()
231
+ self._output_queue = None
232
+ self._shutdown_event = None
233
+
234
+ @staticmethod
235
+ def _stop_workers(executor: loky.ProcessPoolExecutor) -> None:
236
+ """Take ``executor``'s workers out of it, send each SIGTERM, and SIGKILL with its
237
+ descendants any worker still alive after ``worker.TERMINATION_GRACE_S``."""
238
+ # loky exposes no public list of its worker processes. A watch reload may have
239
+ # replaced this executor already, which leaves it no lock and no workers.
240
+ with executor._processes_management_lock or contextlib.nullcontext():
241
+ processes = list(executor._processes.values())
242
+ executor._processes.clear()
243
+ for process in processes:
244
+ if process.pid is not None:
245
+ with contextlib.suppress(ProcessLookupError):
246
+ os.kill(process.pid, signal.SIGTERM)
247
+ deadline = time.monotonic() + worker.TERMINATION_GRACE_S
248
+ for process in processes:
249
+ process.join(max(0.0, deadline - time.monotonic()))
250
+ for process in processes:
251
+ if process.pid is not None and process.exitcode is None:
252
+ worker.kill_descendants(process.pid)
253
+ # loky's process object has no kill(); signal the pid directly.
254
+ with contextlib.suppress(ProcessLookupError):
255
+ os.kill(process.pid, signal.SIGKILL)
256
+ process.join(_KILLED_WORKER_JOIN_S)
@@ -3,9 +3,15 @@
3
3
  Workers execute in separate processes via `loky.get_reusable_executor()`. An Engine keeps its
4
4
  workers and manager alive across its runs; each run receives a fresh manager queue for output,
5
5
  and a manager that cannot be reached when a run starts is replaced then.
6
- The loky initializer and the manager's initializer start the parent-death watcher, so a
7
- worker or manager exits when its Pivot process dies. Keep `timeout=None` and
6
+ The loky initializer (`core._init_worker`, which runs `worker.unwind_when_stopped`) and the manager's initializer
7
+ (`core._exit_when_parent_dies`) start the parent-death watcher, so a worker or manager exits
8
+ when its Pivot process dies; a worker first unwinds its running stage so its `finally` blocks
9
+ run, within `worker.TERMINATION_GRACE_S`. Keep `timeout=None` and
8
10
  `reuse=False if kill_workers else "auto"` in the shared executor policy.
11
+ While a discovery's import session lasts, the worker initializer (`core._init_worker`) also
12
+ has each worker import user code through discovery's checked loader
13
+ (`discovery.install_user_code_loader`); loky replaces a live pool whose initializer arguments
14
+ differ.
9
15
 
10
16
  ## WorkerStageInfo Contract
11
17
 
@@ -37,7 +37,10 @@ def _noop() -> None:
37
37
 
38
38
 
39
39
  def _exit_when_parent_dies(parent_pid: int) -> None:
40
- """Exit a long-lived child (worker or manager) after its owning Pivot process dies."""
40
+ """Exit the output-queue manager after its owning Pivot process dies.
41
+
42
+ Workers use ``worker.unwind_when_stopped`` instead, which lets the running stage unwind.
43
+ """
41
44
 
42
45
  def _watch_parent() -> None:
43
46
  poll = threading.Event()
@@ -74,19 +77,35 @@ def _warm_workers(pool: loky.ProcessPoolExecutor, count: int) -> None:
74
77
 
75
78
 
76
79
  def _get_executor(max_workers: int, *, kill_workers: bool = False) -> loky.ProcessPoolExecutor:
77
- """Return Pivot's reusable executor with one worker-lifetime policy."""
80
+ """Return Pivot's reusable executor with one worker-lifetime policy.
81
+
82
+ Every worker starts with ``_init_worker``. While a discovery's import session lasts, it
83
+ also imports user code as that session does (``discovery.install_user_code_loader``), so a
84
+ worker never runs an edited module's previous code; loky replaces a live pool whose
85
+ initializer arguments differ.
86
+ """
78
87
  _ensure_deterministic_environment()
79
88
  _ensure_cleanup_registered()
89
+ root = discovery.import_session_root()
80
90
  return loky.get_reusable_executor(
81
91
  max_workers=max_workers,
82
92
  timeout=_WORKER_IDLE_TIMEOUT_S,
83
93
  kill_workers=kill_workers,
84
94
  reuse=False if kill_workers else "auto",
85
- initializer=_exit_when_parent_dies,
86
- initargs=(os.getpid(),),
95
+ initializer=_init_worker,
96
+ initargs=(os.getpid(), None if root is None else str(root)),
87
97
  )
88
98
 
89
99
 
100
+ def _init_worker(parent_pid: int, user_code_root: str | None) -> None:
101
+ """Loky worker initializer: stop the worker when it is told to or ``parent_pid`` dies
102
+ (``worker.unwind_when_stopped``), and import user code through discovery's checked loader
103
+ when a discovery's import session for ``user_code_root`` lasts."""
104
+ worker.unwind_when_stopped(parent_pid)
105
+ if user_code_root is not None:
106
+ discovery.install_user_code_loader(user_code_root)
107
+
108
+
90
109
  def create_manager() -> managers.SyncManager:
91
110
  """Start a spawn-context manager that exits when this process dies."""
92
111
  manager = managers.SyncManager(ctx=mp.get_context("spawn"))