pivot 0.3.7__tar.gz → 0.3.9__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pivot-0.3.7 → pivot-0.3.9}/PKG-INFO +3 -1
- {pivot-0.3.7 → pivot-0.3.9}/pyproject.toml +3 -1
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/discovery.py +162 -3
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/engine.py +0 -17
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/worker_pool.py +58 -11
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/AGENTS.md +8 -2
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/core.py +23 -4
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/worker.py +193 -14
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/fingerprint.py +80 -72
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/state.py +9 -16
- {pivot-0.3.7 → pivot-0.3.9}/LICENSE +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/README.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/ast_utils.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/AGENTS.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/CLAUDE.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/_run_common.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/check_ignore.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/checkout.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/commit.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/completion.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/config.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/console.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/dag.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/data.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/decorators.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/doctor.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/errors.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/fingerprint.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/helpers.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/history.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/import_cmd.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/init.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/list.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/metrics.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/params.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/plots.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/remote.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/repro.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/run.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/status.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/targets.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/track.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/update.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/cli/verify.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/compose.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/config/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/config/io.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/config/models.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/dag/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/dag/render.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/decorators.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/agent_rpc.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/graph.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/scheduler.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/sinks.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/sources.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/types.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/engine/watch.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/exceptions.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/CLAUDE.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/executor/commit.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/explain.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/git.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/ignore.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/import_artifact.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/loaders.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/metrics.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/names.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/outputs.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/parameters.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/path_policy.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/project.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/py.typed +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/registry.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/AGENTS.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/config.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/git_archive.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/github.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/storage.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/remote/sync.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/run_history.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/common.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/data.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/metrics.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/params.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/show/plots.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/skip.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/stage_def.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/status.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/AGENTS.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/CLAUDE.md +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/__init__.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/artifact_lock.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/cache.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/lock.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/presentation.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/restore.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/store.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/storage/track.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/types.py +0 -0
- {pivot-0.3.7 → pivot-0.3.9}/src/pivot/yaml_config.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pivot
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.9
|
|
4
4
|
Summary: High-performance Python pipeline tool with automatic code change detection
|
|
5
5
|
Author: Sami Jawhar
|
|
6
6
|
Author-email: Sami Jawhar <sami@metr.org>
|
|
@@ -23,6 +23,7 @@ Requires-Dist: loky>=3.4
|
|
|
23
23
|
Requires-Dist: networkx>=3.0
|
|
24
24
|
Requires-Dist: pathspec>=0.12.0
|
|
25
25
|
Requires-Dist: pandas>=2.0
|
|
26
|
+
Requires-Dist: psutil>=7.0
|
|
26
27
|
Requires-Dist: pydantic>=2.13
|
|
27
28
|
Requires-Dist: pygtrie>=2.5.0
|
|
28
29
|
Requires-Dist: pyyaml>=6.0
|
|
@@ -55,6 +56,7 @@ Requires-Dist: pytest-xdist>=3.8.0 ; extra == 'dev'
|
|
|
55
56
|
Requires-Dist: pytest>=7.0 ; extra == 'dev'
|
|
56
57
|
Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
|
|
57
58
|
Requires-Dist: types-aioboto3[all]>=13.0.0 ; extra == 'dev'
|
|
59
|
+
Requires-Dist: types-psutil>=7.0 ; extra == 'dev'
|
|
58
60
|
Requires-Dist: mkdocs>=1.6.0 ; extra == 'docs'
|
|
59
61
|
Requires-Dist: mkdocs-material>=9.5.0 ; extra == 'docs'
|
|
60
62
|
Requires-Dist: mkdocstrings[python]>=0.26.0 ; extra == 'docs'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "pivot"
|
|
3
|
-
version = "0.3.
|
|
3
|
+
version = "0.3.9"
|
|
4
4
|
description = "High-performance Python pipeline tool with automatic code change detection"
|
|
5
5
|
requires-python = ">=3.13,<3.14"
|
|
6
6
|
readme = "README.md"
|
|
@@ -27,6 +27,7 @@ dependencies = [
|
|
|
27
27
|
"networkx>=3.0",
|
|
28
28
|
"pathspec>=0.12.0",
|
|
29
29
|
"pandas>=2.0",
|
|
30
|
+
"psutil>=7.0",
|
|
30
31
|
"pydantic>=2.13",
|
|
31
32
|
"pygtrie>=2.5.0",
|
|
32
33
|
"pyyaml>=6.0",
|
|
@@ -76,6 +77,7 @@ dev = [
|
|
|
76
77
|
"pytest>=7.0",
|
|
77
78
|
"ruff>=0.8.0",
|
|
78
79
|
"types-aioboto3[all]>=13.0.0",
|
|
80
|
+
"types-psutil>=7.0",
|
|
79
81
|
]
|
|
80
82
|
|
|
81
83
|
[project.scripts]
|
|
@@ -1,21 +1,28 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import _imp
|
|
3
4
|
import contextlib
|
|
4
5
|
import dataclasses
|
|
5
6
|
import importlib
|
|
6
7
|
import importlib.machinery
|
|
7
8
|
import importlib.util
|
|
8
9
|
import logging
|
|
10
|
+
import os
|
|
9
11
|
import pathlib
|
|
12
|
+
import py_compile
|
|
10
13
|
import sys
|
|
11
14
|
import types as _stdlib_types
|
|
12
|
-
from typing import Final
|
|
15
|
+
from typing import TYPE_CHECKING, Final, override
|
|
13
16
|
|
|
14
17
|
import cloudpickle
|
|
15
18
|
|
|
16
19
|
from pivot import exceptions, fingerprint, metrics, project, types
|
|
17
20
|
from pivot.registry import PipelineLike
|
|
18
21
|
|
|
22
|
+
if TYPE_CHECKING:
|
|
23
|
+
from collections.abc import Callable
|
|
24
|
+
from importlib.abc import PathEntryFinder
|
|
25
|
+
|
|
19
26
|
logger = logging.getLogger(__name__)
|
|
20
27
|
|
|
21
28
|
PIPELINE_PY_NAME: Final = "pipeline.py"
|
|
@@ -30,6 +37,133 @@ class DiscoveryError(Exception):
|
|
|
30
37
|
# displaced or inserted, while the active pipeline stays importable for fingerprinting/workers.
|
|
31
38
|
_MISSING_MODULE = object()
|
|
32
39
|
|
|
40
|
+
# A cached bytecode file's flags word (PEP 552): bit 0 marks it hash-based, bit 1 has every
|
|
41
|
+
# import check that hash against the source.
|
|
42
|
+
_CHECKED_HASH_PYC_FLAGS: Final = 0b11
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _is_checked_hash_pyc(path: str) -> bool:
|
|
46
|
+
"""Whether ``path`` holds bytecode for this Python that every import checks by source hash."""
|
|
47
|
+
try:
|
|
48
|
+
with open(path, "rb") as cached:
|
|
49
|
+
header = cached.read(8)
|
|
50
|
+
except OSError:
|
|
51
|
+
return False
|
|
52
|
+
return (
|
|
53
|
+
header[:4] == importlib.util.MAGIC_NUMBER
|
|
54
|
+
and int.from_bytes(header[4:8], "little") == _CHECKED_HASH_PYC_FLAGS
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class _ProjectSourceLoader(importlib.machinery.SourceFileLoader):
|
|
59
|
+
"""Load a user-code file from bytecode checked against its source's content, not its mtime.
|
|
60
|
+
|
|
61
|
+
CPython's default bytecode cache is checked only by the source's size and whole-second mtime,
|
|
62
|
+
so an edit that keeps the size, made in the second the previous version was written and
|
|
63
|
+
cached, imports the previous version's code. A checked-hash cache (PEP 552) is checked by the
|
|
64
|
+
source's content on every import, by this process and by any other that imports the file.
|
|
65
|
+
Other bytecode for the file is replaced with one before the import, or removed when Python
|
|
66
|
+
is told not to write bytecode (``sys.dont_write_bytecode``), so no other process reads it.
|
|
67
|
+
|
|
68
|
+
The file is compiled from source when the cache cannot be written (a read-only
|
|
69
|
+
``__pycache__``), when the source does not compile (which reports its syntax error), under
|
|
70
|
+
``sys.dont_write_bytecode``, and when Python is told never to check hash-based bytecode
|
|
71
|
+
(``--check-hash-based-pycs never``). Its bytecode is replaced in that last case too: pivot's
|
|
72
|
+
workers do not inherit the flag, and they read it.
|
|
73
|
+
|
|
74
|
+
A file whose resolved path is a library's or pivot's, as a library package symlinked into
|
|
75
|
+
the project root, loads as CPython loads it, and its bytecode is left alone.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
@override
|
|
79
|
+
def get_code(self, fullname: str) -> _stdlib_types.CodeType | None:
|
|
80
|
+
source = self.get_filename(fullname)
|
|
81
|
+
if _is_library_or_pivot(pathlib.Path(source).resolve()):
|
|
82
|
+
return super().get_code(fullname)
|
|
83
|
+
cached = importlib.util.cache_from_source(source)
|
|
84
|
+
if not _is_checked_hash_pyc(cached):
|
|
85
|
+
if sys.dont_write_bytecode:
|
|
86
|
+
with contextlib.suppress(OSError):
|
|
87
|
+
os.unlink(cached)
|
|
88
|
+
return self.source_to_code(self.get_data(source), source)
|
|
89
|
+
try:
|
|
90
|
+
py_compile.compile(
|
|
91
|
+
source,
|
|
92
|
+
cfile=cached,
|
|
93
|
+
doraise=True,
|
|
94
|
+
invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH,
|
|
95
|
+
)
|
|
96
|
+
except (py_compile.PyCompileError, OSError):
|
|
97
|
+
return self.source_to_code(self.get_data(source), source)
|
|
98
|
+
if _imp.check_hash_based_pycs == "never":
|
|
99
|
+
return self.source_to_code(self.get_data(source), source)
|
|
100
|
+
return super().get_code(fullname)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
_project_finder_for = importlib.machinery.FileFinder.path_hook(
|
|
104
|
+
(importlib.machinery.ExtensionFileLoader, importlib.machinery.EXTENSION_SUFFIXES),
|
|
105
|
+
(_ProjectSourceLoader, importlib.machinery.SOURCE_SUFFIXES),
|
|
106
|
+
(importlib.machinery.SourcelessFileLoader, importlib.machinery.BYTECODE_SUFFIXES),
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
_PIVOT_PACKAGE_DIR = pathlib.Path(__file__).resolve().parent
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _is_library_or_pivot(resolved: pathlib.Path) -> bool:
|
|
113
|
+
"""Whether resolved file or directory ``resolved`` is a library's
|
|
114
|
+
(``fingerprint.is_library_path``) or pivot's own source, neither of which is user code."""
|
|
115
|
+
return fingerprint.is_library_path(resolved) or resolved.is_relative_to(_PIVOT_PACKAGE_DIR)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _holds_user_code(path: str, root: pathlib.Path) -> bool:
|
|
119
|
+
"""Whether import path entry ``path`` is a directory of code fingerprinting treats as the
|
|
120
|
+
user's, imported through ``_ProjectSourceLoader``.
|
|
121
|
+
|
|
122
|
+
That is every directory but a library's (``fingerprint.is_library_path``: the interpreter's
|
|
123
|
+
prefix, so the stdlib even when the interpreter lives under the root, and site-packages or
|
|
124
|
+
dist-packages) and pivot's own source, minus, under the project root, the library and tooling
|
|
125
|
+
directories discovery skips. A stage's module outside the root, as a package installed from
|
|
126
|
+
a source checkout, is user code.
|
|
127
|
+
"""
|
|
128
|
+
try:
|
|
129
|
+
resolved = pathlib.Path(path or ".").resolve()
|
|
130
|
+
except OSError:
|
|
131
|
+
return False
|
|
132
|
+
if _is_library_or_pivot(resolved):
|
|
133
|
+
return False
|
|
134
|
+
if not resolved.is_relative_to(root):
|
|
135
|
+
return True
|
|
136
|
+
return _SCAN_EXCLUDE_DIRS.isdisjoint(resolved.relative_to(root).parts)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _project_path_hook(root: pathlib.Path) -> Callable[[str], PathEntryFinder]:
|
|
140
|
+
"""A ``sys.path_hooks`` entry that imports user code with ``_ProjectSourceLoader`` and
|
|
141
|
+
declines every other path entry."""
|
|
142
|
+
|
|
143
|
+
def hook(path: str) -> PathEntryFinder:
|
|
144
|
+
if not _holds_user_code(path, root):
|
|
145
|
+
raise ImportError(f"{path} holds no user code")
|
|
146
|
+
return _project_finder_for(path)
|
|
147
|
+
|
|
148
|
+
return hook
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _forget_user_code_finders(root: pathlib.Path) -> None:
|
|
152
|
+
"""Drop the import finders cached for user-code directories, so the next import from one
|
|
153
|
+
asks ``sys.path_hooks`` again."""
|
|
154
|
+
for entry in list(sys.path_importer_cache):
|
|
155
|
+
if _holds_user_code(entry, root):
|
|
156
|
+
del sys.path_importer_cache[entry]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _install_path_hook(root: pathlib.Path) -> Callable[[str], PathEntryFinder]:
|
|
160
|
+
"""Import user code through ``_ProjectSourceLoader`` from now on; return the hook."""
|
|
161
|
+
path_hook = _project_path_hook(root)
|
|
162
|
+
sys.path_hooks.insert(0, path_hook)
|
|
163
|
+
_forget_user_code_finders(root)
|
|
164
|
+
importlib.invalidate_caches()
|
|
165
|
+
return path_hook
|
|
166
|
+
|
|
33
167
|
|
|
34
168
|
@dataclasses.dataclass
|
|
35
169
|
class _ImportSession:
|
|
@@ -37,6 +171,7 @@ class _ImportSession:
|
|
|
37
171
|
path_entry: str
|
|
38
172
|
caller_path_entries: list[tuple[int, str]]
|
|
39
173
|
preexisting_root_modules: dict[str, _stdlib_types.ModuleType]
|
|
174
|
+
path_hook: Callable[[str], PathEntryFinder]
|
|
40
175
|
# Whether each module file lies under root; every scan of sys.modules shares one resolve.
|
|
41
176
|
file_under_root: dict[str, bool] = dataclasses.field(default_factory=dict)
|
|
42
177
|
owned_modules: dict[str, _stdlib_types.ModuleType] = dataclasses.field(default_factory=dict)
|
|
@@ -69,6 +204,9 @@ def _end_import_session(session: _ImportSession) -> None:
|
|
|
69
204
|
sys.path.remove(session.path_entry)
|
|
70
205
|
for index, entry in session.caller_path_entries:
|
|
71
206
|
sys.path.insert(min(index, len(sys.path)), entry)
|
|
207
|
+
if session.path_hook in sys.path_hooks:
|
|
208
|
+
sys.path_hooks.remove(session.path_hook)
|
|
209
|
+
_forget_user_code_finders(session.root)
|
|
72
210
|
importlib.invalidate_caches()
|
|
73
211
|
|
|
74
212
|
|
|
@@ -98,13 +236,30 @@ def _start_import_session(root: pathlib.Path) -> _ImportSession:
|
|
|
98
236
|
root_str,
|
|
99
237
|
caller_path_entries,
|
|
100
238
|
preexisting_root_modules,
|
|
239
|
+
# User code imports through _ProjectSourceLoader while the session lasts.
|
|
240
|
+
_install_path_hook(root),
|
|
101
241
|
file_under_root,
|
|
102
242
|
preexisting_pickle_by_value_modules=cloudpickle.list_registry_pickle_by_value(),
|
|
103
243
|
)
|
|
104
|
-
importlib.invalidate_caches()
|
|
105
244
|
return _import_session
|
|
106
245
|
|
|
107
246
|
|
|
247
|
+
def import_session_root() -> pathlib.Path | None:
|
|
248
|
+
"""The project root of the import session the last discovery started, if it lasts."""
|
|
249
|
+
return None if _import_session is None else _import_session.root
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def install_user_code_loader(root: str) -> None:
|
|
253
|
+
"""Import user code through ``_ProjectSourceLoader`` in this process from now on, as the
|
|
254
|
+
import session for ``root`` does: the initializer of each worker (``executor.core``).
|
|
255
|
+
|
|
256
|
+
A worker imports user code by name: a stage's module that discovery could not send by
|
|
257
|
+
value (one outside the root, or under it in a ``src`` layout) and any module a stage body
|
|
258
|
+
imports. The session's loader is installed only in the process that discovered.
|
|
259
|
+
"""
|
|
260
|
+
_install_path_hook(pathlib.Path(root))
|
|
261
|
+
|
|
262
|
+
|
|
108
263
|
def _displace_module(module_name: str, session: _ImportSession) -> None:
|
|
109
264
|
"""Save one host module before session code claims its name."""
|
|
110
265
|
if module_name in session.displaced_modules:
|
|
@@ -384,7 +539,11 @@ def _load_pipeline_from_module(
|
|
|
384
539
|
try:
|
|
385
540
|
if module is None:
|
|
386
541
|
_ensure_parent_modules(module_name, session)
|
|
387
|
-
spec = importlib.util.spec_from_file_location(
|
|
542
|
+
spec = importlib.util.spec_from_file_location(
|
|
543
|
+
module_name,
|
|
544
|
+
resolved_path,
|
|
545
|
+
loader=_ProjectSourceLoader(module_name, str(resolved_path)),
|
|
546
|
+
)
|
|
388
547
|
if spec is None or spec.loader is None:
|
|
389
548
|
raise DiscoveryError(f"Unable to create import spec for {path}")
|
|
390
549
|
module = importlib.util.module_from_spec(spec)
|
|
@@ -5,7 +5,6 @@ import concurrent.futures
|
|
|
5
5
|
import contextlib
|
|
6
6
|
import contextvars
|
|
7
7
|
import importlib
|
|
8
|
-
import importlib.util
|
|
9
8
|
import itertools
|
|
10
9
|
import linecache
|
|
11
10
|
import logging
|
|
@@ -2480,22 +2479,6 @@ class Engine:
|
|
|
2480
2479
|
# Invalidate manifest cache entries for changed source files
|
|
2481
2480
|
fingerprint.invalidate_manifests_for_paths(event["paths"])
|
|
2482
2481
|
|
|
2483
|
-
# CPython reuses cached bytecode while the source's size and whole-second mtime match,
|
|
2484
|
-
# so a same-size edit within a second of the last write would reload the old code.
|
|
2485
|
-
for path in event["paths"]:
|
|
2486
|
-
if path.endswith(".py"):
|
|
2487
|
-
cached = pathlib.Path(importlib.util.cache_from_source(path))
|
|
2488
|
-
try:
|
|
2489
|
-
cached.unlink(missing_ok=True)
|
|
2490
|
-
except OSError as exc:
|
|
2491
|
-
_logger.warning(
|
|
2492
|
-
"Could not remove %s (%s); its stale bytecode may be reused if %s "
|
|
2493
|
-
+ "kept its size and mtime second",
|
|
2494
|
-
cached,
|
|
2495
|
-
exc.strerror,
|
|
2496
|
-
path,
|
|
2497
|
-
)
|
|
2498
|
-
|
|
2499
2482
|
# Reload registry - returns the old stages on success, the error message on failure
|
|
2500
2483
|
reload_result = await self._reload_registry()
|
|
2501
2484
|
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import contextlib
|
|
3
4
|
import errno
|
|
4
5
|
import logging
|
|
5
6
|
import os
|
|
6
7
|
import pathlib
|
|
8
|
+
import signal
|
|
7
9
|
import threading
|
|
10
|
+
import time
|
|
8
11
|
import typing
|
|
9
12
|
from dataclasses import dataclass, field
|
|
10
13
|
|
|
@@ -17,6 +20,9 @@ _T = typing.TypeVar("_T")
|
|
|
17
20
|
|
|
18
21
|
_logger = logging.getLogger(__name__)
|
|
19
22
|
|
|
23
|
+
# How long hard_cancel waits for a SIGKILLed worker to be reaped; it never blocks pivot's exit.
|
|
24
|
+
_KILLED_WORKER_JOIN_S = 5.0
|
|
25
|
+
|
|
20
26
|
_FileStamp = tuple[int, int, int, int, int]
|
|
21
27
|
|
|
22
28
|
|
|
@@ -194,16 +200,57 @@ class WorkerPool:
|
|
|
194
200
|
self._shutdown_event = None
|
|
195
201
|
|
|
196
202
|
def hard_cancel(self) -> None:
|
|
197
|
-
"""
|
|
203
|
+
"""Stop workers mid-stage and signal the drain thread to stop.
|
|
204
|
+
|
|
205
|
+
Each worker is sent SIGTERM, which unwinds its running stage
|
|
206
|
+
(``worker.unwind_when_stopped``) so the stage's cleanup runs, and is given
|
|
207
|
+
``worker.TERMINATION_GRACE_S`` to exit; a worker still alive then is killed with its
|
|
208
|
+
descendants.
|
|
209
|
+
|
|
210
|
+
The workers are first taken out of loky's executor. Its manager thread takes a worker
|
|
211
|
+
that exits without being asked to for a broken pool and SIGKILLs every worker the
|
|
212
|
+
executor still lists, so the first worker to exit would cut the others' cleanup off.
|
|
213
|
+
"""
|
|
198
214
|
self._accepting = False
|
|
199
215
|
self._in_run = False
|
|
200
|
-
|
|
201
|
-
self._executor
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
216
|
+
try:
|
|
217
|
+
if self._executor is not None:
|
|
218
|
+
executor = self._executor
|
|
219
|
+
self._executor = None
|
|
220
|
+
try:
|
|
221
|
+
self._stop_workers(executor)
|
|
222
|
+
finally:
|
|
223
|
+
# Only shutdown wakes loky's manager thread once its workers are gone.
|
|
224
|
+
executor.shutdown(wait=False, kill_workers=True)
|
|
225
|
+
finally:
|
|
226
|
+
if self._manager is not None:
|
|
227
|
+
self._manager.shutdown()
|
|
228
|
+
self._manager = None
|
|
229
|
+
if self._shutdown_event is not None:
|
|
230
|
+
self._shutdown_event.set()
|
|
231
|
+
self._output_queue = None
|
|
232
|
+
self._shutdown_event = None
|
|
233
|
+
|
|
234
|
+
@staticmethod
|
|
235
|
+
def _stop_workers(executor: loky.ProcessPoolExecutor) -> None:
|
|
236
|
+
"""Take ``executor``'s workers out of it, send each SIGTERM, and SIGKILL with its
|
|
237
|
+
descendants any worker still alive after ``worker.TERMINATION_GRACE_S``."""
|
|
238
|
+
# loky exposes no public list of its worker processes. A watch reload may have
|
|
239
|
+
# replaced this executor already, which leaves it no lock and no workers.
|
|
240
|
+
with executor._processes_management_lock or contextlib.nullcontext():
|
|
241
|
+
processes = list(executor._processes.values())
|
|
242
|
+
executor._processes.clear()
|
|
243
|
+
for process in processes:
|
|
244
|
+
if process.pid is not None:
|
|
245
|
+
with contextlib.suppress(ProcessLookupError):
|
|
246
|
+
os.kill(process.pid, signal.SIGTERM)
|
|
247
|
+
deadline = time.monotonic() + worker.TERMINATION_GRACE_S
|
|
248
|
+
for process in processes:
|
|
249
|
+
process.join(max(0.0, deadline - time.monotonic()))
|
|
250
|
+
for process in processes:
|
|
251
|
+
if process.pid is not None and process.exitcode is None:
|
|
252
|
+
worker.kill_descendants(process.pid)
|
|
253
|
+
# loky's process object has no kill(); signal the pid directly.
|
|
254
|
+
with contextlib.suppress(ProcessLookupError):
|
|
255
|
+
os.kill(process.pid, signal.SIGKILL)
|
|
256
|
+
process.join(_KILLED_WORKER_JOIN_S)
|
|
@@ -3,9 +3,15 @@
|
|
|
3
3
|
Workers execute in separate processes via `loky.get_reusable_executor()`. An Engine keeps its
|
|
4
4
|
workers and manager alive across its runs; each run receives a fresh manager queue for output,
|
|
5
5
|
and a manager that cannot be reached when a run starts is replaced then.
|
|
6
|
-
The loky initializer and the manager's initializer
|
|
7
|
-
|
|
6
|
+
The loky initializer (`core._init_worker`, which runs `worker.unwind_when_stopped`) and the manager's initializer
|
|
7
|
+
(`core._exit_when_parent_dies`) start the parent-death watcher, so a worker or manager exits
|
|
8
|
+
when its Pivot process dies; a worker first unwinds its running stage so its `finally` blocks
|
|
9
|
+
run, within `worker.TERMINATION_GRACE_S`. Keep `timeout=None` and
|
|
8
10
|
`reuse=False if kill_workers else "auto"` in the shared executor policy.
|
|
11
|
+
While a discovery's import session lasts, the worker initializer (`core._init_worker`) also
|
|
12
|
+
has each worker import user code through discovery's checked loader
|
|
13
|
+
(`discovery.install_user_code_loader`); loky replaces a live pool whose initializer arguments
|
|
14
|
+
differ.
|
|
9
15
|
|
|
10
16
|
## WorkerStageInfo Contract
|
|
11
17
|
|
|
@@ -37,7 +37,10 @@ def _noop() -> None:
|
|
|
37
37
|
|
|
38
38
|
|
|
39
39
|
def _exit_when_parent_dies(parent_pid: int) -> None:
|
|
40
|
-
"""Exit
|
|
40
|
+
"""Exit the output-queue manager after its owning Pivot process dies.
|
|
41
|
+
|
|
42
|
+
Workers use ``worker.unwind_when_stopped`` instead, which lets the running stage unwind.
|
|
43
|
+
"""
|
|
41
44
|
|
|
42
45
|
def _watch_parent() -> None:
|
|
43
46
|
poll = threading.Event()
|
|
@@ -74,19 +77,35 @@ def _warm_workers(pool: loky.ProcessPoolExecutor, count: int) -> None:
|
|
|
74
77
|
|
|
75
78
|
|
|
76
79
|
def _get_executor(max_workers: int, *, kill_workers: bool = False) -> loky.ProcessPoolExecutor:
|
|
77
|
-
"""Return Pivot's reusable executor with one worker-lifetime policy.
|
|
80
|
+
"""Return Pivot's reusable executor with one worker-lifetime policy.
|
|
81
|
+
|
|
82
|
+
Every worker starts with ``_init_worker``. While a discovery's import session lasts, it
|
|
83
|
+
also imports user code as that session does (``discovery.install_user_code_loader``), so a
|
|
84
|
+
worker never runs an edited module's previous code; loky replaces a live pool whose
|
|
85
|
+
initializer arguments differ.
|
|
86
|
+
"""
|
|
78
87
|
_ensure_deterministic_environment()
|
|
79
88
|
_ensure_cleanup_registered()
|
|
89
|
+
root = discovery.import_session_root()
|
|
80
90
|
return loky.get_reusable_executor(
|
|
81
91
|
max_workers=max_workers,
|
|
82
92
|
timeout=_WORKER_IDLE_TIMEOUT_S,
|
|
83
93
|
kill_workers=kill_workers,
|
|
84
94
|
reuse=False if kill_workers else "auto",
|
|
85
|
-
initializer=
|
|
86
|
-
initargs=(os.getpid(),),
|
|
95
|
+
initializer=_init_worker,
|
|
96
|
+
initargs=(os.getpid(), None if root is None else str(root)),
|
|
87
97
|
)
|
|
88
98
|
|
|
89
99
|
|
|
100
|
+
def _init_worker(parent_pid: int, user_code_root: str | None) -> None:
|
|
101
|
+
"""Loky worker initializer: stop the worker when it is told to or ``parent_pid`` dies
|
|
102
|
+
(``worker.unwind_when_stopped``), and import user code through discovery's checked loader
|
|
103
|
+
when a discovery's import session for ``user_code_root`` lasts."""
|
|
104
|
+
worker.unwind_when_stopped(parent_pid)
|
|
105
|
+
if user_code_root is not None:
|
|
106
|
+
discovery.install_user_code_loader(user_code_root)
|
|
107
|
+
|
|
108
|
+
|
|
90
109
|
def create_manager() -> managers.SyncManager:
|
|
91
110
|
"""Start a spawn-context manager that exits when this process dies."""
|
|
92
111
|
manager = managers.SyncManager(ctx=mp.get_context("spawn"))
|