pivot 0.3.6__tar.gz → 0.3.8__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. {pivot-0.3.6 → pivot-0.3.8}/PKG-INFO +3 -1
  2. {pivot-0.3.6 → pivot-0.3.8}/pyproject.toml +3 -1
  3. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/engine.py +28 -18
  4. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/watch.py +3 -3
  5. pivot-0.3.8/src/pivot/engine/worker_pool.py +256 -0
  6. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/executor/AGENTS.md +8 -1
  7. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/executor/core.py +49 -17
  8. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/executor/worker.py +211 -2
  9. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/types.py +1 -0
  10. pivot-0.3.6/src/pivot/engine/worker_pool.py +0 -90
  11. {pivot-0.3.6 → pivot-0.3.8}/LICENSE +0 -0
  12. {pivot-0.3.6 → pivot-0.3.8}/README.md +0 -0
  13. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/__init__.py +0 -0
  14. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/ast_utils.py +0 -0
  15. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/AGENTS.md +0 -0
  16. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/CLAUDE.md +0 -0
  17. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/__init__.py +0 -0
  18. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/_run_common.py +0 -0
  19. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/check_ignore.py +0 -0
  20. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/checkout.py +0 -0
  21. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/commit.py +0 -0
  22. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/completion.py +0 -0
  23. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/config.py +0 -0
  24. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/console.py +0 -0
  25. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/dag.py +0 -0
  26. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/data.py +0 -0
  27. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/decorators.py +0 -0
  28. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/doctor.py +0 -0
  29. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/errors.py +0 -0
  30. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/fingerprint.py +0 -0
  31. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/helpers.py +0 -0
  32. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/history.py +0 -0
  33. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/import_cmd.py +0 -0
  34. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/init.py +0 -0
  35. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/list.py +0 -0
  36. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/metrics.py +0 -0
  37. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/params.py +0 -0
  38. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/plots.py +0 -0
  39. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/remote.py +0 -0
  40. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/repro.py +0 -0
  41. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/run.py +0 -0
  42. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/status.py +0 -0
  43. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/targets.py +0 -0
  44. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/track.py +0 -0
  45. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/update.py +0 -0
  46. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/cli/verify.py +0 -0
  47. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/compose.py +0 -0
  48. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/config/__init__.py +0 -0
  49. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/config/io.py +0 -0
  50. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/config/models.py +0 -0
  51. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/dag/__init__.py +0 -0
  52. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/dag/render.py +0 -0
  53. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/decorators.py +0 -0
  54. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/discovery.py +0 -0
  55. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/__init__.py +0 -0
  56. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/agent_rpc.py +0 -0
  57. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/graph.py +0 -0
  58. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/scheduler.py +0 -0
  59. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/sinks.py +0 -0
  60. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/sources.py +0 -0
  61. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/engine/types.py +0 -0
  62. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/exceptions.py +0 -0
  63. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/executor/CLAUDE.md +0 -0
  64. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/executor/__init__.py +0 -0
  65. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/executor/commit.py +0 -0
  66. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/explain.py +0 -0
  67. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/fingerprint.py +0 -0
  68. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/git.py +0 -0
  69. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/ignore.py +0 -0
  70. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/import_artifact.py +0 -0
  71. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/loaders.py +0 -0
  72. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/metrics.py +0 -0
  73. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/names.py +0 -0
  74. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/outputs.py +0 -0
  75. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/parameters.py +0 -0
  76. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/path_policy.py +0 -0
  77. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/project.py +0 -0
  78. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/py.typed +0 -0
  79. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/registry.py +0 -0
  80. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/AGENTS.md +0 -0
  81. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/__init__.py +0 -0
  82. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/config.py +0 -0
  83. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/git_archive.py +0 -0
  84. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/github.py +0 -0
  85. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/storage.py +0 -0
  86. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/remote/sync.py +0 -0
  87. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/run_history.py +0 -0
  88. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/show/__init__.py +0 -0
  89. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/show/common.py +0 -0
  90. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/show/data.py +0 -0
  91. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/show/metrics.py +0 -0
  92. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/show/params.py +0 -0
  93. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/show/plots.py +0 -0
  94. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/skip.py +0 -0
  95. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/stage_def.py +0 -0
  96. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/status.py +0 -0
  97. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/AGENTS.md +0 -0
  98. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/CLAUDE.md +0 -0
  99. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/__init__.py +0 -0
  100. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/artifact_lock.py +0 -0
  101. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/cache.py +0 -0
  102. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/lock.py +0 -0
  103. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/presentation.py +0 -0
  104. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/restore.py +0 -0
  105. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/state.py +0 -0
  106. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/store.py +0 -0
  107. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/storage/track.py +0 -0
  108. {pivot-0.3.6 → pivot-0.3.8}/src/pivot/yaml_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pivot
3
- Version: 0.3.6
3
+ Version: 0.3.8
4
4
  Summary: High-performance Python pipeline tool with automatic code change detection
5
5
  Author: Sami Jawhar
6
6
  Author-email: Sami Jawhar <sami@metr.org>
@@ -23,6 +23,7 @@ Requires-Dist: loky>=3.4
23
23
  Requires-Dist: networkx>=3.0
24
24
  Requires-Dist: pathspec>=0.12.0
25
25
  Requires-Dist: pandas>=2.0
26
+ Requires-Dist: psutil>=7.0
26
27
  Requires-Dist: pydantic>=2.13
27
28
  Requires-Dist: pygtrie>=2.5.0
28
29
  Requires-Dist: pyyaml>=6.0
@@ -55,6 +56,7 @@ Requires-Dist: pytest-xdist>=3.8.0 ; extra == 'dev'
55
56
  Requires-Dist: pytest>=7.0 ; extra == 'dev'
56
57
  Requires-Dist: ruff>=0.8.0 ; extra == 'dev'
57
58
  Requires-Dist: types-aioboto3[all]>=13.0.0 ; extra == 'dev'
59
+ Requires-Dist: types-psutil>=7.0 ; extra == 'dev'
58
60
  Requires-Dist: mkdocs>=1.6.0 ; extra == 'docs'
59
61
  Requires-Dist: mkdocs-material>=9.5.0 ; extra == 'docs'
60
62
  Requires-Dist: mkdocstrings[python]>=0.26.0 ; extra == 'docs'
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pivot"
3
- version = "0.3.6"
3
+ version = "0.3.8"
4
4
  description = "High-performance Python pipeline tool with automatic code change detection"
5
5
  requires-python = ">=3.13,<3.14"
6
6
  readme = "README.md"
@@ -27,6 +27,7 @@ dependencies = [
27
27
  "networkx>=3.0",
28
28
  "pathspec>=0.12.0",
29
29
  "pandas>=2.0",
30
+ "psutil>=7.0",
30
31
  "pydantic>=2.13",
31
32
  "pygtrie>=2.5.0",
32
33
  "pyyaml>=6.0",
@@ -76,6 +77,7 @@ dev = [
76
77
  "pytest>=7.0",
77
78
  "ruff>=0.8.0",
78
79
  "types-aioboto3[all]>=13.0.0",
80
+ "types-psutil>=7.0",
79
81
  ]
80
82
 
81
83
  [project.scripts]
@@ -380,6 +380,20 @@ class Engine:
380
380
  Closes send channels first to signal receivers, then receive channels.
381
381
  This ordering prevents ClosedResourceError in iterating tasks.
382
382
  """
383
+ if self._worker_pool is not None:
384
+ worker_pool = self._worker_pool
385
+ self._worker_pool = None
386
+ try:
387
+ with anyio.CancelScope(shield=True):
388
+ shutdown = (
389
+ worker_pool.hard_cancel
390
+ if _exc and _exc[0] is not None
391
+ else worker_pool.shutdown
392
+ )
393
+ await anyio.to_thread.run_sync(shutdown)
394
+ except (OSError, BrokenPipeError) as exc:
395
+ _logger.debug("WorkerPool shutdown failed (may already be dead): %s", exc)
396
+
383
397
  # Close send channels first to signal end-of-stream to receivers
384
398
  if self._input_send:
385
399
  await self._input_send.aclose()
@@ -1177,9 +1191,8 @@ class Engine:
1177
1191
  results=results,
1178
1192
  )
1179
1193
 
1180
- # Only start drain thread and main loop if workers were dispatched.
1181
- # When all stages skip in the coordinator, the pool is never created.
1182
- if self._worker_pool is not None:
1194
+ # Only start drain thread and main loop if workers were dispatched in this run.
1195
+ if self._worker_pool is not None and self._worker_pool.in_run:
1183
1196
  output_queue = self._worker_pool.output_queue()
1184
1197
  shutdown_event = self._worker_pool.shutdown_event()
1185
1198
 
@@ -1210,6 +1223,10 @@ class Engine:
1210
1223
 
1211
1224
  try:
1212
1225
  result = future.result()
1226
+ if "user_code_files" in result:
1227
+ self._worker_pool.record_user_code_files(
1228
+ result["user_code_files"]
1229
+ )
1213
1230
  duration_ms = await self._handle_stage_completion(
1214
1231
  stage_name,
1215
1232
  result,
@@ -1246,6 +1263,7 @@ class Engine:
1246
1263
 
1247
1264
  except Exception as e:
1248
1265
  _logger.exception("Stage %s failed with exception", stage_name)
1266
+ self._worker_pool.mark_workers_unverified()
1249
1267
  failed_result = StageResult(
1250
1268
  status=StageStatus.FAILED,
1251
1269
  reason=str(e),
@@ -1373,23 +1391,14 @@ class Engine:
1373
1391
  await self._finalize_cancelled_stages(results, run_id)
1374
1392
 
1375
1393
  finally:
1376
- # Close StateDBs first (lightweight), then shut down worker pool.
1394
+ # Close StateDBs first, then release this run's output queue after its drain exits.
1377
1395
  with anyio.CancelScope(shield=True):
1378
1396
  for db in state_dbs.values():
1379
1397
  db.close()
1380
1398
  state_dbs.clear()
1381
1399
 
1382
- # WorkerPool shutdown handles executor + manager cleanup.
1383
- # Must happen after sentinel send so drain thread exits first.
1384
- # Wrapped in outer finally to ensure it always executes, even if task group raises.
1385
- if self._worker_pool is not None:
1386
- worker_pool = self._worker_pool
1387
- self._worker_pool = None
1388
- try:
1389
- with anyio.CancelScope(shield=True):
1390
- await anyio.to_thread.run_sync(worker_pool.shutdown)
1391
- except (OSError, BrokenPipeError) as exc:
1392
- _logger.debug("WorkerPool shutdown failed (may already be dead): %s", exc)
1400
+ if self._worker_pool is not None and self._worker_pool.in_run:
1401
+ self._worker_pool.end_run()
1393
1402
 
1394
1403
  # Write run history after execution completes
1395
1404
  ended_at = datetime.datetime.now(datetime.UTC).isoformat()
@@ -1584,9 +1593,10 @@ class Engine:
1584
1593
  return self._scheduler.can_start(stage_name)
1585
1594
 
1586
1595
  def _ensure_worker_pool(self) -> worker_pool_mod.WorkerPool:
1587
- """Lazily create and start the worker pool on first worker dispatch."""
1596
+ """Lazily begin this run's worker pool on first worker dispatch."""
1588
1597
  if self._worker_pool is None:
1589
1598
  self._worker_pool = worker_pool_mod.WorkerPool()
1599
+ if not self._worker_pool.in_run:
1590
1600
  self._worker_pool.start(max_workers=self._effective_max_workers)
1591
1601
  return self._worker_pool
1592
1602
 
@@ -2524,8 +2534,8 @@ class Engine:
2524
2534
  if stages:
2525
2535
  # Restart worker pool so workers pick up reloaded code.
2526
2536
  # Run in thread to avoid blocking the event loop during process kill/spawn.
2527
- # Catch errors so a failed restart doesn't kill the watch session;
2528
- # create_executor() in _orchestrate_execution will retry pool creation.
2537
+ # Catch errors so dispatch-time source validation can rebuild the pool on its
2538
+ # next use.
2529
2539
  coordinator = self._get_watch_coordinator()
2530
2540
  if coordinator is not None and coordinator.should_restart_workers(
2531
2541
  parallel=self._stored_parallel
@@ -222,9 +222,9 @@ class WatchCoordinator:
222
222
  return engine_graph.get_producer(self._graph, identity)
223
223
 
224
224
  def should_restart_workers(self, *, parallel: bool) -> bool:
225
- """Decide whether worker pool should restart after code/config change.
225
+ """Decide whether an eager reload restart should pre-warm multiple workers.
226
226
 
227
- Workers should restart to pick up reloaded code, but only when
228
- running in parallel mode (sequential mode has no persistent pool).
227
+ A sequential Engine retains one worker too; its dispatch-time source guard replaces
228
+ that worker when needed.
229
229
  """
230
230
  return parallel
@@ -0,0 +1,256 @@
1
+ from __future__ import annotations
2
+
3
+ import contextlib
4
+ import errno
5
+ import logging
6
+ import os
7
+ import pathlib
8
+ import signal
9
+ import threading
10
+ import time
11
+ import typing
12
+ from dataclasses import dataclass, field
13
+
14
+ import xxhash
15
+
16
+ from pivot.executor import core as executor_core
17
+ from pivot.executor import worker
18
+
19
+ _T = typing.TypeVar("_T")
20
+
21
+ _logger = logging.getLogger(__name__)
22
+
23
+ # How long hard_cancel waits for a SIGKILLed worker to be reaped; it never blocks pivot's exit.
24
+ _KILLED_WORKER_JOIN_S = 5.0
25
+
26
+ _FileStamp = tuple[int, int, int, int, int]
27
+
28
+
29
+ def _file_stamp(path: str) -> _FileStamp | None:
30
+ """Return the stat signature and content digest of a user-code file, or None if unreadable.
31
+
32
+ The digest catches an edit that leaves every timestamp unchanged: a filesystem with coarse
33
+ timestamps, or one whose server clock lags, can stamp an edit with the time it already had.
34
+ Running out of file descriptors is not an unreadable file, so it propagates.
35
+ """
36
+ try:
37
+ stat = os.stat(path)
38
+ digest = xxhash.xxh64_intdigest(pathlib.Path(path).read_bytes())
39
+ except OSError as e:
40
+ if e.errno in (errno.EMFILE, errno.ENFILE):
41
+ raise
42
+ return None
43
+ return stat.st_mtime_ns, stat.st_size, stat.st_ino, stat.st_ctime_ns, digest
44
+
45
+
46
+ if typing.TYPE_CHECKING:
47
+ import concurrent.futures
48
+ import multiprocessing as mp
49
+ from multiprocessing.managers import SyncManager
50
+
51
+ import loky
52
+
53
+ from pivot.types import OutputMessage
54
+
55
+
56
+ @dataclass(slots=True)
57
+ class WorkerPool:
58
+ """Manage engine-lifetime workers and the per-run output queue.
59
+
60
+ Workers are reused only while every user-code file they may have imported keeps the stat
61
+ signature and content digest Pivot recorded before they imported it. Stamps are compared
62
+ only with stamps, never with a clock, so a file timestamp that lags the coordinator's clock
63
+ or does not move at all cannot hide an edit.
64
+ """
65
+
66
+ _executor: loky.ProcessPoolExecutor | None = None
67
+ _manager: SyncManager | None = None
68
+ _output_queue: mp.Queue[OutputMessage] | None = None
69
+ _shutdown_event: threading.Event | None = None
70
+ _accepting: bool = True
71
+ _in_run: bool = False
72
+ _max_workers: int = 0
73
+ _user_code_files: dict[str, _FileStamp | None] = field(default_factory=dict)
74
+ _code_changed: bool = False
75
+
76
+ def start(self, *, max_workers: int) -> None:
77
+ """Start a run without shrinking workers warmed by earlier runs."""
78
+ if self._in_run:
79
+ return
80
+ self._max_workers = max(self._max_workers, max_workers)
81
+ executor = typing.cast(
82
+ "loky.ProcessPoolExecutor", executor_core.create_executor(self._max_workers)
83
+ )
84
+ fresh_workers = executor is not self._executor
85
+ if self._executor is executor and self._has_stale_user_code():
86
+ executor_core.restart_workers(self._max_workers, self._max_workers)
87
+ executor = typing.cast(
88
+ "loky.ProcessPoolExecutor",
89
+ executor_core.create_executor(self._max_workers),
90
+ )
91
+ fresh_workers = True
92
+ if fresh_workers:
93
+ self._restamp_user_code_files()
94
+ self._stamp_coordinator_user_code()
95
+ self._executor = executor
96
+ self._output_queue = typing.cast("mp.Queue[OutputMessage]", self._new_output_queue())
97
+ self._shutdown_event = threading.Event()
98
+ self._accepting = True
99
+ self._in_run = True
100
+
101
+ def _new_output_queue(self) -> object:
102
+ """Return a fresh queue from the engine-lifetime manager, replacing it if unreachable."""
103
+ if self._manager is not None:
104
+ try:
105
+ return self._manager.Queue()
106
+ except (OSError, EOFError) as e:
107
+ _logger.warning(
108
+ "Output-queue manager unreachable between runs (%r); starting a new one", e
109
+ )
110
+ self._manager.shutdown()
111
+ self._manager = None
112
+ self._manager = executor_core.create_manager()
113
+ return self._manager.Queue()
114
+
115
+ @property
116
+ def in_run(self) -> bool:
117
+ """Whether this pool has a queue and accepts dispatches for the current run."""
118
+ return self._in_run
119
+
120
+ def record_user_code_files(self, paths: list[str]) -> None:
121
+ """Record the user-code files a worker reports having imported.
122
+
123
+ A file Pivot had not stamped before the worker imported it may have changed in between,
124
+ so its first report marks the workers stale; the next run replaces them after stamping
125
+ it. Files already stamped are compared when the next run starts.
126
+ """
127
+ for path in paths:
128
+ if path not in self._user_code_files:
129
+ self._user_code_files[path] = None
130
+ self._code_changed = True
131
+
132
+ def mark_workers_unverified(self) -> None:
133
+ """Replace the workers before the next run: a task ended without reporting its imports."""
134
+ self._code_changed = True
135
+
136
+ def _has_stale_user_code(self) -> bool:
137
+ return self._code_changed or any(
138
+ _file_stamp(path) != stamp for path, stamp in self._user_code_files.items()
139
+ )
140
+
141
+ def _restamp_user_code_files(self) -> None:
142
+ """Stamp every known user-code file before fresh workers can import it."""
143
+ self._user_code_files = {path: _file_stamp(path) for path in self._user_code_files}
144
+ self._code_changed = False
145
+
146
+ def _stamp_coordinator_user_code(self) -> None:
147
+ """Stamp the user code this process imported, before this run's workers can import it."""
148
+ for path in worker.user_code_files():
149
+ if path not in self._user_code_files:
150
+ self._user_code_files[path] = _file_stamp(path)
151
+
152
+ def output_queue(self) -> mp.Queue[OutputMessage]:
153
+ """Return the worker output queue."""
154
+ if self._output_queue is None:
155
+ raise RuntimeError("WorkerPool not started")
156
+ return self._output_queue
157
+
158
+ def shutdown_event(self) -> threading.Event:
159
+ """Return the shutdown event used by the drain thread."""
160
+ if self._shutdown_event is None:
161
+ raise RuntimeError("WorkerPool not started")
162
+ return self._shutdown_event
163
+
164
+ def submit(
165
+ self,
166
+ fn: typing.Callable[..., _T],
167
+ /,
168
+ *args: object,
169
+ **kwargs: object,
170
+ ) -> concurrent.futures.Future[_T]:
171
+ """Submit work to the underlying executor."""
172
+ if not self._accepting:
173
+ raise RuntimeError("WorkerPool is not accepting new submissions")
174
+ if self._executor is None:
175
+ raise RuntimeError("WorkerPool not started")
176
+ return self._executor.submit(fn, *args, **kwargs)
177
+
178
+ def stop_accepting(self) -> None:
179
+ """Stop accepting new submissions without shutting down workers."""
180
+ self._accepting = False
181
+
182
+ def end_run(self) -> None:
183
+ """Release this run's queue while keeping workers warm for the next run."""
184
+ self._accepting = False
185
+ self._output_queue = None
186
+ self._shutdown_event = None
187
+ self._in_run = False
188
+
189
+ def shutdown(self) -> None:
190
+ """Shut down workers and the manager when the owning Engine exits."""
191
+ self._accepting = False
192
+ self._in_run = False
193
+ if self._executor is not None:
194
+ self._executor.shutdown(wait=True)
195
+ self._executor = None
196
+ if self._manager is not None:
197
+ self._manager.shutdown()
198
+ self._manager = None
199
+ self._output_queue = None
200
+ self._shutdown_event = None
201
+
202
+ def hard_cancel(self) -> None:
203
+ """Stop workers mid-stage and signal the drain thread to stop.
204
+
205
+ Each worker is sent SIGTERM, which unwinds its running stage
206
+ (``worker.unwind_when_stopped``) so the stage's cleanup runs, and is given
207
+ ``worker.TERMINATION_GRACE_S`` to exit; a worker still alive then is killed with its
208
+ descendants.
209
+
210
+ The workers are first taken out of loky's executor. Its manager thread takes a worker
211
+ that exits without being asked to for a broken pool and SIGKILLs every worker the
212
+ executor still lists, so the first worker to exit would cut the others' cleanup off.
213
+ """
214
+ self._accepting = False
215
+ self._in_run = False
216
+ try:
217
+ if self._executor is not None:
218
+ executor = self._executor
219
+ self._executor = None
220
+ try:
221
+ self._stop_workers(executor)
222
+ finally:
223
+ # Only shutdown wakes loky's manager thread once its workers are gone.
224
+ executor.shutdown(wait=False, kill_workers=True)
225
+ finally:
226
+ if self._manager is not None:
227
+ self._manager.shutdown()
228
+ self._manager = None
229
+ if self._shutdown_event is not None:
230
+ self._shutdown_event.set()
231
+ self._output_queue = None
232
+ self._shutdown_event = None
233
+
234
+ @staticmethod
235
+ def _stop_workers(executor: loky.ProcessPoolExecutor) -> None:
236
+ """Take ``executor``'s workers out of it, send each SIGTERM, and SIGKILL with its
237
+ descendants any worker still alive after ``worker.TERMINATION_GRACE_S``."""
238
+ # loky exposes no public list of its worker processes. A watch reload may have
239
+ # replaced this executor already, which leaves it no lock and no workers.
240
+ with executor._processes_management_lock or contextlib.nullcontext():
241
+ processes = list(executor._processes.values())
242
+ executor._processes.clear()
243
+ for process in processes:
244
+ if process.pid is not None:
245
+ with contextlib.suppress(ProcessLookupError):
246
+ os.kill(process.pid, signal.SIGTERM)
247
+ deadline = time.monotonic() + worker.TERMINATION_GRACE_S
248
+ for process in processes:
249
+ process.join(max(0.0, deadline - time.monotonic()))
250
+ for process in processes:
251
+ if process.pid is not None and process.exitcode is None:
252
+ worker.kill_descendants(process.pid)
253
+ # loky's process object has no kill(); signal the pid directly.
254
+ with contextlib.suppress(ProcessLookupError):
255
+ os.kill(process.pid, signal.SIGKILL)
256
+ process.join(_KILLED_WORKER_JOIN_S)
@@ -1,6 +1,13 @@
1
1
  # Pivot Executor - Development Guidelines
2
2
 
3
- Workers execute in separate processes via `loky.get_reusable_executor()`.
3
+ Workers execute in separate processes via `loky.get_reusable_executor()`. An Engine keeps its
4
+ workers and manager alive across its runs; each run receives a fresh manager queue for output,
5
+ and a manager that cannot be reached when a run starts is replaced then.
6
+ The loky initializer (`worker.unwind_when_stopped`) and the manager's initializer
7
+ (`core._exit_when_parent_dies`) start the parent-death watcher, so a worker or manager exits
8
+ when its Pivot process dies; a worker first unwinds its running stage so its `finally` blocks
9
+ run, within `worker.TERMINATION_GRACE_S`. Keep `timeout=None` and
10
+ `reuse=False if kill_workers else "auto"` in the shared executor policy.
4
11
 
5
12
  ## WorkerStageInfo Contract
6
13
 
@@ -4,9 +4,12 @@ import atexit
4
4
  import contextlib
5
5
  import functools
6
6
  import logging
7
+ import multiprocessing as mp
7
8
  import os
8
9
  import pathlib
9
- from typing import TYPE_CHECKING, Literal, TypedDict, cast
10
+ import threading
11
+ from multiprocessing import managers
12
+ from typing import TYPE_CHECKING, Final, Literal, TypedDict, cast
10
13
 
11
14
  import loky
12
15
 
@@ -25,12 +28,29 @@ if TYPE_CHECKING:
25
28
 
26
29
  logger = logging.getLogger(__name__)
27
30
 
31
+ _WORKER_IDLE_TIMEOUT_S: Final[float | None] = None
32
+
28
33
 
29
34
  def _noop() -> None:
30
35
  """Module-level for pickling."""
31
36
  pass
32
37
 
33
38
 
39
+ def _exit_when_parent_dies(parent_pid: int) -> None:
40
+ """Exit the output-queue manager after its owning Pivot process dies.
41
+
42
+ Workers use ``worker.unwind_when_stopped`` instead, which lets the running stage unwind.
43
+ """
44
+
45
+ def _watch_parent() -> None:
46
+ poll = threading.Event()
47
+ while os.getppid() == parent_pid:
48
+ poll.wait(1)
49
+ os._exit(0)
50
+
51
+ threading.Thread(target=_watch_parent, daemon=True, name="pivot-parent-watch").start()
52
+
53
+
34
54
  def configured_worker_count(override: int | None = None) -> int:
35
55
  """The configured worker count: the CLI override, else ``core.max_workers``.
36
56
 
@@ -56,6 +76,27 @@ def _warm_workers(pool: loky.ProcessPoolExecutor, count: int) -> None:
56
76
  f.result()
57
77
 
58
78
 
79
+ def _get_executor(max_workers: int, *, kill_workers: bool = False) -> loky.ProcessPoolExecutor:
80
+ """Return Pivot's reusable executor with one worker-lifetime policy."""
81
+ _ensure_deterministic_environment()
82
+ _ensure_cleanup_registered()
83
+ return loky.get_reusable_executor(
84
+ max_workers=max_workers,
85
+ timeout=_WORKER_IDLE_TIMEOUT_S,
86
+ kill_workers=kill_workers,
87
+ reuse=False if kill_workers else "auto",
88
+ initializer=worker.unwind_when_stopped,
89
+ initargs=(os.getpid(),),
90
+ )
91
+
92
+
93
+ def create_manager() -> managers.SyncManager:
94
+ """Start a spawn-context manager that exits when this process dies."""
95
+ manager = managers.SyncManager(ctx=mp.get_context("spawn"))
96
+ manager.start(initializer=_exit_when_parent_dies, initargs=(os.getpid(),))
97
+ return manager
98
+
99
+
59
100
  def prepare_workers(
60
101
  stage_count: int, *, parallel: bool = True, max_workers: int | None = None
61
102
  ) -> int:
@@ -67,32 +108,25 @@ def prepare_workers(
67
108
  if not parallel or stage_count <= 0:
68
109
  return 1
69
110
  workers = compute_max_workers(stage_count, max_workers)
70
- _ensure_deterministic_environment()
71
- _ensure_cleanup_registered()
72
- pool = loky.get_reusable_executor(max_workers=workers)
73
- _warm_workers(pool, workers)
111
+ _warm_workers(_get_executor(workers), workers)
74
112
  return workers
75
113
 
76
114
 
77
115
  def restart_workers(stage_count: int, max_workers: int | None = None) -> int:
78
- """Kill existing workers and spawn fresh ones. For code reload in watch mode.
116
+ """Kill and warm fresh workers after code reload or stale-source detection."""
79
117
 
80
- Unlike prepare_workers(), this kills existing workers first, then warms the new pool.
81
- """
118
+ # A reusable executor with matching kwargs would otherwise keep live workers.
82
119
  if stage_count <= 0:
83
120
  return 1
84
121
  workers = compute_max_workers(stage_count, max_workers)
85
- _ensure_deterministic_environment()
86
- _ensure_cleanup_registered()
87
- pool = loky.get_reusable_executor(max_workers=workers, kill_workers=True)
88
- _warm_workers(pool, workers)
122
+ _warm_workers(_get_executor(workers, kill_workers=True), workers)
89
123
  return workers
90
124
 
91
125
 
92
126
  def _cleanup_worker_pool() -> None:
93
127
  """Kill loky worker pool on process exit to prevent orphaned workers."""
94
128
  with contextlib.suppress(Exception):
95
- loky.get_reusable_executor(max_workers=1, kill_workers=True)
129
+ _get_executor(1, kill_workers=True)
96
130
 
97
131
 
98
132
  @functools.cache # Ensures single atexit registration across threads
@@ -293,10 +327,8 @@ def _ensure_deterministic_environment() -> None:
293
327
 
294
328
 
295
329
  def create_executor(max_workers: int) -> concurrent.futures.Executor:
296
- """Get reusable loky executor - workers persist across calls for efficiency."""
297
- _ensure_deterministic_environment()
298
- _ensure_cleanup_registered()
299
- return loky.get_reusable_executor(max_workers=max_workers)
330
+ """Get Pivot's reusable executor, keeping its workers warm between runs."""
331
+ return _get_executor(max_workers)
300
332
 
301
333
 
302
334
  def prepare_worker_info(