pivot 0.3.6__tar.gz → 0.3.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. {pivot-0.3.6 → pivot-0.3.7}/PKG-INFO +1 -1
  2. {pivot-0.3.6 → pivot-0.3.7}/pyproject.toml +1 -1
  3. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/engine.py +28 -18
  4. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/watch.py +3 -3
  5. pivot-0.3.7/src/pivot/engine/worker_pool.py +209 -0
  6. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/AGENTS.md +6 -1
  7. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/core.py +46 -17
  8. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/worker.py +34 -1
  9. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/types.py +1 -0
  10. pivot-0.3.6/src/pivot/engine/worker_pool.py +0 -90
  11. {pivot-0.3.6 → pivot-0.3.7}/LICENSE +0 -0
  12. {pivot-0.3.6 → pivot-0.3.7}/README.md +0 -0
  13. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/__init__.py +0 -0
  14. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/ast_utils.py +0 -0
  15. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/AGENTS.md +0 -0
  16. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/CLAUDE.md +0 -0
  17. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/__init__.py +0 -0
  18. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/_run_common.py +0 -0
  19. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/check_ignore.py +0 -0
  20. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/checkout.py +0 -0
  21. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/commit.py +0 -0
  22. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/completion.py +0 -0
  23. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/config.py +0 -0
  24. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/console.py +0 -0
  25. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/dag.py +0 -0
  26. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/data.py +0 -0
  27. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/decorators.py +0 -0
  28. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/doctor.py +0 -0
  29. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/errors.py +0 -0
  30. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/fingerprint.py +0 -0
  31. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/helpers.py +0 -0
  32. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/history.py +0 -0
  33. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/import_cmd.py +0 -0
  34. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/init.py +0 -0
  35. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/list.py +0 -0
  36. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/metrics.py +0 -0
  37. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/params.py +0 -0
  38. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/plots.py +0 -0
  39. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/remote.py +0 -0
  40. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/repro.py +0 -0
  41. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/run.py +0 -0
  42. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/status.py +0 -0
  43. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/targets.py +0 -0
  44. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/track.py +0 -0
  45. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/update.py +0 -0
  46. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/verify.py +0 -0
  47. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/compose.py +0 -0
  48. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/config/__init__.py +0 -0
  49. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/config/io.py +0 -0
  50. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/config/models.py +0 -0
  51. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/dag/__init__.py +0 -0
  52. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/dag/render.py +0 -0
  53. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/decorators.py +0 -0
  54. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/discovery.py +0 -0
  55. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/__init__.py +0 -0
  56. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/agent_rpc.py +0 -0
  57. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/graph.py +0 -0
  58. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/scheduler.py +0 -0
  59. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/sinks.py +0 -0
  60. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/sources.py +0 -0
  61. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/types.py +0 -0
  62. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/exceptions.py +0 -0
  63. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/CLAUDE.md +0 -0
  64. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/__init__.py +0 -0
  65. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/commit.py +0 -0
  66. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/explain.py +0 -0
  67. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/fingerprint.py +0 -0
  68. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/git.py +0 -0
  69. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/ignore.py +0 -0
  70. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/import_artifact.py +0 -0
  71. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/loaders.py +0 -0
  72. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/metrics.py +0 -0
  73. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/names.py +0 -0
  74. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/outputs.py +0 -0
  75. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/parameters.py +0 -0
  76. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/path_policy.py +0 -0
  77. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/project.py +0 -0
  78. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/py.typed +0 -0
  79. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/registry.py +0 -0
  80. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/AGENTS.md +0 -0
  81. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/__init__.py +0 -0
  82. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/config.py +0 -0
  83. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/git_archive.py +0 -0
  84. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/github.py +0 -0
  85. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/storage.py +0 -0
  86. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/sync.py +0 -0
  87. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/run_history.py +0 -0
  88. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/__init__.py +0 -0
  89. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/common.py +0 -0
  90. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/data.py +0 -0
  91. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/metrics.py +0 -0
  92. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/params.py +0 -0
  93. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/plots.py +0 -0
  94. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/skip.py +0 -0
  95. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/stage_def.py +0 -0
  96. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/status.py +0 -0
  97. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/AGENTS.md +0 -0
  98. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/CLAUDE.md +0 -0
  99. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/__init__.py +0 -0
  100. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/artifact_lock.py +0 -0
  101. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/cache.py +0 -0
  102. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/lock.py +0 -0
  103. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/presentation.py +0 -0
  104. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/restore.py +0 -0
  105. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/state.py +0 -0
  106. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/store.py +0 -0
  107. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/track.py +0 -0
  108. {pivot-0.3.6 → pivot-0.3.7}/src/pivot/yaml_config.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pivot
3
- Version: 0.3.6
3
+ Version: 0.3.7
4
4
  Summary: High-performance Python pipeline tool with automatic code change detection
5
5
  Author: Sami Jawhar
6
6
  Author-email: Sami Jawhar <sami@metr.org>
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "pivot"
3
- version = "0.3.6"
3
+ version = "0.3.7"
4
4
  description = "High-performance Python pipeline tool with automatic code change detection"
5
5
  requires-python = ">=3.13,<3.14"
6
6
  readme = "README.md"
@@ -380,6 +380,20 @@ class Engine:
380
380
  Closes send channels first to signal receivers, then receive channels.
381
381
  This ordering prevents ClosedResourceError in iterating tasks.
382
382
  """
383
+ if self._worker_pool is not None:
384
+ worker_pool = self._worker_pool
385
+ self._worker_pool = None
386
+ try:
387
+ with anyio.CancelScope(shield=True):
388
+ shutdown = (
389
+ worker_pool.hard_cancel
390
+ if _exc and _exc[0] is not None
391
+ else worker_pool.shutdown
392
+ )
393
+ await anyio.to_thread.run_sync(shutdown)
394
+ except (OSError, BrokenPipeError) as exc:
395
+ _logger.debug("WorkerPool shutdown failed (may already be dead): %s", exc)
396
+
383
397
  # Close send channels first to signal end-of-stream to receivers
384
398
  if self._input_send:
385
399
  await self._input_send.aclose()
@@ -1177,9 +1191,8 @@ class Engine:
1177
1191
  results=results,
1178
1192
  )
1179
1193
 
1180
- # Only start drain thread and main loop if workers were dispatched.
1181
- # When all stages skip in the coordinator, the pool is never created.
1182
- if self._worker_pool is not None:
1194
+ # Only start drain thread and main loop if workers were dispatched in this run.
1195
+ if self._worker_pool is not None and self._worker_pool.in_run:
1183
1196
  output_queue = self._worker_pool.output_queue()
1184
1197
  shutdown_event = self._worker_pool.shutdown_event()
1185
1198
 
@@ -1210,6 +1223,10 @@ class Engine:
1210
1223
 
1211
1224
  try:
1212
1225
  result = future.result()
1226
+ if "user_code_files" in result:
1227
+ self._worker_pool.record_user_code_files(
1228
+ result["user_code_files"]
1229
+ )
1213
1230
  duration_ms = await self._handle_stage_completion(
1214
1231
  stage_name,
1215
1232
  result,
@@ -1246,6 +1263,7 @@ class Engine:
1246
1263
 
1247
1264
  except Exception as e:
1248
1265
  _logger.exception("Stage %s failed with exception", stage_name)
1266
+ self._worker_pool.mark_workers_unverified()
1249
1267
  failed_result = StageResult(
1250
1268
  status=StageStatus.FAILED,
1251
1269
  reason=str(e),
@@ -1373,23 +1391,14 @@ class Engine:
1373
1391
  await self._finalize_cancelled_stages(results, run_id)
1374
1392
 
1375
1393
  finally:
1376
- # Close StateDBs first (lightweight), then shut down worker pool.
1394
+ # Close StateDBs first, then release this run's output queue after its drain exits.
1377
1395
  with anyio.CancelScope(shield=True):
1378
1396
  for db in state_dbs.values():
1379
1397
  db.close()
1380
1398
  state_dbs.clear()
1381
1399
 
1382
- # WorkerPool shutdown handles executor + manager cleanup.
1383
- # Must happen after sentinel send so drain thread exits first.
1384
- # Wrapped in outer finally to ensure it always executes, even if task group raises.
1385
- if self._worker_pool is not None:
1386
- worker_pool = self._worker_pool
1387
- self._worker_pool = None
1388
- try:
1389
- with anyio.CancelScope(shield=True):
1390
- await anyio.to_thread.run_sync(worker_pool.shutdown)
1391
- except (OSError, BrokenPipeError) as exc:
1392
- _logger.debug("WorkerPool shutdown failed (may already be dead): %s", exc)
1400
+ if self._worker_pool is not None and self._worker_pool.in_run:
1401
+ self._worker_pool.end_run()
1393
1402
 
1394
1403
  # Write run history after execution completes
1395
1404
  ended_at = datetime.datetime.now(datetime.UTC).isoformat()
@@ -1584,9 +1593,10 @@ class Engine:
1584
1593
  return self._scheduler.can_start(stage_name)
1585
1594
 
1586
1595
  def _ensure_worker_pool(self) -> worker_pool_mod.WorkerPool:
1587
- """Lazily create and start the worker pool on first worker dispatch."""
1596
+ """Lazily begin this run's worker pool on first worker dispatch."""
1588
1597
  if self._worker_pool is None:
1589
1598
  self._worker_pool = worker_pool_mod.WorkerPool()
1599
+ if not self._worker_pool.in_run:
1590
1600
  self._worker_pool.start(max_workers=self._effective_max_workers)
1591
1601
  return self._worker_pool
1592
1602
 
@@ -2524,8 +2534,8 @@ class Engine:
2524
2534
  if stages:
2525
2535
  # Restart worker pool so workers pick up reloaded code.
2526
2536
  # Run in thread to avoid blocking the event loop during process kill/spawn.
2527
- # Catch errors so a failed restart doesn't kill the watch session;
2528
- # create_executor() in _orchestrate_execution will retry pool creation.
2537
+ # Catch errors so dispatch-time source validation can rebuild the pool on its
2538
+ # next use.
2529
2539
  coordinator = self._get_watch_coordinator()
2530
2540
  if coordinator is not None and coordinator.should_restart_workers(
2531
2541
  parallel=self._stored_parallel
@@ -222,9 +222,9 @@ class WatchCoordinator:
222
222
  return engine_graph.get_producer(self._graph, identity)
223
223
 
224
224
  def should_restart_workers(self, *, parallel: bool) -> bool:
225
- """Decide whether worker pool should restart after code/config change.
225
+ """Decide whether an eager reload restart should pre-warm multiple workers.
226
226
 
227
- Workers should restart to pick up reloaded code, but only when
228
- running in parallel mode (sequential mode has no persistent pool).
227
+ A sequential Engine retains one worker too; its dispatch-time source guard replaces
228
+ that worker when needed.
229
229
  """
230
230
  return parallel
@@ -0,0 +1,209 @@
1
+ from __future__ import annotations
2
+
3
+ import errno
4
+ import logging
5
+ import os
6
+ import pathlib
7
+ import threading
8
+ import typing
9
+ from dataclasses import dataclass, field
10
+
11
+ import xxhash
12
+
13
+ from pivot.executor import core as executor_core
14
+ from pivot.executor import worker
15
+
16
+ _T = typing.TypeVar("_T")
17
+
18
+ _logger = logging.getLogger(__name__)
19
+
20
+ _FileStamp = tuple[int, int, int, int, int]
21
+
22
+
23
+ def _file_stamp(path: str) -> _FileStamp | None:
24
+ """Return the stat signature and content digest of a user-code file, or None if unreadable.
25
+
26
+ The digest catches an edit that leaves every timestamp unchanged: a filesystem with coarse
27
+ timestamps, or one whose server clock lags, can stamp an edit with the time it already had.
28
+ Running out of file descriptors is not an unreadable file, so it propagates.
29
+ """
30
+ try:
31
+ stat = os.stat(path)
32
+ digest = xxhash.xxh64_intdigest(pathlib.Path(path).read_bytes())
33
+ except OSError as e:
34
+ if e.errno in (errno.EMFILE, errno.ENFILE):
35
+ raise
36
+ return None
37
+ return stat.st_mtime_ns, stat.st_size, stat.st_ino, stat.st_ctime_ns, digest
38
+
39
+
40
+ if typing.TYPE_CHECKING:
41
+ import concurrent.futures
42
+ import multiprocessing as mp
43
+ from multiprocessing.managers import SyncManager
44
+
45
+ import loky
46
+
47
+ from pivot.types import OutputMessage
48
+
49
+
50
+ @dataclass(slots=True)
51
+ class WorkerPool:
52
+ """Manage engine-lifetime workers and the per-run output queue.
53
+
54
+ Workers are reused only while every user-code file they may have imported keeps the stat
55
+ signature and content digest Pivot recorded before they imported it. Stamps are compared
56
+ only with stamps, never with a clock, so a file timestamp that lags the coordinator's clock
57
+ or does not move at all cannot hide an edit.
58
+ """
59
+
60
+ _executor: loky.ProcessPoolExecutor | None = None
61
+ _manager: SyncManager | None = None
62
+ _output_queue: mp.Queue[OutputMessage] | None = None
63
+ _shutdown_event: threading.Event | None = None
64
+ _accepting: bool = True
65
+ _in_run: bool = False
66
+ _max_workers: int = 0
67
+ _user_code_files: dict[str, _FileStamp | None] = field(default_factory=dict)
68
+ _code_changed: bool = False
69
+
70
+ def start(self, *, max_workers: int) -> None:
71
+ """Start a run without shrinking workers warmed by earlier runs."""
72
+ if self._in_run:
73
+ return
74
+ self._max_workers = max(self._max_workers, max_workers)
75
+ executor = typing.cast(
76
+ "loky.ProcessPoolExecutor", executor_core.create_executor(self._max_workers)
77
+ )
78
+ fresh_workers = executor is not self._executor
79
+ if self._executor is executor and self._has_stale_user_code():
80
+ executor_core.restart_workers(self._max_workers, self._max_workers)
81
+ executor = typing.cast(
82
+ "loky.ProcessPoolExecutor",
83
+ executor_core.create_executor(self._max_workers),
84
+ )
85
+ fresh_workers = True
86
+ if fresh_workers:
87
+ self._restamp_user_code_files()
88
+ self._stamp_coordinator_user_code()
89
+ self._executor = executor
90
+ self._output_queue = typing.cast("mp.Queue[OutputMessage]", self._new_output_queue())
91
+ self._shutdown_event = threading.Event()
92
+ self._accepting = True
93
+ self._in_run = True
94
+
95
+ def _new_output_queue(self) -> object:
96
+ """Return a fresh queue from the engine-lifetime manager, replacing it if unreachable."""
97
+ if self._manager is not None:
98
+ try:
99
+ return self._manager.Queue()
100
+ except (OSError, EOFError) as e:
101
+ _logger.warning(
102
+ "Output-queue manager unreachable between runs (%r); starting a new one", e
103
+ )
104
+ self._manager.shutdown()
105
+ self._manager = None
106
+ self._manager = executor_core.create_manager()
107
+ return self._manager.Queue()
108
+
109
+ @property
110
+ def in_run(self) -> bool:
111
+ """Whether this pool has a queue and accepts dispatches for the current run."""
112
+ return self._in_run
113
+
114
+ def record_user_code_files(self, paths: list[str]) -> None:
115
+ """Record the user-code files a worker reports having imported.
116
+
117
+ A file Pivot had not stamped before the worker imported it may have changed in between,
118
+ so its first report marks the workers stale; the next run replaces them after stamping
119
+ it. Files already stamped are compared when the next run starts.
120
+ """
121
+ for path in paths:
122
+ if path not in self._user_code_files:
123
+ self._user_code_files[path] = None
124
+ self._code_changed = True
125
+
126
+ def mark_workers_unverified(self) -> None:
127
+ """Replace the workers before the next run: a task ended without reporting its imports."""
128
+ self._code_changed = True
129
+
130
+ def _has_stale_user_code(self) -> bool:
131
+ return self._code_changed or any(
132
+ _file_stamp(path) != stamp for path, stamp in self._user_code_files.items()
133
+ )
134
+
135
+ def _restamp_user_code_files(self) -> None:
136
+ """Stamp every known user-code file before fresh workers can import it."""
137
+ self._user_code_files = {path: _file_stamp(path) for path in self._user_code_files}
138
+ self._code_changed = False
139
+
140
+ def _stamp_coordinator_user_code(self) -> None:
141
+ """Stamp the user code this process imported, before this run's workers can import it."""
142
+ for path in worker.user_code_files():
143
+ if path not in self._user_code_files:
144
+ self._user_code_files[path] = _file_stamp(path)
145
+
146
+ def output_queue(self) -> mp.Queue[OutputMessage]:
147
+ """Return the worker output queue."""
148
+ if self._output_queue is None:
149
+ raise RuntimeError("WorkerPool not started")
150
+ return self._output_queue
151
+
152
+ def shutdown_event(self) -> threading.Event:
153
+ """Return the shutdown event used by the drain thread."""
154
+ if self._shutdown_event is None:
155
+ raise RuntimeError("WorkerPool not started")
156
+ return self._shutdown_event
157
+
158
+ def submit(
159
+ self,
160
+ fn: typing.Callable[..., _T],
161
+ /,
162
+ *args: object,
163
+ **kwargs: object,
164
+ ) -> concurrent.futures.Future[_T]:
165
+ """Submit work to the underlying executor."""
166
+ if not self._accepting:
167
+ raise RuntimeError("WorkerPool is not accepting new submissions")
168
+ if self._executor is None:
169
+ raise RuntimeError("WorkerPool not started")
170
+ return self._executor.submit(fn, *args, **kwargs)
171
+
172
+ def stop_accepting(self) -> None:
173
+ """Stop accepting new submissions without shutting down workers."""
174
+ self._accepting = False
175
+
176
+ def end_run(self) -> None:
177
+ """Release this run's queue while keeping workers warm for the next run."""
178
+ self._accepting = False
179
+ self._output_queue = None
180
+ self._shutdown_event = None
181
+ self._in_run = False
182
+
183
+ def shutdown(self) -> None:
184
+ """Shut down workers and the manager when the owning Engine exits."""
185
+ self._accepting = False
186
+ self._in_run = False
187
+ if self._executor is not None:
188
+ self._executor.shutdown(wait=True)
189
+ self._executor = None
190
+ if self._manager is not None:
191
+ self._manager.shutdown()
192
+ self._manager = None
193
+ self._output_queue = None
194
+ self._shutdown_event = None
195
+
196
+ def hard_cancel(self) -> None:
197
+ """Forcefully shut down workers and signal drain thread to stop."""
198
+ self._accepting = False
199
+ self._in_run = False
200
+ if self._executor is not None:
201
+ self._executor.shutdown(wait=False, kill_workers=True)
202
+ self._executor = None
203
+ if self._manager is not None:
204
+ self._manager.shutdown()
205
+ self._manager = None
206
+ if self._shutdown_event is not None:
207
+ self._shutdown_event.set()
208
+ self._output_queue = None
209
+ self._shutdown_event = None
@@ -1,6 +1,11 @@
1
1
  # Pivot Executor - Development Guidelines
2
2
 
3
- Workers execute in separate processes via `loky.get_reusable_executor()`.
3
+ Workers execute in separate processes via `loky.get_reusable_executor()`. An Engine keeps its
4
+ workers and manager alive across its runs; each run receives a fresh manager queue for output,
5
+ and a manager that cannot be reached when a run starts is replaced then.
6
+ The loky initializer and the manager's initializer start the parent-death watcher, so a
7
+ worker or manager exits when its Pivot process dies. Keep `timeout=None` and
8
+ `reuse=False if kill_workers else "auto"` in the shared executor policy.
4
9
 
5
10
  ## WorkerStageInfo Contract
6
11
 
@@ -4,9 +4,12 @@ import atexit
4
4
  import contextlib
5
5
  import functools
6
6
  import logging
7
+ import multiprocessing as mp
7
8
  import os
8
9
  import pathlib
9
- from typing import TYPE_CHECKING, Literal, TypedDict, cast
10
+ import threading
11
+ from multiprocessing import managers
12
+ from typing import TYPE_CHECKING, Final, Literal, TypedDict, cast
10
13
 
11
14
  import loky
12
15
 
@@ -25,12 +28,26 @@ if TYPE_CHECKING:
25
28
 
26
29
  logger = logging.getLogger(__name__)
27
30
 
31
+ _WORKER_IDLE_TIMEOUT_S: Final[float | None] = None
32
+
28
33
 
29
34
  def _noop() -> None:
30
35
  """Module-level for pickling."""
31
36
  pass
32
37
 
33
38
 
39
+ def _exit_when_parent_dies(parent_pid: int) -> None:
40
+ """Exit a long-lived child (worker or manager) after its owning Pivot process dies."""
41
+
42
+ def _watch_parent() -> None:
43
+ poll = threading.Event()
44
+ while os.getppid() == parent_pid:
45
+ poll.wait(1)
46
+ os._exit(0)
47
+
48
+ threading.Thread(target=_watch_parent, daemon=True, name="pivot-parent-watch").start()
49
+
50
+
34
51
  def configured_worker_count(override: int | None = None) -> int:
35
52
  """The configured worker count: the CLI override, else ``core.max_workers``.
36
53
 
@@ -56,6 +73,27 @@ def _warm_workers(pool: loky.ProcessPoolExecutor, count: int) -> None:
56
73
  f.result()
57
74
 
58
75
 
76
+ def _get_executor(max_workers: int, *, kill_workers: bool = False) -> loky.ProcessPoolExecutor:
77
+ """Return Pivot's reusable executor with one worker-lifetime policy."""
78
+ _ensure_deterministic_environment()
79
+ _ensure_cleanup_registered()
80
+ return loky.get_reusable_executor(
81
+ max_workers=max_workers,
82
+ timeout=_WORKER_IDLE_TIMEOUT_S,
83
+ kill_workers=kill_workers,
84
+ reuse=False if kill_workers else "auto",
85
+ initializer=_exit_when_parent_dies,
86
+ initargs=(os.getpid(),),
87
+ )
88
+
89
+
90
+ def create_manager() -> managers.SyncManager:
91
+ """Start a spawn-context manager that exits when this process dies."""
92
+ manager = managers.SyncManager(ctx=mp.get_context("spawn"))
93
+ manager.start(initializer=_exit_when_parent_dies, initargs=(os.getpid(),))
94
+ return manager
95
+
96
+
59
97
  def prepare_workers(
60
98
  stage_count: int, *, parallel: bool = True, max_workers: int | None = None
61
99
  ) -> int:
@@ -67,32 +105,25 @@ def prepare_workers(
67
105
  if not parallel or stage_count <= 0:
68
106
  return 1
69
107
  workers = compute_max_workers(stage_count, max_workers)
70
- _ensure_deterministic_environment()
71
- _ensure_cleanup_registered()
72
- pool = loky.get_reusable_executor(max_workers=workers)
73
- _warm_workers(pool, workers)
108
+ _warm_workers(_get_executor(workers), workers)
74
109
  return workers
75
110
 
76
111
 
77
112
  def restart_workers(stage_count: int, max_workers: int | None = None) -> int:
78
- """Kill existing workers and spawn fresh ones. For code reload in watch mode.
113
+ """Kill and warm fresh workers after code reload or stale-source detection."""
79
114
 
80
- Unlike prepare_workers(), this kills existing workers first, then warms the new pool.
81
- """
115
+ # A reusable executor with matching kwargs would otherwise keep live workers.
82
116
  if stage_count <= 0:
83
117
  return 1
84
118
  workers = compute_max_workers(stage_count, max_workers)
85
- _ensure_deterministic_environment()
86
- _ensure_cleanup_registered()
87
- pool = loky.get_reusable_executor(max_workers=workers, kill_workers=True)
88
- _warm_workers(pool, workers)
119
+ _warm_workers(_get_executor(workers, kill_workers=True), workers)
89
120
  return workers
90
121
 
91
122
 
92
123
  def _cleanup_worker_pool() -> None:
93
124
  """Kill loky worker pool on process exit to prevent orphaned workers."""
94
125
  with contextlib.suppress(Exception):
95
- loky.get_reusable_executor(max_workers=1, kill_workers=True)
126
+ _get_executor(1, kill_workers=True)
96
127
 
97
128
 
98
129
  @functools.cache # Ensures single atexit registration across threads
@@ -293,10 +324,8 @@ def _ensure_deterministic_environment() -> None:
293
324
 
294
325
 
295
326
  def create_executor(max_workers: int) -> concurrent.futures.Executor:
296
- """Get reusable loky executor - workers persist across calls for efficiency."""
297
- _ensure_deterministic_environment()
298
- _ensure_cleanup_registered()
299
- return loky.get_reusable_executor(max_workers=max_workers)
327
+ """Get Pivot's reusable executor, keeping its workers warm between runs."""
328
+ return _get_executor(max_workers)
300
329
 
301
330
 
302
331
  def prepare_worker_info(
@@ -26,6 +26,7 @@ import pydantic
26
26
  from pivot import (
27
27
  config,
28
28
  exceptions,
29
+ fingerprint,
29
30
  loaders,
30
31
  metrics,
31
32
  parameters,
@@ -178,7 +179,7 @@ def _make_result(
178
179
  return result
179
180
 
180
181
 
181
- def execute_stage(
182
+ def _execute_stage(
182
183
  stage_name: str,
183
184
  stage_info: WorkerStageInfo,
184
185
  cache_dir: pathlib.Path,
@@ -488,6 +489,38 @@ def execute_stage(
488
489
  )
489
490
 
490
491
 
492
+ def user_code_files() -> list[str]:
493
+ """Return source paths for user modules this process has loaded.
494
+
495
+ A path that was deleted after import is still reported: the coordinator must restart
496
+ instead of letting the worker keep executing the cached module.
497
+ """
498
+ files = set[str]()
499
+ for module in tuple(sys.modules.values()):
500
+ if not fingerprint.is_user_code(module):
501
+ continue
502
+ module_file = getattr(module, "__file__", None)
503
+ if not isinstance(module_file, str):
504
+ continue
505
+ try:
506
+ files.add(str(pathlib.Path(module_file).resolve(strict=False)))
507
+ except OSError:
508
+ files.add(str(pathlib.Path(module_file).absolute()))
509
+ return sorted(files)
510
+
511
+
512
+ def execute_stage(
513
+ stage_name: str,
514
+ stage_info: WorkerStageInfo,
515
+ cache_dir: pathlib.Path,
516
+ output_queue: Queue[OutputMessage],
517
+ ) -> StageResult:
518
+ """Execute a stage and report every user-code module resident in this worker."""
519
+ result = _execute_stage(stage_name, stage_info, cache_dir, output_queue)
520
+ result["user_code_files"] = user_code_files()
521
+ return result
522
+
523
+
491
524
  def _to_identity_hashes(hashes: dict[str, HashInfo]) -> dict[types.ArtifactIdentity, HashInfo]:
492
525
  return {types.identity_from_key(key): info for key, info in hashes.items()}
493
526
 
@@ -172,6 +172,7 @@ class StageResult(TypedDict):
172
172
  output_lines: list[tuple[str, bool]]
173
173
  metrics: NotRequired[list[tuple[str, float]]] # (name, duration_ms) for cross-process
174
174
  deferred_writes: NotRequired[DeferredWrites]
175
+ user_code_files: NotRequired[list[str]] # Absolute source paths workers imported
175
176
 
176
177
 
177
178
  # =============================================================================
@@ -1,90 +0,0 @@
1
- from __future__ import annotations
2
-
3
- import multiprocessing as mp
4
- import threading
5
- import typing
6
- from dataclasses import dataclass
7
-
8
- from pivot.executor import core as executor_core
9
-
10
- _T = typing.TypeVar("_T")
11
-
12
- if typing.TYPE_CHECKING:
13
- import concurrent.futures
14
- from multiprocessing.managers import SyncManager
15
-
16
- import loky
17
-
18
- from pivot.types import OutputMessage
19
-
20
-
21
- @dataclass(slots=True)
22
- class WorkerPool:
23
- """Manage worker executor, manager queue, and shutdown signaling."""
24
-
25
- _executor: loky.ProcessPoolExecutor | None = None
26
- _manager: SyncManager | None = None
27
- _output_queue: mp.Queue[OutputMessage] | None = None
28
- _shutdown_event: threading.Event | None = None
29
- _accepting: bool = True
30
-
31
- def start(self, *, max_workers: int) -> None:
32
- """Initialize executor, manager, output queue, and shutdown event."""
33
- self._executor = typing.cast(
34
- "loky.ProcessPoolExecutor", executor_core.create_executor(max_workers)
35
- )
36
- spawn_ctx = mp.get_context("spawn")
37
- self._manager = spawn_ctx.Manager()
38
- raw_queue = self._manager.Queue()
39
- queue_obj = typing.cast("object", raw_queue)
40
- self._output_queue = typing.cast("mp.Queue[OutputMessage]", queue_obj)
41
- self._shutdown_event = threading.Event()
42
- self._accepting = True
43
-
44
- def output_queue(self) -> mp.Queue[OutputMessage]:
45
- """Return the worker output queue."""
46
- if self._output_queue is None:
47
- raise RuntimeError("WorkerPool not started")
48
- return self._output_queue
49
-
50
- def shutdown_event(self) -> threading.Event:
51
- """Return the shutdown event used by the drain thread."""
52
- if self._shutdown_event is None:
53
- raise RuntimeError("WorkerPool not started")
54
- return self._shutdown_event
55
-
56
- def submit(
57
- self,
58
- fn: typing.Callable[..., _T],
59
- /,
60
- *args: object,
61
- **kwargs: object,
62
- ) -> concurrent.futures.Future[_T]:
63
- """Submit work to the underlying executor."""
64
- if not self._accepting:
65
- raise RuntimeError("WorkerPool is not accepting new submissions")
66
- if self._executor is None:
67
- raise RuntimeError("WorkerPool not started")
68
- return self._executor.submit(fn, *args, **kwargs)
69
-
70
- def stop_accepting(self) -> None:
71
- """Stop accepting new submissions without shutting down workers."""
72
- self._accepting = False
73
-
74
- def shutdown(self) -> None:
75
- """Shut down executor and manager, waiting for completion."""
76
- self._accepting = False
77
- if self._executor is not None:
78
- self._executor.shutdown(wait=True)
79
- if self._manager is not None:
80
- self._manager.shutdown()
81
-
82
- def hard_cancel(self) -> None:
83
- """Forcefully shut down workers and signal drain thread to stop."""
84
- self._accepting = False
85
- if self._executor is not None:
86
- self._executor.shutdown(wait=False, kill_workers=True)
87
- if self._manager is not None:
88
- self._manager.shutdown()
89
- if self._shutdown_event is not None:
90
- self._shutdown_event.set()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes