pivot 0.3.6__tar.gz → 0.3.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pivot-0.3.6 → pivot-0.3.7}/PKG-INFO +1 -1
- {pivot-0.3.6 → pivot-0.3.7}/pyproject.toml +1 -1
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/engine.py +28 -18
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/watch.py +3 -3
- pivot-0.3.7/src/pivot/engine/worker_pool.py +209 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/AGENTS.md +6 -1
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/core.py +46 -17
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/worker.py +34 -1
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/types.py +1 -0
- pivot-0.3.6/src/pivot/engine/worker_pool.py +0 -90
- {pivot-0.3.6 → pivot-0.3.7}/LICENSE +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/README.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/ast_utils.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/AGENTS.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/CLAUDE.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/_run_common.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/check_ignore.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/checkout.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/commit.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/completion.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/config.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/console.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/dag.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/data.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/decorators.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/doctor.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/errors.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/fingerprint.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/helpers.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/history.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/import_cmd.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/init.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/list.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/metrics.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/params.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/plots.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/remote.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/repro.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/run.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/status.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/targets.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/track.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/update.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/cli/verify.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/compose.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/config/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/config/io.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/config/models.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/dag/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/dag/render.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/decorators.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/discovery.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/agent_rpc.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/graph.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/scheduler.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/sinks.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/sources.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/engine/types.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/exceptions.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/CLAUDE.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/executor/commit.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/explain.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/fingerprint.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/git.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/ignore.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/import_artifact.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/loaders.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/metrics.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/names.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/outputs.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/parameters.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/path_policy.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/project.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/py.typed +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/registry.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/AGENTS.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/config.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/git_archive.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/github.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/storage.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/remote/sync.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/run_history.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/common.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/data.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/metrics.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/params.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/show/plots.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/skip.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/stage_def.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/status.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/AGENTS.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/CLAUDE.md +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/__init__.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/artifact_lock.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/cache.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/lock.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/presentation.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/restore.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/state.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/store.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/storage/track.py +0 -0
- {pivot-0.3.6 → pivot-0.3.7}/src/pivot/yaml_config.py +0 -0
|
@@ -380,6 +380,20 @@ class Engine:
|
|
|
380
380
|
Closes send channels first to signal receivers, then receive channels.
|
|
381
381
|
This ordering prevents ClosedResourceError in iterating tasks.
|
|
382
382
|
"""
|
|
383
|
+
if self._worker_pool is not None:
|
|
384
|
+
worker_pool = self._worker_pool
|
|
385
|
+
self._worker_pool = None
|
|
386
|
+
try:
|
|
387
|
+
with anyio.CancelScope(shield=True):
|
|
388
|
+
shutdown = (
|
|
389
|
+
worker_pool.hard_cancel
|
|
390
|
+
if _exc and _exc[0] is not None
|
|
391
|
+
else worker_pool.shutdown
|
|
392
|
+
)
|
|
393
|
+
await anyio.to_thread.run_sync(shutdown)
|
|
394
|
+
except (OSError, BrokenPipeError) as exc:
|
|
395
|
+
_logger.debug("WorkerPool shutdown failed (may already be dead): %s", exc)
|
|
396
|
+
|
|
383
397
|
# Close send channels first to signal end-of-stream to receivers
|
|
384
398
|
if self._input_send:
|
|
385
399
|
await self._input_send.aclose()
|
|
@@ -1177,9 +1191,8 @@ class Engine:
|
|
|
1177
1191
|
results=results,
|
|
1178
1192
|
)
|
|
1179
1193
|
|
|
1180
|
-
# Only start drain thread and main loop if workers were dispatched.
|
|
1181
|
-
|
|
1182
|
-
if self._worker_pool is not None:
|
|
1194
|
+
# Only start drain thread and main loop if workers were dispatched in this run.
|
|
1195
|
+
if self._worker_pool is not None and self._worker_pool.in_run:
|
|
1183
1196
|
output_queue = self._worker_pool.output_queue()
|
|
1184
1197
|
shutdown_event = self._worker_pool.shutdown_event()
|
|
1185
1198
|
|
|
@@ -1210,6 +1223,10 @@ class Engine:
|
|
|
1210
1223
|
|
|
1211
1224
|
try:
|
|
1212
1225
|
result = future.result()
|
|
1226
|
+
if "user_code_files" in result:
|
|
1227
|
+
self._worker_pool.record_user_code_files(
|
|
1228
|
+
result["user_code_files"]
|
|
1229
|
+
)
|
|
1213
1230
|
duration_ms = await self._handle_stage_completion(
|
|
1214
1231
|
stage_name,
|
|
1215
1232
|
result,
|
|
@@ -1246,6 +1263,7 @@ class Engine:
|
|
|
1246
1263
|
|
|
1247
1264
|
except Exception as e:
|
|
1248
1265
|
_logger.exception("Stage %s failed with exception", stage_name)
|
|
1266
|
+
self._worker_pool.mark_workers_unverified()
|
|
1249
1267
|
failed_result = StageResult(
|
|
1250
1268
|
status=StageStatus.FAILED,
|
|
1251
1269
|
reason=str(e),
|
|
@@ -1373,23 +1391,14 @@ class Engine:
|
|
|
1373
1391
|
await self._finalize_cancelled_stages(results, run_id)
|
|
1374
1392
|
|
|
1375
1393
|
finally:
|
|
1376
|
-
# Close StateDBs first
|
|
1394
|
+
# Close StateDBs first, then release this run's output queue after its drain exits.
|
|
1377
1395
|
with anyio.CancelScope(shield=True):
|
|
1378
1396
|
for db in state_dbs.values():
|
|
1379
1397
|
db.close()
|
|
1380
1398
|
state_dbs.clear()
|
|
1381
1399
|
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
# Wrapped in outer finally to ensure it always executes, even if task group raises.
|
|
1385
|
-
if self._worker_pool is not None:
|
|
1386
|
-
worker_pool = self._worker_pool
|
|
1387
|
-
self._worker_pool = None
|
|
1388
|
-
try:
|
|
1389
|
-
with anyio.CancelScope(shield=True):
|
|
1390
|
-
await anyio.to_thread.run_sync(worker_pool.shutdown)
|
|
1391
|
-
except (OSError, BrokenPipeError) as exc:
|
|
1392
|
-
_logger.debug("WorkerPool shutdown failed (may already be dead): %s", exc)
|
|
1400
|
+
if self._worker_pool is not None and self._worker_pool.in_run:
|
|
1401
|
+
self._worker_pool.end_run()
|
|
1393
1402
|
|
|
1394
1403
|
# Write run history after execution completes
|
|
1395
1404
|
ended_at = datetime.datetime.now(datetime.UTC).isoformat()
|
|
@@ -1584,9 +1593,10 @@ class Engine:
|
|
|
1584
1593
|
return self._scheduler.can_start(stage_name)
|
|
1585
1594
|
|
|
1586
1595
|
def _ensure_worker_pool(self) -> worker_pool_mod.WorkerPool:
|
|
1587
|
-
"""Lazily
|
|
1596
|
+
"""Lazily begin this run's worker pool on first worker dispatch."""
|
|
1588
1597
|
if self._worker_pool is None:
|
|
1589
1598
|
self._worker_pool = worker_pool_mod.WorkerPool()
|
|
1599
|
+
if not self._worker_pool.in_run:
|
|
1590
1600
|
self._worker_pool.start(max_workers=self._effective_max_workers)
|
|
1591
1601
|
return self._worker_pool
|
|
1592
1602
|
|
|
@@ -2524,8 +2534,8 @@ class Engine:
|
|
|
2524
2534
|
if stages:
|
|
2525
2535
|
# Restart worker pool so workers pick up reloaded code.
|
|
2526
2536
|
# Run in thread to avoid blocking the event loop during process kill/spawn.
|
|
2527
|
-
# Catch errors so
|
|
2528
|
-
#
|
|
2537
|
+
# Catch errors so dispatch-time source validation can rebuild the pool on its
|
|
2538
|
+
# next use.
|
|
2529
2539
|
coordinator = self._get_watch_coordinator()
|
|
2530
2540
|
if coordinator is not None and coordinator.should_restart_workers(
|
|
2531
2541
|
parallel=self._stored_parallel
|
|
@@ -222,9 +222,9 @@ class WatchCoordinator:
|
|
|
222
222
|
return engine_graph.get_producer(self._graph, identity)
|
|
223
223
|
|
|
224
224
|
def should_restart_workers(self, *, parallel: bool) -> bool:
|
|
225
|
-
"""Decide whether
|
|
225
|
+
"""Decide whether an eager reload restart should pre-warm multiple workers.
|
|
226
226
|
|
|
227
|
-
|
|
228
|
-
|
|
227
|
+
A sequential Engine retains one worker too; its dispatch-time source guard replaces
|
|
228
|
+
that worker when needed.
|
|
229
229
|
"""
|
|
230
230
|
return parallel
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import errno
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import pathlib
|
|
7
|
+
import threading
|
|
8
|
+
import typing
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
|
|
11
|
+
import xxhash
|
|
12
|
+
|
|
13
|
+
from pivot.executor import core as executor_core
|
|
14
|
+
from pivot.executor import worker
|
|
15
|
+
|
|
16
|
+
_T = typing.TypeVar("_T")
|
|
17
|
+
|
|
18
|
+
_logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
_FileStamp = tuple[int, int, int, int, int]
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _file_stamp(path: str) -> _FileStamp | None:
|
|
24
|
+
"""Return the stat signature and content digest of a user-code file, or None if unreadable.
|
|
25
|
+
|
|
26
|
+
The digest catches an edit that leaves every timestamp unchanged: a filesystem with coarse
|
|
27
|
+
timestamps, or one whose server clock lags, can stamp an edit with the time it already had.
|
|
28
|
+
Running out of file descriptors is not an unreadable file, so it propagates.
|
|
29
|
+
"""
|
|
30
|
+
try:
|
|
31
|
+
stat = os.stat(path)
|
|
32
|
+
digest = xxhash.xxh64_intdigest(pathlib.Path(path).read_bytes())
|
|
33
|
+
except OSError as e:
|
|
34
|
+
if e.errno in (errno.EMFILE, errno.ENFILE):
|
|
35
|
+
raise
|
|
36
|
+
return None
|
|
37
|
+
return stat.st_mtime_ns, stat.st_size, stat.st_ino, stat.st_ctime_ns, digest
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
if typing.TYPE_CHECKING:
|
|
41
|
+
import concurrent.futures
|
|
42
|
+
import multiprocessing as mp
|
|
43
|
+
from multiprocessing.managers import SyncManager
|
|
44
|
+
|
|
45
|
+
import loky
|
|
46
|
+
|
|
47
|
+
from pivot.types import OutputMessage
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(slots=True)
|
|
51
|
+
class WorkerPool:
|
|
52
|
+
"""Manage engine-lifetime workers and the per-run output queue.
|
|
53
|
+
|
|
54
|
+
Workers are reused only while every user-code file they may have imported keeps the stat
|
|
55
|
+
signature and content digest Pivot recorded before they imported it. Stamps are compared
|
|
56
|
+
only with stamps, never with a clock, so a file timestamp that lags the coordinator's clock
|
|
57
|
+
or does not move at all cannot hide an edit.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
_executor: loky.ProcessPoolExecutor | None = None
|
|
61
|
+
_manager: SyncManager | None = None
|
|
62
|
+
_output_queue: mp.Queue[OutputMessage] | None = None
|
|
63
|
+
_shutdown_event: threading.Event | None = None
|
|
64
|
+
_accepting: bool = True
|
|
65
|
+
_in_run: bool = False
|
|
66
|
+
_max_workers: int = 0
|
|
67
|
+
_user_code_files: dict[str, _FileStamp | None] = field(default_factory=dict)
|
|
68
|
+
_code_changed: bool = False
|
|
69
|
+
|
|
70
|
+
def start(self, *, max_workers: int) -> None:
|
|
71
|
+
"""Start a run without shrinking workers warmed by earlier runs."""
|
|
72
|
+
if self._in_run:
|
|
73
|
+
return
|
|
74
|
+
self._max_workers = max(self._max_workers, max_workers)
|
|
75
|
+
executor = typing.cast(
|
|
76
|
+
"loky.ProcessPoolExecutor", executor_core.create_executor(self._max_workers)
|
|
77
|
+
)
|
|
78
|
+
fresh_workers = executor is not self._executor
|
|
79
|
+
if self._executor is executor and self._has_stale_user_code():
|
|
80
|
+
executor_core.restart_workers(self._max_workers, self._max_workers)
|
|
81
|
+
executor = typing.cast(
|
|
82
|
+
"loky.ProcessPoolExecutor",
|
|
83
|
+
executor_core.create_executor(self._max_workers),
|
|
84
|
+
)
|
|
85
|
+
fresh_workers = True
|
|
86
|
+
if fresh_workers:
|
|
87
|
+
self._restamp_user_code_files()
|
|
88
|
+
self._stamp_coordinator_user_code()
|
|
89
|
+
self._executor = executor
|
|
90
|
+
self._output_queue = typing.cast("mp.Queue[OutputMessage]", self._new_output_queue())
|
|
91
|
+
self._shutdown_event = threading.Event()
|
|
92
|
+
self._accepting = True
|
|
93
|
+
self._in_run = True
|
|
94
|
+
|
|
95
|
+
def _new_output_queue(self) -> object:
|
|
96
|
+
"""Return a fresh queue from the engine-lifetime manager, replacing it if unreachable."""
|
|
97
|
+
if self._manager is not None:
|
|
98
|
+
try:
|
|
99
|
+
return self._manager.Queue()
|
|
100
|
+
except (OSError, EOFError) as e:
|
|
101
|
+
_logger.warning(
|
|
102
|
+
"Output-queue manager unreachable between runs (%r); starting a new one", e
|
|
103
|
+
)
|
|
104
|
+
self._manager.shutdown()
|
|
105
|
+
self._manager = None
|
|
106
|
+
self._manager = executor_core.create_manager()
|
|
107
|
+
return self._manager.Queue()
|
|
108
|
+
|
|
109
|
+
@property
|
|
110
|
+
def in_run(self) -> bool:
|
|
111
|
+
"""Whether this pool has a queue and accepts dispatches for the current run."""
|
|
112
|
+
return self._in_run
|
|
113
|
+
|
|
114
|
+
def record_user_code_files(self, paths: list[str]) -> None:
|
|
115
|
+
"""Record the user-code files a worker reports having imported.
|
|
116
|
+
|
|
117
|
+
A file Pivot had not stamped before the worker imported it may have changed in between,
|
|
118
|
+
so its first report marks the workers stale; the next run replaces them after stamping
|
|
119
|
+
it. Files already stamped are compared when the next run starts.
|
|
120
|
+
"""
|
|
121
|
+
for path in paths:
|
|
122
|
+
if path not in self._user_code_files:
|
|
123
|
+
self._user_code_files[path] = None
|
|
124
|
+
self._code_changed = True
|
|
125
|
+
|
|
126
|
+
def mark_workers_unverified(self) -> None:
|
|
127
|
+
"""Replace the workers before the next run: a task ended without reporting its imports."""
|
|
128
|
+
self._code_changed = True
|
|
129
|
+
|
|
130
|
+
def _has_stale_user_code(self) -> bool:
|
|
131
|
+
return self._code_changed or any(
|
|
132
|
+
_file_stamp(path) != stamp for path, stamp in self._user_code_files.items()
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
def _restamp_user_code_files(self) -> None:
|
|
136
|
+
"""Stamp every known user-code file before fresh workers can import it."""
|
|
137
|
+
self._user_code_files = {path: _file_stamp(path) for path in self._user_code_files}
|
|
138
|
+
self._code_changed = False
|
|
139
|
+
|
|
140
|
+
def _stamp_coordinator_user_code(self) -> None:
|
|
141
|
+
"""Stamp the user code this process imported, before this run's workers can import it."""
|
|
142
|
+
for path in worker.user_code_files():
|
|
143
|
+
if path not in self._user_code_files:
|
|
144
|
+
self._user_code_files[path] = _file_stamp(path)
|
|
145
|
+
|
|
146
|
+
def output_queue(self) -> mp.Queue[OutputMessage]:
|
|
147
|
+
"""Return the worker output queue."""
|
|
148
|
+
if self._output_queue is None:
|
|
149
|
+
raise RuntimeError("WorkerPool not started")
|
|
150
|
+
return self._output_queue
|
|
151
|
+
|
|
152
|
+
def shutdown_event(self) -> threading.Event:
|
|
153
|
+
"""Return the shutdown event used by the drain thread."""
|
|
154
|
+
if self._shutdown_event is None:
|
|
155
|
+
raise RuntimeError("WorkerPool not started")
|
|
156
|
+
return self._shutdown_event
|
|
157
|
+
|
|
158
|
+
def submit(
|
|
159
|
+
self,
|
|
160
|
+
fn: typing.Callable[..., _T],
|
|
161
|
+
/,
|
|
162
|
+
*args: object,
|
|
163
|
+
**kwargs: object,
|
|
164
|
+
) -> concurrent.futures.Future[_T]:
|
|
165
|
+
"""Submit work to the underlying executor."""
|
|
166
|
+
if not self._accepting:
|
|
167
|
+
raise RuntimeError("WorkerPool is not accepting new submissions")
|
|
168
|
+
if self._executor is None:
|
|
169
|
+
raise RuntimeError("WorkerPool not started")
|
|
170
|
+
return self._executor.submit(fn, *args, **kwargs)
|
|
171
|
+
|
|
172
|
+
def stop_accepting(self) -> None:
|
|
173
|
+
"""Stop accepting new submissions without shutting down workers."""
|
|
174
|
+
self._accepting = False
|
|
175
|
+
|
|
176
|
+
def end_run(self) -> None:
|
|
177
|
+
"""Release this run's queue while keeping workers warm for the next run."""
|
|
178
|
+
self._accepting = False
|
|
179
|
+
self._output_queue = None
|
|
180
|
+
self._shutdown_event = None
|
|
181
|
+
self._in_run = False
|
|
182
|
+
|
|
183
|
+
def shutdown(self) -> None:
|
|
184
|
+
"""Shut down workers and the manager when the owning Engine exits."""
|
|
185
|
+
self._accepting = False
|
|
186
|
+
self._in_run = False
|
|
187
|
+
if self._executor is not None:
|
|
188
|
+
self._executor.shutdown(wait=True)
|
|
189
|
+
self._executor = None
|
|
190
|
+
if self._manager is not None:
|
|
191
|
+
self._manager.shutdown()
|
|
192
|
+
self._manager = None
|
|
193
|
+
self._output_queue = None
|
|
194
|
+
self._shutdown_event = None
|
|
195
|
+
|
|
196
|
+
def hard_cancel(self) -> None:
|
|
197
|
+
"""Forcefully shut down workers and signal drain thread to stop."""
|
|
198
|
+
self._accepting = False
|
|
199
|
+
self._in_run = False
|
|
200
|
+
if self._executor is not None:
|
|
201
|
+
self._executor.shutdown(wait=False, kill_workers=True)
|
|
202
|
+
self._executor = None
|
|
203
|
+
if self._manager is not None:
|
|
204
|
+
self._manager.shutdown()
|
|
205
|
+
self._manager = None
|
|
206
|
+
if self._shutdown_event is not None:
|
|
207
|
+
self._shutdown_event.set()
|
|
208
|
+
self._output_queue = None
|
|
209
|
+
self._shutdown_event = None
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
# Pivot Executor - Development Guidelines
|
|
2
2
|
|
|
3
|
-
Workers execute in separate processes via `loky.get_reusable_executor()`.
|
|
3
|
+
Workers execute in separate processes via `loky.get_reusable_executor()`. An Engine keeps its
|
|
4
|
+
workers and manager alive across its runs; each run receives a fresh manager queue for output,
|
|
5
|
+
and a manager that cannot be reached when a run starts is replaced then.
|
|
6
|
+
The loky initializer and the manager's initializer start the parent-death watcher, so a
|
|
7
|
+
worker or manager exits when its Pivot process dies. Keep `timeout=None` and
|
|
8
|
+
`reuse=False if kill_workers else "auto"` in the shared executor policy.
|
|
4
9
|
|
|
5
10
|
## WorkerStageInfo Contract
|
|
6
11
|
|
|
@@ -4,9 +4,12 @@ import atexit
|
|
|
4
4
|
import contextlib
|
|
5
5
|
import functools
|
|
6
6
|
import logging
|
|
7
|
+
import multiprocessing as mp
|
|
7
8
|
import os
|
|
8
9
|
import pathlib
|
|
9
|
-
|
|
10
|
+
import threading
|
|
11
|
+
from multiprocessing import managers
|
|
12
|
+
from typing import TYPE_CHECKING, Final, Literal, TypedDict, cast
|
|
10
13
|
|
|
11
14
|
import loky
|
|
12
15
|
|
|
@@ -25,12 +28,26 @@ if TYPE_CHECKING:
|
|
|
25
28
|
|
|
26
29
|
logger = logging.getLogger(__name__)
|
|
27
30
|
|
|
31
|
+
_WORKER_IDLE_TIMEOUT_S: Final[float | None] = None
|
|
32
|
+
|
|
28
33
|
|
|
29
34
|
def _noop() -> None:
|
|
30
35
|
"""Module-level for pickling."""
|
|
31
36
|
pass
|
|
32
37
|
|
|
33
38
|
|
|
39
|
+
def _exit_when_parent_dies(parent_pid: int) -> None:
|
|
40
|
+
"""Exit a long-lived child (worker or manager) after its owning Pivot process dies."""
|
|
41
|
+
|
|
42
|
+
def _watch_parent() -> None:
|
|
43
|
+
poll = threading.Event()
|
|
44
|
+
while os.getppid() == parent_pid:
|
|
45
|
+
poll.wait(1)
|
|
46
|
+
os._exit(0)
|
|
47
|
+
|
|
48
|
+
threading.Thread(target=_watch_parent, daemon=True, name="pivot-parent-watch").start()
|
|
49
|
+
|
|
50
|
+
|
|
34
51
|
def configured_worker_count(override: int | None = None) -> int:
|
|
35
52
|
"""The configured worker count: the CLI override, else ``core.max_workers``.
|
|
36
53
|
|
|
@@ -56,6 +73,27 @@ def _warm_workers(pool: loky.ProcessPoolExecutor, count: int) -> None:
|
|
|
56
73
|
f.result()
|
|
57
74
|
|
|
58
75
|
|
|
76
|
+
def _get_executor(max_workers: int, *, kill_workers: bool = False) -> loky.ProcessPoolExecutor:
|
|
77
|
+
"""Return Pivot's reusable executor with one worker-lifetime policy."""
|
|
78
|
+
_ensure_deterministic_environment()
|
|
79
|
+
_ensure_cleanup_registered()
|
|
80
|
+
return loky.get_reusable_executor(
|
|
81
|
+
max_workers=max_workers,
|
|
82
|
+
timeout=_WORKER_IDLE_TIMEOUT_S,
|
|
83
|
+
kill_workers=kill_workers,
|
|
84
|
+
reuse=False if kill_workers else "auto",
|
|
85
|
+
initializer=_exit_when_parent_dies,
|
|
86
|
+
initargs=(os.getpid(),),
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def create_manager() -> managers.SyncManager:
|
|
91
|
+
"""Start a spawn-context manager that exits when this process dies."""
|
|
92
|
+
manager = managers.SyncManager(ctx=mp.get_context("spawn"))
|
|
93
|
+
manager.start(initializer=_exit_when_parent_dies, initargs=(os.getpid(),))
|
|
94
|
+
return manager
|
|
95
|
+
|
|
96
|
+
|
|
59
97
|
def prepare_workers(
|
|
60
98
|
stage_count: int, *, parallel: bool = True, max_workers: int | None = None
|
|
61
99
|
) -> int:
|
|
@@ -67,32 +105,25 @@ def prepare_workers(
|
|
|
67
105
|
if not parallel or stage_count <= 0:
|
|
68
106
|
return 1
|
|
69
107
|
workers = compute_max_workers(stage_count, max_workers)
|
|
70
|
-
|
|
71
|
-
_ensure_cleanup_registered()
|
|
72
|
-
pool = loky.get_reusable_executor(max_workers=workers)
|
|
73
|
-
_warm_workers(pool, workers)
|
|
108
|
+
_warm_workers(_get_executor(workers), workers)
|
|
74
109
|
return workers
|
|
75
110
|
|
|
76
111
|
|
|
77
112
|
def restart_workers(stage_count: int, max_workers: int | None = None) -> int:
|
|
78
|
-
"""Kill
|
|
113
|
+
"""Kill and warm fresh workers after code reload or stale-source detection."""
|
|
79
114
|
|
|
80
|
-
|
|
81
|
-
"""
|
|
115
|
+
# A reusable executor with matching kwargs would otherwise keep live workers.
|
|
82
116
|
if stage_count <= 0:
|
|
83
117
|
return 1
|
|
84
118
|
workers = compute_max_workers(stage_count, max_workers)
|
|
85
|
-
|
|
86
|
-
_ensure_cleanup_registered()
|
|
87
|
-
pool = loky.get_reusable_executor(max_workers=workers, kill_workers=True)
|
|
88
|
-
_warm_workers(pool, workers)
|
|
119
|
+
_warm_workers(_get_executor(workers, kill_workers=True), workers)
|
|
89
120
|
return workers
|
|
90
121
|
|
|
91
122
|
|
|
92
123
|
def _cleanup_worker_pool() -> None:
|
|
93
124
|
"""Kill loky worker pool on process exit to prevent orphaned workers."""
|
|
94
125
|
with contextlib.suppress(Exception):
|
|
95
|
-
|
|
126
|
+
_get_executor(1, kill_workers=True)
|
|
96
127
|
|
|
97
128
|
|
|
98
129
|
@functools.cache # Ensures single atexit registration across threads
|
|
@@ -293,10 +324,8 @@ def _ensure_deterministic_environment() -> None:
|
|
|
293
324
|
|
|
294
325
|
|
|
295
326
|
def create_executor(max_workers: int) -> concurrent.futures.Executor:
|
|
296
|
-
"""Get reusable
|
|
297
|
-
|
|
298
|
-
_ensure_cleanup_registered()
|
|
299
|
-
return loky.get_reusable_executor(max_workers=max_workers)
|
|
327
|
+
"""Get Pivot's reusable executor, keeping its workers warm between runs."""
|
|
328
|
+
return _get_executor(max_workers)
|
|
300
329
|
|
|
301
330
|
|
|
302
331
|
def prepare_worker_info(
|
|
@@ -26,6 +26,7 @@ import pydantic
|
|
|
26
26
|
from pivot import (
|
|
27
27
|
config,
|
|
28
28
|
exceptions,
|
|
29
|
+
fingerprint,
|
|
29
30
|
loaders,
|
|
30
31
|
metrics,
|
|
31
32
|
parameters,
|
|
@@ -178,7 +179,7 @@ def _make_result(
|
|
|
178
179
|
return result
|
|
179
180
|
|
|
180
181
|
|
|
181
|
-
def
|
|
182
|
+
def _execute_stage(
|
|
182
183
|
stage_name: str,
|
|
183
184
|
stage_info: WorkerStageInfo,
|
|
184
185
|
cache_dir: pathlib.Path,
|
|
@@ -488,6 +489,38 @@ def execute_stage(
|
|
|
488
489
|
)
|
|
489
490
|
|
|
490
491
|
|
|
492
|
+
def user_code_files() -> list[str]:
|
|
493
|
+
"""Return source paths for user modules this process has loaded.
|
|
494
|
+
|
|
495
|
+
A path that was deleted after import is still reported: the coordinator must restart
|
|
496
|
+
instead of letting the worker keep executing the cached module.
|
|
497
|
+
"""
|
|
498
|
+
files = set[str]()
|
|
499
|
+
for module in tuple(sys.modules.values()):
|
|
500
|
+
if not fingerprint.is_user_code(module):
|
|
501
|
+
continue
|
|
502
|
+
module_file = getattr(module, "__file__", None)
|
|
503
|
+
if not isinstance(module_file, str):
|
|
504
|
+
continue
|
|
505
|
+
try:
|
|
506
|
+
files.add(str(pathlib.Path(module_file).resolve(strict=False)))
|
|
507
|
+
except OSError:
|
|
508
|
+
files.add(str(pathlib.Path(module_file).absolute()))
|
|
509
|
+
return sorted(files)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def execute_stage(
|
|
513
|
+
stage_name: str,
|
|
514
|
+
stage_info: WorkerStageInfo,
|
|
515
|
+
cache_dir: pathlib.Path,
|
|
516
|
+
output_queue: Queue[OutputMessage],
|
|
517
|
+
) -> StageResult:
|
|
518
|
+
"""Execute a stage and report every user-code module resident in this worker."""
|
|
519
|
+
result = _execute_stage(stage_name, stage_info, cache_dir, output_queue)
|
|
520
|
+
result["user_code_files"] = user_code_files()
|
|
521
|
+
return result
|
|
522
|
+
|
|
523
|
+
|
|
491
524
|
def _to_identity_hashes(hashes: dict[str, HashInfo]) -> dict[types.ArtifactIdentity, HashInfo]:
|
|
492
525
|
return {types.identity_from_key(key): info for key, info in hashes.items()}
|
|
493
526
|
|
|
@@ -172,6 +172,7 @@ class StageResult(TypedDict):
|
|
|
172
172
|
output_lines: list[tuple[str, bool]]
|
|
173
173
|
metrics: NotRequired[list[tuple[str, float]]] # (name, duration_ms) for cross-process
|
|
174
174
|
deferred_writes: NotRequired[DeferredWrites]
|
|
175
|
+
user_code_files: NotRequired[list[str]] # Absolute source paths workers imported
|
|
175
176
|
|
|
176
177
|
|
|
177
178
|
# =============================================================================
|
|
@@ -1,90 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import multiprocessing as mp
|
|
4
|
-
import threading
|
|
5
|
-
import typing
|
|
6
|
-
from dataclasses import dataclass
|
|
7
|
-
|
|
8
|
-
from pivot.executor import core as executor_core
|
|
9
|
-
|
|
10
|
-
_T = typing.TypeVar("_T")
|
|
11
|
-
|
|
12
|
-
if typing.TYPE_CHECKING:
|
|
13
|
-
import concurrent.futures
|
|
14
|
-
from multiprocessing.managers import SyncManager
|
|
15
|
-
|
|
16
|
-
import loky
|
|
17
|
-
|
|
18
|
-
from pivot.types import OutputMessage
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
@dataclass(slots=True)
|
|
22
|
-
class WorkerPool:
|
|
23
|
-
"""Manage worker executor, manager queue, and shutdown signaling."""
|
|
24
|
-
|
|
25
|
-
_executor: loky.ProcessPoolExecutor | None = None
|
|
26
|
-
_manager: SyncManager | None = None
|
|
27
|
-
_output_queue: mp.Queue[OutputMessage] | None = None
|
|
28
|
-
_shutdown_event: threading.Event | None = None
|
|
29
|
-
_accepting: bool = True
|
|
30
|
-
|
|
31
|
-
def start(self, *, max_workers: int) -> None:
|
|
32
|
-
"""Initialize executor, manager, output queue, and shutdown event."""
|
|
33
|
-
self._executor = typing.cast(
|
|
34
|
-
"loky.ProcessPoolExecutor", executor_core.create_executor(max_workers)
|
|
35
|
-
)
|
|
36
|
-
spawn_ctx = mp.get_context("spawn")
|
|
37
|
-
self._manager = spawn_ctx.Manager()
|
|
38
|
-
raw_queue = self._manager.Queue()
|
|
39
|
-
queue_obj = typing.cast("object", raw_queue)
|
|
40
|
-
self._output_queue = typing.cast("mp.Queue[OutputMessage]", queue_obj)
|
|
41
|
-
self._shutdown_event = threading.Event()
|
|
42
|
-
self._accepting = True
|
|
43
|
-
|
|
44
|
-
def output_queue(self) -> mp.Queue[OutputMessage]:
|
|
45
|
-
"""Return the worker output queue."""
|
|
46
|
-
if self._output_queue is None:
|
|
47
|
-
raise RuntimeError("WorkerPool not started")
|
|
48
|
-
return self._output_queue
|
|
49
|
-
|
|
50
|
-
def shutdown_event(self) -> threading.Event:
|
|
51
|
-
"""Return the shutdown event used by the drain thread."""
|
|
52
|
-
if self._shutdown_event is None:
|
|
53
|
-
raise RuntimeError("WorkerPool not started")
|
|
54
|
-
return self._shutdown_event
|
|
55
|
-
|
|
56
|
-
def submit(
|
|
57
|
-
self,
|
|
58
|
-
fn: typing.Callable[..., _T],
|
|
59
|
-
/,
|
|
60
|
-
*args: object,
|
|
61
|
-
**kwargs: object,
|
|
62
|
-
) -> concurrent.futures.Future[_T]:
|
|
63
|
-
"""Submit work to the underlying executor."""
|
|
64
|
-
if not self._accepting:
|
|
65
|
-
raise RuntimeError("WorkerPool is not accepting new submissions")
|
|
66
|
-
if self._executor is None:
|
|
67
|
-
raise RuntimeError("WorkerPool not started")
|
|
68
|
-
return self._executor.submit(fn, *args, **kwargs)
|
|
69
|
-
|
|
70
|
-
def stop_accepting(self) -> None:
|
|
71
|
-
"""Stop accepting new submissions without shutting down workers."""
|
|
72
|
-
self._accepting = False
|
|
73
|
-
|
|
74
|
-
def shutdown(self) -> None:
|
|
75
|
-
"""Shut down executor and manager, waiting for completion."""
|
|
76
|
-
self._accepting = False
|
|
77
|
-
if self._executor is not None:
|
|
78
|
-
self._executor.shutdown(wait=True)
|
|
79
|
-
if self._manager is not None:
|
|
80
|
-
self._manager.shutdown()
|
|
81
|
-
|
|
82
|
-
def hard_cancel(self) -> None:
|
|
83
|
-
"""Forcefully shut down workers and signal drain thread to stop."""
|
|
84
|
-
self._accepting = False
|
|
85
|
-
if self._executor is not None:
|
|
86
|
-
self._executor.shutdown(wait=False, kill_workers=True)
|
|
87
|
-
if self._manager is not None:
|
|
88
|
-
self._manager.shutdown()
|
|
89
|
-
if self._shutdown_event is not None:
|
|
90
|
-
self._shutdown_event.set()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|