ttasks 0.3.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ttasks-0.3.0 → ttasks-0.3.1}/PKG-INFO +1 -1
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_executor.py +93 -15
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_graph.py +8 -1
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_version.py +2 -2
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_executor.py +240 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_workflow.py +139 -14
- {ttasks-0.3.0 → ttasks-0.3.1}/.github/workflows/docs.yml +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/.github/workflows/publish.yml +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/.gitignore +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/.python-version +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/.vscode/launch.json +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/.vscode/settings.json +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/README.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/index.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/patterns/finally-tasks.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/patterns/progress-and-output.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/patterns/retries-and-cancellation.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/quickstart.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/reference/api.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/tutorials/async-execution.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/tutorials/graph-workflows.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/docs/tutorials/task-execution.md +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/examples/finally_tasks.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/main.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/mkdocs.yml +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/pyproject.toml +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/scripts/preflight.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/__init__.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_events.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_exceptions.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_sqlite.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_store.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/_task.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/src/ttasks/py.typed +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/conftest.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_e2e.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_events.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_public_api.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_sqlite_store.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_store.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_task.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/tests/test_task_factories.py +0 -0
- {ttasks-0.3.0 → ttasks-0.3.1}/uv.lock +0 -0
|
@@ -11,9 +11,10 @@ import time
|
|
|
11
11
|
import warnings
|
|
12
12
|
from collections.abc import Callable, Mapping
|
|
13
13
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
14
|
+
from contextlib import suppress
|
|
14
15
|
from dataclasses import dataclass
|
|
15
16
|
from datetime import datetime
|
|
16
|
-
from threading import RLock, Thread
|
|
17
|
+
from threading import RLock, Thread, current_thread
|
|
17
18
|
from types import MappingProxyType
|
|
18
19
|
from typing import TYPE_CHECKING, Any, TextIO, cast
|
|
19
20
|
|
|
@@ -236,6 +237,8 @@ class TaskExecutor:
|
|
|
236
237
|
:meth:`Future.cancel` only cancels work that has not started yet; cancel
|
|
237
238
|
running tasks through :meth:`cancel`.
|
|
238
239
|
"""
|
|
240
|
+
self._validate_task(task)
|
|
241
|
+
policy = self._resolve_retry_policy(retry_policy)
|
|
239
242
|
# Shallow-copy the mapping so caller mutation cannot race the worker;
|
|
240
243
|
# Task refs themselves intentionally remain shared.
|
|
241
244
|
upstream_snapshot = dict(upstream or {})
|
|
@@ -244,12 +247,27 @@ class TaskExecutor:
|
|
|
244
247
|
raise RuntimeError("executor is shut down")
|
|
245
248
|
if self._pool is None:
|
|
246
249
|
self._pool = ThreadPoolExecutor(thread_name_prefix="ttasks")
|
|
247
|
-
|
|
248
|
-
self.
|
|
250
|
+
future = self._pool.submit(
|
|
251
|
+
self._execute_submitted,
|
|
249
252
|
task,
|
|
250
253
|
upstream_snapshot,
|
|
251
|
-
|
|
254
|
+
policy,
|
|
252
255
|
)
|
|
256
|
+
future.add_done_callback(
|
|
257
|
+
lambda submitted: self.cancel(task) if submitted.cancelled() else None
|
|
258
|
+
)
|
|
259
|
+
return future
|
|
260
|
+
|
|
261
|
+
def _execute_submitted(
|
|
262
|
+
self,
|
|
263
|
+
task: Task,
|
|
264
|
+
upstream: Mapping[str, Task],
|
|
265
|
+
retry_policy: RetryPolicy,
|
|
266
|
+
) -> TaskResult:
|
|
267
|
+
"""Execute submitted work, preserving queued cancellation semantics."""
|
|
268
|
+
if task.status == TaskStatus.CANCELLED:
|
|
269
|
+
raise TaskCancelled(f"Task {task.id!r} was cancelled")
|
|
270
|
+
return self.execute(task, upstream, retry_policy=retry_policy)
|
|
253
271
|
|
|
254
272
|
def shutdown(self) -> None:
|
|
255
273
|
"""Shut down async submission, waiting for submitted work to finish.
|
|
@@ -265,7 +283,15 @@ class TaskExecutor:
|
|
|
265
283
|
pool = self._pool
|
|
266
284
|
self._pool = None
|
|
267
285
|
if pool is not None:
|
|
268
|
-
|
|
286
|
+
current = current_thread()
|
|
287
|
+
pool_threads = list(getattr(pool, "_threads", ()))
|
|
288
|
+
if current in pool_threads:
|
|
289
|
+
pool.shutdown(wait=False)
|
|
290
|
+
for thread in pool_threads:
|
|
291
|
+
if thread is not current:
|
|
292
|
+
thread.join()
|
|
293
|
+
else:
|
|
294
|
+
pool.shutdown(wait=True)
|
|
269
295
|
|
|
270
296
|
def close(self) -> None:
|
|
271
297
|
"""Alias for :meth:`shutdown` for resource-cleanup contexts."""
|
|
@@ -417,10 +443,11 @@ class TaskExecutor:
|
|
|
417
443
|
self.store.graphs.save(graph)
|
|
418
444
|
except BaseException as error:
|
|
419
445
|
self.graph_persistence_errors.append((graph.id, error))
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
446
|
+
with suppress(Warning):
|
|
447
|
+
warnings.warn(
|
|
448
|
+
f"graph persistence failed for graph {graph.id!r}: {error}",
|
|
449
|
+
stacklevel=2,
|
|
450
|
+
)
|
|
424
451
|
|
|
425
452
|
def cancel(self, task: Task) -> None:
|
|
426
453
|
"""Cancel a task and terminate its subprocess if one is active.
|
|
@@ -495,7 +522,8 @@ class TaskExecutor:
|
|
|
495
522
|
``retry_policy`` retries failed attempts for this single task only.
|
|
496
523
|
Cancellation is never retried.
|
|
497
524
|
"""
|
|
498
|
-
|
|
525
|
+
self._validate_task(task)
|
|
526
|
+
policy = self._resolve_retry_policy(retry_policy)
|
|
499
527
|
if policy.max_attempts == 1 or self._handlers.get(task.type) is None:
|
|
500
528
|
return self._execute_once(task, upstream)
|
|
501
529
|
|
|
@@ -513,7 +541,7 @@ class TaskExecutor:
|
|
|
513
541
|
if out_of_attempts or task.status != TaskStatus.FAILED:
|
|
514
542
|
raise
|
|
515
543
|
if policy.backoff:
|
|
516
|
-
|
|
544
|
+
self._sleep_retry_backoff(task, policy.backoff)
|
|
517
545
|
if task.status == TaskStatus.CANCELLED:
|
|
518
546
|
raise TaskCancelled(
|
|
519
547
|
f"Task {task.id!r} was cancelled",
|
|
@@ -521,6 +549,35 @@ class TaskExecutor:
|
|
|
521
549
|
|
|
522
550
|
raise AssertionError("unreachable retry loop exit") # pragma: no cover
|
|
523
551
|
|
|
552
|
+
@staticmethod
|
|
553
|
+
def _validate_task(task: Task) -> None:
|
|
554
|
+
"""Reject malformed task arguments before accessing Task internals."""
|
|
555
|
+
if not isinstance(task, Task):
|
|
556
|
+
raise TypeError("task must be a Task")
|
|
557
|
+
|
|
558
|
+
@staticmethod
|
|
559
|
+
def _resolve_retry_policy(retry_policy: RetryPolicy | None) -> RetryPolicy:
|
|
560
|
+
"""Return a concrete RetryPolicy, rejecting malformed public input."""
|
|
561
|
+
if retry_policy is None:
|
|
562
|
+
return RetryPolicy()
|
|
563
|
+
if not isinstance(retry_policy, RetryPolicy):
|
|
564
|
+
raise TypeError("retry_policy must be a RetryPolicy")
|
|
565
|
+
return retry_policy
|
|
566
|
+
|
|
567
|
+
@staticmethod
|
|
568
|
+
def _sleep_retry_backoff(task: Task, backoff: float) -> None:
|
|
569
|
+
"""Sleep between retry attempts while periodically observing cancel()."""
|
|
570
|
+
if backoff <= 0.5:
|
|
571
|
+
time.sleep(backoff)
|
|
572
|
+
return
|
|
573
|
+
|
|
574
|
+
deadline = time.monotonic() + backoff
|
|
575
|
+
while task.status != TaskStatus.CANCELLED:
|
|
576
|
+
remaining = deadline - time.monotonic()
|
|
577
|
+
if remaining <= 0:
|
|
578
|
+
return
|
|
579
|
+
time.sleep(min(remaining, 0.05))
|
|
580
|
+
|
|
524
581
|
def _execute_once(
|
|
525
582
|
self,
|
|
526
583
|
task: Task,
|
|
@@ -714,6 +771,16 @@ class TaskExecutor:
|
|
|
714
771
|
stderr_thread.start()
|
|
715
772
|
timed_out = False
|
|
716
773
|
timeout_error: subprocess.TimeoutExpired | None = None
|
|
774
|
+
deadline = (
|
|
775
|
+
None if context.timeout is None else time.monotonic() + context.timeout
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
def remaining_timeout() -> float | None:
|
|
779
|
+
"""Return remaining wall-clock timeout for process/output draining."""
|
|
780
|
+
if deadline is None:
|
|
781
|
+
return None
|
|
782
|
+
return max(0.0, deadline - time.monotonic())
|
|
783
|
+
|
|
717
784
|
if context.cancelled:
|
|
718
785
|
self._terminate_process(process)
|
|
719
786
|
try:
|
|
@@ -724,14 +791,23 @@ class TaskExecutor:
|
|
|
724
791
|
timed_out = True
|
|
725
792
|
timeout_error = e
|
|
726
793
|
finally:
|
|
727
|
-
stdout_thread
|
|
728
|
-
|
|
794
|
+
for thread in (stdout_thread, stderr_thread):
|
|
795
|
+
thread.join(remaining_timeout())
|
|
796
|
+
if (
|
|
797
|
+
not timed_out
|
|
798
|
+
and deadline is not None
|
|
799
|
+
and (stdout_thread.is_alive() or stderr_thread.is_alive())
|
|
800
|
+
):
|
|
801
|
+
self._terminate_process(process)
|
|
802
|
+
timed_out = True
|
|
803
|
+
if timed_out:
|
|
804
|
+
stdout_thread.join()
|
|
805
|
+
stderr_thread.join()
|
|
729
806
|
self._running_processes.pop(context.id, None)
|
|
730
807
|
|
|
731
808
|
stdout = "".join(stdout_chunks)
|
|
732
809
|
stderr = "".join(stderr_chunks)
|
|
733
810
|
if timed_out:
|
|
734
|
-
assert timeout_error is not None
|
|
735
811
|
message = f"Task timed out after {context.timeout} seconds"
|
|
736
812
|
completed = subprocess.CompletedProcess(
|
|
737
813
|
args=args,
|
|
@@ -739,7 +815,9 @@ class TaskExecutor:
|
|
|
739
815
|
stdout=stdout,
|
|
740
816
|
stderr=stderr,
|
|
741
817
|
)
|
|
742
|
-
|
|
818
|
+
if timeout_error is not None:
|
|
819
|
+
raise TaskTimeoutError(message, completed) from timeout_error
|
|
820
|
+
raise TaskTimeoutError(message, completed)
|
|
743
821
|
|
|
744
822
|
result = subprocess.CompletedProcess(
|
|
745
823
|
args=args,
|
|
@@ -72,6 +72,8 @@ class TaskGraph:
|
|
|
72
72
|
self._tasks[task.id] = task
|
|
73
73
|
deps: list[str] = []
|
|
74
74
|
for dep in after:
|
|
75
|
+
if not isinstance(dep, Task):
|
|
76
|
+
raise TypeError(f"Expected Task dependency, got {type(dep).__name__}")
|
|
75
77
|
if dep.id not in deps:
|
|
76
78
|
deps.append(dep.id)
|
|
77
79
|
self._deps[task.id] = deps
|
|
@@ -349,11 +351,12 @@ class TaskGraph:
|
|
|
349
351
|
|
|
350
352
|
def inactive(tid: str) -> bool:
|
|
351
353
|
"""Return whether tid can no longer change in this run."""
|
|
354
|
+
if tid in futures:
|
|
355
|
+
return futures[tid].done()
|
|
352
356
|
task = self._tasks[tid]
|
|
353
357
|
return (
|
|
354
358
|
(task.is_terminal and not retryable_this_run(tid))
|
|
355
359
|
or tid in self._errors
|
|
356
|
-
or (tid in futures and futures[tid].done())
|
|
357
360
|
)
|
|
358
361
|
|
|
359
362
|
def ready(tid: str) -> bool:
|
|
@@ -373,6 +376,10 @@ class TaskGraph:
|
|
|
373
376
|
authoritative.
|
|
374
377
|
"""
|
|
375
378
|
for d in self._deps[tid]:
|
|
379
|
+
if d in futures and not futures[d].done():
|
|
380
|
+
continue
|
|
381
|
+
if d in self._errors:
|
|
382
|
+
return d
|
|
376
383
|
if self._tasks[d].status.is_bad and not retryable_this_run(d):
|
|
377
384
|
return d
|
|
378
385
|
return None
|
|
@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
|
|
|
18
18
|
commit_id: str | None
|
|
19
19
|
__commit_id__: str | None
|
|
20
20
|
|
|
21
|
-
__version__ = version = '0.3.
|
|
22
|
-
__version_tuple__ = version_tuple = (0, 3,
|
|
21
|
+
__version__ = version = '0.3.1'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 1)
|
|
23
23
|
|
|
24
24
|
__commit_id__ = commit_id = None
|
|
@@ -357,6 +357,51 @@ def test_retry_policy_rejects_invalid_backoff() -> None:
|
|
|
357
357
|
RetryPolicy(backoff=float("nan"))
|
|
358
358
|
|
|
359
359
|
|
|
360
|
+
def test_execute_rejects_invalid_retry_policy() -> None:
|
|
361
|
+
"""execute() rejects malformed retry_policy values clearly."""
|
|
362
|
+
executor = TaskExecutor()
|
|
363
|
+
bad_policy: Any = object()
|
|
364
|
+
|
|
365
|
+
with pytest.raises(TypeError, match="retry_policy must be a RetryPolicy"):
|
|
366
|
+
executor.execute(Task.bash("", title="Example"), retry_policy=bad_policy)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def test_execute_rejects_non_task() -> None:
|
|
370
|
+
"""execute() rejects malformed task objects before accessing internals."""
|
|
371
|
+
executor = TaskExecutor()
|
|
372
|
+
not_task: Any = object()
|
|
373
|
+
|
|
374
|
+
with pytest.raises(TypeError, match="task must be a Task"):
|
|
375
|
+
executor.execute(not_task)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def test_submit_rejects_invalid_retry_policy_synchronously() -> None:
|
|
379
|
+
"""submit() rejects malformed retry_policy values before queueing work."""
|
|
380
|
+
executor = TaskExecutor()
|
|
381
|
+
task = Task.bash("", title="Example")
|
|
382
|
+
bad_policy: Any = object()
|
|
383
|
+
|
|
384
|
+
with pytest.raises(TypeError, match="retry_policy must be a RetryPolicy"):
|
|
385
|
+
executor.submit(task, retry_policy=bad_policy)
|
|
386
|
+
|
|
387
|
+
assert task.status == TaskStatus.PENDING
|
|
388
|
+
assert task.result is None
|
|
389
|
+
assert executor.is_shutdown is False
|
|
390
|
+
executor.close()
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def test_submit_rejects_non_task_synchronously() -> None:
|
|
394
|
+
"""submit() rejects malformed task objects before creating async work."""
|
|
395
|
+
executor = TaskExecutor()
|
|
396
|
+
not_task: Any = object()
|
|
397
|
+
|
|
398
|
+
with pytest.raises(TypeError, match="task must be a Task"):
|
|
399
|
+
executor.submit(not_task)
|
|
400
|
+
|
|
401
|
+
assert executor.is_shutdown is False
|
|
402
|
+
executor.close()
|
|
403
|
+
|
|
404
|
+
|
|
360
405
|
def test_execute_retry_policy_recovers_after_handler_failure() -> None:
|
|
361
406
|
"""A retry policy re-runs a failed single task until it succeeds."""
|
|
362
407
|
executor = TaskExecutor()
|
|
@@ -503,6 +548,44 @@ def test_execute_retry_policy_honors_cancellation_during_backoff(
|
|
|
503
548
|
assert task.status == TaskStatus.CANCELLED
|
|
504
549
|
|
|
505
550
|
|
|
551
|
+
def test_retry_backoff_observes_external_cancellation_promptly() -> None:
|
|
552
|
+
"""A real backoff sleep wakes promptly when another thread cancels the task."""
|
|
553
|
+
executor = TaskExecutor()
|
|
554
|
+
task = Task.bash("", title="Example")
|
|
555
|
+
failed = threading.Event()
|
|
556
|
+
attempts = 0
|
|
557
|
+
|
|
558
|
+
def handler(context: TaskContext) -> None:
|
|
559
|
+
"""Fail each attempt; cancellation should stop retries during backoff."""
|
|
560
|
+
nonlocal attempts
|
|
561
|
+
attempts += 1
|
|
562
|
+
raise RuntimeError("boom")
|
|
563
|
+
|
|
564
|
+
def note_failure(event: TaskEvent) -> None:
|
|
565
|
+
if event.type is TaskEventType.FAILED:
|
|
566
|
+
failed.set()
|
|
567
|
+
|
|
568
|
+
executor.register(TaskType.BASH, handler)
|
|
569
|
+
executor.events.subscribe(note_failure)
|
|
570
|
+
|
|
571
|
+
future = executor.submit(
|
|
572
|
+
task,
|
|
573
|
+
retry_policy=RetryPolicy(max_attempts=2, backoff=1.0),
|
|
574
|
+
)
|
|
575
|
+
assert failed.wait(timeout=1)
|
|
576
|
+
start = time.monotonic()
|
|
577
|
+
executor.cancel(task)
|
|
578
|
+
|
|
579
|
+
with pytest.raises(TaskCancelled, match="was cancelled"):
|
|
580
|
+
future.result(timeout=0.5)
|
|
581
|
+
elapsed = time.monotonic() - start
|
|
582
|
+
executor.close()
|
|
583
|
+
|
|
584
|
+
assert attempts == 1
|
|
585
|
+
assert task.status == TaskStatus.CANCELLED
|
|
586
|
+
assert elapsed < 0.5
|
|
587
|
+
|
|
588
|
+
|
|
506
589
|
def test_execute_retry_policy_honors_zero_backoff_cancellation() -> None:
|
|
507
590
|
"""Cancellation between immediate retry attempts is still observed."""
|
|
508
591
|
executor = TaskExecutor()
|
|
@@ -715,6 +798,80 @@ def test_submit_missing_handler_future_raises_after_failed_event() -> None:
|
|
|
715
798
|
assert [event.type for event in events] == [TaskEventType.FAILED]
|
|
716
799
|
|
|
717
800
|
|
|
801
|
+
def test_executor_cancelled_queued_submit_raises_task_cancelled() -> None:
|
|
802
|
+
"""executor.cancel() for queued async work is reflected on the future."""
|
|
803
|
+
executor = TaskExecutor()
|
|
804
|
+
blocker = Task.bash("", title="Blocker")
|
|
805
|
+
queued = Task.bash("", title="Queued")
|
|
806
|
+
blocker_started = threading.Event()
|
|
807
|
+
release_blocker = threading.Event()
|
|
808
|
+
|
|
809
|
+
def handler(context: TaskContext) -> str:
|
|
810
|
+
"""Keep one worker occupied so the second future remains queued."""
|
|
811
|
+
if context.id == blocker.id:
|
|
812
|
+
blocker_started.set()
|
|
813
|
+
assert release_blocker.wait(timeout=1)
|
|
814
|
+
return context.title
|
|
815
|
+
|
|
816
|
+
executor.register(TaskType.BASH, handler)
|
|
817
|
+
with executor._pool_lock:
|
|
818
|
+
executor._pool = ThreadPoolExecutor(max_workers=1)
|
|
819
|
+
|
|
820
|
+
blocker_future = executor.submit(blocker)
|
|
821
|
+
assert blocker_started.wait(timeout=1)
|
|
822
|
+
queued_future = executor.submit(queued)
|
|
823
|
+
executor.cancel(queued)
|
|
824
|
+
release_blocker.set()
|
|
825
|
+
|
|
826
|
+
assert blocker_future.result(timeout=1).output == "Blocker"
|
|
827
|
+
with pytest.raises(TaskCancelled, match="was cancelled"):
|
|
828
|
+
queued_future.result(timeout=1)
|
|
829
|
+
executor.close()
|
|
830
|
+
|
|
831
|
+
assert queued.status == TaskStatus.CANCELLED
|
|
832
|
+
assert queued.result is not None
|
|
833
|
+
assert queued.result.termination_reason == "cancelled"
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def test_future_cancel_marks_queued_task_cancelled() -> None:
|
|
837
|
+
"""Future.cancel() on queued work is reflected in task state and events."""
|
|
838
|
+
executor = TaskExecutor()
|
|
839
|
+
blocker = Task.bash("", title="Blocker")
|
|
840
|
+
queued = Task.bash("", title="Queued")
|
|
841
|
+
blocker_started = threading.Event()
|
|
842
|
+
release_blocker = threading.Event()
|
|
843
|
+
events: list[TaskEvent] = []
|
|
844
|
+
|
|
845
|
+
def handler(context: TaskContext) -> str:
|
|
846
|
+
"""Keep one worker occupied so the second future remains queued."""
|
|
847
|
+
if context.id == blocker.id:
|
|
848
|
+
blocker_started.set()
|
|
849
|
+
assert release_blocker.wait(timeout=1)
|
|
850
|
+
return context.title
|
|
851
|
+
|
|
852
|
+
executor.register(TaskType.BASH, handler)
|
|
853
|
+
executor.events.subscribe(events.append)
|
|
854
|
+
with executor._pool_lock:
|
|
855
|
+
executor._pool = ThreadPoolExecutor(max_workers=1)
|
|
856
|
+
|
|
857
|
+
blocker_future = executor.submit(blocker)
|
|
858
|
+
assert blocker_started.wait(timeout=1)
|
|
859
|
+
queued_future = executor.submit(queued)
|
|
860
|
+
|
|
861
|
+
assert queued_future.cancel() is True
|
|
862
|
+
release_blocker.set()
|
|
863
|
+
executor.close()
|
|
864
|
+
|
|
865
|
+
assert blocker_future.result(timeout=1).output == "Blocker"
|
|
866
|
+
assert queued.status == TaskStatus.CANCELLED
|
|
867
|
+
assert queued.result is not None
|
|
868
|
+
assert queued.result.termination_reason == "cancelled"
|
|
869
|
+
cancelled = [event for event in events if event.type is TaskEventType.CANCELLED]
|
|
870
|
+
assert len(cancelled) == 1
|
|
871
|
+
assert cancelled[0].task is queued
|
|
872
|
+
assert cancelled[0].previous_status == TaskStatus.PENDING
|
|
873
|
+
|
|
874
|
+
|
|
718
875
|
def test_future_cancel_does_not_cancel_running_task() -> None:
|
|
719
876
|
"""Running submitted tasks are cancelled through executor.cancel(), not Future."""
|
|
720
877
|
executor = TaskExecutor()
|
|
@@ -818,6 +975,67 @@ def test_shutdown_allows_queued_submissions_to_finish() -> None:
|
|
|
818
975
|
assert queued.status == TaskStatus.SUCCEEDED
|
|
819
976
|
|
|
820
977
|
|
|
978
|
+
def test_shutdown_from_worker_waits_for_other_running_workers() -> None:
|
|
979
|
+
"""shutdown() from a worker still waits for other in-flight work."""
|
|
980
|
+
executor = TaskExecutor()
|
|
981
|
+
shutdown_task = Task.bash("", title="Shutdown")
|
|
982
|
+
other_task = Task.bash("", title="Other")
|
|
983
|
+
other_started = threading.Event()
|
|
984
|
+
release_other = threading.Event()
|
|
985
|
+
other_finished = threading.Event()
|
|
986
|
+
|
|
987
|
+
def handler(context: TaskContext) -> str:
|
|
988
|
+
"""One worker shuts down while the other is still running."""
|
|
989
|
+
if context.id == other_task.id:
|
|
990
|
+
other_started.set()
|
|
991
|
+
assert release_other.wait(timeout=1)
|
|
992
|
+
other_finished.set()
|
|
993
|
+
return "other done"
|
|
994
|
+
assert other_started.wait(timeout=1)
|
|
995
|
+
timer = threading.Timer(0.1, release_other.set)
|
|
996
|
+
timer.start()
|
|
997
|
+
try:
|
|
998
|
+
executor.shutdown()
|
|
999
|
+
finally:
|
|
1000
|
+
timer.cancel()
|
|
1001
|
+
return "waited" if other_finished.is_set() else "returned early"
|
|
1002
|
+
|
|
1003
|
+
executor.register(TaskType.BASH, handler)
|
|
1004
|
+
with executor._pool_lock:
|
|
1005
|
+
executor._pool = ThreadPoolExecutor(max_workers=2)
|
|
1006
|
+
|
|
1007
|
+
other_future = executor.submit(other_task)
|
|
1008
|
+
assert other_started.wait(timeout=1)
|
|
1009
|
+
shutdown_future = executor.submit(shutdown_task)
|
|
1010
|
+
|
|
1011
|
+
assert shutdown_future.result(timeout=1).output == "waited"
|
|
1012
|
+
assert other_future.result(timeout=1).output == "other done"
|
|
1013
|
+
assert executor.is_shutdown is True
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def test_submitted_task_can_shutdown_its_executor() -> None:
|
|
1017
|
+
"""shutdown() from an executor worker should not fail the running task."""
|
|
1018
|
+
executor = TaskExecutor()
|
|
1019
|
+
task = Task.bash("", title="Self shutdown")
|
|
1020
|
+
|
|
1021
|
+
def handler(context: TaskContext) -> str:
|
|
1022
|
+
"""Close async submission from inside the running submitted task."""
|
|
1023
|
+
executor.shutdown()
|
|
1024
|
+
return "ok"
|
|
1025
|
+
|
|
1026
|
+
executor.register(TaskType.BASH, handler)
|
|
1027
|
+
|
|
1028
|
+
future = executor.submit(task)
|
|
1029
|
+
result = future.result(timeout=1)
|
|
1030
|
+
|
|
1031
|
+
assert result.status == TaskStatus.SUCCEEDED
|
|
1032
|
+
assert result.output == "ok"
|
|
1033
|
+
assert task.status == TaskStatus.SUCCEEDED
|
|
1034
|
+
assert executor.is_shutdown is True
|
|
1035
|
+
with pytest.raises(RuntimeError, match="executor is shut down"):
|
|
1036
|
+
executor.submit(Task.bash("", title="Later"))
|
|
1037
|
+
|
|
1038
|
+
|
|
821
1039
|
def test_close_aliases_shutdown() -> None:
|
|
822
1040
|
"""close() remains a resource-cleanup alias for graceful shutdown."""
|
|
823
1041
|
executor = TaskExecutor()
|
|
@@ -1390,6 +1608,28 @@ def test_real_subprocess_timeout_kills_within_wall_budget() -> None:
|
|
|
1390
1608
|
assert not executor.is_running(task.id)
|
|
1391
1609
|
|
|
1392
1610
|
|
|
1611
|
+
def test_timeout_applies_while_draining_output_from_background_children() -> None:
|
|
1612
|
+
"""A child inheriting stdout cannot make streaming output bypass timeout."""
|
|
1613
|
+
executor = TaskExecutor()
|
|
1614
|
+
task = Task.bash(
|
|
1615
|
+
"printf 'before\\n'; sleep 2 &",
|
|
1616
|
+
title="Background output holder",
|
|
1617
|
+
timeout=0.1,
|
|
1618
|
+
)
|
|
1619
|
+
|
|
1620
|
+
start = time.monotonic()
|
|
1621
|
+
with pytest.raises(TaskTimeoutError, match="Task timed out after 0.1 seconds"):
|
|
1622
|
+
executor.execute(task)
|
|
1623
|
+
elapsed = time.monotonic() - start
|
|
1624
|
+
|
|
1625
|
+
assert task.status == TaskStatus.FAILED
|
|
1626
|
+
assert task.result is not None
|
|
1627
|
+
assert task.result.termination_reason == "timeout"
|
|
1628
|
+
assert task.result.output == "before\n"
|
|
1629
|
+
assert elapsed < 1.0, f"timeout waited for background child (took {elapsed:.2f}s)"
|
|
1630
|
+
assert not executor.is_running(task.id)
|
|
1631
|
+
|
|
1632
|
+
|
|
1393
1633
|
def test_timed_out_subprocess_result_preserves_partial_output() -> None:
|
|
1394
1634
|
"""Timeout results retain output captured before termination."""
|
|
1395
1635
|
executor = TaskExecutor()
|
|
@@ -287,6 +287,28 @@ def test_run_no_progress_guard_raises_runtime_error() -> None:
|
|
|
287
287
|
graph._run_inner(TaskExecutor(), max_workers=1)
|
|
288
288
|
|
|
289
289
|
|
|
290
|
+
def test_executor_setup_error_blocks_descendants_without_deadlock() -> None:
|
|
291
|
+
"""A future error before task terminalization still blocks descendants."""
|
|
292
|
+
parent = _bash("Parent", "")
|
|
293
|
+
child = _bash("Child", "")
|
|
294
|
+
|
|
295
|
+
class ExplodingExecutor(TaskExecutor):
|
|
296
|
+
def execute(self, *args: Any, **kwargs: Any) -> Any:
|
|
297
|
+
raise RuntimeError("setup boom")
|
|
298
|
+
|
|
299
|
+
graph = TaskGraph()
|
|
300
|
+
graph[parent] = []
|
|
301
|
+
graph[child] = [parent]
|
|
302
|
+
|
|
303
|
+
graph.run(ExplodingExecutor.empty())
|
|
304
|
+
|
|
305
|
+
assert parent.status == TaskStatus.PENDING
|
|
306
|
+
assert child.status == TaskStatus.BLOCKED
|
|
307
|
+
assert child.blocked_by == parent.id
|
|
308
|
+
assert isinstance(graph.errors[parent.id], RuntimeError)
|
|
309
|
+
assert graph.ok is False
|
|
310
|
+
|
|
311
|
+
|
|
290
312
|
# ---- Execution ---------------------------------------------------------------
|
|
291
313
|
|
|
292
314
|
|
|
@@ -1033,6 +1055,16 @@ def test_add_rejects_non_task_and_non_bool_finally() -> None:
|
|
|
1033
1055
|
graph.add(_bash("A", "echo a"), finally_=bad_finally)
|
|
1034
1056
|
|
|
1035
1057
|
|
|
1058
|
+
def test_add_rejects_non_task_dependency() -> None:
|
|
1059
|
+
"""add(after=...) validates dependencies instead of leaking AttributeError."""
|
|
1060
|
+
graph = TaskGraph()
|
|
1061
|
+
task = _bash("A", "echo a")
|
|
1062
|
+
not_task: Any = object()
|
|
1063
|
+
|
|
1064
|
+
with pytest.raises(TypeError, match="Expected Task dependency, got object"):
|
|
1065
|
+
graph.add(task, after=[not_task])
|
|
1066
|
+
|
|
1067
|
+
|
|
1036
1068
|
def test_add_runs_like_setitem_form() -> None:
|
|
1037
1069
|
"""A graph built with .add() executes to the same end state as the mapping form."""
|
|
1038
1070
|
a = _bash("A", "echo a")
|
|
@@ -1129,6 +1161,38 @@ def test_carryover_blocked_with_failed_parent_stays_blocked() -> None:
|
|
|
1129
1161
|
assert b.status == TaskStatus.BLOCKED
|
|
1130
1162
|
|
|
1131
1163
|
|
|
1164
|
+
def test_descendant_waits_for_submitted_carryover_blocked_parent() -> None:
|
|
1165
|
+
"""A queued carryover-BLOCKED parent must not immediately block its child."""
|
|
1166
|
+
a = _bash("A", "")
|
|
1167
|
+
b = _bash("B", "")
|
|
1168
|
+
c = _bash("C", "")
|
|
1169
|
+
a.transition_to(TaskStatus.RUNNING)
|
|
1170
|
+
a.transition_to(TaskStatus.FAILED, error="old failure")
|
|
1171
|
+
b.transition_to(TaskStatus.BLOCKED)
|
|
1172
|
+
b._set_blocked_by(a.id)
|
|
1173
|
+
seen: list[str] = []
|
|
1174
|
+
|
|
1175
|
+
executor = TaskExecutor.empty()
|
|
1176
|
+
|
|
1177
|
+
def handler(context: Any) -> str:
|
|
1178
|
+
seen.append(context.title)
|
|
1179
|
+
return context.title
|
|
1180
|
+
|
|
1181
|
+
executor.register(TaskType.BASH, handler)
|
|
1182
|
+
graph = TaskGraph()
|
|
1183
|
+
graph[a] = []
|
|
1184
|
+
graph[b] = [a]
|
|
1185
|
+
graph[c] = [b]
|
|
1186
|
+
|
|
1187
|
+
graph.run(executor, max_workers=1)
|
|
1188
|
+
|
|
1189
|
+
assert seen == ["A", "B", "C"]
|
|
1190
|
+
assert graph.ok
|
|
1191
|
+
assert graph.blocked == []
|
|
1192
|
+
assert b.blocked_by is None
|
|
1193
|
+
assert c.status == TaskStatus.SUCCEEDED
|
|
1194
|
+
|
|
1195
|
+
|
|
1132
1196
|
def test_within_run_blocked_is_not_retried_same_run() -> None:
|
|
1133
1197
|
"""A task BLOCKED during this run is terminal-for-the-run, not retried."""
|
|
1134
1198
|
attempts: dict[str, int] = {"B": 0}
|
|
@@ -1181,6 +1245,46 @@ def test_finally_waits_for_retryable_failed_dependency_added_later() -> None:
|
|
|
1181
1245
|
assert cleanup.status == TaskStatus.SUCCEEDED
|
|
1182
1246
|
|
|
1183
1247
|
|
|
1248
|
+
def test_finally_waits_for_inflight_carryover_retry() -> None:
|
|
1249
|
+
"""Finally tasks must not run while a carryover-failed parent is retrying."""
|
|
1250
|
+
import threading
|
|
1251
|
+
|
|
1252
|
+
parent = _bash("parent", "")
|
|
1253
|
+
parent.transition_to(TaskStatus.RUNNING)
|
|
1254
|
+
parent.transition_to(TaskStatus.FAILED, error="old failure")
|
|
1255
|
+
cleanup = _bash("cleanup", "")
|
|
1256
|
+
release_parent = threading.Event()
|
|
1257
|
+
|
|
1258
|
+
class SlowRetryExecutor(TaskExecutor):
|
|
1259
|
+
def execute(self, task: Any, *args: Any, **kwargs: Any) -> Any:
|
|
1260
|
+
if task is parent:
|
|
1261
|
+
assert release_parent.wait(timeout=1)
|
|
1262
|
+
return super().execute(task, *args, **kwargs)
|
|
1263
|
+
|
|
1264
|
+
executor = SlowRetryExecutor.empty()
|
|
1265
|
+
|
|
1266
|
+
def handler(context: Any) -> str:
|
|
1267
|
+
if context.id == cleanup.id:
|
|
1268
|
+
return context.upstream[parent.id].status.value
|
|
1269
|
+
return "parent retried"
|
|
1270
|
+
|
|
1271
|
+
executor.register(TaskType.BASH, handler)
|
|
1272
|
+
graph = TaskGraph()
|
|
1273
|
+
graph.add(cleanup, after=[parent], finally_=True)
|
|
1274
|
+
graph[parent] = []
|
|
1275
|
+
|
|
1276
|
+
timer = threading.Timer(0.1, release_parent.set)
|
|
1277
|
+
timer.start()
|
|
1278
|
+
try:
|
|
1279
|
+
graph.run(executor, max_workers=2)
|
|
1280
|
+
finally:
|
|
1281
|
+
timer.cancel()
|
|
1282
|
+
|
|
1283
|
+
assert parent.status == TaskStatus.SUCCEEDED
|
|
1284
|
+
assert cleanup.result is not None
|
|
1285
|
+
assert cleanup.result.output == "succeeded"
|
|
1286
|
+
|
|
1287
|
+
|
|
1184
1288
|
def test_finally_runs_after_carryover_blocked_recovers() -> None:
|
|
1185
1289
|
"""A finally task fires after a carryover-BLOCKED task recovers to SUCCEEDED."""
|
|
1186
1290
|
finally_ran: list[str] = []
|
|
@@ -1235,28 +1339,30 @@ def test_graph_persistence_errors_initially_empty() -> None:
|
|
|
1235
1339
|
assert executor.graph_persistence_errors == []
|
|
1236
1340
|
|
|
1237
1341
|
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1342
|
+
class _BrokenGraphs:
|
|
1343
|
+
def save(self, graph: Any) -> None:
|
|
1344
|
+
raise RuntimeError("disk full")
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
class _BrokenGraphStore:
|
|
1348
|
+
@property
|
|
1349
|
+
def tasks(self) -> Any:
|
|
1350
|
+
return None
|
|
1241
1351
|
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1352
|
+
@property
|
|
1353
|
+
def graphs(self) -> Any:
|
|
1354
|
+
return _BrokenGraphs()
|
|
1245
1355
|
|
|
1246
|
-
class _BrokenStore:
|
|
1247
|
-
@property
|
|
1248
|
-
def tasks(self) -> Any:
|
|
1249
|
-
return None
|
|
1250
1356
|
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1357
|
+
def test_graph_save_failure_does_not_break_run_and_records_error() -> None:
|
|
1358
|
+
"""A failing graph save records on graph_persistence_errors and warns."""
|
|
1359
|
+
import warnings
|
|
1254
1360
|
|
|
1255
1361
|
a = _bash("A", "echo a")
|
|
1256
1362
|
graph = TaskGraph()
|
|
1257
1363
|
graph[a] = []
|
|
1258
1364
|
|
|
1259
|
-
executor = TaskExecutor(store=
|
|
1365
|
+
executor = TaskExecutor(store=_BrokenGraphStore())
|
|
1260
1366
|
with warnings.catch_warnings(record=True) as captured:
|
|
1261
1367
|
warnings.simplefilter("always")
|
|
1262
1368
|
graph.run(executor)
|
|
@@ -1267,6 +1373,25 @@ def test_graph_save_failure_does_not_break_run_and_records_error() -> None:
|
|
|
1267
1373
|
assert any("graph persistence failed" in str(w.message) for w in captured)
|
|
1268
1374
|
|
|
1269
1375
|
|
|
1376
|
+
def test_graph_save_failure_does_not_propagate_when_warnings_are_errors() -> None:
|
|
1377
|
+
"""Graph persistence warnings remain non-fatal under strict warning filters."""
|
|
1378
|
+
import warnings
|
|
1379
|
+
|
|
1380
|
+
a = _bash("A", "")
|
|
1381
|
+
graph = TaskGraph()
|
|
1382
|
+
graph[a] = []
|
|
1383
|
+
|
|
1384
|
+
executor = TaskExecutor.empty(store=_BrokenGraphStore())
|
|
1385
|
+
executor.register(TaskType.BASH, lambda _context: "ok")
|
|
1386
|
+
with warnings.catch_warnings():
|
|
1387
|
+
warnings.simplefilter("error")
|
|
1388
|
+
graph.run(executor)
|
|
1389
|
+
|
|
1390
|
+
assert a.status == TaskStatus.SUCCEEDED
|
|
1391
|
+
assert executor.graph_persistence_errors
|
|
1392
|
+
assert all(gid == graph.id for gid, _ in executor.graph_persistence_errors)
|
|
1393
|
+
|
|
1394
|
+
|
|
1270
1395
|
def test_persist_graph_no_store_is_noop() -> None:
|
|
1271
1396
|
"""Without a configured store, _persist_graph silently does nothing."""
|
|
1272
1397
|
executor = TaskExecutor()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|