optio-core 0.4.2__tar.gz → 0.4.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {optio_core-0.4.2/src/optio_core.egg-info → optio_core-0.4.4}/PKG-INFO +1 -1
- {optio_core-0.4.2 → optio_core-0.4.4}/pyproject.toml +1 -1
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/executor.py +249 -63
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/lifecycle.py +65 -1
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/store.py +38 -0
- {optio_core-0.4.2 → optio_core-0.4.4/src/optio_core.egg-info}/PKG-INFO +1 -1
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/SOURCES.txt +1 -0
- optio_core-0.4.4/tests/test_lost_final_writes.py +373 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/LICENSE +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/README.md +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/setup.cfg +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/__init__.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_engine_service.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_force_cancel.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_generated/optio_engine.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_launch_block_store.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/context.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/exceptions.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/__init__.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m001_status_subdocument.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m002_backfill_child_metadata.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m003_backfill_has_saved_state.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m004_create_expire_at_ttl_index.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/models.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/progress_helpers.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/scheduler.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/state_machine.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/dependency_links.txt +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/requires.txt +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/top_level.txt +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_auto_resume.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_cancel_propagation.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_cancel_race_parent_overwrite.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_failure_cancel_distinction.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_failure_structured.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_progress.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_result_channel.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_client_directed_events.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_context_blob.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_context_resume.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_control_upstream.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_deadline_cancel.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_deadline_cancel_launchguard.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_engine_service.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_engine_service_resolve.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_engine_service_upload.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_executor.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_group_cancel.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_group_cancel_purge.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_heartbeat.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_launch_guard.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_lifecycle_reconciliation.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_migration_m003.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_migration_m004.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_models.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_no_redis.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_outcomes.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_parallel.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_persistent_launch_blocks.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_progress_helpers.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_progress_throttle.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_publish_result.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_resync.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_resync_cancel_stale.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_scheduler.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_shutdown_drain_completion.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_state_machine.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_store.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_task_ttl.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_upload_registry.py +0 -0
- {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_widget_primitives.py +0 -0
|
@@ -3,13 +3,16 @@
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import logging
|
|
5
5
|
import os as _os
|
|
6
|
+
import sys
|
|
6
7
|
import time
|
|
7
8
|
from dataclasses import dataclass
|
|
8
9
|
from datetime import datetime, timezone
|
|
9
10
|
from typing import Any, Callable, Awaitable
|
|
10
11
|
from bson import ObjectId
|
|
11
12
|
from motor.motor_asyncio import AsyncIOMotorDatabase
|
|
13
|
+
from pymongo.errors import PyMongoError
|
|
12
14
|
|
|
15
|
+
logger = logging.getLogger(__name__)
|
|
13
16
|
_trace_logger = logging.getLogger("optio_core.cancel_trace")
|
|
14
17
|
_CANCEL_TRACE = _os.environ.get("OPTIO_CANCEL_TRACE", "0").lower() in ("1", "true", "yes")
|
|
15
18
|
|
|
@@ -29,7 +32,7 @@ from optio_core.store import (
|
|
|
29
32
|
update_status, clear_result_fields,
|
|
30
33
|
create_child_process, append_log,
|
|
31
34
|
clear_widget_upstream, compute_expire_at,
|
|
32
|
-
_collection,
|
|
35
|
+
finalize_if_active, _collection,
|
|
33
36
|
)
|
|
34
37
|
from optio_core.context import ProcessContext
|
|
35
38
|
from optio_core.exceptions import ChildProcessFailed
|
|
@@ -47,6 +50,26 @@ class _CancelEntry:
|
|
|
47
50
|
deadline: float | None = None
|
|
48
51
|
|
|
49
52
|
|
|
53
|
+
# Rule 3 retry backoff: first retry on the next supervisor tick, then doubling
|
|
54
|
+
# from this delay up to the cap. No attempt limit.
|
|
55
|
+
_FINAL_RETRY_FIRST_DELAY = 1.0
|
|
56
|
+
_FINAL_RETRY_MAX_DELAY = 30.0
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class _UnrecordedFinal:
|
|
61
|
+
"""A final state that could not be written to Mongo, kept for retry.
|
|
62
|
+
|
|
63
|
+
`status` is the terminal status still to write (None once written).
|
|
64
|
+
`cascade` asks for the force-cancel cascade to direct active children to
|
|
65
|
+
be re-run. Spec: docs/2026-10-03-lost-final-state-writes-design.md
|
|
66
|
+
"""
|
|
67
|
+
status: ProcessStatus | None
|
|
68
|
+
cascade: bool = False
|
|
69
|
+
attempts: int = 0
|
|
70
|
+
next_attempt: float = 0.0
|
|
71
|
+
|
|
72
|
+
|
|
50
73
|
class Executor:
|
|
51
74
|
"""Executes task functions with lifecycle management."""
|
|
52
75
|
|
|
@@ -67,6 +90,11 @@ class Executor:
|
|
|
67
90
|
self._notify_parent_failure = notify_parent_failure
|
|
68
91
|
self._cancellation_flags: dict[ObjectId, _CancelEntry] = {}
|
|
69
92
|
self._running_tasks: dict[ObjectId, asyncio.Task] = {}
|
|
93
|
+
# Rules 3 and 4 of docs/2026-10-03-lost-final-state-writes-design.md:
|
|
94
|
+
# final states that could not be written (retried by the supervisor
|
|
95
|
+
# loop), and OIDs whose force_cancel is still in flight.
|
|
96
|
+
self._unrecorded_finals: dict[ObjectId, _UnrecordedFinal] = {}
|
|
97
|
+
self._force_cancelling: set[ObjectId] = set()
|
|
70
98
|
self._task_registry: dict[str, TaskInstance] = {}
|
|
71
99
|
# Task→launcher return channel (in-memory only, same-process).
|
|
72
100
|
# Keyed by processId string. Registry holds published objects for
|
|
@@ -189,6 +217,17 @@ class Executor:
|
|
|
189
217
|
if current is None:
|
|
190
218
|
raise RuntimeError("_execute_process must be called from within an asyncio Task")
|
|
191
219
|
self._running_tasks[oid] = current
|
|
220
|
+
# A new run of this row supersedes a final state still parked from an
|
|
221
|
+
# earlier run (Rule 3): the row was launchable, so that write is moot.
|
|
222
|
+
self._unrecorded_finals.pop(oid, None)
|
|
223
|
+
|
|
224
|
+
# Rule 3 (docs/2026-10-03-lost-final-state-writes-design.md): `final`
|
|
225
|
+
# is the terminal status this run means to write, noted as soon as
|
|
226
|
+
# the outcome is known; `final_recorded` flips once that write has
|
|
227
|
+
# landed. If the finally below is reached without it, the status is
|
|
228
|
+
# parked for the supervisor loop to retry.
|
|
229
|
+
final: ProcessStatus | None = None
|
|
230
|
+
final_recorded = False
|
|
192
231
|
|
|
193
232
|
try:
|
|
194
233
|
now = datetime.now(timezone.utc)
|
|
@@ -231,14 +270,15 @@ class Executor:
|
|
|
231
270
|
ctx._parent_listener = _listener
|
|
232
271
|
|
|
233
272
|
if execute_fn is None:
|
|
273
|
+
final = ProcessStatus(
|
|
274
|
+
state="failed", error="No execute function found",
|
|
275
|
+
failed_at=datetime.now(timezone.utc),
|
|
276
|
+
)
|
|
234
277
|
await update_status(
|
|
235
|
-
self._db, self._prefix, oid,
|
|
236
|
-
ProcessStatus(
|
|
237
|
-
state="failed", error="No execute function found",
|
|
238
|
-
failed_at=datetime.now(timezone.utc),
|
|
239
|
-
),
|
|
278
|
+
self._db, self._prefix, oid, final,
|
|
240
279
|
expire_at=compute_expire_at(ttl_seconds),
|
|
241
280
|
)
|
|
281
|
+
final_recorded = True
|
|
242
282
|
return ("failed", None)
|
|
243
283
|
|
|
244
284
|
start_time = time.monotonic()
|
|
@@ -286,15 +326,16 @@ class Executor:
|
|
|
286
326
|
"CANCEL-TRACE %s: executor write cancelled (raised CancelledError)",
|
|
287
327
|
proc["processId"],
|
|
288
328
|
)
|
|
329
|
+
final = ProcessStatus(
|
|
330
|
+
state="cancelled",
|
|
331
|
+
stopped_at=datetime.now(timezone.utc),
|
|
332
|
+
)
|
|
289
333
|
await ctx.flush_final_progress()
|
|
290
334
|
await update_status(
|
|
291
|
-
self._db, self._prefix, oid,
|
|
292
|
-
ProcessStatus(
|
|
293
|
-
state="cancelled",
|
|
294
|
-
stopped_at=datetime.now(timezone.utc),
|
|
295
|
-
),
|
|
335
|
+
self._db, self._prefix, oid, final,
|
|
296
336
|
expire_at=compute_expire_at(ttl_seconds),
|
|
297
337
|
)
|
|
338
|
+
final_recorded = True
|
|
298
339
|
await append_log(
|
|
299
340
|
self._db, self._prefix, oid, "event",
|
|
300
341
|
"State changed to cancelled (raised CancelledError)",
|
|
@@ -303,36 +344,37 @@ class Executor:
|
|
|
303
344
|
await self._cleanup_ephemeral(str(oid))
|
|
304
345
|
raise
|
|
305
346
|
except Exception as e:
|
|
347
|
+
final = ProcessStatus(
|
|
348
|
+
state="failed", error=str(e),
|
|
349
|
+
failed_at=datetime.now(timezone.utc),
|
|
350
|
+
)
|
|
306
351
|
await ctx.flush_final_progress()
|
|
307
352
|
await update_status(
|
|
308
|
-
self._db, self._prefix, oid,
|
|
309
|
-
ProcessStatus(
|
|
310
|
-
state="failed", error=str(e),
|
|
311
|
-
failed_at=datetime.now(timezone.utc),
|
|
312
|
-
),
|
|
353
|
+
self._db, self._prefix, oid, final,
|
|
313
354
|
expire_at=compute_expire_at(ttl_seconds),
|
|
314
355
|
)
|
|
356
|
+
final_recorded = True
|
|
315
357
|
await append_log(self._db, self._prefix, oid, "error", str(e))
|
|
316
358
|
await clear_widget_upstream(self._db, self._prefix, oid)
|
|
317
359
|
await self._cleanup_ephemeral(str(oid))
|
|
318
360
|
return ("failed", e)
|
|
319
361
|
|
|
362
|
+
# Note the outcome before flushing, so a flush that fails still
|
|
363
|
+
# leaves the right state to retry; rebuilt after the flush so the
|
|
364
|
+
# recorded timing matches what it was before.
|
|
365
|
+
final = self._terminal_status(end_state, start_time)
|
|
320
366
|
await ctx.flush_final_progress()
|
|
321
|
-
|
|
367
|
+
final = self._terminal_status(end_state, start_time)
|
|
322
368
|
|
|
323
369
|
if end_state == "done":
|
|
324
370
|
_trace(
|
|
325
371
|
"CANCEL-TRACE %s: executor write done", proc["processId"],
|
|
326
372
|
)
|
|
327
373
|
await update_status(
|
|
328
|
-
self._db, self._prefix, oid,
|
|
329
|
-
ProcessStatus(
|
|
330
|
-
state="done",
|
|
331
|
-
done_at=datetime.now(timezone.utc),
|
|
332
|
-
duration=round(elapsed, 2),
|
|
333
|
-
),
|
|
374
|
+
self._db, self._prefix, oid, final,
|
|
334
375
|
expire_at=compute_expire_at(ttl_seconds),
|
|
335
376
|
)
|
|
377
|
+
final_recorded = True
|
|
336
378
|
await append_log(self._db, self._prefix, oid, "event", "State changed to done")
|
|
337
379
|
elif end_state == "cancelled":
|
|
338
380
|
_trace(
|
|
@@ -340,13 +382,10 @@ class Executor:
|
|
|
340
382
|
proc["processId"],
|
|
341
383
|
)
|
|
342
384
|
await update_status(
|
|
343
|
-
self._db, self._prefix, oid,
|
|
344
|
-
ProcessStatus(
|
|
345
|
-
state="cancelled",
|
|
346
|
-
stopped_at=datetime.now(timezone.utc),
|
|
347
|
-
),
|
|
385
|
+
self._db, self._prefix, oid, final,
|
|
348
386
|
expire_at=compute_expire_at(ttl_seconds),
|
|
349
387
|
)
|
|
388
|
+
final_recorded = True
|
|
350
389
|
await append_log(self._db, self._prefix, oid, "event", "State changed to cancelled")
|
|
351
390
|
|
|
352
391
|
await clear_widget_upstream(self._db, self._prefix, oid)
|
|
@@ -363,6 +402,30 @@ class Executor:
|
|
|
363
402
|
if _fut is not None and not _fut.done():
|
|
364
403
|
from optio_core.exceptions import ResultNotPublished
|
|
365
404
|
_fut.set_exception(ResultNotPublished(_pid))
|
|
405
|
+
# Rule 3: the final write never landed (Mongo unreachable, or an
|
|
406
|
+
# error before the outcome was known). A CancelledError unwind is
|
|
407
|
+
# left alone: force_cancel, shutdown (Rule 2) or the next start
|
|
408
|
+
# (Rule 1) owns that row.
|
|
409
|
+
_exc = sys.exc_info()[1]
|
|
410
|
+
if not final_recorded and not isinstance(_exc, asyncio.CancelledError):
|
|
411
|
+
if final is None:
|
|
412
|
+
final = ProcessStatus(
|
|
413
|
+
state="failed",
|
|
414
|
+
error=f"Final state could not be recorded: {_exc!r}",
|
|
415
|
+
failed_at=datetime.now(timezone.utc),
|
|
416
|
+
)
|
|
417
|
+
self._park_final(oid, final, cause=_exc)
|
|
418
|
+
|
|
419
|
+
@staticmethod
|
|
420
|
+
def _terminal_status(end_state: str, start_time: float) -> ProcessStatus:
|
|
421
|
+
"""Terminal status for a task body that returned ('done' or 'cancelled')."""
|
|
422
|
+
now = datetime.now(timezone.utc)
|
|
423
|
+
if end_state == "done":
|
|
424
|
+
return ProcessStatus(
|
|
425
|
+
state="done", done_at=now,
|
|
426
|
+
duration=round(time.monotonic() - start_time, 2),
|
|
427
|
+
)
|
|
428
|
+
return ProcessStatus(state="cancelled", stopped_at=now)
|
|
366
429
|
|
|
367
430
|
async def execute_child(
|
|
368
431
|
self,
|
|
@@ -476,6 +539,118 @@ class Executor:
|
|
|
476
539
|
)
|
|
477
540
|
return True
|
|
478
541
|
|
|
542
|
+
def owns(self, oid: ObjectId) -> bool:
|
|
543
|
+
"""True while this executor still has a task, a cancel entry or a
|
|
544
|
+
force-cancel in flight for `oid`.
|
|
545
|
+
|
|
546
|
+
A task registers its entry before it writes `running`, and writes its
|
|
547
|
+
final state before it drops the entry. So a row in `running`,
|
|
548
|
+
`cancel_requested` or `cancelling` for which this returns False has
|
|
549
|
+
lost its final write (Rule 4 of
|
|
550
|
+
docs/2026-10-03-lost-final-state-writes-design.md).
|
|
551
|
+
"""
|
|
552
|
+
return (
|
|
553
|
+
oid in self._cancellation_flags
|
|
554
|
+
or oid in self._running_tasks
|
|
555
|
+
or oid in self._force_cancelling
|
|
556
|
+
)
|
|
557
|
+
|
|
558
|
+
def _park_final(
|
|
559
|
+
self, oid: ObjectId, status: ProcessStatus | None, *,
|
|
560
|
+
cascade: bool = False, cause: BaseException | None = None,
|
|
561
|
+
) -> None:
|
|
562
|
+
"""Keep a final state (and/or a force-cancel cascade) that could not
|
|
563
|
+
be written, for retry_unrecorded_finals. A status already parked for
|
|
564
|
+
the same OID wins; cascade requests accumulate."""
|
|
565
|
+
entry = self._unrecorded_finals.get(oid)
|
|
566
|
+
if entry is None:
|
|
567
|
+
self._unrecorded_finals[oid] = _UnrecordedFinal(status=status, cascade=cascade)
|
|
568
|
+
logger.warning(
|
|
569
|
+
"Could not record the final state (%s) of process %s: %r. "
|
|
570
|
+
"Retrying in the background.",
|
|
571
|
+
status.state if status is not None else "child cascade", oid, cause,
|
|
572
|
+
)
|
|
573
|
+
return
|
|
574
|
+
if entry.status is None:
|
|
575
|
+
entry.status = status
|
|
576
|
+
entry.cascade = entry.cascade or cascade
|
|
577
|
+
|
|
578
|
+
async def _record_final(self, oid: ObjectId, entry: _UnrecordedFinal) -> None:
|
|
579
|
+
"""Write a parked final state, then run a parked cascade. Raises
|
|
580
|
+
PyMongoError while Mongo is still unreachable; the entry then keeps
|
|
581
|
+
whatever is left to do."""
|
|
582
|
+
if entry.status is not None:
|
|
583
|
+
state = entry.status.state
|
|
584
|
+
written = await finalize_if_active(self._db, self._prefix, oid, entry.status)
|
|
585
|
+
entry.status = None
|
|
586
|
+
if written:
|
|
587
|
+
await append_log(
|
|
588
|
+
self._db, self._prefix, oid, "event",
|
|
589
|
+
f"State recorded late: {state} (database was unavailable)",
|
|
590
|
+
)
|
|
591
|
+
await self._cleanup_ephemeral(str(oid))
|
|
592
|
+
if entry.cascade:
|
|
593
|
+
await self._force_cancel_children(oid)
|
|
594
|
+
entry.cascade = False
|
|
595
|
+
|
|
596
|
+
async def retry_unrecorded_finals(self) -> None:
|
|
597
|
+
"""Retry the parked final states that are due (Rule 3). Called by the
|
|
598
|
+
supervisor loop on every tick. A row that is no longer active is left
|
|
599
|
+
alone and its entry dropped; while Mongo stays unreachable the next
|
|
600
|
+
attempt backs off, doubling up to _FINAL_RETRY_MAX_DELAY."""
|
|
601
|
+
now = time.monotonic()
|
|
602
|
+
for oid, entry in list(self._unrecorded_finals.items()):
|
|
603
|
+
if entry.next_attempt > now or self.owns(oid):
|
|
604
|
+
continue
|
|
605
|
+
try:
|
|
606
|
+
await self._record_final(oid, entry)
|
|
607
|
+
except PyMongoError as e:
|
|
608
|
+
entry.attempts += 1
|
|
609
|
+
entry.next_attempt = time.monotonic() + min(
|
|
610
|
+
_FINAL_RETRY_MAX_DELAY,
|
|
611
|
+
_FINAL_RETRY_FIRST_DELAY * 2 ** (entry.attempts - 1),
|
|
612
|
+
)
|
|
613
|
+
logger.debug(
|
|
614
|
+
"Final state of process %s still not recorded (retry %d): %r",
|
|
615
|
+
oid, entry.attempts, e,
|
|
616
|
+
)
|
|
617
|
+
continue
|
|
618
|
+
self._unrecorded_finals.pop(oid, None)
|
|
619
|
+
logger.warning(
|
|
620
|
+
"Final state of process %s recorded late, after %d failed retries",
|
|
621
|
+
oid, entry.attempts,
|
|
622
|
+
)
|
|
623
|
+
|
|
624
|
+
async def record_parked_final_now(self, oid: ObjectId) -> bool:
|
|
625
|
+
"""If a final state is parked for `oid`, write it now, ignoring the
|
|
626
|
+
backoff. Returns True if one was parked. Raises PyMongoError while
|
|
627
|
+
Mongo is still unreachable; the entry then stays parked."""
|
|
628
|
+
entry = self._unrecorded_finals.get(oid)
|
|
629
|
+
if entry is None:
|
|
630
|
+
return False
|
|
631
|
+
await self._record_final(oid, entry)
|
|
632
|
+
self._unrecorded_finals.pop(oid, None)
|
|
633
|
+
return True
|
|
634
|
+
|
|
635
|
+
async def _force_cancel_children(self, oid: ObjectId) -> None:
|
|
636
|
+
"""Cascade a force-cancel to direct active children. Unconditional --
|
|
637
|
+
force is force."""
|
|
638
|
+
from optio_core.store import list_direct_children
|
|
639
|
+
from optio_core.state_machine import ACTIVE_STATES
|
|
640
|
+
|
|
641
|
+
children = await list_direct_children(
|
|
642
|
+
self._db, self._prefix, oid, states=ACTIVE_STATES,
|
|
643
|
+
)
|
|
644
|
+
if children:
|
|
645
|
+
_trace(
|
|
646
|
+
"force_cancel oid=%s: cascading to children=%s",
|
|
647
|
+
oid, [str(c["_id"]) for c in children],
|
|
648
|
+
)
|
|
649
|
+
await asyncio.gather(
|
|
650
|
+
*(self.force_cancel(c["_id"]) for c in children),
|
|
651
|
+
return_exceptions=True,
|
|
652
|
+
)
|
|
653
|
+
|
|
479
654
|
async def force_cancel(self, oid: ObjectId) -> None:
|
|
480
655
|
"""Hard-cancel a process whose cooperative deadline has expired.
|
|
481
656
|
|
|
@@ -486,47 +661,58 @@ class Executor:
|
|
|
486
661
|
captures both auto-propagate descendants (already in supervisor
|
|
487
662
|
map; idempotent) and opt-out descendants (only this cascade
|
|
488
663
|
reaches them).
|
|
664
|
+
|
|
665
|
+
If Mongo is unreachable, the write and the cascade are parked for
|
|
666
|
+
retry_unrecorded_finals instead of raising (Rule 3 of
|
|
667
|
+
docs/2026-10-03-lost-final-state-writes-design.md).
|
|
489
668
|
"""
|
|
490
|
-
from optio_core._force_cancel import
|
|
491
|
-
|
|
492
|
-
|
|
669
|
+
from optio_core._force_cancel import (
|
|
670
|
+
FORCE_CANCEL_ERROR, _write_force_cancelled_state,
|
|
671
|
+
)
|
|
493
672
|
|
|
494
673
|
task = self._running_tasks.get(oid)
|
|
495
674
|
_trace(
|
|
496
675
|
"force_cancel oid=%s task=%s done=%s",
|
|
497
676
|
oid, task, task.done() if task else None,
|
|
498
677
|
)
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
678
|
+
# The row stays owned by this engine until our own write is done,
|
|
679
|
+
# even after the task's finally has dropped its entries (Rule 4).
|
|
680
|
+
self._force_cancelling.add(oid)
|
|
681
|
+
try:
|
|
682
|
+
if task is not None and not task.done():
|
|
683
|
+
_trace("force_cancel oid=%s: calling task.cancel() + 2s shield wait", oid)
|
|
684
|
+
task.cancel()
|
|
685
|
+
try:
|
|
686
|
+
await asyncio.wait_for(
|
|
687
|
+
asyncio.shield(task),
|
|
688
|
+
timeout=self._optio._config.force_cancel_shield_seconds,
|
|
689
|
+
)
|
|
690
|
+
_trace("force_cancel oid=%s: task unwound within shield window", oid)
|
|
691
|
+
except asyncio.TimeoutError:
|
|
692
|
+
_trace("force_cancel oid=%s: 2s shield TIMEOUT — task still running", oid)
|
|
693
|
+
pass
|
|
694
|
+
except asyncio.CancelledError:
|
|
695
|
+
_trace("force_cancel oid=%s: task acknowledged Cancel", oid)
|
|
696
|
+
pass
|
|
697
|
+
except Exception as _e:
|
|
698
|
+
_trace("force_cancel oid=%s: task raised %s: %s",
|
|
699
|
+
oid, type(_e).__name__, _e)
|
|
700
|
+
pass
|
|
502
701
|
try:
|
|
503
|
-
await
|
|
504
|
-
|
|
505
|
-
|
|
702
|
+
await _write_force_cancelled_state(self._db, self._prefix, oid)
|
|
703
|
+
except PyMongoError as e:
|
|
704
|
+
self._park_final(
|
|
705
|
+
oid,
|
|
706
|
+
ProcessStatus(
|
|
707
|
+
state="failed", error=FORCE_CANCEL_ERROR,
|
|
708
|
+
failed_at=datetime.now(timezone.utc),
|
|
709
|
+
),
|
|
710
|
+
cascade=True, cause=e,
|
|
506
711
|
)
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
except Exception as _e:
|
|
515
|
-
_trace("force_cancel oid=%s: task raised %s: %s",
|
|
516
|
-
oid, type(_e).__name__, _e)
|
|
517
|
-
pass
|
|
518
|
-
await _write_force_cancelled_state(self._db, self._prefix, oid)
|
|
519
|
-
|
|
520
|
-
# Cascade to direct active children. Unconditional — force is force.
|
|
521
|
-
children = await list_direct_children(
|
|
522
|
-
self._db, self._prefix, oid, states=ACTIVE_STATES,
|
|
523
|
-
)
|
|
524
|
-
if children:
|
|
525
|
-
_trace(
|
|
526
|
-
"force_cancel oid=%s: cascading to children=%s",
|
|
527
|
-
oid, [str(c["_id"]) for c in children],
|
|
528
|
-
)
|
|
529
|
-
await asyncio.gather(
|
|
530
|
-
*(self.force_cancel(c["_id"]) for c in children),
|
|
531
|
-
return_exceptions=True,
|
|
532
|
-
)
|
|
712
|
+
return
|
|
713
|
+
try:
|
|
714
|
+
await self._force_cancel_children(oid)
|
|
715
|
+
except PyMongoError as e:
|
|
716
|
+
self._park_final(oid, None, cascade=True, cause=e)
|
|
717
|
+
finally:
|
|
718
|
+
self._force_cancelling.discard(oid)
|
|
@@ -34,6 +34,7 @@ from optio_core.store import (
|
|
|
34
34
|
upsert_process, remove_stale_processes, find_stale_process_ids,
|
|
35
35
|
get_process_by_process_id, update_status, clear_result_fields,
|
|
36
36
|
append_log, compute_expire_at, purge_processes, set_auto_resume_scheduled,
|
|
37
|
+
finalize_if_active,
|
|
37
38
|
)
|
|
38
39
|
from optio_core.state_machine import (
|
|
39
40
|
ACTIVE_STATES, CANCELLABLE_STATES, DISMISSABLE_STATES, END_STATES,
|
|
@@ -47,6 +48,14 @@ from optio_core.scheduler import ProcessScheduler
|
|
|
47
48
|
|
|
48
49
|
logger = logging.getLogger("optio_core_core")
|
|
49
50
|
|
|
51
|
+
# Rule 4 of docs/2026-10-03-lost-final-state-writes-design.md: states in which
|
|
52
|
+
# a row with no owning task has lost its final write, and what it settles to.
|
|
53
|
+
_ORPHAN_STATES = frozenset({"running", "cancel_requested", "cancelling"})
|
|
54
|
+
ORPHAN_ERROR = (
|
|
55
|
+
"Process had no running task in this engine; "
|
|
56
|
+
"its final state was never recorded"
|
|
57
|
+
)
|
|
58
|
+
|
|
50
59
|
|
|
51
60
|
def _trace(fmt: str, *args: object) -> None:
|
|
52
61
|
"""Cancel-trace log helper, gated on OPTIO_CANCEL_TRACE env var."""
|
|
@@ -674,6 +683,10 @@ class Optio:
|
|
|
674
683
|
"CANCEL-TRACE %s: request_cancel_with_deadline found=%s",
|
|
675
684
|
process_id, found,
|
|
676
685
|
)
|
|
686
|
+
if not found:
|
|
687
|
+
# No task in this engine owns the row we just moved to
|
|
688
|
+
# cancel_requested: its final write was lost (Rule 4).
|
|
689
|
+
await self._settle_if_orphaned(proc, "cancel_requested")
|
|
677
690
|
if found:
|
|
678
691
|
# Conditional: only advance to 'cancelling' if the row is
|
|
679
692
|
# still in 'cancel_requested'. If the executor has already
|
|
@@ -739,6 +752,8 @@ class Optio:
|
|
|
739
752
|
state = proc["status"]["state"]
|
|
740
753
|
if state not in ACTIVE_STATES:
|
|
741
754
|
return state
|
|
755
|
+
if await self._settle_if_orphaned(proc, state):
|
|
756
|
+
continue # Rule 4: settled; the re-read returns its state
|
|
742
757
|
if time.monotonic() >= deadline:
|
|
743
758
|
raise asyncio.TimeoutError(
|
|
744
759
|
f"Process {process_id} did not reach terminal state within {ceiling}s"
|
|
@@ -883,6 +898,8 @@ class Optio:
|
|
|
883
898
|
if proc is None or proc["status"]["state"] not in ACTIVE_STATES:
|
|
884
899
|
i += 1
|
|
885
900
|
continue
|
|
901
|
+
if await self._settle_if_orphaned(proc, proc["status"]["state"]):
|
|
902
|
+
continue # Rule 4: settled; the re-read sees it terminal
|
|
886
903
|
if time.monotonic() >= deadline:
|
|
887
904
|
remaining = len(pending) - i
|
|
888
905
|
raise asyncio.TimeoutError(
|
|
@@ -897,6 +914,46 @@ class Optio:
|
|
|
897
914
|
)
|
|
898
915
|
return len(pending)
|
|
899
916
|
|
|
917
|
+
async def _settle_if_orphaned(self, proc: dict, observed_state: str) -> bool:
|
|
918
|
+
"""Settle a row whose final write was lost, instead of waiting on it.
|
|
919
|
+
|
|
920
|
+
Applies to a row seen in `running`, `cancel_requested` or
|
|
921
|
+
`cancelling` that no task, cancel entry or force-cancel in this
|
|
922
|
+
engine owns (see Executor.owns). A final state parked for it
|
|
923
|
+
(Rule 3) is written as is; otherwise the row becomes `failed` with
|
|
924
|
+
ORPHAN_ERROR, but only while it is still in `observed_state`.
|
|
925
|
+
Returns True if the row is now settled. `scheduled` never
|
|
926
|
+
qualifies: launch_process writes it before the task registers.
|
|
927
|
+
|
|
928
|
+
Assumes one engine per (database, prefix), as Rule 1 does.
|
|
929
|
+
Spec: docs/2026-10-03-lost-final-state-writes-design.md (Rule 4)
|
|
930
|
+
"""
|
|
931
|
+
if observed_state not in _ORPHAN_STATES or self._executor is None:
|
|
932
|
+
return False
|
|
933
|
+
oid = proc["_id"]
|
|
934
|
+
if self._executor.owns(oid):
|
|
935
|
+
return False
|
|
936
|
+
if await self._executor.record_parked_final_now(oid):
|
|
937
|
+
return True
|
|
938
|
+
db, prefix = self._config.mongo_db, self._config.prefix
|
|
939
|
+
status = ProcessStatus(
|
|
940
|
+
state="failed", error=ORPHAN_ERROR,
|
|
941
|
+
failed_at=datetime.now(timezone.utc),
|
|
942
|
+
)
|
|
943
|
+
if not await finalize_if_active(
|
|
944
|
+
db, prefix, oid, status, expected_state=observed_state,
|
|
945
|
+
):
|
|
946
|
+
return False
|
|
947
|
+
await append_log(
|
|
948
|
+
db, prefix, oid, "event",
|
|
949
|
+
f"State reconciled: {observed_state} -> failed (no running task)",
|
|
950
|
+
)
|
|
951
|
+
logger.warning(
|
|
952
|
+
"Settled process %s (%s): it was %s with no running task in this engine",
|
|
953
|
+
proc.get("processId"), oid, observed_state,
|
|
954
|
+
)
|
|
955
|
+
return True
|
|
956
|
+
|
|
900
957
|
async def dismiss(self, process_id: str) -> DismissOutcome:
|
|
901
958
|
"""Dismiss a completed process (reset to idle). Returns DismissOutcome
|
|
902
959
|
with `proc` populated from a post-dismiss re-read on success."""
|
|
@@ -1210,7 +1267,9 @@ class Optio:
|
|
|
1210
1267
|
logger.info(f"Reconciled {len(stale)} interrupted process(es) to 'failed'")
|
|
1211
1268
|
|
|
1212
1269
|
async def _supervisor_loop(self) -> None:
|
|
1213
|
-
"""Scan for past-deadline cancellations every 500 ms; force-cancel them.
|
|
1270
|
+
"""Scan for past-deadline cancellations every 500 ms; force-cancel them.
|
|
1271
|
+
Also retries final states that could not be written (Rule 3 of
|
|
1272
|
+
docs/2026-10-03-lost-final-state-writes-design.md)."""
|
|
1214
1273
|
while self._running:
|
|
1215
1274
|
try:
|
|
1216
1275
|
now = time.monotonic()
|
|
@@ -1227,6 +1286,11 @@ class Optio:
|
|
|
1227
1286
|
await self._executor.force_cancel(oid)
|
|
1228
1287
|
except Exception as e:
|
|
1229
1288
|
logger.exception(f"Supervisor loop error: {e}")
|
|
1289
|
+
try:
|
|
1290
|
+
if self._executor is not None:
|
|
1291
|
+
await self._executor.retry_unrecorded_finals()
|
|
1292
|
+
except Exception as e:
|
|
1293
|
+
logger.exception(f"Supervisor loop error while retrying final states: {e}")
|
|
1230
1294
|
await asyncio.sleep(0.5)
|
|
1231
1295
|
|
|
1232
1296
|
async def _auto_resume_timer(self) -> None:
|
|
@@ -191,6 +191,44 @@ async def update_status(
|
|
|
191
191
|
)
|
|
192
192
|
|
|
193
193
|
|
|
194
|
+
async def finalize_if_active(
|
|
195
|
+
db: AsyncIOMotorDatabase, prefix: str, process_oid: ObjectId,
|
|
196
|
+
status: ProcessStatus, *, expected_state: str | None = None,
|
|
197
|
+
) -> bool:
|
|
198
|
+
"""Conditionally write a terminal status onto a row that is still active.
|
|
199
|
+
|
|
200
|
+
Matches only while `status.state` is in ACTIVE_STATES -- or, with
|
|
201
|
+
`expected_state`, only while it is exactly that state -- so a writer that
|
|
202
|
+
already moved the row on is never overwritten. Also clears
|
|
203
|
+
`widgetUpstream`, clears `autoResumeScheduled` for `failed`, and sets
|
|
204
|
+
`expireAt` from the row's `ttlSeconds`. Returns True if the row was
|
|
205
|
+
updated.
|
|
206
|
+
|
|
207
|
+
Used to record a final state late, and to settle a row whose final write
|
|
208
|
+
was lost. Spec: docs/2026-10-03-lost-final-state-writes-design.md
|
|
209
|
+
"""
|
|
210
|
+
from optio_core.state_machine import ACTIVE_STATES
|
|
211
|
+
|
|
212
|
+
coll = _collection(db, prefix)
|
|
213
|
+
now = datetime.now(timezone.utc)
|
|
214
|
+
ttl_doc = await coll.find_one({"_id": process_oid}, {"ttlSeconds": 1})
|
|
215
|
+
expire_at = compute_expire_at((ttl_doc or {}).get("ttlSeconds"), now=now)
|
|
216
|
+
set_doc: dict[str, Any] = {"status": status.to_dict(), "widgetUpstream": None}
|
|
217
|
+
if status.state == "failed":
|
|
218
|
+
set_doc["autoResumeScheduled"] = False
|
|
219
|
+
if expire_at is not None:
|
|
220
|
+
set_doc["expireAt"] = expire_at
|
|
221
|
+
state_match: Any = (
|
|
222
|
+
expected_state if expected_state is not None
|
|
223
|
+
else {"$in": list(ACTIVE_STATES)}
|
|
224
|
+
)
|
|
225
|
+
result = await coll.update_one(
|
|
226
|
+
{"_id": process_oid, "status.state": state_match},
|
|
227
|
+
{"$set": set_doc},
|
|
228
|
+
)
|
|
229
|
+
return bool(result.modified_count)
|
|
230
|
+
|
|
231
|
+
|
|
194
232
|
async def set_auto_resume_scheduled(
|
|
195
233
|
db: AsyncIOMotorDatabase, prefix: str, process_oid: ObjectId, value: bool,
|
|
196
234
|
) -> None:
|
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
"""Tests for lost final-state writes.
|
|
2
|
+
|
|
3
|
+
Spec: docs/2026-10-03-lost-final-state-writes-design.md
|
|
4
|
+
|
|
5
|
+
Rule 3: a final-state write that fails because Mongo is unreachable is kept
|
|
6
|
+
and retried by the supervisor loop. Rule 4: cancel / cancel_and_wait /
|
|
7
|
+
group_cancel_and_wait settle an active row that has no task behind it instead
|
|
8
|
+
of waiting for it.
|
|
9
|
+
|
|
10
|
+
"Mongo is unreachable" is simulated by making chosen writes raise
|
|
11
|
+
AutoReconnect. No test depends on wall-clock time: every wait is on a
|
|
12
|
+
condition, with a 60 s hang ceiling.
|
|
13
|
+
"""
|
|
14
|
+
import asyncio
|
|
15
|
+
import time
|
|
16
|
+
|
|
17
|
+
import pytest
|
|
18
|
+
from pymongo.errors import AutoReconnect
|
|
19
|
+
|
|
20
|
+
from optio_core.lifecycle import Optio
|
|
21
|
+
from optio_core.models import ProcessStatus, TaskInstance
|
|
22
|
+
from optio_core.store import get_process_by_process_id, upsert_process
|
|
23
|
+
|
|
24
|
+
pytestmark = pytest.mark.asyncio
|
|
25
|
+
|
|
26
|
+
HANG_CEILING = 60.0
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
async def _start_optio(mongo_db, prefix, tasks, cancel_grace_seconds=2.0, run=True):
|
|
30
|
+
async def gen(_s, _f):
|
|
31
|
+
return list(tasks)
|
|
32
|
+
optio = Optio()
|
|
33
|
+
await optio.init(
|
|
34
|
+
mongo_db=mongo_db, prefix=prefix,
|
|
35
|
+
get_task_definitions=gen,
|
|
36
|
+
cancel_grace_seconds=cancel_grace_seconds,
|
|
37
|
+
)
|
|
38
|
+
run_task = asyncio.create_task(optio.run()) if run else None
|
|
39
|
+
return optio, run_task
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
async def _stop_optio(optio, run_task):
|
|
43
|
+
await optio.shutdown()
|
|
44
|
+
if run_task is not None:
|
|
45
|
+
run_task.cancel()
|
|
46
|
+
try:
|
|
47
|
+
await run_task
|
|
48
|
+
except (asyncio.CancelledError, Exception):
|
|
49
|
+
pass
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def _wait_until(pred, what):
|
|
53
|
+
deadline = time.monotonic() + HANG_CEILING
|
|
54
|
+
while True:
|
|
55
|
+
value = await pred()
|
|
56
|
+
if value:
|
|
57
|
+
return value
|
|
58
|
+
if time.monotonic() > deadline:
|
|
59
|
+
raise AssertionError(f"timed out waiting for {what}")
|
|
60
|
+
await asyncio.sleep(0.02)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def _wait_state(mongo_db, prefix, process_id, states, log=None):
|
|
64
|
+
"""Wait for the row to reach one of `states` and, with `log`, also carry
|
|
65
|
+
that log entry. The status and the log line are separate writes, so a
|
|
66
|
+
test must wait for both rather than read the log right after the state."""
|
|
67
|
+
async def pred():
|
|
68
|
+
proc = await get_process_by_process_id(mongo_db, prefix, process_id)
|
|
69
|
+
if proc is None or proc["status"]["state"] not in states:
|
|
70
|
+
return None
|
|
71
|
+
if log is not None and not _logged(proc, log):
|
|
72
|
+
return None
|
|
73
|
+
return proc
|
|
74
|
+
what = f"{process_id} to reach {sorted(states)}" + (f" with log {log!r}" if log else "")
|
|
75
|
+
return await _wait_until(pred, what)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _fail_update_status(monkeypatch, states, times=1):
|
|
79
|
+
"""Make the executor's next `times` writes of a status in `states` raise."""
|
|
80
|
+
import optio_core.executor as ex
|
|
81
|
+
real = ex.update_status
|
|
82
|
+
left = {"n": times}
|
|
83
|
+
|
|
84
|
+
async def flaky(db, prefix, oid, status, expire_at=None):
|
|
85
|
+
if status.state in states and left["n"] > 0:
|
|
86
|
+
left["n"] -= 1
|
|
87
|
+
raise AutoReconnect("injected: mongo unreachable")
|
|
88
|
+
return await real(db, prefix, oid, status, expire_at=expire_at)
|
|
89
|
+
|
|
90
|
+
monkeypatch.setattr(ex, "update_status", flaky)
|
|
91
|
+
return left
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _logged(proc, needle):
|
|
95
|
+
return any(needle in (e.get("message") or "") for e in proc.get("log", []))
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
async def _done(ctx): # noqa: ARG001
|
|
99
|
+
return None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
# ---------- Rule 3: retry a final-state write that did not land ----------
|
|
103
|
+
|
|
104
|
+
async def test_lost_done_write_is_recorded_late(mongo_db, monkeypatch):
|
|
105
|
+
prefix = "lfw_done"
|
|
106
|
+
task = TaskInstance(process_id="p.done", name="Done", params={}, execute=_done)
|
|
107
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task])
|
|
108
|
+
try:
|
|
109
|
+
_fail_update_status(monkeypatch, {"done"})
|
|
110
|
+
await optio.launch("p.done", session_id=None)
|
|
111
|
+
proc = await _wait_state(
|
|
112
|
+
mongo_db, prefix, "p.done", {"done"}, log="State recorded late: done",
|
|
113
|
+
)
|
|
114
|
+
assert proc["status"]["doneAt"] is not None
|
|
115
|
+
finally:
|
|
116
|
+
await _stop_optio(optio, run_task)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
async def test_lost_failed_write_is_recorded_late(mongo_db, monkeypatch):
|
|
120
|
+
prefix = "lfw_failed"
|
|
121
|
+
|
|
122
|
+
async def boom(ctx): # noqa: ARG001
|
|
123
|
+
raise ValueError("boom")
|
|
124
|
+
|
|
125
|
+
task = TaskInstance(process_id="p.fail", name="Fail", params={}, execute=boom)
|
|
126
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task])
|
|
127
|
+
try:
|
|
128
|
+
_fail_update_status(monkeypatch, {"failed"})
|
|
129
|
+
await optio.launch("p.fail", session_id=None)
|
|
130
|
+
proc = await _wait_state(
|
|
131
|
+
mongo_db, prefix, "p.fail", {"failed"}, log="State recorded late: failed",
|
|
132
|
+
)
|
|
133
|
+
assert proc["status"]["error"] == "boom"
|
|
134
|
+
finally:
|
|
135
|
+
await _stop_optio(optio, run_task)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def test_lost_cooperative_cancel_write_is_recorded_late(mongo_db, monkeypatch):
|
|
139
|
+
prefix = "lfw_coop"
|
|
140
|
+
started = asyncio.Event()
|
|
141
|
+
|
|
142
|
+
async def cooperative(ctx):
|
|
143
|
+
started.set()
|
|
144
|
+
while not ctx.cancellation_flag.is_set():
|
|
145
|
+
await asyncio.sleep(0.02)
|
|
146
|
+
|
|
147
|
+
task = TaskInstance(process_id="p.coop", name="Coop", params={}, execute=cooperative)
|
|
148
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task])
|
|
149
|
+
try:
|
|
150
|
+
_fail_update_status(monkeypatch, {"cancelled"})
|
|
151
|
+
await optio.launch("p.coop", session_id=None)
|
|
152
|
+
await started.wait()
|
|
153
|
+
await optio.cancel("p.coop")
|
|
154
|
+
await _wait_state(
|
|
155
|
+
mongo_db, prefix, "p.coop", {"cancelled"}, log="State recorded late: cancelled",
|
|
156
|
+
)
|
|
157
|
+
finally:
|
|
158
|
+
await _stop_optio(optio, run_task)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
async def test_lost_force_cancel_write_is_recorded_late(mongo_db, monkeypatch):
|
|
162
|
+
"""The bobcat incident: the force-cancel write raised and was never retried."""
|
|
163
|
+
import optio_core._force_cancel as fc
|
|
164
|
+
|
|
165
|
+
prefix = "lfw_force"
|
|
166
|
+
started = asyncio.Event()
|
|
167
|
+
|
|
168
|
+
async def stubborn(ctx): # noqa: ARG001
|
|
169
|
+
started.set()
|
|
170
|
+
while True:
|
|
171
|
+
await asyncio.sleep(0.05)
|
|
172
|
+
|
|
173
|
+
real = fc._write_force_cancelled_state
|
|
174
|
+
left = {"n": 1}
|
|
175
|
+
|
|
176
|
+
async def flaky(db, prefix_, oid, *args, **kwargs):
|
|
177
|
+
if left["n"] > 0:
|
|
178
|
+
left["n"] -= 1
|
|
179
|
+
raise AutoReconnect("injected: mongo unreachable")
|
|
180
|
+
return await real(db, prefix_, oid, *args, **kwargs)
|
|
181
|
+
|
|
182
|
+
monkeypatch.setattr(fc, "_write_force_cancelled_state", flaky)
|
|
183
|
+
|
|
184
|
+
task = TaskInstance(process_id="p.stub", name="Stub", params={}, execute=stubborn)
|
|
185
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task], cancel_grace_seconds=0.2)
|
|
186
|
+
try:
|
|
187
|
+
await optio.launch("p.stub", session_id=None)
|
|
188
|
+
await started.wait()
|
|
189
|
+
await optio.cancel("p.stub")
|
|
190
|
+
proc = await _wait_state(mongo_db, prefix, "p.stub", {"failed"})
|
|
191
|
+
assert proc["status"]["error"] == fc.FORCE_CANCEL_ERROR
|
|
192
|
+
assert left["n"] == 0 # the injected failure did happen
|
|
193
|
+
finally:
|
|
194
|
+
await _stop_optio(optio, run_task)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
async def test_lost_running_write_settles_the_scheduled_row(mongo_db, monkeypatch):
|
|
198
|
+
prefix = "lfw_start"
|
|
199
|
+
task = TaskInstance(process_id="p.start", name="Start", params={}, execute=_done)
|
|
200
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task])
|
|
201
|
+
try:
|
|
202
|
+
_fail_update_status(monkeypatch, {"running"})
|
|
203
|
+
await optio.launch("p.start", session_id=None)
|
|
204
|
+
proc = await _wait_state(mongo_db, prefix, "p.start", {"failed"})
|
|
205
|
+
assert proc["status"]["error"].startswith("Final state could not be recorded")
|
|
206
|
+
finally:
|
|
207
|
+
await _stop_optio(optio, run_task)
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
async def test_retry_keeps_trying_and_never_overwrites_a_settled_row(mongo_db, monkeypatch):
|
|
211
|
+
"""Executor-level: a failed retry stays queued; a row that left the active
|
|
212
|
+
states in the meantime is left alone and the entry is dropped."""
|
|
213
|
+
import optio_core.executor as ex
|
|
214
|
+
from optio_core.executor import Executor
|
|
215
|
+
|
|
216
|
+
prefix = "lfw_retry"
|
|
217
|
+
tasks = [
|
|
218
|
+
TaskInstance(process_id=pid, name=pid, params={}, execute=_done)
|
|
219
|
+
for pid in ("p.keep", "p.moved")
|
|
220
|
+
]
|
|
221
|
+
for t in tasks:
|
|
222
|
+
await upsert_process(mongo_db, prefix, t)
|
|
223
|
+
executor = Executor(mongo_db, prefix, services={})
|
|
224
|
+
executor.register_tasks(tasks)
|
|
225
|
+
|
|
226
|
+
_fail_update_status(monkeypatch, {"done"}, times=2)
|
|
227
|
+
for pid in ("p.keep", "p.moved"):
|
|
228
|
+
with pytest.raises(AutoReconnect):
|
|
229
|
+
await executor.launch_process(pid, session_id=None)
|
|
230
|
+
keep = await get_process_by_process_id(mongo_db, prefix, "p.keep")
|
|
231
|
+
moved = await get_process_by_process_id(mongo_db, prefix, "p.moved")
|
|
232
|
+
assert keep["status"]["state"] == "running"
|
|
233
|
+
assert set(executor._unrecorded_finals) == {keep["_id"], moved["_id"]}
|
|
234
|
+
|
|
235
|
+
# Something else settles p.moved meanwhile.
|
|
236
|
+
await mongo_db[f"{prefix}_processes"].update_one(
|
|
237
|
+
{"_id": moved["_id"]}, {"$set": {"status": ProcessStatus(state="idle").to_dict()}},
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
# First retry: Mongo still unavailable for p.keep.
|
|
241
|
+
real_finalize = ex.finalize_if_active
|
|
242
|
+
left = {"n": 1}
|
|
243
|
+
|
|
244
|
+
async def flaky_finalize(db, prefix_, oid, status, **kw):
|
|
245
|
+
if oid == keep["_id"] and left["n"] > 0:
|
|
246
|
+
left["n"] -= 1
|
|
247
|
+
raise AutoReconnect("injected: mongo unreachable")
|
|
248
|
+
return await real_finalize(db, prefix_, oid, status, **kw)
|
|
249
|
+
|
|
250
|
+
monkeypatch.setattr(ex, "finalize_if_active", flaky_finalize)
|
|
251
|
+
await executor.retry_unrecorded_finals()
|
|
252
|
+
assert keep["_id"] in executor._unrecorded_finals
|
|
253
|
+
assert moved["_id"] not in executor._unrecorded_finals
|
|
254
|
+
moved = await get_process_by_process_id(mongo_db, prefix, "p.moved")
|
|
255
|
+
assert moved["status"]["state"] == "idle"
|
|
256
|
+
|
|
257
|
+
# Make the backed-off entry due, then retry: it lands.
|
|
258
|
+
executor._unrecorded_finals[keep["_id"]].next_attempt = 0.0
|
|
259
|
+
await executor.retry_unrecorded_finals()
|
|
260
|
+
assert executor._unrecorded_finals == {}
|
|
261
|
+
keep = await get_process_by_process_id(mongo_db, prefix, "p.keep")
|
|
262
|
+
assert keep["status"]["state"] == "done"
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
# ---------- Rule 4: settle a row that has no task behind it ----------
|
|
266
|
+
|
|
267
|
+
async def _orphan(mongo_db, prefix, process_id, state):
|
|
268
|
+
"""Leave a row active with no task behind it, as a lost final write does."""
|
|
269
|
+
await mongo_db[f"{prefix}_processes"].update_one(
|
|
270
|
+
{"processId": process_id},
|
|
271
|
+
{"$set": {"status": ProcessStatus(state=state).to_dict()}},
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _assert_settled_orphan(proc, prev_state):
|
|
276
|
+
assert proc["status"]["state"] == "failed", proc["status"]
|
|
277
|
+
assert "no running task" in proc["status"]["error"]
|
|
278
|
+
assert _logged(proc, f"State reconciled: {prev_state} -> failed (no running task)")
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
async def test_group_cancel_and_wait_settles_orphans(mongo_db):
|
|
282
|
+
prefix = "lfw_group"
|
|
283
|
+
states = {"p.run": "running", "p.req": "cancel_requested", "p.cing": "cancelling"}
|
|
284
|
+
tasks = [
|
|
285
|
+
TaskInstance(process_id=pid, name=pid, params={}, execute=_done,
|
|
286
|
+
metadata={"grp": "g"})
|
|
287
|
+
for pid in states
|
|
288
|
+
]
|
|
289
|
+
optio, run_task = await _start_optio(mongo_db, prefix, tasks, cancel_grace_seconds=0.2)
|
|
290
|
+
try:
|
|
291
|
+
for pid, state in states.items():
|
|
292
|
+
await _orphan(mongo_db, prefix, pid, state)
|
|
293
|
+
await optio.group_cancel_and_wait({"grp": "g"})
|
|
294
|
+
for pid, state in states.items():
|
|
295
|
+
proc = await get_process_by_process_id(mongo_db, prefix, pid)
|
|
296
|
+
# cancel() moves a running orphan to cancel_requested first.
|
|
297
|
+
prev = "cancel_requested" if state == "running" else state
|
|
298
|
+
_assert_settled_orphan(proc, prev)
|
|
299
|
+
finally:
|
|
300
|
+
await _stop_optio(optio, run_task)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
async def test_cancel_and_wait_settles_orphan(mongo_db):
|
|
304
|
+
prefix = "lfw_caw"
|
|
305
|
+
task = TaskInstance(process_id="p.cing", name="Cing", params={}, execute=_done)
|
|
306
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task], cancel_grace_seconds=0.2)
|
|
307
|
+
try:
|
|
308
|
+
await _orphan(mongo_db, prefix, "p.cing", "cancelling")
|
|
309
|
+
assert await optio.cancel_and_wait("p.cing") == "failed"
|
|
310
|
+
proc = await get_process_by_process_id(mongo_db, prefix, "p.cing")
|
|
311
|
+
_assert_settled_orphan(proc, "cancelling")
|
|
312
|
+
finally:
|
|
313
|
+
await _stop_optio(optio, run_task)
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
async def test_cancel_settles_running_orphan(mongo_db):
|
|
317
|
+
"""Before: the row was left in cancel_requested for good."""
|
|
318
|
+
prefix = "lfw_cancel"
|
|
319
|
+
task = TaskInstance(process_id="p.run", name="Run", params={}, execute=_done)
|
|
320
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task])
|
|
321
|
+
try:
|
|
322
|
+
await _orphan(mongo_db, prefix, "p.run", "running")
|
|
323
|
+
outcome = await optio.cancel("p.run")
|
|
324
|
+
assert outcome.ok
|
|
325
|
+
proc = await get_process_by_process_id(mongo_db, prefix, "p.run")
|
|
326
|
+
_assert_settled_orphan(proc, "cancel_requested")
|
|
327
|
+
finally:
|
|
328
|
+
await _stop_optio(optio, run_task)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
async def test_orphan_with_parked_final_gets_the_parked_state(mongo_db, monkeypatch):
|
|
332
|
+
"""A row whose final write is queued (Rule 3) settles to that state, not
|
|
333
|
+
to the generic orphan failure. No run(): the supervisor must not retry first."""
|
|
334
|
+
prefix = "lfw_parked"
|
|
335
|
+
task = TaskInstance(process_id="p.done", name="Done", params={}, execute=_done)
|
|
336
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [task], run=False)
|
|
337
|
+
try:
|
|
338
|
+
_fail_update_status(monkeypatch, {"done"})
|
|
339
|
+
await optio.launch("p.done", session_id=None)
|
|
340
|
+
|
|
341
|
+
async def parked():
|
|
342
|
+
return bool(optio._executor._unrecorded_finals)
|
|
343
|
+
await _wait_until(parked, "the done write to be parked")
|
|
344
|
+
|
|
345
|
+
assert await optio.cancel_and_wait("p.done") == "done"
|
|
346
|
+
finally:
|
|
347
|
+
await _stop_optio(optio, run_task)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
async def test_live_task_and_scheduled_row_are_not_orphans(mongo_db):
|
|
351
|
+
prefix = "lfw_live"
|
|
352
|
+
started = asyncio.Event()
|
|
353
|
+
|
|
354
|
+
async def cooperative(ctx):
|
|
355
|
+
started.set()
|
|
356
|
+
while not ctx.cancellation_flag.is_set():
|
|
357
|
+
await asyncio.sleep(0.02)
|
|
358
|
+
|
|
359
|
+
live = TaskInstance(process_id="p.live", name="Live", params={}, execute=cooperative,
|
|
360
|
+
metadata={"grp": "live"})
|
|
361
|
+
sched = TaskInstance(process_id="p.sched", name="Sched", params={}, execute=_done)
|
|
362
|
+
optio, run_task = await _start_optio(mongo_db, prefix, [live, sched])
|
|
363
|
+
try:
|
|
364
|
+
await optio.launch("p.live", session_id=None)
|
|
365
|
+
await started.wait()
|
|
366
|
+
await optio.group_cancel_and_wait({"grp": "live"})
|
|
367
|
+
proc = await get_process_by_process_id(mongo_db, prefix, "p.live")
|
|
368
|
+
assert proc["status"]["state"] == "cancelled"
|
|
369
|
+
|
|
370
|
+
await _orphan(mongo_db, prefix, "p.sched", "scheduled")
|
|
371
|
+
assert await optio.cancel_and_wait("p.sched") == "cancelled"
|
|
372
|
+
finally:
|
|
373
|
+
await _stop_optio(optio, run_task)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m002_backfill_child_metadata.py
RENAMED
|
File without changes
|
{optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m003_backfill_has_saved_state.py
RENAMED
|
File without changes
|
{optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m004_create_expire_at_ttl_index.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|