optio-core 0.4.2__tar.gz → 0.4.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {optio_core-0.4.2/src/optio_core.egg-info → optio_core-0.4.4}/PKG-INFO +1 -1
  2. {optio_core-0.4.2 → optio_core-0.4.4}/pyproject.toml +1 -1
  3. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/executor.py +249 -63
  4. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/lifecycle.py +65 -1
  5. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/store.py +38 -0
  6. {optio_core-0.4.2 → optio_core-0.4.4/src/optio_core.egg-info}/PKG-INFO +1 -1
  7. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/SOURCES.txt +1 -0
  8. optio_core-0.4.4/tests/test_lost_final_writes.py +373 -0
  9. {optio_core-0.4.2 → optio_core-0.4.4}/LICENSE +0 -0
  10. {optio_core-0.4.2 → optio_core-0.4.4}/README.md +0 -0
  11. {optio_core-0.4.2 → optio_core-0.4.4}/setup.cfg +0 -0
  12. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/__init__.py +0 -0
  13. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_engine_service.py +0 -0
  14. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_force_cancel.py +0 -0
  15. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_generated/optio_engine.py +0 -0
  16. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/_launch_block_store.py +0 -0
  17. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/context.py +0 -0
  18. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/exceptions.py +0 -0
  19. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/__init__.py +0 -0
  20. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m001_status_subdocument.py +0 -0
  21. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m002_backfill_child_metadata.py +0 -0
  22. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m003_backfill_has_saved_state.py +0 -0
  23. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/migrations/m004_create_expire_at_ttl_index.py +0 -0
  24. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/models.py +0 -0
  25. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/progress_helpers.py +0 -0
  26. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/scheduler.py +0 -0
  27. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core/state_machine.py +0 -0
  28. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/dependency_links.txt +0 -0
  29. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/requires.txt +0 -0
  30. {optio_core-0.4.2 → optio_core-0.4.4}/src/optio_core.egg-info/top_level.txt +0 -0
  31. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_auto_resume.py +0 -0
  32. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_cancel_propagation.py +0 -0
  33. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_cancel_race_parent_overwrite.py +0 -0
  34. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_failure_cancel_distinction.py +0 -0
  35. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_failure_structured.py +0 -0
  36. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_progress.py +0 -0
  37. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_child_result_channel.py +0 -0
  38. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_client_directed_events.py +0 -0
  39. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_context_blob.py +0 -0
  40. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_context_resume.py +0 -0
  41. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_control_upstream.py +0 -0
  42. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_deadline_cancel.py +0 -0
  43. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_deadline_cancel_launchguard.py +0 -0
  44. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_engine_service.py +0 -0
  45. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_engine_service_resolve.py +0 -0
  46. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_engine_service_upload.py +0 -0
  47. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_executor.py +0 -0
  48. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_group_cancel.py +0 -0
  49. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_group_cancel_purge.py +0 -0
  50. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_heartbeat.py +0 -0
  51. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_launch_guard.py +0 -0
  52. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_lifecycle_reconciliation.py +0 -0
  53. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_migration_m003.py +0 -0
  54. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_migration_m004.py +0 -0
  55. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_models.py +0 -0
  56. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_no_redis.py +0 -0
  57. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_outcomes.py +0 -0
  58. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_parallel.py +0 -0
  59. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_persistent_launch_blocks.py +0 -0
  60. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_progress_helpers.py +0 -0
  61. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_progress_throttle.py +0 -0
  62. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_publish_result.py +0 -0
  63. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_resync.py +0 -0
  64. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_resync_cancel_stale.py +0 -0
  65. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_scheduler.py +0 -0
  66. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_shutdown_drain_completion.py +0 -0
  67. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_state_machine.py +0 -0
  68. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_store.py +0 -0
  69. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_task_ttl.py +0 -0
  70. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_upload_registry.py +0 -0
  71. {optio_core-0.4.2 → optio_core-0.4.4}/tests/test_widget_primitives.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: optio-core
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: Reusable async process management library
5
5
  Author-email: Kristof Csillag <kristof.csillag@deai-labs.com>
6
6
  License-Expression: Apache-2.0
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "optio-core"
7
- version = "0.4.2"
7
+ version = "0.4.4"
8
8
  description = "Reusable async process management library"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -3,13 +3,16 @@
3
3
  import asyncio
4
4
  import logging
5
5
  import os as _os
6
+ import sys
6
7
  import time
7
8
  from dataclasses import dataclass
8
9
  from datetime import datetime, timezone
9
10
  from typing import Any, Callable, Awaitable
10
11
  from bson import ObjectId
11
12
  from motor.motor_asyncio import AsyncIOMotorDatabase
13
+ from pymongo.errors import PyMongoError
12
14
 
15
+ logger = logging.getLogger(__name__)
13
16
  _trace_logger = logging.getLogger("optio_core.cancel_trace")
14
17
  _CANCEL_TRACE = _os.environ.get("OPTIO_CANCEL_TRACE", "0").lower() in ("1", "true", "yes")
15
18
 
@@ -29,7 +32,7 @@ from optio_core.store import (
29
32
  update_status, clear_result_fields,
30
33
  create_child_process, append_log,
31
34
  clear_widget_upstream, compute_expire_at,
32
- _collection,
35
+ finalize_if_active, _collection,
33
36
  )
34
37
  from optio_core.context import ProcessContext
35
38
  from optio_core.exceptions import ChildProcessFailed
@@ -47,6 +50,26 @@ class _CancelEntry:
47
50
  deadline: float | None = None
48
51
 
49
52
 
53
+ # Rule 3 retry backoff: first retry on the next supervisor tick, then doubling
54
+ # from this delay up to the cap. No attempt limit.
55
+ _FINAL_RETRY_FIRST_DELAY = 1.0
56
+ _FINAL_RETRY_MAX_DELAY = 30.0
57
+
58
+
59
+ @dataclass
60
+ class _UnrecordedFinal:
61
+ """A final state that could not be written to Mongo, kept for retry.
62
+
63
+ `status` is the terminal status still to write (None once written).
64
+ `cascade` asks for the force-cancel cascade to direct active children to
65
+ be re-run. Spec: docs/2026-10-03-lost-final-state-writes-design.md
66
+ """
67
+ status: ProcessStatus | None
68
+ cascade: bool = False
69
+ attempts: int = 0
70
+ next_attempt: float = 0.0
71
+
72
+
50
73
  class Executor:
51
74
  """Executes task functions with lifecycle management."""
52
75
 
@@ -67,6 +90,11 @@ class Executor:
67
90
  self._notify_parent_failure = notify_parent_failure
68
91
  self._cancellation_flags: dict[ObjectId, _CancelEntry] = {}
69
92
  self._running_tasks: dict[ObjectId, asyncio.Task] = {}
93
+ # Rules 3 and 4 of docs/2026-10-03-lost-final-state-writes-design.md:
94
+ # final states that could not be written (retried by the supervisor
95
+ # loop), and OIDs whose force_cancel is still in flight.
96
+ self._unrecorded_finals: dict[ObjectId, _UnrecordedFinal] = {}
97
+ self._force_cancelling: set[ObjectId] = set()
70
98
  self._task_registry: dict[str, TaskInstance] = {}
71
99
  # Task→launcher return channel (in-memory only, same-process).
72
100
  # Keyed by processId string. Registry holds published objects for
@@ -189,6 +217,17 @@ class Executor:
189
217
  if current is None:
190
218
  raise RuntimeError("_execute_process must be called from within an asyncio Task")
191
219
  self._running_tasks[oid] = current
220
+ # A new run of this row supersedes a final state still parked from an
221
+ # earlier run (Rule 3): the row was launchable, so that write is moot.
222
+ self._unrecorded_finals.pop(oid, None)
223
+
224
+ # Rule 3 (docs/2026-10-03-lost-final-state-writes-design.md): `final`
225
+ # is the terminal status this run means to write, noted as soon as
226
+ # the outcome is known; `final_recorded` flips once that write has
227
+ # landed. If the finally below is reached without it, the status is
228
+ # parked for the supervisor loop to retry.
229
+ final: ProcessStatus | None = None
230
+ final_recorded = False
192
231
 
193
232
  try:
194
233
  now = datetime.now(timezone.utc)
@@ -231,14 +270,15 @@ class Executor:
231
270
  ctx._parent_listener = _listener
232
271
 
233
272
  if execute_fn is None:
273
+ final = ProcessStatus(
274
+ state="failed", error="No execute function found",
275
+ failed_at=datetime.now(timezone.utc),
276
+ )
234
277
  await update_status(
235
- self._db, self._prefix, oid,
236
- ProcessStatus(
237
- state="failed", error="No execute function found",
238
- failed_at=datetime.now(timezone.utc),
239
- ),
278
+ self._db, self._prefix, oid, final,
240
279
  expire_at=compute_expire_at(ttl_seconds),
241
280
  )
281
+ final_recorded = True
242
282
  return ("failed", None)
243
283
 
244
284
  start_time = time.monotonic()
@@ -286,15 +326,16 @@ class Executor:
286
326
  "CANCEL-TRACE %s: executor write cancelled (raised CancelledError)",
287
327
  proc["processId"],
288
328
  )
329
+ final = ProcessStatus(
330
+ state="cancelled",
331
+ stopped_at=datetime.now(timezone.utc),
332
+ )
289
333
  await ctx.flush_final_progress()
290
334
  await update_status(
291
- self._db, self._prefix, oid,
292
- ProcessStatus(
293
- state="cancelled",
294
- stopped_at=datetime.now(timezone.utc),
295
- ),
335
+ self._db, self._prefix, oid, final,
296
336
  expire_at=compute_expire_at(ttl_seconds),
297
337
  )
338
+ final_recorded = True
298
339
  await append_log(
299
340
  self._db, self._prefix, oid, "event",
300
341
  "State changed to cancelled (raised CancelledError)",
@@ -303,36 +344,37 @@ class Executor:
303
344
  await self._cleanup_ephemeral(str(oid))
304
345
  raise
305
346
  except Exception as e:
347
+ final = ProcessStatus(
348
+ state="failed", error=str(e),
349
+ failed_at=datetime.now(timezone.utc),
350
+ )
306
351
  await ctx.flush_final_progress()
307
352
  await update_status(
308
- self._db, self._prefix, oid,
309
- ProcessStatus(
310
- state="failed", error=str(e),
311
- failed_at=datetime.now(timezone.utc),
312
- ),
353
+ self._db, self._prefix, oid, final,
313
354
  expire_at=compute_expire_at(ttl_seconds),
314
355
  )
356
+ final_recorded = True
315
357
  await append_log(self._db, self._prefix, oid, "error", str(e))
316
358
  await clear_widget_upstream(self._db, self._prefix, oid)
317
359
  await self._cleanup_ephemeral(str(oid))
318
360
  return ("failed", e)
319
361
 
362
+ # Note the outcome before flushing, so a flush that fails still
363
+ # leaves the right state to retry; rebuilt after the flush so the
364
+ # recorded timing matches what it was before.
365
+ final = self._terminal_status(end_state, start_time)
320
366
  await ctx.flush_final_progress()
321
- elapsed = time.monotonic() - start_time
367
+ final = self._terminal_status(end_state, start_time)
322
368
 
323
369
  if end_state == "done":
324
370
  _trace(
325
371
  "CANCEL-TRACE %s: executor write done", proc["processId"],
326
372
  )
327
373
  await update_status(
328
- self._db, self._prefix, oid,
329
- ProcessStatus(
330
- state="done",
331
- done_at=datetime.now(timezone.utc),
332
- duration=round(elapsed, 2),
333
- ),
374
+ self._db, self._prefix, oid, final,
334
375
  expire_at=compute_expire_at(ttl_seconds),
335
376
  )
377
+ final_recorded = True
336
378
  await append_log(self._db, self._prefix, oid, "event", "State changed to done")
337
379
  elif end_state == "cancelled":
338
380
  _trace(
@@ -340,13 +382,10 @@ class Executor:
340
382
  proc["processId"],
341
383
  )
342
384
  await update_status(
343
- self._db, self._prefix, oid,
344
- ProcessStatus(
345
- state="cancelled",
346
- stopped_at=datetime.now(timezone.utc),
347
- ),
385
+ self._db, self._prefix, oid, final,
348
386
  expire_at=compute_expire_at(ttl_seconds),
349
387
  )
388
+ final_recorded = True
350
389
  await append_log(self._db, self._prefix, oid, "event", "State changed to cancelled")
351
390
 
352
391
  await clear_widget_upstream(self._db, self._prefix, oid)
@@ -363,6 +402,30 @@ class Executor:
363
402
  if _fut is not None and not _fut.done():
364
403
  from optio_core.exceptions import ResultNotPublished
365
404
  _fut.set_exception(ResultNotPublished(_pid))
405
+ # Rule 3: the final write never landed (Mongo unreachable, or an
406
+ # error before the outcome was known). A CancelledError unwind is
407
+ # left alone: force_cancel, shutdown (Rule 2) or the next start
408
+ # (Rule 1) owns that row.
409
+ _exc = sys.exc_info()[1]
410
+ if not final_recorded and not isinstance(_exc, asyncio.CancelledError):
411
+ if final is None:
412
+ final = ProcessStatus(
413
+ state="failed",
414
+ error=f"Final state could not be recorded: {_exc!r}",
415
+ failed_at=datetime.now(timezone.utc),
416
+ )
417
+ self._park_final(oid, final, cause=_exc)
418
+
419
+ @staticmethod
420
+ def _terminal_status(end_state: str, start_time: float) -> ProcessStatus:
421
+ """Terminal status for a task body that returned ('done' or 'cancelled')."""
422
+ now = datetime.now(timezone.utc)
423
+ if end_state == "done":
424
+ return ProcessStatus(
425
+ state="done", done_at=now,
426
+ duration=round(time.monotonic() - start_time, 2),
427
+ )
428
+ return ProcessStatus(state="cancelled", stopped_at=now)
366
429
 
367
430
  async def execute_child(
368
431
  self,
@@ -476,6 +539,118 @@ class Executor:
476
539
  )
477
540
  return True
478
541
 
542
+ def owns(self, oid: ObjectId) -> bool:
543
+ """True while this executor still has a task, a cancel entry or a
544
+ force-cancel in flight for `oid`.
545
+
546
+ A task registers its entry before it writes `running`, and writes its
547
+ final state before it drops the entry. So a row in `running`,
548
+ `cancel_requested` or `cancelling` for which this returns False has
549
+ lost its final write (Rule 4 of
550
+ docs/2026-10-03-lost-final-state-writes-design.md).
551
+ """
552
+ return (
553
+ oid in self._cancellation_flags
554
+ or oid in self._running_tasks
555
+ or oid in self._force_cancelling
556
+ )
557
+
558
+ def _park_final(
559
+ self, oid: ObjectId, status: ProcessStatus | None, *,
560
+ cascade: bool = False, cause: BaseException | None = None,
561
+ ) -> None:
562
+ """Keep a final state (and/or a force-cancel cascade) that could not
563
+ be written, for retry_unrecorded_finals. A status already parked for
564
+ the same OID wins; cascade requests accumulate."""
565
+ entry = self._unrecorded_finals.get(oid)
566
+ if entry is None:
567
+ self._unrecorded_finals[oid] = _UnrecordedFinal(status=status, cascade=cascade)
568
+ logger.warning(
569
+ "Could not record the final state (%s) of process %s: %r. "
570
+ "Retrying in the background.",
571
+ status.state if status is not None else "child cascade", oid, cause,
572
+ )
573
+ return
574
+ if entry.status is None:
575
+ entry.status = status
576
+ entry.cascade = entry.cascade or cascade
577
+
578
+ async def _record_final(self, oid: ObjectId, entry: _UnrecordedFinal) -> None:
579
+ """Write a parked final state, then run a parked cascade. Raises
580
+ PyMongoError while Mongo is still unreachable; the entry then keeps
581
+ whatever is left to do."""
582
+ if entry.status is not None:
583
+ state = entry.status.state
584
+ written = await finalize_if_active(self._db, self._prefix, oid, entry.status)
585
+ entry.status = None
586
+ if written:
587
+ await append_log(
588
+ self._db, self._prefix, oid, "event",
589
+ f"State recorded late: {state} (database was unavailable)",
590
+ )
591
+ await self._cleanup_ephemeral(str(oid))
592
+ if entry.cascade:
593
+ await self._force_cancel_children(oid)
594
+ entry.cascade = False
595
+
596
+ async def retry_unrecorded_finals(self) -> None:
597
+ """Retry the parked final states that are due (Rule 3). Called by the
598
+ supervisor loop on every tick. A row that is no longer active is left
599
+ alone and its entry dropped; while Mongo stays unreachable the next
600
+ attempt backs off, doubling up to _FINAL_RETRY_MAX_DELAY."""
601
+ now = time.monotonic()
602
+ for oid, entry in list(self._unrecorded_finals.items()):
603
+ if entry.next_attempt > now or self.owns(oid):
604
+ continue
605
+ try:
606
+ await self._record_final(oid, entry)
607
+ except PyMongoError as e:
608
+ entry.attempts += 1
609
+ entry.next_attempt = time.monotonic() + min(
610
+ _FINAL_RETRY_MAX_DELAY,
611
+ _FINAL_RETRY_FIRST_DELAY * 2 ** (entry.attempts - 1),
612
+ )
613
+ logger.debug(
614
+ "Final state of process %s still not recorded (retry %d): %r",
615
+ oid, entry.attempts, e,
616
+ )
617
+ continue
618
+ self._unrecorded_finals.pop(oid, None)
619
+ logger.warning(
620
+ "Final state of process %s recorded late, after %d failed retries",
621
+ oid, entry.attempts,
622
+ )
623
+
624
+ async def record_parked_final_now(self, oid: ObjectId) -> bool:
625
+ """If a final state is parked for `oid`, write it now, ignoring the
626
+ backoff. Returns True if one was parked. Raises PyMongoError while
627
+ Mongo is still unreachable; the entry then stays parked."""
628
+ entry = self._unrecorded_finals.get(oid)
629
+ if entry is None:
630
+ return False
631
+ await self._record_final(oid, entry)
632
+ self._unrecorded_finals.pop(oid, None)
633
+ return True
634
+
635
+ async def _force_cancel_children(self, oid: ObjectId) -> None:
636
+ """Cascade a force-cancel to direct active children. Unconditional --
637
+ force is force."""
638
+ from optio_core.store import list_direct_children
639
+ from optio_core.state_machine import ACTIVE_STATES
640
+
641
+ children = await list_direct_children(
642
+ self._db, self._prefix, oid, states=ACTIVE_STATES,
643
+ )
644
+ if children:
645
+ _trace(
646
+ "force_cancel oid=%s: cascading to children=%s",
647
+ oid, [str(c["_id"]) for c in children],
648
+ )
649
+ await asyncio.gather(
650
+ *(self.force_cancel(c["_id"]) for c in children),
651
+ return_exceptions=True,
652
+ )
653
+
479
654
  async def force_cancel(self, oid: ObjectId) -> None:
480
655
  """Hard-cancel a process whose cooperative deadline has expired.
481
656
 
@@ -486,47 +661,58 @@ class Executor:
486
661
  captures both auto-propagate descendants (already in supervisor
487
662
  map; idempotent) and opt-out descendants (only this cascade
488
663
  reaches them).
664
+
665
+ If Mongo is unreachable, the write and the cascade are parked for
666
+ retry_unrecorded_finals instead of raising (Rule 3 of
667
+ docs/2026-10-03-lost-final-state-writes-design.md).
489
668
  """
490
- from optio_core._force_cancel import _write_force_cancelled_state
491
- from optio_core.store import list_direct_children
492
- from optio_core.state_machine import ACTIVE_STATES
669
+ from optio_core._force_cancel import (
670
+ FORCE_CANCEL_ERROR, _write_force_cancelled_state,
671
+ )
493
672
 
494
673
  task = self._running_tasks.get(oid)
495
674
  _trace(
496
675
  "force_cancel oid=%s task=%s done=%s",
497
676
  oid, task, task.done() if task else None,
498
677
  )
499
- if task is not None and not task.done():
500
- _trace("force_cancel oid=%s: calling task.cancel() + 2s shield wait", oid)
501
- task.cancel()
678
+ # The row stays owned by this engine until our own write is done,
679
+ # even after the task's finally has dropped its entries (Rule 4).
680
+ self._force_cancelling.add(oid)
681
+ try:
682
+ if task is not None and not task.done():
683
+ _trace("force_cancel oid=%s: calling task.cancel() + 2s shield wait", oid)
684
+ task.cancel()
685
+ try:
686
+ await asyncio.wait_for(
687
+ asyncio.shield(task),
688
+ timeout=self._optio._config.force_cancel_shield_seconds,
689
+ )
690
+ _trace("force_cancel oid=%s: task unwound within shield window", oid)
691
+ except asyncio.TimeoutError:
692
+ _trace("force_cancel oid=%s: 2s shield TIMEOUT — task still running", oid)
693
+ pass
694
+ except asyncio.CancelledError:
695
+ _trace("force_cancel oid=%s: task acknowledged Cancel", oid)
696
+ pass
697
+ except Exception as _e:
698
+ _trace("force_cancel oid=%s: task raised %s: %s",
699
+ oid, type(_e).__name__, _e)
700
+ pass
502
701
  try:
503
- await asyncio.wait_for(
504
- asyncio.shield(task),
505
- timeout=self._optio._config.force_cancel_shield_seconds,
702
+ await _write_force_cancelled_state(self._db, self._prefix, oid)
703
+ except PyMongoError as e:
704
+ self._park_final(
705
+ oid,
706
+ ProcessStatus(
707
+ state="failed", error=FORCE_CANCEL_ERROR,
708
+ failed_at=datetime.now(timezone.utc),
709
+ ),
710
+ cascade=True, cause=e,
506
711
  )
507
- _trace("force_cancel oid=%s: task unwound within shield window", oid)
508
- except asyncio.TimeoutError:
509
- _trace("force_cancel oid=%s: 2s shield TIMEOUT — task still running", oid)
510
- pass
511
- except asyncio.CancelledError:
512
- _trace("force_cancel oid=%s: task acknowledged Cancel", oid)
513
- pass
514
- except Exception as _e:
515
- _trace("force_cancel oid=%s: task raised %s: %s",
516
- oid, type(_e).__name__, _e)
517
- pass
518
- await _write_force_cancelled_state(self._db, self._prefix, oid)
519
-
520
- # Cascade to direct active children. Unconditional — force is force.
521
- children = await list_direct_children(
522
- self._db, self._prefix, oid, states=ACTIVE_STATES,
523
- )
524
- if children:
525
- _trace(
526
- "force_cancel oid=%s: cascading to children=%s",
527
- oid, [str(c["_id"]) for c in children],
528
- )
529
- await asyncio.gather(
530
- *(self.force_cancel(c["_id"]) for c in children),
531
- return_exceptions=True,
532
- )
712
+ return
713
+ try:
714
+ await self._force_cancel_children(oid)
715
+ except PyMongoError as e:
716
+ self._park_final(oid, None, cascade=True, cause=e)
717
+ finally:
718
+ self._force_cancelling.discard(oid)
@@ -34,6 +34,7 @@ from optio_core.store import (
34
34
  upsert_process, remove_stale_processes, find_stale_process_ids,
35
35
  get_process_by_process_id, update_status, clear_result_fields,
36
36
  append_log, compute_expire_at, purge_processes, set_auto_resume_scheduled,
37
+ finalize_if_active,
37
38
  )
38
39
  from optio_core.state_machine import (
39
40
  ACTIVE_STATES, CANCELLABLE_STATES, DISMISSABLE_STATES, END_STATES,
@@ -47,6 +48,14 @@ from optio_core.scheduler import ProcessScheduler
47
48
 
48
49
  logger = logging.getLogger("optio_core_core")
49
50
 
51
+ # Rule 4 of docs/2026-10-03-lost-final-state-writes-design.md: states in which
52
+ # a row with no owning task has lost its final write, and what it settles to.
53
+ _ORPHAN_STATES = frozenset({"running", "cancel_requested", "cancelling"})
54
+ ORPHAN_ERROR = (
55
+ "Process had no running task in this engine; "
56
+ "its final state was never recorded"
57
+ )
58
+
50
59
 
51
60
  def _trace(fmt: str, *args: object) -> None:
52
61
  """Cancel-trace log helper, gated on OPTIO_CANCEL_TRACE env var."""
@@ -674,6 +683,10 @@ class Optio:
674
683
  "CANCEL-TRACE %s: request_cancel_with_deadline found=%s",
675
684
  process_id, found,
676
685
  )
686
+ if not found:
687
+ # No task in this engine owns the row we just moved to
688
+ # cancel_requested: its final write was lost (Rule 4).
689
+ await self._settle_if_orphaned(proc, "cancel_requested")
677
690
  if found:
678
691
  # Conditional: only advance to 'cancelling' if the row is
679
692
  # still in 'cancel_requested'. If the executor has already
@@ -739,6 +752,8 @@ class Optio:
739
752
  state = proc["status"]["state"]
740
753
  if state not in ACTIVE_STATES:
741
754
  return state
755
+ if await self._settle_if_orphaned(proc, state):
756
+ continue # Rule 4: settled; the re-read returns its state
742
757
  if time.monotonic() >= deadline:
743
758
  raise asyncio.TimeoutError(
744
759
  f"Process {process_id} did not reach terminal state within {ceiling}s"
@@ -883,6 +898,8 @@ class Optio:
883
898
  if proc is None or proc["status"]["state"] not in ACTIVE_STATES:
884
899
  i += 1
885
900
  continue
901
+ if await self._settle_if_orphaned(proc, proc["status"]["state"]):
902
+ continue # Rule 4: settled; the re-read sees it terminal
886
903
  if time.monotonic() >= deadline:
887
904
  remaining = len(pending) - i
888
905
  raise asyncio.TimeoutError(
@@ -897,6 +914,46 @@ class Optio:
897
914
  )
898
915
  return len(pending)
899
916
 
917
+ async def _settle_if_orphaned(self, proc: dict, observed_state: str) -> bool:
918
+ """Settle a row whose final write was lost, instead of waiting on it.
919
+
920
+ Applies to a row seen in `running`, `cancel_requested` or
921
+ `cancelling` that no task, cancel entry or force-cancel in this
922
+ engine owns (see Executor.owns). A final state parked for it
923
+ (Rule 3) is written as is; otherwise the row becomes `failed` with
924
+ ORPHAN_ERROR, but only while it is still in `observed_state`.
925
+ Returns True if the row is now settled. `scheduled` never
926
+ qualifies: launch_process writes it before the task registers.
927
+
928
+ Assumes one engine per (database, prefix), as Rule 1 does.
929
+ Spec: docs/2026-10-03-lost-final-state-writes-design.md (Rule 4)
930
+ """
931
+ if observed_state not in _ORPHAN_STATES or self._executor is None:
932
+ return False
933
+ oid = proc["_id"]
934
+ if self._executor.owns(oid):
935
+ return False
936
+ if await self._executor.record_parked_final_now(oid):
937
+ return True
938
+ db, prefix = self._config.mongo_db, self._config.prefix
939
+ status = ProcessStatus(
940
+ state="failed", error=ORPHAN_ERROR,
941
+ failed_at=datetime.now(timezone.utc),
942
+ )
943
+ if not await finalize_if_active(
944
+ db, prefix, oid, status, expected_state=observed_state,
945
+ ):
946
+ return False
947
+ await append_log(
948
+ db, prefix, oid, "event",
949
+ f"State reconciled: {observed_state} -> failed (no running task)",
950
+ )
951
+ logger.warning(
952
+ "Settled process %s (%s): it was %s with no running task in this engine",
953
+ proc.get("processId"), oid, observed_state,
954
+ )
955
+ return True
956
+
900
957
  async def dismiss(self, process_id: str) -> DismissOutcome:
901
958
  """Dismiss a completed process (reset to idle). Returns DismissOutcome
902
959
  with `proc` populated from a post-dismiss re-read on success."""
@@ -1210,7 +1267,9 @@ class Optio:
1210
1267
  logger.info(f"Reconciled {len(stale)} interrupted process(es) to 'failed'")
1211
1268
 
1212
1269
  async def _supervisor_loop(self) -> None:
1213
- """Scan for past-deadline cancellations every 500 ms; force-cancel them."""
1270
+ """Scan for past-deadline cancellations every 500 ms; force-cancel them.
1271
+ Also retries final states that could not be written (Rule 3 of
1272
+ docs/2026-10-03-lost-final-state-writes-design.md)."""
1214
1273
  while self._running:
1215
1274
  try:
1216
1275
  now = time.monotonic()
@@ -1227,6 +1286,11 @@ class Optio:
1227
1286
  await self._executor.force_cancel(oid)
1228
1287
  except Exception as e:
1229
1288
  logger.exception(f"Supervisor loop error: {e}")
1289
+ try:
1290
+ if self._executor is not None:
1291
+ await self._executor.retry_unrecorded_finals()
1292
+ except Exception as e:
1293
+ logger.exception(f"Supervisor loop error while retrying final states: {e}")
1230
1294
  await asyncio.sleep(0.5)
1231
1295
 
1232
1296
  async def _auto_resume_timer(self) -> None:
@@ -191,6 +191,44 @@ async def update_status(
191
191
  )
192
192
 
193
193
 
194
+ async def finalize_if_active(
195
+ db: AsyncIOMotorDatabase, prefix: str, process_oid: ObjectId,
196
+ status: ProcessStatus, *, expected_state: str | None = None,
197
+ ) -> bool:
198
+ """Conditionally write a terminal status onto a row that is still active.
199
+
200
+ Matches only while `status.state` is in ACTIVE_STATES -- or, with
201
+ `expected_state`, only while it is exactly that state -- so a writer that
202
+ already moved the row on is never overwritten. Also clears
203
+ `widgetUpstream`, clears `autoResumeScheduled` for `failed`, and sets
204
+ `expireAt` from the row's `ttlSeconds`. Returns True if the row was
205
+ updated.
206
+
207
+ Used to record a final state late, and to settle a row whose final write
208
+ was lost. Spec: docs/2026-10-03-lost-final-state-writes-design.md
209
+ """
210
+ from optio_core.state_machine import ACTIVE_STATES
211
+
212
+ coll = _collection(db, prefix)
213
+ now = datetime.now(timezone.utc)
214
+ ttl_doc = await coll.find_one({"_id": process_oid}, {"ttlSeconds": 1})
215
+ expire_at = compute_expire_at((ttl_doc or {}).get("ttlSeconds"), now=now)
216
+ set_doc: dict[str, Any] = {"status": status.to_dict(), "widgetUpstream": None}
217
+ if status.state == "failed":
218
+ set_doc["autoResumeScheduled"] = False
219
+ if expire_at is not None:
220
+ set_doc["expireAt"] = expire_at
221
+ state_match: Any = (
222
+ expected_state if expected_state is not None
223
+ else {"$in": list(ACTIVE_STATES)}
224
+ )
225
+ result = await coll.update_one(
226
+ {"_id": process_oid, "status.state": state_match},
227
+ {"$set": set_doc},
228
+ )
229
+ return bool(result.modified_count)
230
+
231
+
194
232
  async def set_auto_resume_scheduled(
195
233
  db: AsyncIOMotorDatabase, prefix: str, process_oid: ObjectId, value: bool,
196
234
  ) -> None:
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: optio-core
3
- Version: 0.4.2
3
+ Version: 0.4.4
4
4
  Summary: Reusable async process management library
5
5
  Author-email: Kristof Csillag <kristof.csillag@deai-labs.com>
6
6
  License-Expression: Apache-2.0
@@ -47,6 +47,7 @@ tests/test_group_cancel_purge.py
47
47
  tests/test_heartbeat.py
48
48
  tests/test_launch_guard.py
49
49
  tests/test_lifecycle_reconciliation.py
50
+ tests/test_lost_final_writes.py
50
51
  tests/test_migration_m003.py
51
52
  tests/test_migration_m004.py
52
53
  tests/test_models.py
@@ -0,0 +1,373 @@
1
+ """Tests for lost final-state writes.
2
+
3
+ Spec: docs/2026-10-03-lost-final-state-writes-design.md
4
+
5
+ Rule 3: a final-state write that fails because Mongo is unreachable is kept
6
+ and retried by the supervisor loop. Rule 4: cancel / cancel_and_wait /
7
+ group_cancel_and_wait settle an active row that has no task behind it instead
8
+ of waiting for it.
9
+
10
+ "Mongo is unreachable" is simulated by making chosen writes raise
11
+ AutoReconnect. No test depends on wall-clock time: every wait is on a
12
+ condition, with a 60 s hang ceiling.
13
+ """
14
+ import asyncio
15
+ import time
16
+
17
+ import pytest
18
+ from pymongo.errors import AutoReconnect
19
+
20
+ from optio_core.lifecycle import Optio
21
+ from optio_core.models import ProcessStatus, TaskInstance
22
+ from optio_core.store import get_process_by_process_id, upsert_process
23
+
24
+ pytestmark = pytest.mark.asyncio
25
+
26
+ HANG_CEILING = 60.0
27
+
28
+
29
+ async def _start_optio(mongo_db, prefix, tasks, cancel_grace_seconds=2.0, run=True):
30
+ async def gen(_s, _f):
31
+ return list(tasks)
32
+ optio = Optio()
33
+ await optio.init(
34
+ mongo_db=mongo_db, prefix=prefix,
35
+ get_task_definitions=gen,
36
+ cancel_grace_seconds=cancel_grace_seconds,
37
+ )
38
+ run_task = asyncio.create_task(optio.run()) if run else None
39
+ return optio, run_task
40
+
41
+
42
+ async def _stop_optio(optio, run_task):
43
+ await optio.shutdown()
44
+ if run_task is not None:
45
+ run_task.cancel()
46
+ try:
47
+ await run_task
48
+ except (asyncio.CancelledError, Exception):
49
+ pass
50
+
51
+
52
+ async def _wait_until(pred, what):
53
+ deadline = time.monotonic() + HANG_CEILING
54
+ while True:
55
+ value = await pred()
56
+ if value:
57
+ return value
58
+ if time.monotonic() > deadline:
59
+ raise AssertionError(f"timed out waiting for {what}")
60
+ await asyncio.sleep(0.02)
61
+
62
+
63
+ async def _wait_state(mongo_db, prefix, process_id, states, log=None):
64
+ """Wait for the row to reach one of `states` and, with `log`, also carry
65
+ that log entry. The status and the log line are separate writes, so a
66
+ test must wait for both rather than read the log right after the state."""
67
+ async def pred():
68
+ proc = await get_process_by_process_id(mongo_db, prefix, process_id)
69
+ if proc is None or proc["status"]["state"] not in states:
70
+ return None
71
+ if log is not None and not _logged(proc, log):
72
+ return None
73
+ return proc
74
+ what = f"{process_id} to reach {sorted(states)}" + (f" with log {log!r}" if log else "")
75
+ return await _wait_until(pred, what)
76
+
77
+
78
+ def _fail_update_status(monkeypatch, states, times=1):
79
+ """Make the executor's next `times` writes of a status in `states` raise."""
80
+ import optio_core.executor as ex
81
+ real = ex.update_status
82
+ left = {"n": times}
83
+
84
+ async def flaky(db, prefix, oid, status, expire_at=None):
85
+ if status.state in states and left["n"] > 0:
86
+ left["n"] -= 1
87
+ raise AutoReconnect("injected: mongo unreachable")
88
+ return await real(db, prefix, oid, status, expire_at=expire_at)
89
+
90
+ monkeypatch.setattr(ex, "update_status", flaky)
91
+ return left
92
+
93
+
94
+ def _logged(proc, needle):
95
+ return any(needle in (e.get("message") or "") for e in proc.get("log", []))
96
+
97
+
98
+ async def _done(ctx): # noqa: ARG001
99
+ return None
100
+
101
+
102
+ # ---------- Rule 3: retry a final-state write that did not land ----------
103
+
104
+ async def test_lost_done_write_is_recorded_late(mongo_db, monkeypatch):
105
+ prefix = "lfw_done"
106
+ task = TaskInstance(process_id="p.done", name="Done", params={}, execute=_done)
107
+ optio, run_task = await _start_optio(mongo_db, prefix, [task])
108
+ try:
109
+ _fail_update_status(monkeypatch, {"done"})
110
+ await optio.launch("p.done", session_id=None)
111
+ proc = await _wait_state(
112
+ mongo_db, prefix, "p.done", {"done"}, log="State recorded late: done",
113
+ )
114
+ assert proc["status"]["doneAt"] is not None
115
+ finally:
116
+ await _stop_optio(optio, run_task)
117
+
118
+
119
+ async def test_lost_failed_write_is_recorded_late(mongo_db, monkeypatch):
120
+ prefix = "lfw_failed"
121
+
122
+ async def boom(ctx): # noqa: ARG001
123
+ raise ValueError("boom")
124
+
125
+ task = TaskInstance(process_id="p.fail", name="Fail", params={}, execute=boom)
126
+ optio, run_task = await _start_optio(mongo_db, prefix, [task])
127
+ try:
128
+ _fail_update_status(monkeypatch, {"failed"})
129
+ await optio.launch("p.fail", session_id=None)
130
+ proc = await _wait_state(
131
+ mongo_db, prefix, "p.fail", {"failed"}, log="State recorded late: failed",
132
+ )
133
+ assert proc["status"]["error"] == "boom"
134
+ finally:
135
+ await _stop_optio(optio, run_task)
136
+
137
+
138
+ async def test_lost_cooperative_cancel_write_is_recorded_late(mongo_db, monkeypatch):
139
+ prefix = "lfw_coop"
140
+ started = asyncio.Event()
141
+
142
+ async def cooperative(ctx):
143
+ started.set()
144
+ while not ctx.cancellation_flag.is_set():
145
+ await asyncio.sleep(0.02)
146
+
147
+ task = TaskInstance(process_id="p.coop", name="Coop", params={}, execute=cooperative)
148
+ optio, run_task = await _start_optio(mongo_db, prefix, [task])
149
+ try:
150
+ _fail_update_status(monkeypatch, {"cancelled"})
151
+ await optio.launch("p.coop", session_id=None)
152
+ await started.wait()
153
+ await optio.cancel("p.coop")
154
+ await _wait_state(
155
+ mongo_db, prefix, "p.coop", {"cancelled"}, log="State recorded late: cancelled",
156
+ )
157
+ finally:
158
+ await _stop_optio(optio, run_task)
159
+
160
+
161
+ async def test_lost_force_cancel_write_is_recorded_late(mongo_db, monkeypatch):
162
+ """The bobcat incident: the force-cancel write raised and was never retried."""
163
+ import optio_core._force_cancel as fc
164
+
165
+ prefix = "lfw_force"
166
+ started = asyncio.Event()
167
+
168
+ async def stubborn(ctx): # noqa: ARG001
169
+ started.set()
170
+ while True:
171
+ await asyncio.sleep(0.05)
172
+
173
+ real = fc._write_force_cancelled_state
174
+ left = {"n": 1}
175
+
176
+ async def flaky(db, prefix_, oid, *args, **kwargs):
177
+ if left["n"] > 0:
178
+ left["n"] -= 1
179
+ raise AutoReconnect("injected: mongo unreachable")
180
+ return await real(db, prefix_, oid, *args, **kwargs)
181
+
182
+ monkeypatch.setattr(fc, "_write_force_cancelled_state", flaky)
183
+
184
+ task = TaskInstance(process_id="p.stub", name="Stub", params={}, execute=stubborn)
185
+ optio, run_task = await _start_optio(mongo_db, prefix, [task], cancel_grace_seconds=0.2)
186
+ try:
187
+ await optio.launch("p.stub", session_id=None)
188
+ await started.wait()
189
+ await optio.cancel("p.stub")
190
+ proc = await _wait_state(mongo_db, prefix, "p.stub", {"failed"})
191
+ assert proc["status"]["error"] == fc.FORCE_CANCEL_ERROR
192
+ assert left["n"] == 0 # the injected failure did happen
193
+ finally:
194
+ await _stop_optio(optio, run_task)
195
+
196
+
197
+ async def test_lost_running_write_settles_the_scheduled_row(mongo_db, monkeypatch):
198
+ prefix = "lfw_start"
199
+ task = TaskInstance(process_id="p.start", name="Start", params={}, execute=_done)
200
+ optio, run_task = await _start_optio(mongo_db, prefix, [task])
201
+ try:
202
+ _fail_update_status(monkeypatch, {"running"})
203
+ await optio.launch("p.start", session_id=None)
204
+ proc = await _wait_state(mongo_db, prefix, "p.start", {"failed"})
205
+ assert proc["status"]["error"].startswith("Final state could not be recorded")
206
+ finally:
207
+ await _stop_optio(optio, run_task)
208
+
209
+
210
+ async def test_retry_keeps_trying_and_never_overwrites_a_settled_row(mongo_db, monkeypatch):
211
+ """Executor-level: a failed retry stays queued; a row that left the active
212
+ states in the meantime is left alone and the entry is dropped."""
213
+ import optio_core.executor as ex
214
+ from optio_core.executor import Executor
215
+
216
+ prefix = "lfw_retry"
217
+ tasks = [
218
+ TaskInstance(process_id=pid, name=pid, params={}, execute=_done)
219
+ for pid in ("p.keep", "p.moved")
220
+ ]
221
+ for t in tasks:
222
+ await upsert_process(mongo_db, prefix, t)
223
+ executor = Executor(mongo_db, prefix, services={})
224
+ executor.register_tasks(tasks)
225
+
226
+ _fail_update_status(monkeypatch, {"done"}, times=2)
227
+ for pid in ("p.keep", "p.moved"):
228
+ with pytest.raises(AutoReconnect):
229
+ await executor.launch_process(pid, session_id=None)
230
+ keep = await get_process_by_process_id(mongo_db, prefix, "p.keep")
231
+ moved = await get_process_by_process_id(mongo_db, prefix, "p.moved")
232
+ assert keep["status"]["state"] == "running"
233
+ assert set(executor._unrecorded_finals) == {keep["_id"], moved["_id"]}
234
+
235
+ # Something else settles p.moved meanwhile.
236
+ await mongo_db[f"{prefix}_processes"].update_one(
237
+ {"_id": moved["_id"]}, {"$set": {"status": ProcessStatus(state="idle").to_dict()}},
238
+ )
239
+
240
+ # First retry: Mongo still unavailable for p.keep.
241
+ real_finalize = ex.finalize_if_active
242
+ left = {"n": 1}
243
+
244
+ async def flaky_finalize(db, prefix_, oid, status, **kw):
245
+ if oid == keep["_id"] and left["n"] > 0:
246
+ left["n"] -= 1
247
+ raise AutoReconnect("injected: mongo unreachable")
248
+ return await real_finalize(db, prefix_, oid, status, **kw)
249
+
250
+ monkeypatch.setattr(ex, "finalize_if_active", flaky_finalize)
251
+ await executor.retry_unrecorded_finals()
252
+ assert keep["_id"] in executor._unrecorded_finals
253
+ assert moved["_id"] not in executor._unrecorded_finals
254
+ moved = await get_process_by_process_id(mongo_db, prefix, "p.moved")
255
+ assert moved["status"]["state"] == "idle"
256
+
257
+ # Make the backed-off entry due, then retry: it lands.
258
+ executor._unrecorded_finals[keep["_id"]].next_attempt = 0.0
259
+ await executor.retry_unrecorded_finals()
260
+ assert executor._unrecorded_finals == {}
261
+ keep = await get_process_by_process_id(mongo_db, prefix, "p.keep")
262
+ assert keep["status"]["state"] == "done"
263
+
264
+
265
+ # ---------- Rule 4: settle a row that has no task behind it ----------
266
+
267
+ async def _orphan(mongo_db, prefix, process_id, state):
268
+ """Leave a row active with no task behind it, as a lost final write does."""
269
+ await mongo_db[f"{prefix}_processes"].update_one(
270
+ {"processId": process_id},
271
+ {"$set": {"status": ProcessStatus(state=state).to_dict()}},
272
+ )
273
+
274
+
275
+ def _assert_settled_orphan(proc, prev_state):
276
+ assert proc["status"]["state"] == "failed", proc["status"]
277
+ assert "no running task" in proc["status"]["error"]
278
+ assert _logged(proc, f"State reconciled: {prev_state} -> failed (no running task)")
279
+
280
+
281
+ async def test_group_cancel_and_wait_settles_orphans(mongo_db):
282
+ prefix = "lfw_group"
283
+ states = {"p.run": "running", "p.req": "cancel_requested", "p.cing": "cancelling"}
284
+ tasks = [
285
+ TaskInstance(process_id=pid, name=pid, params={}, execute=_done,
286
+ metadata={"grp": "g"})
287
+ for pid in states
288
+ ]
289
+ optio, run_task = await _start_optio(mongo_db, prefix, tasks, cancel_grace_seconds=0.2)
290
+ try:
291
+ for pid, state in states.items():
292
+ await _orphan(mongo_db, prefix, pid, state)
293
+ await optio.group_cancel_and_wait({"grp": "g"})
294
+ for pid, state in states.items():
295
+ proc = await get_process_by_process_id(mongo_db, prefix, pid)
296
+ # cancel() moves a running orphan to cancel_requested first.
297
+ prev = "cancel_requested" if state == "running" else state
298
+ _assert_settled_orphan(proc, prev)
299
+ finally:
300
+ await _stop_optio(optio, run_task)
301
+
302
+
303
+ async def test_cancel_and_wait_settles_orphan(mongo_db):
304
+ prefix = "lfw_caw"
305
+ task = TaskInstance(process_id="p.cing", name="Cing", params={}, execute=_done)
306
+ optio, run_task = await _start_optio(mongo_db, prefix, [task], cancel_grace_seconds=0.2)
307
+ try:
308
+ await _orphan(mongo_db, prefix, "p.cing", "cancelling")
309
+ assert await optio.cancel_and_wait("p.cing") == "failed"
310
+ proc = await get_process_by_process_id(mongo_db, prefix, "p.cing")
311
+ _assert_settled_orphan(proc, "cancelling")
312
+ finally:
313
+ await _stop_optio(optio, run_task)
314
+
315
+
316
+ async def test_cancel_settles_running_orphan(mongo_db):
317
+ """Before: the row was left in cancel_requested for good."""
318
+ prefix = "lfw_cancel"
319
+ task = TaskInstance(process_id="p.run", name="Run", params={}, execute=_done)
320
+ optio, run_task = await _start_optio(mongo_db, prefix, [task])
321
+ try:
322
+ await _orphan(mongo_db, prefix, "p.run", "running")
323
+ outcome = await optio.cancel("p.run")
324
+ assert outcome.ok
325
+ proc = await get_process_by_process_id(mongo_db, prefix, "p.run")
326
+ _assert_settled_orphan(proc, "cancel_requested")
327
+ finally:
328
+ await _stop_optio(optio, run_task)
329
+
330
+
331
+ async def test_orphan_with_parked_final_gets_the_parked_state(mongo_db, monkeypatch):
332
+ """A row whose final write is queued (Rule 3) settles to that state, not
333
+ to the generic orphan failure. No run(): the supervisor must not retry first."""
334
+ prefix = "lfw_parked"
335
+ task = TaskInstance(process_id="p.done", name="Done", params={}, execute=_done)
336
+ optio, run_task = await _start_optio(mongo_db, prefix, [task], run=False)
337
+ try:
338
+ _fail_update_status(monkeypatch, {"done"})
339
+ await optio.launch("p.done", session_id=None)
340
+
341
+ async def parked():
342
+ return bool(optio._executor._unrecorded_finals)
343
+ await _wait_until(parked, "the done write to be parked")
344
+
345
+ assert await optio.cancel_and_wait("p.done") == "done"
346
+ finally:
347
+ await _stop_optio(optio, run_task)
348
+
349
+
350
+ async def test_live_task_and_scheduled_row_are_not_orphans(mongo_db):
351
+ prefix = "lfw_live"
352
+ started = asyncio.Event()
353
+
354
+ async def cooperative(ctx):
355
+ started.set()
356
+ while not ctx.cancellation_flag.is_set():
357
+ await asyncio.sleep(0.02)
358
+
359
+ live = TaskInstance(process_id="p.live", name="Live", params={}, execute=cooperative,
360
+ metadata={"grp": "live"})
361
+ sched = TaskInstance(process_id="p.sched", name="Sched", params={}, execute=_done)
362
+ optio, run_task = await _start_optio(mongo_db, prefix, [live, sched])
363
+ try:
364
+ await optio.launch("p.live", session_id=None)
365
+ await started.wait()
366
+ await optio.group_cancel_and_wait({"grp": "live"})
367
+ proc = await get_process_by_process_id(mongo_db, prefix, "p.live")
368
+ assert proc["status"]["state"] == "cancelled"
369
+
370
+ await _orphan(mongo_db, prefix, "p.sched", "scheduled")
371
+ assert await optio.cancel_and_wait("p.sched") == "cancelled"
372
+ finally:
373
+ await _stop_optio(optio, run_task)
File without changes
File without changes
File without changes