pyworkflow-engine 0.2.11__tar.gz → 0.2.13__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/PKG-INFO +1 -1
  2. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyproject.toml +1 -1
  3. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/__init__.py +1 -1
  4. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/app.py +12 -5
  5. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/tasks.py +1 -1
  6. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/base.py +8 -1
  7. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/cassandra.py +88 -0
  8. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/citus.py +1 -2
  9. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/config.py +49 -0
  10. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/dynamodb.py +31 -0
  11. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/file.py +47 -0
  12. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/memory.py +50 -0
  13. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/mysql.py +23 -0
  14. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/postgres.py +45 -4
  15. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/sqlite.py +20 -0
  16. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/__init__.py +4 -0
  17. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/context.py +73 -0
  18. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/dispatcher.py +13 -2
  19. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/CLAUDE.md +0 -0
  20. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/DISTRIBUTED.md +0 -0
  21. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/LICENSE +0 -0
  22. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/MANIFEST.in +0 -0
  23. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/README.md +0 -0
  24. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/RELEASING.md +0 -0
  25. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/architecture.md +0 -0
  26. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/cancellation.mdx +0 -0
  27. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/continue-as-new.mdx +0 -0
  28. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/events.mdx +0 -0
  29. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/fault-tolerance.mdx +0 -0
  30. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/hooks.mdx +0 -0
  31. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/limitations.mdx +0 -0
  32. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/schedules.mdx +0 -0
  33. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/sleep.mdx +0 -0
  34. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/step-context.mdx +0 -0
  35. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/steps.mdx +0 -0
  36. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/stream-steps.mdx +0 -0
  37. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/stream-workflows.mdx +0 -0
  38. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/workflows.mdx +0 -0
  39. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/conventions.md +0 -0
  40. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/guides/brokers.mdx +0 -0
  41. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/guides/cli.mdx +0 -0
  42. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/guides/configuration.mdx +0 -0
  43. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/harness-gaps.md +0 -0
  44. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/introduction.mdx +0 -0
  45. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/layers.md +0 -0
  46. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/quickstart.mdx +0 -0
  47. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/__init__.py +0 -0
  48. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/__init__.py +0 -0
  49. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/docker-compose.yml +0 -0
  50. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/pyworkflow.config.yaml +0 -0
  51. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/__init__.py +0 -0
  52. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/basic.py +0 -0
  53. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/batch_processing.py +0 -0
  54. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/cancellation.py +0 -0
  55. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/child_workflow_from_step.py +0 -0
  56. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/child_workflow_patterns.py +0 -0
  57. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/child_workflows.py +0 -0
  58. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/continue_as_new.py +0 -0
  59. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/fault_tolerance.py +0 -0
  60. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/hooks.py +0 -0
  61. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/idempotency.py +0 -0
  62. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/long_running.py +0 -0
  63. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/retries.py +0 -0
  64. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/schedules.py +0 -0
  65. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/sleep_in_step.py +0 -0
  66. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/step_context.py +0 -0
  67. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/01_basic_workflow.py +0 -0
  68. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/02_fault_tolerance.py +0 -0
  69. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/__init__.py +0 -0
  70. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/pyworkflow.config.yaml +0 -0
  71. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/__init__.py +0 -0
  72. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/01_basic_workflow.py +0 -0
  73. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/02_file_storage.py +0 -0
  74. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/03_retries.py +0 -0
  75. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/04_long_running.py +0 -0
  76. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/05_event_log.py +0 -0
  77. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/06_idempotency.py +0 -0
  78. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/07_hooks.py +0 -0
  79. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/08_cancellation.py +0 -0
  80. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/09_child_workflows.py +0 -0
  81. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/10_child_workflow_patterns.py +0 -0
  82. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/11_continue_as_new.py +0 -0
  83. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/12_schedules.py +0 -0
  84. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/13_step_context.py +0 -0
  85. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/14_child_workflow_from_step.py +0 -0
  86. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/__init__.py +0 -0
  87. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/01_quick_tasks.py +0 -0
  88. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/02_retries.py +0 -0
  89. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/03_sleep.py +0 -0
  90. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/__init__.py +0 -0
  91. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/__init__.py +0 -0
  92. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/context.py +0 -0
  93. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/handler.py +0 -0
  94. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/testing.py +0 -0
  95. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/__init__.py +0 -0
  96. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/loop.py +0 -0
  97. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/scheduler.py +0 -0
  98. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/singleton.py +0 -0
  99. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/__init__.py +0 -0
  100. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/__main__.py +0 -0
  101. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/__init__.py +0 -0
  102. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/hooks.py +0 -0
  103. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/quickstart.py +0 -0
  104. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/runs.py +0 -0
  105. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/scheduler.py +0 -0
  106. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/schedules.py +0 -0
  107. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/setup.py +0 -0
  108. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/worker.py +0 -0
  109. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/workflows.py +0 -0
  110. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/output/__init__.py +0 -0
  111. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/output/formatters.py +0 -0
  112. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/output/styles.py +0 -0
  113. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/__init__.py +0 -0
  114. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/async_helpers.py +0 -0
  115. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/config.py +0 -0
  116. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/config_generator.py +0 -0
  117. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/discovery.py +0 -0
  118. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/docker_manager.py +0 -0
  119. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/interactive.py +0 -0
  120. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/storage.py +0 -0
  121. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/config.py +0 -0
  122. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/__init__.py +0 -0
  123. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/aws.py +0 -0
  124. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/base.py +0 -0
  125. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/local.py +0 -0
  126. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/mock.py +0 -0
  127. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/step_context.py +0 -0
  128. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/__init__.py +0 -0
  129. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/exceptions.py +0 -0
  130. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/registry.py +0 -0
  131. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/scheduled.py +0 -0
  132. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/step.py +0 -0
  133. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/validation.py +0 -0
  134. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/workflow.py +0 -0
  135. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/discovery.py +0 -0
  136. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/__init__.py +0 -0
  137. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/events.py +0 -0
  138. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/executor.py +0 -0
  139. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/replay.py +0 -0
  140. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/observability/__init__.py +0 -0
  141. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/observability/logging.py +0 -0
  142. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/__init__.py +0 -0
  143. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/child_handle.py +0 -0
  144. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/child_workflow.py +0 -0
  145. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/continue_as_new.py +0 -0
  146. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/define_hook.py +0 -0
  147. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/hooks.py +0 -0
  148. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/resume_hook.py +0 -0
  149. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/schedule.py +0 -0
  150. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/shield.py +0 -0
  151. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/sleep.py +0 -0
  152. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/step_checkpoint.py +0 -0
  153. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/step_hook.py +0 -0
  154. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/__init__.py +0 -0
  155. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/base.py +0 -0
  156. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/celery.py +0 -0
  157. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/factory.py +0 -0
  158. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/local.py +0 -0
  159. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/scheduler/__init__.py +0 -0
  160. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/scheduler/local.py +0 -0
  161. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/serialization/__init__.py +0 -0
  162. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/serialization/decoder.py +0 -0
  163. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/serialization/encoder.py +0 -0
  164. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/__init__.py +0 -0
  165. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/migrations/__init__.py +0 -0
  166. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/migrations/base.py +0 -0
  167. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/schemas.py +0 -0
  168. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/checkpoint.py +0 -0
  169. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/consumer.py +0 -0
  170. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/decorator.py +0 -0
  171. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/emit.py +0 -0
  172. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/registry.py +0 -0
  173. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/runtime.py +0 -0
  174. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/signal.py +0 -0
  175. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/step_context.py +0 -0
  176. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/__init__.py +0 -0
  177. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/duration.py +0 -0
  178. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/helpers.py +0 -0
  179. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/schedule.py +0 -0
  180. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow_engine.egg-info/SOURCES.txt +0 -0
  181. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/setup.cfg +0 -0
  182. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/__init__.py +0 -0
  183. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_cancellation.py +0 -0
  184. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_cassandra_storage.py +0 -0
  185. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_child_workflows.py +0 -0
  186. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_continue_as_new.py +0 -0
  187. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_dynamodb_storage.py +0 -0
  188. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_fault_tolerance.py +0 -0
  189. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_schedule_storage.py +0 -0
  190. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_schema_migrations.py +0 -0
  191. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_singleton.py +0 -0
  192. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_stream_e2e.py +0 -0
  193. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_workflow_suspended.py +0 -0
  194. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/__init__.py +0 -0
  195. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/__init__.py +0 -0
  196. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_cassandra_storage.py +0 -0
  197. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_citus_storage.py +0 -0
  198. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_dynamodb_storage.py +0 -0
  199. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_postgres_storage.py +0 -0
  200. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_sqlite_storage.py +0 -0
  201. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/conftest.py +0 -0
  202. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/storage/__init__.py +0 -0
  203. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/storage/test_migrations.py +0 -0
  204. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_cancellation.py +0 -0
  205. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_child_workflows.py +0 -0
  206. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_cli_worker.py +0 -0
  207. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_continue_as_new.py +0 -0
  208. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_emit.py +0 -0
  209. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_event_limits.py +0 -0
  210. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_executor.py +0 -0
  211. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_fault_tolerance.py +0 -0
  212. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_force_local.py +0 -0
  213. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_hooks.py +0 -0
  214. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_parent_run_id.py +0 -0
  215. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_primitives_from_steps.py +0 -0
  216. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_registry.py +0 -0
  217. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_replay.py +0 -0
  218. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_retention.py +0 -0
  219. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_schedule_schemas.py +0 -0
  220. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_schedule_utils.py +0 -0
  221. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_scheduled_workflow.py +0 -0
  222. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_signal.py +0 -0
  223. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_singleton.py +0 -0
  224. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step.py +0 -0
  225. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step_checkpoint.py +0 -0
  226. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step_context.py +0 -0
  227. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step_hook.py +0 -0
  228. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_stream_storage.py +0 -0
  229. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_stream_workflow.py +0 -0
  230. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_validation.py +0 -0
  231. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_workflow.py +0 -0
  232. {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_workflow_suspended.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pyworkflow-engine
3
- Version: 0.2.11
3
+ Version: 0.2.13
4
4
  Summary: A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow
5
5
  Author: PyWorkflow Contributors
6
6
  License: MIT
@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
7
7
 
8
8
  [project]
9
9
  name = "pyworkflow-engine"
10
- version = "0.2.11"
10
+ version = "0.2.13"
11
11
  description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
12
12
  readme = "README.md"
13
13
  requires-python = ">=3.11"
@@ -29,7 +29,7 @@ Quick Start:
29
29
  >>> run_id = await start(my_workflow, "Alice")
30
30
  """
31
31
 
32
- __version__ = "0.2.11"
32
+ __version__ = "0.2.13"
33
33
 
34
34
  # Configuration
35
35
  from pyworkflow.config import (
@@ -84,9 +84,7 @@ def _run_startup_migrations() -> None:
84
84
  from loguru import logger as loguru_logger
85
85
 
86
86
  if os.getenv("PYWORKFLOW_SKIP_STARTUP_MIGRATIONS", "").lower() in ("1", "true", "yes"):
87
- loguru_logger.info(
88
- "STARTUP_MIGRATIONS: skipped (PYWORKFLOW_SKIP_STARTUP_MIGRATIONS set)"
89
- )
87
+ loguru_logger.info("STARTUP_MIGRATIONS: skipped (PYWORKFLOW_SKIP_STARTUP_MIGRATIONS set)")
90
88
  return
91
89
 
92
90
  from pyworkflow.config import _load_env_storage_config
@@ -134,7 +132,16 @@ def _run_startup_migrations() -> None:
134
132
  )
135
133
  raise
136
134
 
137
- loguru_logger.info("STARTUP_MIGRATIONS: completed successfully")
135
+ # Declare that the schema is up-to-date in this process. Forked worker
136
+ # children inherit the flag via copy-on-write, so every runtime connect()
137
+ # in this pod will skip _initialize_schema() / run_migrations() entirely.
138
+ from pyworkflow.storage.config import mark_schema_ensured
139
+
140
+ mark_schema_ensured()
141
+
142
+ loguru_logger.info(
143
+ "STARTUP_MIGRATIONS: completed successfully — runtime connects will skip schema init"
144
+ )
138
145
 
139
146
 
140
147
  def is_sentinel_url(url: str) -> bool:
@@ -413,7 +420,7 @@ def create_celery_app(
413
420
  },
414
421
  "pyworkflow-streams-drain-scheduled": {
415
422
  "task": "pyworkflow.streams.drain_scheduled_signals",
416
- "schedule": timedelta(seconds=2),
423
+ "schedule": timedelta(seconds=30),
417
424
  "options": {"queue": "pyworkflow.default"},
418
425
  },
419
426
  },
@@ -2832,7 +2832,7 @@ def run_data_retention_task(self: SingletonWorkflowTask) -> dict[str, Any]:
2832
2832
 
2833
2833
  cutoff = datetime.now(UTC) - timedelta(days=config.data_retention_days)
2834
2834
  logger.info(
2835
- "Running data retention: deleting runs updated before {}",
2835
+ "Running data retention: deleting runs and orphaned stream data older than {}",
2836
2836
  cutoff.isoformat(),
2837
2837
  )
2838
2838
  storage_config = storage_to_config(storage)
@@ -707,7 +707,14 @@ class StorageBackend(ABC):
707
707
  Delete workflow runs in terminal states last updated before `older_than`.
708
708
 
709
709
  Terminal states: COMPLETED, FAILED, CANCELLED, CONTINUED_AS_NEW, INTERRUPTED.
710
- Associated events, steps, hooks, and cancellation flags are deleted too.
710
+
711
+ Deletes the following associated data:
712
+ - events, steps, checkpoints, hooks, cancellation_flags (by run_id)
713
+ - signals (published_at < older_than)
714
+ - signal_acknowledgments (acknowledged_at < older_than)
715
+ - scheduled_signals (delivered + created_at < older_than)
716
+ - stream_subscriptions (terminated/completed + created_at < older_than)
717
+ - schedules (soft-deleted + deleted_at/updated_at < older_than)
711
718
 
712
719
  Args:
713
720
  older_than: Delete runs where updated_at < older_than
@@ -1876,6 +1876,94 @@ class CassandraStorageBackend(StorageBackend):
1876
1876
  )
1877
1877
  count += 1
1878
1878
 
1879
+ # --- Stream / schedule cleanup ---
1880
+ # ALLOW FILTERING is acceptable here: this is a background retention
1881
+ # job that runs at most once per day.
1882
+ sig_rows = session.execute(
1883
+ SimpleStatement(
1884
+ "SELECT stream_id, stream_run_id, sequence FROM signals "
1885
+ "WHERE published_at < %s ALLOW FILTERING",
1886
+ consistency_level=self.read_consistency,
1887
+ ),
1888
+ (older_than,),
1889
+ )
1890
+ for sr in sig_rows:
1891
+ session.execute(
1892
+ SimpleStatement(
1893
+ "DELETE FROM signals WHERE stream_id = %s "
1894
+ "AND stream_run_id = %s AND sequence = %s",
1895
+ consistency_level=self.write_consistency,
1896
+ ),
1897
+ (sr.stream_id, sr.stream_run_id, sr.sequence),
1898
+ )
1899
+
1900
+ ack_rows = session.execute(
1901
+ SimpleStatement(
1902
+ "SELECT signal_id, step_run_id FROM signal_acknowledgments "
1903
+ "WHERE acknowledged_at < %s ALLOW FILTERING",
1904
+ consistency_level=self.read_consistency,
1905
+ ),
1906
+ (older_than,),
1907
+ )
1908
+ for ar in ack_rows:
1909
+ session.execute(
1910
+ SimpleStatement(
1911
+ "DELETE FROM signal_acknowledgments WHERE signal_id = %s AND step_run_id = %s",
1912
+ consistency_level=self.write_consistency,
1913
+ ),
1914
+ (ar.signal_id, ar.step_run_id),
1915
+ )
1916
+
1917
+ sub_rows = session.execute(
1918
+ SimpleStatement(
1919
+ "SELECT stream_id, step_run_id FROM stream_subscriptions "
1920
+ "WHERE status IN ('terminated','completed') "
1921
+ "AND created_at < %s ALLOW FILTERING",
1922
+ consistency_level=self.read_consistency,
1923
+ ),
1924
+ (older_than,),
1925
+ )
1926
+ for sub in sub_rows:
1927
+ session.execute(
1928
+ SimpleStatement(
1929
+ "DELETE FROM stream_subscriptions WHERE stream_id = %s AND step_run_id = %s",
1930
+ consistency_level=self.write_consistency,
1931
+ ),
1932
+ (sub.stream_id, sub.step_run_id),
1933
+ )
1934
+
1935
+ # Purge soft-deleted schedules
1936
+ del_sched_rows = session.execute(
1937
+ SimpleStatement(
1938
+ "SELECT schedule_id FROM schedules_by_status WHERE status = 'DELETED'",
1939
+ consistency_level=self.read_consistency,
1940
+ ),
1941
+ )
1942
+ for ds in del_sched_rows:
1943
+ sched_row = session.execute(
1944
+ SimpleStatement(
1945
+ "SELECT updated_at FROM schedules WHERE schedule_id = %s",
1946
+ consistency_level=self.read_consistency,
1947
+ ),
1948
+ (ds.schedule_id,),
1949
+ ).one()
1950
+ if sched_row and sched_row.updated_at < older_than:
1951
+ session.execute(
1952
+ SimpleStatement(
1953
+ "DELETE FROM schedules WHERE schedule_id = %s",
1954
+ consistency_level=self.write_consistency,
1955
+ ),
1956
+ (ds.schedule_id,),
1957
+ )
1958
+ session.execute(
1959
+ SimpleStatement(
1960
+ "DELETE FROM schedules_by_status "
1961
+ "WHERE status = 'DELETED' AND schedule_id = %s",
1962
+ consistency_level=self.write_consistency,
1963
+ ),
1964
+ (ds.schedule_id,),
1965
+ )
1966
+
1879
1967
  return count
1880
1968
 
1881
1969
  # Helper methods for converting Cassandra rows to domain objects
@@ -189,8 +189,7 @@ class CitusMigrationRunner(PostgresMigrationRunner):
189
189
  # V7: Add result column. Top-level ALTER for Citus propagation (see V5 note).
190
190
  logger.info("CITUS_MIGRATION v7: ALTER stream_subscriptions ADD result")
191
191
  await conn.execute(
192
- "ALTER TABLE stream_subscriptions "
193
- "ADD COLUMN IF NOT EXISTS result JSONB NULL"
192
+ "ALTER TABLE stream_subscriptions ADD COLUMN IF NOT EXISTS result JSONB NULL"
194
193
  )
195
194
  elif migration.up_func:
196
195
  await migration.up_func(conn)
@@ -25,6 +25,55 @@ from pyworkflow.storage.base import StorageBackend
25
25
  _storage_cache: dict[str, tuple[StorageBackend, None]] = {}
26
26
 
27
27
 
28
+ # ---------------------------------------------------------------------------
29
+ # Schema-ensured signal
30
+ # ---------------------------------------------------------------------------
31
+ #
32
+ # Process-global flag set by the Celery worker startup hook
33
+ # (``pyworkflow.celery.app._run_startup_migrations``) after it has successfully
34
+ # run pending migrations. When True, runtime ``connect()`` calls SKIP the
35
+ # ``_initialize_schema()`` step entirely — no CREATE TABLE IF NOT EXISTS
36
+ # sweep, no ``run_migrations()`` probe on the ``schema_versions`` table.
37
+ #
38
+ # Why a module-level flag rather than instance state or an env var:
39
+ # * Forked worker children inherit the flag via copy-on-write, so the
40
+ # parent process's "I already migrated" decision naturally propagates.
41
+ # * It's not an env var so child processes that shell out or subprocess
42
+ # don't accidentally carry the claim into an unrelated context.
43
+ # * Per-instance state would be reset on every loop change (pool recreate),
44
+ # so the flag must outlive backend instances.
45
+ #
46
+ # Paths that do NOT set the flag still get lazy initialization:
47
+ # * Tests that instantiate backends directly
48
+ # * CLI commands
49
+ # * Celery workers started with ``PYWORKFLOW_SKIP_STARTUP_MIGRATIONS=1``
50
+ # * Any caller that bypasses the startup hook
51
+ _schema_ensured: bool = False
52
+
53
+
54
+ def mark_schema_ensured() -> None:
55
+ """Declare that migrations are up-to-date in this process.
56
+
57
+ Called by the startup migration hook once migrations have been applied
58
+ successfully. After this, ``connect()`` on any Postgres/Citus backend
59
+ in this process (and in its fork children) will skip
60
+ ``_initialize_schema()``.
61
+ """
62
+ global _schema_ensured
63
+ _schema_ensured = True
64
+
65
+
66
+ def is_schema_ensured() -> bool:
67
+ """Return True if the schema has already been ensured in this process."""
68
+ return _schema_ensured
69
+
70
+
71
+ def reset_schema_ensured() -> None:
72
+ """Reset the flag. Intended for tests only."""
73
+ global _schema_ensured
74
+ _schema_ensured = False
75
+
76
+
28
77
  def _config_to_cache_key(config: dict[str, Any] | None) -> str:
29
78
  """
30
79
  Create a cache key from config dict.
@@ -1426,6 +1426,37 @@ class DynamoDBStorageBackend(StorageBackend):
1426
1426
 
1427
1427
  count += 1
1428
1428
 
1429
+ # --- Schedule cleanup: purge soft-deleted schedules ---
1430
+ sched_params: dict[str, Any] = {
1431
+ "TableName": self.table_name,
1432
+ "IndexName": "GSI1",
1433
+ "KeyConditionExpression": "GSI1PK = :pk",
1434
+ "FilterExpression": "#st = :deleted AND updated_at < :cutoff",
1435
+ "ExpressionAttributeNames": {"#st": "status"},
1436
+ "ExpressionAttributeValues": {
1437
+ ":pk": {"S": "SCHEDULES"},
1438
+ ":deleted": {"S": "DELETED"},
1439
+ ":cutoff": {"S": cutoff_iso},
1440
+ },
1441
+ "ProjectionExpression": "PK, SK",
1442
+ }
1443
+ while True:
1444
+ resp = await client.query(**sched_params)
1445
+ items_to_delete = resp.get("Items", [])
1446
+ for i in range(0, len(items_to_delete), 25):
1447
+ batch = items_to_delete[i : i + 25]
1448
+ request_items = {
1449
+ self.table_name: [
1450
+ {"DeleteRequest": {"Key": {"PK": it["PK"], "SK": it["SK"]}}}
1451
+ for it in batch
1452
+ ]
1453
+ }
1454
+ await client.batch_write_item(RequestItems=request_items)
1455
+ lk = resp.get("LastEvaluatedKey")
1456
+ if not lk:
1457
+ break
1458
+ sched_params["ExclusiveStartKey"] = lk
1459
+
1429
1460
  return count
1430
1461
 
1431
1462
  # Helper methods for converting DynamoDB items to domain objects
@@ -973,6 +973,7 @@ class FileStorageBackend(StorageBackend):
973
973
  async def delete_old_runs(self, older_than: datetime) -> int:
974
974
  """Delete terminal runs (and all related files) updated before older_than."""
975
975
  terminal = {"completed", "failed", "cancelled", "continued_as_new", "interrupted"}
976
+ cutoff_iso = older_than.isoformat()
976
977
 
977
978
  def _sync_delete() -> int:
978
979
  count = 0
@@ -994,6 +995,10 @@ class FileStorageBackend(StorageBackend):
994
995
  try:
995
996
  sd = json.loads(sf.read_text())
996
997
  if sd.get("run_id") == run_id:
998
+ # Also delete checkpoint for this step
999
+ (self.checkpoints_dir / f"{sd['step_id']}.json").unlink(
1000
+ missing_ok=True
1001
+ )
997
1002
  sf.unlink(missing_ok=True)
998
1003
  except Exception:
999
1004
  pass
@@ -1003,6 +1008,48 @@ class FileStorageBackend(StorageBackend):
1003
1008
  count += 1
1004
1009
  except Exception:
1005
1010
  continue
1011
+
1012
+ # --- Stream / schedule cleanup ---
1013
+ # Prune old signals from each stream's signal file
1014
+ for sig_file in list(self.signals_dir.glob("*.json")):
1015
+ try:
1016
+ signals = json.loads(sig_file.read_text())
1017
+ kept = [s for s in signals if s.get("published_at", "") >= cutoff_iso]
1018
+ if len(kept) < len(signals):
1019
+ sig_file.write_text(json.dumps(kept, indent=2))
1020
+ except Exception:
1021
+ pass
1022
+
1023
+ # Delete old terminal subscriptions
1024
+ for sub_file in list(self.subscriptions_dir.glob("*.json")):
1025
+ try:
1026
+ sub = json.loads(sub_file.read_text())
1027
+ if sub.get("status") in ("terminated", "completed"):
1028
+ if sub.get("created_at", "") < cutoff_iso:
1029
+ sub_file.unlink(missing_ok=True)
1030
+ except Exception:
1031
+ pass
1032
+
1033
+ # Delete old acknowledgments
1034
+ for ack_file in list(self.acknowledgments_dir.glob("*.json")):
1035
+ try:
1036
+ ack = json.loads(ack_file.read_text())
1037
+ if ack.get("acknowledged_at", "") < cutoff_iso:
1038
+ ack_file.unlink(missing_ok=True)
1039
+ except Exception:
1040
+ pass
1041
+
1042
+ # Purge soft-deleted schedules
1043
+ for sched_file in list(self.schedules_dir.glob("*.json")):
1044
+ try:
1045
+ sched = json.loads(sched_file.read_text())
1046
+ if sched.get("status") == "DELETED":
1047
+ deleted_at = sched.get("deleted_at") or sched.get("updated_at", "")
1048
+ if deleted_at and deleted_at < cutoff_iso:
1049
+ sched_file.unlink(missing_ok=True)
1050
+ except Exception:
1051
+ pass
1052
+
1006
1053
  return count
1007
1054
 
1008
1055
  return await asyncio.to_thread(_sync_delete)
@@ -615,6 +615,7 @@ class InMemoryStorageBackend(StorageBackend):
615
615
  RunStatus.CONTINUED_AS_NEW,
616
616
  RunStatus.INTERRUPTED,
617
617
  }
618
+ cutoff_iso = older_than.isoformat()
618
619
  with self._lock:
619
620
  to_delete = [
620
621
  run_id
@@ -629,12 +630,61 @@ class InMemoryStorageBackend(StorageBackend):
629
630
  self._idempotency_index.pop(run.idempotency_key, None)
630
631
  step_ids = [sid for sid, s in self._steps.items() if s.run_id == run_id]
631
632
  for sid in step_ids:
633
+ self._checkpoints.pop(sid, None)
632
634
  del self._steps[sid]
633
635
  hook_keys = [k for k in self._hooks if k[0] == run_id]
634
636
  for k in hook_keys:
635
637
  hook = self._hooks.pop(k)
636
638
  self._token_index.pop(hook.token, None)
637
639
  self._cancellation_flags.pop(run_id, None)
640
+
641
+ # --- Stream / schedule cleanup ---
642
+ # Purge old signals
643
+ for stream_id in list(self._signals):
644
+ self._signals[stream_id] = [
645
+ s for s in self._signals[stream_id] if s.get("published_at", "") >= cutoff_iso
646
+ ]
647
+
648
+ # Purge delivered scheduled signals
649
+ if hasattr(self, "_scheduled_signals"):
650
+ to_remove = [
651
+ sid for sid, row in self._scheduled_signals.items() if row["delivered"]
652
+ ]
653
+ for sid in to_remove:
654
+ del self._scheduled_signals[sid]
655
+
656
+ # Purge terminal subscriptions
657
+ sub_to_remove = [
658
+ key
659
+ for key, sub in self._subscriptions.items()
660
+ if sub["status"] in ("terminated", "completed")
661
+ and sub.get("created_at", "") < cutoff_iso
662
+ ]
663
+ for key in sub_to_remove:
664
+ del self._subscriptions[key]
665
+
666
+ # Purge acknowledgments for signals that no longer exist
667
+ existing_signal_ids = set()
668
+ for sigs in self._signals.values():
669
+ for s in sigs:
670
+ existing_signal_ids.add(s["signal_id"])
671
+ self._acknowledgments = {
672
+ (sig_id, step_id)
673
+ for sig_id, step_id in self._acknowledgments
674
+ if sig_id in existing_signal_ids
675
+ }
676
+
677
+ # Purge soft-deleted schedules
678
+ sched_to_remove = [
679
+ sid
680
+ for sid, sched in self._schedules.items()
681
+ if sched.status == ScheduleStatus.DELETED
682
+ and sched.updated_at is not None
683
+ and sched.updated_at < older_than
684
+ ]
685
+ for sid in sched_to_remove:
686
+ del self._schedules[sid]
687
+
638
688
  return len(to_delete)
639
689
 
640
690
  # Stream Operations
@@ -1422,11 +1422,34 @@ class MySQLStorageBackend(StorageBackend):
1422
1422
  async with pool.acquire() as conn:
1423
1423
  async with conn.cursor() as cur:
1424
1424
  await cur.execute(f"DELETE FROM events WHERE run_id IN ({subq})", params)
1425
+ step_subq = f"SELECT step_id FROM steps WHERE run_id IN ({subq})"
1426
+ await cur.execute(
1427
+ f"DELETE FROM checkpoints WHERE step_run_id IN ({step_subq})", params
1428
+ )
1425
1429
  await cur.execute(f"DELETE FROM steps WHERE run_id IN ({subq})", params)
1426
1430
  await cur.execute(f"DELETE FROM hooks WHERE run_id IN ({subq})", params)
1427
1431
  await cur.execute(
1428
1432
  f"DELETE FROM cancellation_flags WHERE run_id IN ({subq})", params
1429
1433
  )
1434
+
1435
+ # --- Stream / schedule cleanup ---
1436
+ await cur.execute(
1437
+ "DELETE FROM signals WHERE published_at < %s",
1438
+ (older_than,),
1439
+ )
1440
+ await cur.execute(
1441
+ "DELETE FROM signal_acknowledgments WHERE acknowledged_at < %s",
1442
+ (older_than,),
1443
+ )
1444
+ await cur.execute(
1445
+ "DELETE FROM stream_subscriptions WHERE status IN (%s,%s) AND created_at < %s",
1446
+ ("terminated", "completed", older_than),
1447
+ )
1448
+ await cur.execute(
1449
+ "DELETE FROM schedules WHERE status = 'DELETED' AND deleted_at < %s",
1450
+ (older_than,),
1451
+ )
1452
+
1430
1453
  await cur.execute(
1431
1454
  f"DELETE FROM workflow_runs WHERE status IN ({placeholders}) AND updated_at < %s",
1432
1455
  params,
@@ -229,8 +229,7 @@ class PostgresMigrationRunner(MigrationRunner):
229
229
  # V7: result column on stream_subscriptions for step output payloads.
230
230
  logger.info("PG_MIGRATION v7: ALTER stream_subscriptions ADD result")
231
231
  await conn.execute(
232
- "ALTER TABLE stream_subscriptions "
233
- "ADD COLUMN IF NOT EXISTS result JSONB NULL"
232
+ "ALTER TABLE stream_subscriptions ADD COLUMN IF NOT EXISTS result JSONB NULL"
234
233
  )
235
234
  elif migration.up_func:
236
235
  await migration.up_func(conn)
@@ -358,7 +357,19 @@ class PostgresStorageBackend(StorageBackend):
358
357
  self._pool_loop_id = current_loop_id
359
358
 
360
359
  if not self._initialized:
361
- await self._initialize_schema()
360
+ # When the Celery worker startup hook has already run migrations
361
+ # for this process, skip _initialize_schema() entirely. The hook
362
+ # sets a module-level flag (see pyworkflow.storage.config) that
363
+ # survives fork via copy-on-write, so every runtime connect in
364
+ # this process and its children is a pure pool-create.
365
+ #
366
+ # Paths that do NOT go through the startup hook (tests, CLI,
367
+ # PYWORKFLOW_SKIP_STARTUP_MIGRATIONS=1) still initialize lazily
368
+ # here, preserving backwards compatibility.
369
+ from pyworkflow.storage.config import is_schema_ensured
370
+
371
+ if not is_schema_ensured():
372
+ await self._initialize_schema()
362
373
  self._initialized = True
363
374
 
364
375
  async def disconnect(self) -> None:
@@ -1651,8 +1662,14 @@ class PostgresStorageBackend(StorageBackend):
1651
1662
  await self.update_schedule(schedule)
1652
1663
 
1653
1664
  async def delete_old_runs(self, older_than: datetime) -> int:
1654
- """Delete terminal runs (and all related data) updated before older_than."""
1665
+ """Delete terminal runs (and all related data) updated before older_than.
1666
+
1667
+ Also purges orphaned stream data (signals, acknowledgments, delivered
1668
+ scheduled signals, terminal subscriptions) and soft-deleted schedules
1669
+ older than the cutoff.
1670
+ """
1655
1671
  terminal = ["completed", "failed", "cancelled", "continued_as_new", "interrupted"]
1672
+ sub_terminal = ["terminated", "completed"]
1656
1673
  pool = await self._get_pool()
1657
1674
  async with pool.acquire() as conn, conn.transaction():
1658
1675
  subq = "SELECT run_id FROM workflow_runs WHERE status = ANY($1) AND updated_at < $2"
@@ -1670,6 +1687,30 @@ class PostgresStorageBackend(StorageBackend):
1670
1687
  terminal,
1671
1688
  older_than,
1672
1689
  )
1690
+
1691
+ # --- Stream / schedule cleanup ---
1692
+ await conn.execute(
1693
+ "DELETE FROM scheduled_signals WHERE delivered = TRUE AND created_at < $1",
1694
+ older_than,
1695
+ )
1696
+ await conn.execute(
1697
+ "DELETE FROM signals WHERE published_at < $1",
1698
+ older_than,
1699
+ )
1700
+ await conn.execute(
1701
+ "DELETE FROM signal_acknowledgments WHERE acknowledged_at < $1",
1702
+ older_than,
1703
+ )
1704
+ await conn.execute(
1705
+ "DELETE FROM stream_subscriptions WHERE status = ANY($1) AND created_at < $2",
1706
+ sub_terminal,
1707
+ older_than,
1708
+ )
1709
+ await conn.execute(
1710
+ "DELETE FROM schedules WHERE status = 'DELETED' AND deleted_at < $1",
1711
+ older_than,
1712
+ )
1713
+
1673
1714
  result = await conn.execute(
1674
1715
  "DELETE FROM workflow_runs WHERE status = ANY($1) AND updated_at < $2",
1675
1716
  terminal,
@@ -1399,12 +1399,32 @@ class SQLiteStorageBackend(StorageBackend):
1399
1399
  f"SELECT run_id FROM workflow_runs WHERE status IN ({placeholders}) AND updated_at < ?"
1400
1400
  )
1401
1401
  params = (*terminal, older_than.isoformat())
1402
+ cutoff_iso = older_than.isoformat()
1402
1403
  async with db.cursor() as cur:
1403
1404
  await cur.execute(f"DELETE FROM events WHERE run_id IN ({subq})", params)
1404
1405
  await cur.execute(f"DELETE FROM steps WHERE run_id IN ({subq})", params)
1405
1406
  await cur.execute(f"DELETE FROM hooks WHERE run_id IN ({subq})", params)
1406
1407
  await cur.execute(f"DELETE FROM cancellation_flags WHERE run_id IN ({subq})", params)
1407
1408
  await cur.execute(f"DELETE FROM checkpoints WHERE step_run_id IN ({subq})", params)
1409
+
1410
+ # --- Stream / schedule cleanup ---
1411
+ await cur.execute(
1412
+ "DELETE FROM signals WHERE published_at < ?",
1413
+ (cutoff_iso,),
1414
+ )
1415
+ await cur.execute(
1416
+ "DELETE FROM signal_acknowledgments WHERE acknowledged_at < ?",
1417
+ (cutoff_iso,),
1418
+ )
1419
+ await cur.execute(
1420
+ "DELETE FROM stream_subscriptions WHERE status IN (?,?) AND created_at < ?",
1421
+ ("terminated", "completed", cutoff_iso),
1422
+ )
1423
+ await cur.execute(
1424
+ "DELETE FROM schedules WHERE status = 'DELETED' AND deleted_at < ?",
1425
+ (cutoff_iso,),
1426
+ )
1427
+
1408
1428
  await cur.execute(
1409
1429
  f"DELETE FROM workflow_runs WHERE status IN ({placeholders}) AND updated_at < ?",
1410
1430
  params,
@@ -28,8 +28,10 @@ from pyworkflow.streams.checkpoint import ( # noqa: F401
28
28
  )
29
29
  from pyworkflow.streams.consumer import StreamConsumer, poll_once # noqa: F401
30
30
  from pyworkflow.streams.context import ( # noqa: F401
31
+ acknowledge_inbox_signal,
31
32
  get_checkpoint,
32
33
  get_current_signal,
34
+ poll_inbox,
33
35
  save_checkpoint,
34
36
  set_result,
35
37
  suspend,
@@ -73,6 +75,8 @@ __all__ = [
73
75
  "get_current_signal",
74
76
  "get_checkpoint",
75
77
  "save_checkpoint",
78
+ "poll_inbox",
79
+ "acknowledge_inbox_signal",
76
80
  # Checkpoint
77
81
  "CheckpointBackend",
78
82
  "DefaultCheckpointBackend",