pyworkflow-engine 0.2.11__tar.gz → 0.2.13__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/PKG-INFO +1 -1
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyproject.toml +1 -1
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/__init__.py +1 -1
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/app.py +12 -5
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/tasks.py +1 -1
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/base.py +8 -1
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/cassandra.py +88 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/citus.py +1 -2
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/config.py +49 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/dynamodb.py +31 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/file.py +47 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/memory.py +50 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/mysql.py +23 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/postgres.py +45 -4
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/sqlite.py +20 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/__init__.py +4 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/context.py +73 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/dispatcher.py +13 -2
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/CLAUDE.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/DISTRIBUTED.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/LICENSE +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/MANIFEST.in +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/README.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/RELEASING.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/architecture.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/cancellation.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/continue-as-new.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/events.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/fault-tolerance.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/hooks.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/limitations.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/schedules.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/sleep.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/step-context.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/steps.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/stream-steps.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/stream-workflows.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/concepts/workflows.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/conventions.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/guides/brokers.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/guides/cli.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/guides/configuration.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/harness-gaps.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/introduction.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/layers.md +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/docs/quickstart.mdx +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/docker-compose.yml +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/pyworkflow.config.yaml +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/basic.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/batch_processing.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/cancellation.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/child_workflow_from_step.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/child_workflow_patterns.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/child_workflows.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/continue_as_new.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/fault_tolerance.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/hooks.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/idempotency.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/long_running.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/retries.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/schedules.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/sleep_in_step.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/durable/workflows/step_context.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/01_basic_workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/02_fault_tolerance.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/celery/transient/pyworkflow.config.yaml +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/01_basic_workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/02_file_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/03_retries.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/04_long_running.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/05_event_log.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/06_idempotency.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/07_hooks.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/08_cancellation.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/09_child_workflows.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/10_child_workflow_patterns.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/11_continue_as_new.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/12_schedules.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/13_step_context.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/14_child_workflow_from_step.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/durable/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/01_quick_tasks.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/02_retries.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/03_sleep.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/examples/local/transient/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/context.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/handler.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/aws/testing.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/loop.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/scheduler.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/celery/singleton.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/__main__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/hooks.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/quickstart.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/runs.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/scheduler.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/schedules.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/setup.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/worker.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/commands/workflows.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/output/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/output/formatters.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/output/styles.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/async_helpers.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/config.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/config_generator.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/discovery.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/docker_manager.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/interactive.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/cli/utils/storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/config.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/aws.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/base.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/local.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/mock.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/context/step_context.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/exceptions.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/registry.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/scheduled.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/step.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/validation.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/core/workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/discovery.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/events.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/executor.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/engine/replay.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/observability/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/observability/logging.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/child_handle.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/child_workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/continue_as_new.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/define_hook.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/hooks.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/resume_hook.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/schedule.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/shield.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/sleep.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/step_checkpoint.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/primitives/step_hook.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/base.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/celery.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/factory.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/runtime/local.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/scheduler/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/scheduler/local.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/serialization/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/serialization/decoder.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/serialization/encoder.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/migrations/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/migrations/base.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/storage/schemas.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/checkpoint.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/consumer.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/decorator.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/emit.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/registry.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/runtime.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/signal.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/streams/step_context.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/duration.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/helpers.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow/utils/schedule.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/pyworkflow_engine.egg-info/SOURCES.txt +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/setup.cfg +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_cancellation.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_cassandra_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_child_workflows.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_continue_as_new.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_dynamodb_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_fault_tolerance.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_schedule_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_schema_migrations.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_singleton.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_stream_e2e.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/integration/test_workflow_suspended.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_cassandra_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_citus_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_dynamodb_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_postgres_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/backends/test_sqlite_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/conftest.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/storage/__init__.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/storage/test_migrations.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_cancellation.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_child_workflows.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_cli_worker.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_continue_as_new.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_emit.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_event_limits.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_executor.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_fault_tolerance.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_force_local.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_hooks.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_parent_run_id.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_primitives_from_steps.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_registry.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_replay.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_retention.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_schedule_schemas.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_schedule_utils.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_scheduled_workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_signal.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_singleton.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step_checkpoint.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step_context.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_step_hook.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_stream_storage.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_stream_workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_validation.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_workflow.py +0 -0
- {pyworkflow_engine-0.2.11 → pyworkflow_engine-0.2.13}/tests/unit/test_workflow_suspended.py +0 -0
|
@@ -7,7 +7,7 @@ packages = [{include = "pyworkflow"}]
|
|
|
7
7
|
|
|
8
8
|
[project]
|
|
9
9
|
name = "pyworkflow-engine"
|
|
10
|
-
version = "0.2.
|
|
10
|
+
version = "0.2.13"
|
|
11
11
|
description = "A Python implementation of durable, event-sourced workflows inspired by Vercel Workflow"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
requires-python = ">=3.11"
|
|
@@ -84,9 +84,7 @@ def _run_startup_migrations() -> None:
|
|
|
84
84
|
from loguru import logger as loguru_logger
|
|
85
85
|
|
|
86
86
|
if os.getenv("PYWORKFLOW_SKIP_STARTUP_MIGRATIONS", "").lower() in ("1", "true", "yes"):
|
|
87
|
-
loguru_logger.info(
|
|
88
|
-
"STARTUP_MIGRATIONS: skipped (PYWORKFLOW_SKIP_STARTUP_MIGRATIONS set)"
|
|
89
|
-
)
|
|
87
|
+
loguru_logger.info("STARTUP_MIGRATIONS: skipped (PYWORKFLOW_SKIP_STARTUP_MIGRATIONS set)")
|
|
90
88
|
return
|
|
91
89
|
|
|
92
90
|
from pyworkflow.config import _load_env_storage_config
|
|
@@ -134,7 +132,16 @@ def _run_startup_migrations() -> None:
|
|
|
134
132
|
)
|
|
135
133
|
raise
|
|
136
134
|
|
|
137
|
-
|
|
135
|
+
# Declare that the schema is up-to-date in this process. Forked worker
|
|
136
|
+
# children inherit the flag via copy-on-write, so every runtime connect()
|
|
137
|
+
# in this pod will skip _initialize_schema() / run_migrations() entirely.
|
|
138
|
+
from pyworkflow.storage.config import mark_schema_ensured
|
|
139
|
+
|
|
140
|
+
mark_schema_ensured()
|
|
141
|
+
|
|
142
|
+
loguru_logger.info(
|
|
143
|
+
"STARTUP_MIGRATIONS: completed successfully — runtime connects will skip schema init"
|
|
144
|
+
)
|
|
138
145
|
|
|
139
146
|
|
|
140
147
|
def is_sentinel_url(url: str) -> bool:
|
|
@@ -413,7 +420,7 @@ def create_celery_app(
|
|
|
413
420
|
},
|
|
414
421
|
"pyworkflow-streams-drain-scheduled": {
|
|
415
422
|
"task": "pyworkflow.streams.drain_scheduled_signals",
|
|
416
|
-
"schedule": timedelta(seconds=
|
|
423
|
+
"schedule": timedelta(seconds=30),
|
|
417
424
|
"options": {"queue": "pyworkflow.default"},
|
|
418
425
|
},
|
|
419
426
|
},
|
|
@@ -2832,7 +2832,7 @@ def run_data_retention_task(self: SingletonWorkflowTask) -> dict[str, Any]:
|
|
|
2832
2832
|
|
|
2833
2833
|
cutoff = datetime.now(UTC) - timedelta(days=config.data_retention_days)
|
|
2834
2834
|
logger.info(
|
|
2835
|
-
"Running data retention: deleting runs
|
|
2835
|
+
"Running data retention: deleting runs and orphaned stream data older than {}",
|
|
2836
2836
|
cutoff.isoformat(),
|
|
2837
2837
|
)
|
|
2838
2838
|
storage_config = storage_to_config(storage)
|
|
@@ -707,7 +707,14 @@ class StorageBackend(ABC):
|
|
|
707
707
|
Delete workflow runs in terminal states last updated before `older_than`.
|
|
708
708
|
|
|
709
709
|
Terminal states: COMPLETED, FAILED, CANCELLED, CONTINUED_AS_NEW, INTERRUPTED.
|
|
710
|
-
|
|
710
|
+
|
|
711
|
+
Deletes the following associated data:
|
|
712
|
+
- events, steps, checkpoints, hooks, cancellation_flags (by run_id)
|
|
713
|
+
- signals (published_at < older_than)
|
|
714
|
+
- signal_acknowledgments (acknowledged_at < older_than)
|
|
715
|
+
- scheduled_signals (delivered + created_at < older_than)
|
|
716
|
+
- stream_subscriptions (terminated/completed + created_at < older_than)
|
|
717
|
+
- schedules (soft-deleted + deleted_at/updated_at < older_than)
|
|
711
718
|
|
|
712
719
|
Args:
|
|
713
720
|
older_than: Delete runs where updated_at < older_than
|
|
@@ -1876,6 +1876,94 @@ class CassandraStorageBackend(StorageBackend):
|
|
|
1876
1876
|
)
|
|
1877
1877
|
count += 1
|
|
1878
1878
|
|
|
1879
|
+
# --- Stream / schedule cleanup ---
|
|
1880
|
+
# ALLOW FILTERING is acceptable here: this is a background retention
|
|
1881
|
+
# job that runs at most once per day.
|
|
1882
|
+
sig_rows = session.execute(
|
|
1883
|
+
SimpleStatement(
|
|
1884
|
+
"SELECT stream_id, stream_run_id, sequence FROM signals "
|
|
1885
|
+
"WHERE published_at < %s ALLOW FILTERING",
|
|
1886
|
+
consistency_level=self.read_consistency,
|
|
1887
|
+
),
|
|
1888
|
+
(older_than,),
|
|
1889
|
+
)
|
|
1890
|
+
for sr in sig_rows:
|
|
1891
|
+
session.execute(
|
|
1892
|
+
SimpleStatement(
|
|
1893
|
+
"DELETE FROM signals WHERE stream_id = %s "
|
|
1894
|
+
"AND stream_run_id = %s AND sequence = %s",
|
|
1895
|
+
consistency_level=self.write_consistency,
|
|
1896
|
+
),
|
|
1897
|
+
(sr.stream_id, sr.stream_run_id, sr.sequence),
|
|
1898
|
+
)
|
|
1899
|
+
|
|
1900
|
+
ack_rows = session.execute(
|
|
1901
|
+
SimpleStatement(
|
|
1902
|
+
"SELECT signal_id, step_run_id FROM signal_acknowledgments "
|
|
1903
|
+
"WHERE acknowledged_at < %s ALLOW FILTERING",
|
|
1904
|
+
consistency_level=self.read_consistency,
|
|
1905
|
+
),
|
|
1906
|
+
(older_than,),
|
|
1907
|
+
)
|
|
1908
|
+
for ar in ack_rows:
|
|
1909
|
+
session.execute(
|
|
1910
|
+
SimpleStatement(
|
|
1911
|
+
"DELETE FROM signal_acknowledgments WHERE signal_id = %s AND step_run_id = %s",
|
|
1912
|
+
consistency_level=self.write_consistency,
|
|
1913
|
+
),
|
|
1914
|
+
(ar.signal_id, ar.step_run_id),
|
|
1915
|
+
)
|
|
1916
|
+
|
|
1917
|
+
sub_rows = session.execute(
|
|
1918
|
+
SimpleStatement(
|
|
1919
|
+
"SELECT stream_id, step_run_id FROM stream_subscriptions "
|
|
1920
|
+
"WHERE status IN ('terminated','completed') "
|
|
1921
|
+
"AND created_at < %s ALLOW FILTERING",
|
|
1922
|
+
consistency_level=self.read_consistency,
|
|
1923
|
+
),
|
|
1924
|
+
(older_than,),
|
|
1925
|
+
)
|
|
1926
|
+
for sub in sub_rows:
|
|
1927
|
+
session.execute(
|
|
1928
|
+
SimpleStatement(
|
|
1929
|
+
"DELETE FROM stream_subscriptions WHERE stream_id = %s AND step_run_id = %s",
|
|
1930
|
+
consistency_level=self.write_consistency,
|
|
1931
|
+
),
|
|
1932
|
+
(sub.stream_id, sub.step_run_id),
|
|
1933
|
+
)
|
|
1934
|
+
|
|
1935
|
+
# Purge soft-deleted schedules
|
|
1936
|
+
del_sched_rows = session.execute(
|
|
1937
|
+
SimpleStatement(
|
|
1938
|
+
"SELECT schedule_id FROM schedules_by_status WHERE status = 'DELETED'",
|
|
1939
|
+
consistency_level=self.read_consistency,
|
|
1940
|
+
),
|
|
1941
|
+
)
|
|
1942
|
+
for ds in del_sched_rows:
|
|
1943
|
+
sched_row = session.execute(
|
|
1944
|
+
SimpleStatement(
|
|
1945
|
+
"SELECT updated_at FROM schedules WHERE schedule_id = %s",
|
|
1946
|
+
consistency_level=self.read_consistency,
|
|
1947
|
+
),
|
|
1948
|
+
(ds.schedule_id,),
|
|
1949
|
+
).one()
|
|
1950
|
+
if sched_row and sched_row.updated_at < older_than:
|
|
1951
|
+
session.execute(
|
|
1952
|
+
SimpleStatement(
|
|
1953
|
+
"DELETE FROM schedules WHERE schedule_id = %s",
|
|
1954
|
+
consistency_level=self.write_consistency,
|
|
1955
|
+
),
|
|
1956
|
+
(ds.schedule_id,),
|
|
1957
|
+
)
|
|
1958
|
+
session.execute(
|
|
1959
|
+
SimpleStatement(
|
|
1960
|
+
"DELETE FROM schedules_by_status "
|
|
1961
|
+
"WHERE status = 'DELETED' AND schedule_id = %s",
|
|
1962
|
+
consistency_level=self.write_consistency,
|
|
1963
|
+
),
|
|
1964
|
+
(ds.schedule_id,),
|
|
1965
|
+
)
|
|
1966
|
+
|
|
1879
1967
|
return count
|
|
1880
1968
|
|
|
1881
1969
|
# Helper methods for converting Cassandra rows to domain objects
|
|
@@ -189,8 +189,7 @@ class CitusMigrationRunner(PostgresMigrationRunner):
|
|
|
189
189
|
# V7: Add result column. Top-level ALTER for Citus propagation (see V5 note).
|
|
190
190
|
logger.info("CITUS_MIGRATION v7: ALTER stream_subscriptions ADD result")
|
|
191
191
|
await conn.execute(
|
|
192
|
-
"ALTER TABLE stream_subscriptions "
|
|
193
|
-
"ADD COLUMN IF NOT EXISTS result JSONB NULL"
|
|
192
|
+
"ALTER TABLE stream_subscriptions ADD COLUMN IF NOT EXISTS result JSONB NULL"
|
|
194
193
|
)
|
|
195
194
|
elif migration.up_func:
|
|
196
195
|
await migration.up_func(conn)
|
|
@@ -25,6 +25,55 @@ from pyworkflow.storage.base import StorageBackend
|
|
|
25
25
|
_storage_cache: dict[str, tuple[StorageBackend, None]] = {}
|
|
26
26
|
|
|
27
27
|
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
# Schema-ensured signal
|
|
30
|
+
# ---------------------------------------------------------------------------
|
|
31
|
+
#
|
|
32
|
+
# Process-global flag set by the Celery worker startup hook
|
|
33
|
+
# (``pyworkflow.celery.app._run_startup_migrations``) after it has successfully
|
|
34
|
+
# run pending migrations. When True, runtime ``connect()`` calls SKIP the
|
|
35
|
+
# ``_initialize_schema()`` step entirely — no CREATE TABLE IF NOT EXISTS
|
|
36
|
+
# sweep, no ``run_migrations()`` probe on the ``schema_versions`` table.
|
|
37
|
+
#
|
|
38
|
+
# Why a module-level flag rather than instance state or an env var:
|
|
39
|
+
# * Forked worker children inherit the flag via copy-on-write, so the
|
|
40
|
+
# parent process's "I already migrated" decision naturally propagates.
|
|
41
|
+
# * It's not an env var so child processes that shell out or subprocess
|
|
42
|
+
# don't accidentally carry the claim into an unrelated context.
|
|
43
|
+
# * Per-instance state would be reset on every loop change (pool recreate),
|
|
44
|
+
# so the flag must outlive backend instances.
|
|
45
|
+
#
|
|
46
|
+
# Paths that do NOT set the flag still get lazy initialization:
|
|
47
|
+
# * Tests that instantiate backends directly
|
|
48
|
+
# * CLI commands
|
|
49
|
+
# * Celery workers started with ``PYWORKFLOW_SKIP_STARTUP_MIGRATIONS=1``
|
|
50
|
+
# * Any caller that bypasses the startup hook
|
|
51
|
+
_schema_ensured: bool = False
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def mark_schema_ensured() -> None:
|
|
55
|
+
"""Declare that migrations are up-to-date in this process.
|
|
56
|
+
|
|
57
|
+
Called by the startup migration hook once migrations have been applied
|
|
58
|
+
successfully. After this, ``connect()`` on any Postgres/Citus backend
|
|
59
|
+
in this process (and in its fork children) will skip
|
|
60
|
+
``_initialize_schema()``.
|
|
61
|
+
"""
|
|
62
|
+
global _schema_ensured
|
|
63
|
+
_schema_ensured = True
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def is_schema_ensured() -> bool:
|
|
67
|
+
"""Return True if the schema has already been ensured in this process."""
|
|
68
|
+
return _schema_ensured
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def reset_schema_ensured() -> None:
|
|
72
|
+
"""Reset the flag. Intended for tests only."""
|
|
73
|
+
global _schema_ensured
|
|
74
|
+
_schema_ensured = False
|
|
75
|
+
|
|
76
|
+
|
|
28
77
|
def _config_to_cache_key(config: dict[str, Any] | None) -> str:
|
|
29
78
|
"""
|
|
30
79
|
Create a cache key from config dict.
|
|
@@ -1426,6 +1426,37 @@ class DynamoDBStorageBackend(StorageBackend):
|
|
|
1426
1426
|
|
|
1427
1427
|
count += 1
|
|
1428
1428
|
|
|
1429
|
+
# --- Schedule cleanup: purge soft-deleted schedules ---
|
|
1430
|
+
sched_params: dict[str, Any] = {
|
|
1431
|
+
"TableName": self.table_name,
|
|
1432
|
+
"IndexName": "GSI1",
|
|
1433
|
+
"KeyConditionExpression": "GSI1PK = :pk",
|
|
1434
|
+
"FilterExpression": "#st = :deleted AND updated_at < :cutoff",
|
|
1435
|
+
"ExpressionAttributeNames": {"#st": "status"},
|
|
1436
|
+
"ExpressionAttributeValues": {
|
|
1437
|
+
":pk": {"S": "SCHEDULES"},
|
|
1438
|
+
":deleted": {"S": "DELETED"},
|
|
1439
|
+
":cutoff": {"S": cutoff_iso},
|
|
1440
|
+
},
|
|
1441
|
+
"ProjectionExpression": "PK, SK",
|
|
1442
|
+
}
|
|
1443
|
+
while True:
|
|
1444
|
+
resp = await client.query(**sched_params)
|
|
1445
|
+
items_to_delete = resp.get("Items", [])
|
|
1446
|
+
for i in range(0, len(items_to_delete), 25):
|
|
1447
|
+
batch = items_to_delete[i : i + 25]
|
|
1448
|
+
request_items = {
|
|
1449
|
+
self.table_name: [
|
|
1450
|
+
{"DeleteRequest": {"Key": {"PK": it["PK"], "SK": it["SK"]}}}
|
|
1451
|
+
for it in batch
|
|
1452
|
+
]
|
|
1453
|
+
}
|
|
1454
|
+
await client.batch_write_item(RequestItems=request_items)
|
|
1455
|
+
lk = resp.get("LastEvaluatedKey")
|
|
1456
|
+
if not lk:
|
|
1457
|
+
break
|
|
1458
|
+
sched_params["ExclusiveStartKey"] = lk
|
|
1459
|
+
|
|
1429
1460
|
return count
|
|
1430
1461
|
|
|
1431
1462
|
# Helper methods for converting DynamoDB items to domain objects
|
|
@@ -973,6 +973,7 @@ class FileStorageBackend(StorageBackend):
|
|
|
973
973
|
async def delete_old_runs(self, older_than: datetime) -> int:
|
|
974
974
|
"""Delete terminal runs (and all related files) updated before older_than."""
|
|
975
975
|
terminal = {"completed", "failed", "cancelled", "continued_as_new", "interrupted"}
|
|
976
|
+
cutoff_iso = older_than.isoformat()
|
|
976
977
|
|
|
977
978
|
def _sync_delete() -> int:
|
|
978
979
|
count = 0
|
|
@@ -994,6 +995,10 @@ class FileStorageBackend(StorageBackend):
|
|
|
994
995
|
try:
|
|
995
996
|
sd = json.loads(sf.read_text())
|
|
996
997
|
if sd.get("run_id") == run_id:
|
|
998
|
+
# Also delete checkpoint for this step
|
|
999
|
+
(self.checkpoints_dir / f"{sd['step_id']}.json").unlink(
|
|
1000
|
+
missing_ok=True
|
|
1001
|
+
)
|
|
997
1002
|
sf.unlink(missing_ok=True)
|
|
998
1003
|
except Exception:
|
|
999
1004
|
pass
|
|
@@ -1003,6 +1008,48 @@ class FileStorageBackend(StorageBackend):
|
|
|
1003
1008
|
count += 1
|
|
1004
1009
|
except Exception:
|
|
1005
1010
|
continue
|
|
1011
|
+
|
|
1012
|
+
# --- Stream / schedule cleanup ---
|
|
1013
|
+
# Prune old signals from each stream's signal file
|
|
1014
|
+
for sig_file in list(self.signals_dir.glob("*.json")):
|
|
1015
|
+
try:
|
|
1016
|
+
signals = json.loads(sig_file.read_text())
|
|
1017
|
+
kept = [s for s in signals if s.get("published_at", "") >= cutoff_iso]
|
|
1018
|
+
if len(kept) < len(signals):
|
|
1019
|
+
sig_file.write_text(json.dumps(kept, indent=2))
|
|
1020
|
+
except Exception:
|
|
1021
|
+
pass
|
|
1022
|
+
|
|
1023
|
+
# Delete old terminal subscriptions
|
|
1024
|
+
for sub_file in list(self.subscriptions_dir.glob("*.json")):
|
|
1025
|
+
try:
|
|
1026
|
+
sub = json.loads(sub_file.read_text())
|
|
1027
|
+
if sub.get("status") in ("terminated", "completed"):
|
|
1028
|
+
if sub.get("created_at", "") < cutoff_iso:
|
|
1029
|
+
sub_file.unlink(missing_ok=True)
|
|
1030
|
+
except Exception:
|
|
1031
|
+
pass
|
|
1032
|
+
|
|
1033
|
+
# Delete old acknowledgments
|
|
1034
|
+
for ack_file in list(self.acknowledgments_dir.glob("*.json")):
|
|
1035
|
+
try:
|
|
1036
|
+
ack = json.loads(ack_file.read_text())
|
|
1037
|
+
if ack.get("acknowledged_at", "") < cutoff_iso:
|
|
1038
|
+
ack_file.unlink(missing_ok=True)
|
|
1039
|
+
except Exception:
|
|
1040
|
+
pass
|
|
1041
|
+
|
|
1042
|
+
# Purge soft-deleted schedules
|
|
1043
|
+
for sched_file in list(self.schedules_dir.glob("*.json")):
|
|
1044
|
+
try:
|
|
1045
|
+
sched = json.loads(sched_file.read_text())
|
|
1046
|
+
if sched.get("status") == "DELETED":
|
|
1047
|
+
deleted_at = sched.get("deleted_at") or sched.get("updated_at", "")
|
|
1048
|
+
if deleted_at and deleted_at < cutoff_iso:
|
|
1049
|
+
sched_file.unlink(missing_ok=True)
|
|
1050
|
+
except Exception:
|
|
1051
|
+
pass
|
|
1052
|
+
|
|
1006
1053
|
return count
|
|
1007
1054
|
|
|
1008
1055
|
return await asyncio.to_thread(_sync_delete)
|
|
@@ -615,6 +615,7 @@ class InMemoryStorageBackend(StorageBackend):
|
|
|
615
615
|
RunStatus.CONTINUED_AS_NEW,
|
|
616
616
|
RunStatus.INTERRUPTED,
|
|
617
617
|
}
|
|
618
|
+
cutoff_iso = older_than.isoformat()
|
|
618
619
|
with self._lock:
|
|
619
620
|
to_delete = [
|
|
620
621
|
run_id
|
|
@@ -629,12 +630,61 @@ class InMemoryStorageBackend(StorageBackend):
|
|
|
629
630
|
self._idempotency_index.pop(run.idempotency_key, None)
|
|
630
631
|
step_ids = [sid for sid, s in self._steps.items() if s.run_id == run_id]
|
|
631
632
|
for sid in step_ids:
|
|
633
|
+
self._checkpoints.pop(sid, None)
|
|
632
634
|
del self._steps[sid]
|
|
633
635
|
hook_keys = [k for k in self._hooks if k[0] == run_id]
|
|
634
636
|
for k in hook_keys:
|
|
635
637
|
hook = self._hooks.pop(k)
|
|
636
638
|
self._token_index.pop(hook.token, None)
|
|
637
639
|
self._cancellation_flags.pop(run_id, None)
|
|
640
|
+
|
|
641
|
+
# --- Stream / schedule cleanup ---
|
|
642
|
+
# Purge old signals
|
|
643
|
+
for stream_id in list(self._signals):
|
|
644
|
+
self._signals[stream_id] = [
|
|
645
|
+
s for s in self._signals[stream_id] if s.get("published_at", "") >= cutoff_iso
|
|
646
|
+
]
|
|
647
|
+
|
|
648
|
+
# Purge delivered scheduled signals
|
|
649
|
+
if hasattr(self, "_scheduled_signals"):
|
|
650
|
+
to_remove = [
|
|
651
|
+
sid for sid, row in self._scheduled_signals.items() if row["delivered"]
|
|
652
|
+
]
|
|
653
|
+
for sid in to_remove:
|
|
654
|
+
del self._scheduled_signals[sid]
|
|
655
|
+
|
|
656
|
+
# Purge terminal subscriptions
|
|
657
|
+
sub_to_remove = [
|
|
658
|
+
key
|
|
659
|
+
for key, sub in self._subscriptions.items()
|
|
660
|
+
if sub["status"] in ("terminated", "completed")
|
|
661
|
+
and sub.get("created_at", "") < cutoff_iso
|
|
662
|
+
]
|
|
663
|
+
for key in sub_to_remove:
|
|
664
|
+
del self._subscriptions[key]
|
|
665
|
+
|
|
666
|
+
# Purge acknowledgments for signals that no longer exist
|
|
667
|
+
existing_signal_ids = set()
|
|
668
|
+
for sigs in self._signals.values():
|
|
669
|
+
for s in sigs:
|
|
670
|
+
existing_signal_ids.add(s["signal_id"])
|
|
671
|
+
self._acknowledgments = {
|
|
672
|
+
(sig_id, step_id)
|
|
673
|
+
for sig_id, step_id in self._acknowledgments
|
|
674
|
+
if sig_id in existing_signal_ids
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
# Purge soft-deleted schedules
|
|
678
|
+
sched_to_remove = [
|
|
679
|
+
sid
|
|
680
|
+
for sid, sched in self._schedules.items()
|
|
681
|
+
if sched.status == ScheduleStatus.DELETED
|
|
682
|
+
and sched.updated_at is not None
|
|
683
|
+
and sched.updated_at < older_than
|
|
684
|
+
]
|
|
685
|
+
for sid in sched_to_remove:
|
|
686
|
+
del self._schedules[sid]
|
|
687
|
+
|
|
638
688
|
return len(to_delete)
|
|
639
689
|
|
|
640
690
|
# Stream Operations
|
|
@@ -1422,11 +1422,34 @@ class MySQLStorageBackend(StorageBackend):
|
|
|
1422
1422
|
async with pool.acquire() as conn:
|
|
1423
1423
|
async with conn.cursor() as cur:
|
|
1424
1424
|
await cur.execute(f"DELETE FROM events WHERE run_id IN ({subq})", params)
|
|
1425
|
+
step_subq = f"SELECT step_id FROM steps WHERE run_id IN ({subq})"
|
|
1426
|
+
await cur.execute(
|
|
1427
|
+
f"DELETE FROM checkpoints WHERE step_run_id IN ({step_subq})", params
|
|
1428
|
+
)
|
|
1425
1429
|
await cur.execute(f"DELETE FROM steps WHERE run_id IN ({subq})", params)
|
|
1426
1430
|
await cur.execute(f"DELETE FROM hooks WHERE run_id IN ({subq})", params)
|
|
1427
1431
|
await cur.execute(
|
|
1428
1432
|
f"DELETE FROM cancellation_flags WHERE run_id IN ({subq})", params
|
|
1429
1433
|
)
|
|
1434
|
+
|
|
1435
|
+
# --- Stream / schedule cleanup ---
|
|
1436
|
+
await cur.execute(
|
|
1437
|
+
"DELETE FROM signals WHERE published_at < %s",
|
|
1438
|
+
(older_than,),
|
|
1439
|
+
)
|
|
1440
|
+
await cur.execute(
|
|
1441
|
+
"DELETE FROM signal_acknowledgments WHERE acknowledged_at < %s",
|
|
1442
|
+
(older_than,),
|
|
1443
|
+
)
|
|
1444
|
+
await cur.execute(
|
|
1445
|
+
"DELETE FROM stream_subscriptions WHERE status IN (%s,%s) AND created_at < %s",
|
|
1446
|
+
("terminated", "completed", older_than),
|
|
1447
|
+
)
|
|
1448
|
+
await cur.execute(
|
|
1449
|
+
"DELETE FROM schedules WHERE status = 'DELETED' AND deleted_at < %s",
|
|
1450
|
+
(older_than,),
|
|
1451
|
+
)
|
|
1452
|
+
|
|
1430
1453
|
await cur.execute(
|
|
1431
1454
|
f"DELETE FROM workflow_runs WHERE status IN ({placeholders}) AND updated_at < %s",
|
|
1432
1455
|
params,
|
|
@@ -229,8 +229,7 @@ class PostgresMigrationRunner(MigrationRunner):
|
|
|
229
229
|
# V7: result column on stream_subscriptions for step output payloads.
|
|
230
230
|
logger.info("PG_MIGRATION v7: ALTER stream_subscriptions ADD result")
|
|
231
231
|
await conn.execute(
|
|
232
|
-
"ALTER TABLE stream_subscriptions "
|
|
233
|
-
"ADD COLUMN IF NOT EXISTS result JSONB NULL"
|
|
232
|
+
"ALTER TABLE stream_subscriptions ADD COLUMN IF NOT EXISTS result JSONB NULL"
|
|
234
233
|
)
|
|
235
234
|
elif migration.up_func:
|
|
236
235
|
await migration.up_func(conn)
|
|
@@ -358,7 +357,19 @@ class PostgresStorageBackend(StorageBackend):
|
|
|
358
357
|
self._pool_loop_id = current_loop_id
|
|
359
358
|
|
|
360
359
|
if not self._initialized:
|
|
361
|
-
|
|
360
|
+
# When the Celery worker startup hook has already run migrations
|
|
361
|
+
# for this process, skip _initialize_schema() entirely. The hook
|
|
362
|
+
# sets a module-level flag (see pyworkflow.storage.config) that
|
|
363
|
+
# survives fork via copy-on-write, so every runtime connect in
|
|
364
|
+
# this process and its children is a pure pool-create.
|
|
365
|
+
#
|
|
366
|
+
# Paths that do NOT go through the startup hook (tests, CLI,
|
|
367
|
+
# PYWORKFLOW_SKIP_STARTUP_MIGRATIONS=1) still initialize lazily
|
|
368
|
+
# here, preserving backwards compatibility.
|
|
369
|
+
from pyworkflow.storage.config import is_schema_ensured
|
|
370
|
+
|
|
371
|
+
if not is_schema_ensured():
|
|
372
|
+
await self._initialize_schema()
|
|
362
373
|
self._initialized = True
|
|
363
374
|
|
|
364
375
|
async def disconnect(self) -> None:
|
|
@@ -1651,8 +1662,14 @@ class PostgresStorageBackend(StorageBackend):
|
|
|
1651
1662
|
await self.update_schedule(schedule)
|
|
1652
1663
|
|
|
1653
1664
|
async def delete_old_runs(self, older_than: datetime) -> int:
|
|
1654
|
-
"""Delete terminal runs (and all related data) updated before older_than.
|
|
1665
|
+
"""Delete terminal runs (and all related data) updated before older_than.
|
|
1666
|
+
|
|
1667
|
+
Also purges orphaned stream data (signals, acknowledgments, delivered
|
|
1668
|
+
scheduled signals, terminal subscriptions) and soft-deleted schedules
|
|
1669
|
+
older than the cutoff.
|
|
1670
|
+
"""
|
|
1655
1671
|
terminal = ["completed", "failed", "cancelled", "continued_as_new", "interrupted"]
|
|
1672
|
+
sub_terminal = ["terminated", "completed"]
|
|
1656
1673
|
pool = await self._get_pool()
|
|
1657
1674
|
async with pool.acquire() as conn, conn.transaction():
|
|
1658
1675
|
subq = "SELECT run_id FROM workflow_runs WHERE status = ANY($1) AND updated_at < $2"
|
|
@@ -1670,6 +1687,30 @@ class PostgresStorageBackend(StorageBackend):
|
|
|
1670
1687
|
terminal,
|
|
1671
1688
|
older_than,
|
|
1672
1689
|
)
|
|
1690
|
+
|
|
1691
|
+
# --- Stream / schedule cleanup ---
|
|
1692
|
+
await conn.execute(
|
|
1693
|
+
"DELETE FROM scheduled_signals WHERE delivered = TRUE AND created_at < $1",
|
|
1694
|
+
older_than,
|
|
1695
|
+
)
|
|
1696
|
+
await conn.execute(
|
|
1697
|
+
"DELETE FROM signals WHERE published_at < $1",
|
|
1698
|
+
older_than,
|
|
1699
|
+
)
|
|
1700
|
+
await conn.execute(
|
|
1701
|
+
"DELETE FROM signal_acknowledgments WHERE acknowledged_at < $1",
|
|
1702
|
+
older_than,
|
|
1703
|
+
)
|
|
1704
|
+
await conn.execute(
|
|
1705
|
+
"DELETE FROM stream_subscriptions WHERE status = ANY($1) AND created_at < $2",
|
|
1706
|
+
sub_terminal,
|
|
1707
|
+
older_than,
|
|
1708
|
+
)
|
|
1709
|
+
await conn.execute(
|
|
1710
|
+
"DELETE FROM schedules WHERE status = 'DELETED' AND deleted_at < $1",
|
|
1711
|
+
older_than,
|
|
1712
|
+
)
|
|
1713
|
+
|
|
1673
1714
|
result = await conn.execute(
|
|
1674
1715
|
"DELETE FROM workflow_runs WHERE status = ANY($1) AND updated_at < $2",
|
|
1675
1716
|
terminal,
|
|
@@ -1399,12 +1399,32 @@ class SQLiteStorageBackend(StorageBackend):
|
|
|
1399
1399
|
f"SELECT run_id FROM workflow_runs WHERE status IN ({placeholders}) AND updated_at < ?"
|
|
1400
1400
|
)
|
|
1401
1401
|
params = (*terminal, older_than.isoformat())
|
|
1402
|
+
cutoff_iso = older_than.isoformat()
|
|
1402
1403
|
async with db.cursor() as cur:
|
|
1403
1404
|
await cur.execute(f"DELETE FROM events WHERE run_id IN ({subq})", params)
|
|
1404
1405
|
await cur.execute(f"DELETE FROM steps WHERE run_id IN ({subq})", params)
|
|
1405
1406
|
await cur.execute(f"DELETE FROM hooks WHERE run_id IN ({subq})", params)
|
|
1406
1407
|
await cur.execute(f"DELETE FROM cancellation_flags WHERE run_id IN ({subq})", params)
|
|
1407
1408
|
await cur.execute(f"DELETE FROM checkpoints WHERE step_run_id IN ({subq})", params)
|
|
1409
|
+
|
|
1410
|
+
# --- Stream / schedule cleanup ---
|
|
1411
|
+
await cur.execute(
|
|
1412
|
+
"DELETE FROM signals WHERE published_at < ?",
|
|
1413
|
+
(cutoff_iso,),
|
|
1414
|
+
)
|
|
1415
|
+
await cur.execute(
|
|
1416
|
+
"DELETE FROM signal_acknowledgments WHERE acknowledged_at < ?",
|
|
1417
|
+
(cutoff_iso,),
|
|
1418
|
+
)
|
|
1419
|
+
await cur.execute(
|
|
1420
|
+
"DELETE FROM stream_subscriptions WHERE status IN (?,?) AND created_at < ?",
|
|
1421
|
+
("terminated", "completed", cutoff_iso),
|
|
1422
|
+
)
|
|
1423
|
+
await cur.execute(
|
|
1424
|
+
"DELETE FROM schedules WHERE status = 'DELETED' AND deleted_at < ?",
|
|
1425
|
+
(cutoff_iso,),
|
|
1426
|
+
)
|
|
1427
|
+
|
|
1408
1428
|
await cur.execute(
|
|
1409
1429
|
f"DELETE FROM workflow_runs WHERE status IN ({placeholders}) AND updated_at < ?",
|
|
1410
1430
|
params,
|
|
@@ -28,8 +28,10 @@ from pyworkflow.streams.checkpoint import ( # noqa: F401
|
|
|
28
28
|
)
|
|
29
29
|
from pyworkflow.streams.consumer import StreamConsumer, poll_once # noqa: F401
|
|
30
30
|
from pyworkflow.streams.context import ( # noqa: F401
|
|
31
|
+
acknowledge_inbox_signal,
|
|
31
32
|
get_checkpoint,
|
|
32
33
|
get_current_signal,
|
|
34
|
+
poll_inbox,
|
|
33
35
|
save_checkpoint,
|
|
34
36
|
set_result,
|
|
35
37
|
suspend,
|
|
@@ -73,6 +75,8 @@ __all__ = [
|
|
|
73
75
|
"get_current_signal",
|
|
74
76
|
"get_checkpoint",
|
|
75
77
|
"save_checkpoint",
|
|
78
|
+
"poll_inbox",
|
|
79
|
+
"acknowledge_inbox_signal",
|
|
76
80
|
# Checkpoint
|
|
77
81
|
"CheckpointBackend",
|
|
78
82
|
"DefaultCheckpointBackend",
|