pyworkflow-engine 0.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dashboard/backend/app/__init__.py +1 -0
- dashboard/backend/app/config.py +32 -0
- dashboard/backend/app/controllers/__init__.py +6 -0
- dashboard/backend/app/controllers/run_controller.py +86 -0
- dashboard/backend/app/controllers/workflow_controller.py +33 -0
- dashboard/backend/app/dependencies/__init__.py +5 -0
- dashboard/backend/app/dependencies/storage.py +50 -0
- dashboard/backend/app/repositories/__init__.py +6 -0
- dashboard/backend/app/repositories/run_repository.py +80 -0
- dashboard/backend/app/repositories/workflow_repository.py +27 -0
- dashboard/backend/app/rest/__init__.py +8 -0
- dashboard/backend/app/rest/v1/__init__.py +12 -0
- dashboard/backend/app/rest/v1/health.py +33 -0
- dashboard/backend/app/rest/v1/runs.py +133 -0
- dashboard/backend/app/rest/v1/workflows.py +41 -0
- dashboard/backend/app/schemas/__init__.py +23 -0
- dashboard/backend/app/schemas/common.py +16 -0
- dashboard/backend/app/schemas/event.py +24 -0
- dashboard/backend/app/schemas/hook.py +25 -0
- dashboard/backend/app/schemas/run.py +54 -0
- dashboard/backend/app/schemas/step.py +28 -0
- dashboard/backend/app/schemas/workflow.py +31 -0
- dashboard/backend/app/server.py +87 -0
- dashboard/backend/app/services/__init__.py +6 -0
- dashboard/backend/app/services/run_service.py +240 -0
- dashboard/backend/app/services/workflow_service.py +155 -0
- dashboard/backend/main.py +18 -0
- docs/concepts/cancellation.mdx +362 -0
- docs/concepts/continue-as-new.mdx +434 -0
- docs/concepts/events.mdx +266 -0
- docs/concepts/fault-tolerance.mdx +370 -0
- docs/concepts/hooks.mdx +552 -0
- docs/concepts/limitations.mdx +167 -0
- docs/concepts/schedules.mdx +775 -0
- docs/concepts/sleep.mdx +312 -0
- docs/concepts/steps.mdx +301 -0
- docs/concepts/workflows.mdx +255 -0
- docs/guides/cli.mdx +942 -0
- docs/guides/configuration.mdx +560 -0
- docs/introduction.mdx +155 -0
- docs/quickstart.mdx +279 -0
- examples/__init__.py +1 -0
- examples/celery/__init__.py +1 -0
- examples/celery/durable/docker-compose.yml +55 -0
- examples/celery/durable/pyworkflow.config.yaml +12 -0
- examples/celery/durable/workflows/__init__.py +122 -0
- examples/celery/durable/workflows/basic.py +87 -0
- examples/celery/durable/workflows/batch_processing.py +102 -0
- examples/celery/durable/workflows/cancellation.py +273 -0
- examples/celery/durable/workflows/child_workflow_patterns.py +240 -0
- examples/celery/durable/workflows/child_workflows.py +202 -0
- examples/celery/durable/workflows/continue_as_new.py +260 -0
- examples/celery/durable/workflows/fault_tolerance.py +210 -0
- examples/celery/durable/workflows/hooks.py +211 -0
- examples/celery/durable/workflows/idempotency.py +112 -0
- examples/celery/durable/workflows/long_running.py +99 -0
- examples/celery/durable/workflows/retries.py +101 -0
- examples/celery/durable/workflows/schedules.py +209 -0
- examples/celery/transient/01_basic_workflow.py +91 -0
- examples/celery/transient/02_fault_tolerance.py +257 -0
- examples/celery/transient/__init__.py +20 -0
- examples/celery/transient/pyworkflow.config.yaml +25 -0
- examples/local/__init__.py +1 -0
- examples/local/durable/01_basic_workflow.py +94 -0
- examples/local/durable/02_file_storage.py +132 -0
- examples/local/durable/03_retries.py +169 -0
- examples/local/durable/04_long_running.py +119 -0
- examples/local/durable/05_event_log.py +145 -0
- examples/local/durable/06_idempotency.py +148 -0
- examples/local/durable/07_hooks.py +334 -0
- examples/local/durable/08_cancellation.py +233 -0
- examples/local/durable/09_child_workflows.py +198 -0
- examples/local/durable/10_child_workflow_patterns.py +265 -0
- examples/local/durable/11_continue_as_new.py +249 -0
- examples/local/durable/12_schedules.py +198 -0
- examples/local/durable/__init__.py +1 -0
- examples/local/transient/01_quick_tasks.py +87 -0
- examples/local/transient/02_retries.py +130 -0
- examples/local/transient/03_sleep.py +141 -0
- examples/local/transient/__init__.py +1 -0
- pyworkflow/__init__.py +256 -0
- pyworkflow/aws/__init__.py +68 -0
- pyworkflow/aws/context.py +234 -0
- pyworkflow/aws/handler.py +184 -0
- pyworkflow/aws/testing.py +310 -0
- pyworkflow/celery/__init__.py +41 -0
- pyworkflow/celery/app.py +198 -0
- pyworkflow/celery/scheduler.py +315 -0
- pyworkflow/celery/tasks.py +1746 -0
- pyworkflow/cli/__init__.py +132 -0
- pyworkflow/cli/__main__.py +6 -0
- pyworkflow/cli/commands/__init__.py +1 -0
- pyworkflow/cli/commands/hooks.py +640 -0
- pyworkflow/cli/commands/quickstart.py +495 -0
- pyworkflow/cli/commands/runs.py +773 -0
- pyworkflow/cli/commands/scheduler.py +130 -0
- pyworkflow/cli/commands/schedules.py +794 -0
- pyworkflow/cli/commands/setup.py +703 -0
- pyworkflow/cli/commands/worker.py +413 -0
- pyworkflow/cli/commands/workflows.py +1257 -0
- pyworkflow/cli/output/__init__.py +1 -0
- pyworkflow/cli/output/formatters.py +321 -0
- pyworkflow/cli/output/styles.py +121 -0
- pyworkflow/cli/utils/__init__.py +1 -0
- pyworkflow/cli/utils/async_helpers.py +30 -0
- pyworkflow/cli/utils/config.py +130 -0
- pyworkflow/cli/utils/config_generator.py +344 -0
- pyworkflow/cli/utils/discovery.py +53 -0
- pyworkflow/cli/utils/docker_manager.py +651 -0
- pyworkflow/cli/utils/interactive.py +364 -0
- pyworkflow/cli/utils/storage.py +115 -0
- pyworkflow/config.py +329 -0
- pyworkflow/context/__init__.py +63 -0
- pyworkflow/context/aws.py +230 -0
- pyworkflow/context/base.py +416 -0
- pyworkflow/context/local.py +930 -0
- pyworkflow/context/mock.py +381 -0
- pyworkflow/core/__init__.py +0 -0
- pyworkflow/core/exceptions.py +353 -0
- pyworkflow/core/registry.py +313 -0
- pyworkflow/core/scheduled.py +328 -0
- pyworkflow/core/step.py +494 -0
- pyworkflow/core/workflow.py +294 -0
- pyworkflow/discovery.py +248 -0
- pyworkflow/engine/__init__.py +0 -0
- pyworkflow/engine/events.py +879 -0
- pyworkflow/engine/executor.py +682 -0
- pyworkflow/engine/replay.py +273 -0
- pyworkflow/observability/__init__.py +19 -0
- pyworkflow/observability/logging.py +234 -0
- pyworkflow/primitives/__init__.py +33 -0
- pyworkflow/primitives/child_handle.py +174 -0
- pyworkflow/primitives/child_workflow.py +372 -0
- pyworkflow/primitives/continue_as_new.py +101 -0
- pyworkflow/primitives/define_hook.py +150 -0
- pyworkflow/primitives/hooks.py +97 -0
- pyworkflow/primitives/resume_hook.py +210 -0
- pyworkflow/primitives/schedule.py +545 -0
- pyworkflow/primitives/shield.py +96 -0
- pyworkflow/primitives/sleep.py +100 -0
- pyworkflow/runtime/__init__.py +21 -0
- pyworkflow/runtime/base.py +179 -0
- pyworkflow/runtime/celery.py +310 -0
- pyworkflow/runtime/factory.py +101 -0
- pyworkflow/runtime/local.py +706 -0
- pyworkflow/scheduler/__init__.py +9 -0
- pyworkflow/scheduler/local.py +248 -0
- pyworkflow/serialization/__init__.py +0 -0
- pyworkflow/serialization/decoder.py +146 -0
- pyworkflow/serialization/encoder.py +162 -0
- pyworkflow/storage/__init__.py +54 -0
- pyworkflow/storage/base.py +612 -0
- pyworkflow/storage/config.py +185 -0
- pyworkflow/storage/dynamodb.py +1315 -0
- pyworkflow/storage/file.py +827 -0
- pyworkflow/storage/memory.py +549 -0
- pyworkflow/storage/postgres.py +1161 -0
- pyworkflow/storage/schemas.py +486 -0
- pyworkflow/storage/sqlite.py +1136 -0
- pyworkflow/utils/__init__.py +0 -0
- pyworkflow/utils/duration.py +177 -0
- pyworkflow/utils/schedule.py +391 -0
- pyworkflow_engine-0.1.7.dist-info/METADATA +687 -0
- pyworkflow_engine-0.1.7.dist-info/RECORD +196 -0
- pyworkflow_engine-0.1.7.dist-info/WHEEL +5 -0
- pyworkflow_engine-0.1.7.dist-info/entry_points.txt +2 -0
- pyworkflow_engine-0.1.7.dist-info/licenses/LICENSE +21 -0
- pyworkflow_engine-0.1.7.dist-info/top_level.txt +5 -0
- tests/examples/__init__.py +0 -0
- tests/integration/__init__.py +0 -0
- tests/integration/test_cancellation.py +330 -0
- tests/integration/test_child_workflows.py +439 -0
- tests/integration/test_continue_as_new.py +428 -0
- tests/integration/test_dynamodb_storage.py +1146 -0
- tests/integration/test_fault_tolerance.py +369 -0
- tests/integration/test_schedule_storage.py +484 -0
- tests/unit/__init__.py +0 -0
- tests/unit/backends/__init__.py +1 -0
- tests/unit/backends/test_dynamodb_storage.py +1554 -0
- tests/unit/backends/test_postgres_storage.py +1281 -0
- tests/unit/backends/test_sqlite_storage.py +1460 -0
- tests/unit/conftest.py +41 -0
- tests/unit/test_cancellation.py +364 -0
- tests/unit/test_child_workflows.py +680 -0
- tests/unit/test_continue_as_new.py +441 -0
- tests/unit/test_event_limits.py +316 -0
- tests/unit/test_executor.py +320 -0
- tests/unit/test_fault_tolerance.py +334 -0
- tests/unit/test_hooks.py +495 -0
- tests/unit/test_registry.py +261 -0
- tests/unit/test_replay.py +420 -0
- tests/unit/test_schedule_schemas.py +285 -0
- tests/unit/test_schedule_utils.py +286 -0
- tests/unit/test_scheduled_workflow.py +274 -0
- tests/unit/test_step.py +353 -0
- tests/unit/test_workflow.py +243 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
"""CLI commands for running the local scheduler."""
|
|
2
|
+
|
|
3
|
+
import signal
|
|
4
|
+
import sys
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
|
|
8
|
+
from pyworkflow.cli.utils.async_helpers import async_command
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@click.group()
|
|
12
|
+
def scheduler():
|
|
13
|
+
"""
|
|
14
|
+
Run the schedule executor.
|
|
15
|
+
|
|
16
|
+
The scheduler polls storage for due schedules and triggers workflows.
|
|
17
|
+
Use this command when running with local runtime (no Celery).
|
|
18
|
+
|
|
19
|
+
For Celery runtime, use 'celery beat' instead:
|
|
20
|
+
|
|
21
|
+
celery -A pyworkflow.celery.app beat \\
|
|
22
|
+
--scheduler pyworkflow.celery.scheduler:PyWorkflowScheduler
|
|
23
|
+
"""
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@scheduler.command("run")
|
|
28
|
+
@click.option(
|
|
29
|
+
"--poll-interval",
|
|
30
|
+
type=float,
|
|
31
|
+
default=5.0,
|
|
32
|
+
help="Seconds between storage polls (default: 5.0)",
|
|
33
|
+
)
|
|
34
|
+
@click.option(
|
|
35
|
+
"--duration",
|
|
36
|
+
type=float,
|
|
37
|
+
default=None,
|
|
38
|
+
help="Run for specified seconds then exit (default: run forever)",
|
|
39
|
+
)
|
|
40
|
+
@click.pass_context
|
|
41
|
+
@async_command
|
|
42
|
+
async def run_scheduler(ctx: click.Context, poll_interval: float, duration: float | None) -> None:
|
|
43
|
+
"""
|
|
44
|
+
Run the local scheduler.
|
|
45
|
+
|
|
46
|
+
This polls storage for due schedules and triggers them using
|
|
47
|
+
the configured runtime. Use this when running with local runtime.
|
|
48
|
+
|
|
49
|
+
For Celery runtime, use 'celery beat' instead.
|
|
50
|
+
|
|
51
|
+
Examples:
|
|
52
|
+
|
|
53
|
+
# Run scheduler with default settings
|
|
54
|
+
pyworkflow scheduler run
|
|
55
|
+
|
|
56
|
+
# Custom poll interval
|
|
57
|
+
pyworkflow scheduler run --poll-interval 10
|
|
58
|
+
|
|
59
|
+
# Run for 60 seconds (useful for testing)
|
|
60
|
+
pyworkflow scheduler run --duration 60
|
|
61
|
+
|
|
62
|
+
# With a specific module
|
|
63
|
+
pyworkflow --module myapp.workflows scheduler run
|
|
64
|
+
"""
|
|
65
|
+
from pyworkflow.cli.utils.discovery import discover_workflows
|
|
66
|
+
from pyworkflow.cli.utils.storage import create_storage
|
|
67
|
+
from pyworkflow.config import configure, reset_config
|
|
68
|
+
from pyworkflow.scheduler import LocalScheduler
|
|
69
|
+
|
|
70
|
+
# Get config from context
|
|
71
|
+
module = ctx.obj.get("module")
|
|
72
|
+
storage_type = ctx.obj.get("storage_type")
|
|
73
|
+
storage_path = ctx.obj.get("storage_path")
|
|
74
|
+
runtime = ctx.obj.get("runtime", "local")
|
|
75
|
+
|
|
76
|
+
# Discover workflows if module specified
|
|
77
|
+
if module:
|
|
78
|
+
click.echo(f"Discovering workflows from: {module}")
|
|
79
|
+
try:
|
|
80
|
+
discover_workflows(module)
|
|
81
|
+
except Exception as e:
|
|
82
|
+
raise click.ClickException(f"Failed to discover workflows: {e}")
|
|
83
|
+
|
|
84
|
+
# Create storage backend
|
|
85
|
+
try:
|
|
86
|
+
storage = create_storage(storage_type, storage_path)
|
|
87
|
+
except Exception as e:
|
|
88
|
+
raise click.ClickException(f"Failed to create storage: {e}")
|
|
89
|
+
|
|
90
|
+
# Configure pyworkflow
|
|
91
|
+
reset_config()
|
|
92
|
+
configure(
|
|
93
|
+
storage=storage,
|
|
94
|
+
default_runtime=runtime,
|
|
95
|
+
default_durable=True,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
# Create and run scheduler
|
|
99
|
+
local_scheduler = LocalScheduler(
|
|
100
|
+
storage=storage,
|
|
101
|
+
poll_interval=poll_interval,
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
click.echo("Starting local scheduler...")
|
|
105
|
+
click.echo(f" Poll interval: {poll_interval}s")
|
|
106
|
+
click.echo(f" Runtime: {runtime}")
|
|
107
|
+
if duration:
|
|
108
|
+
click.echo(f" Duration: {duration}s")
|
|
109
|
+
else:
|
|
110
|
+
click.echo(" Duration: indefinite (Ctrl+C to stop)")
|
|
111
|
+
click.echo()
|
|
112
|
+
|
|
113
|
+
# Handle graceful shutdown
|
|
114
|
+
def signal_handler(signum, frame):
|
|
115
|
+
click.echo("\nReceived shutdown signal, stopping scheduler...")
|
|
116
|
+
local_scheduler.stop()
|
|
117
|
+
|
|
118
|
+
# Register signal handlers
|
|
119
|
+
if sys.platform != "win32":
|
|
120
|
+
signal.signal(signal.SIGTERM, signal_handler)
|
|
121
|
+
signal.signal(signal.SIGINT, signal_handler)
|
|
122
|
+
|
|
123
|
+
try:
|
|
124
|
+
await local_scheduler.run(duration=duration)
|
|
125
|
+
except KeyboardInterrupt:
|
|
126
|
+
click.echo("\nScheduler stopped by user")
|
|
127
|
+
except Exception as e:
|
|
128
|
+
raise click.ClickException(f"Scheduler error: {e}")
|
|
129
|
+
|
|
130
|
+
click.echo("Scheduler stopped")
|