pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/context/__init__.py +13 -0
  9. pyworkflow/context/base.py +26 -0
  10. pyworkflow/context/local.py +80 -0
  11. pyworkflow/context/step_context.py +295 -0
  12. pyworkflow/core/registry.py +6 -1
  13. pyworkflow/core/step.py +141 -0
  14. pyworkflow/core/workflow.py +56 -0
  15. pyworkflow/engine/events.py +30 -0
  16. pyworkflow/engine/replay.py +39 -0
  17. pyworkflow/primitives/child_workflow.py +1 -1
  18. pyworkflow/runtime/local.py +1 -1
  19. pyworkflow/storage/__init__.py +14 -0
  20. pyworkflow/storage/base.py +35 -0
  21. pyworkflow/storage/cassandra.py +1747 -0
  22. pyworkflow/storage/config.py +69 -0
  23. pyworkflow/storage/dynamodb.py +31 -2
  24. pyworkflow/storage/file.py +28 -0
  25. pyworkflow/storage/memory.py +18 -0
  26. pyworkflow/storage/mysql.py +1159 -0
  27. pyworkflow/storage/postgres.py +27 -2
  28. pyworkflow/storage/schemas.py +4 -3
  29. pyworkflow/storage/sqlite.py +25 -2
  30. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/METADATA +7 -4
  31. pyworkflow_engine-0.1.9.dist-info/RECORD +91 -0
  32. pyworkflow_engine-0.1.9.dist-info/top_level.txt +1 -0
  33. dashboard/backend/app/__init__.py +0 -1
  34. dashboard/backend/app/config.py +0 -32
  35. dashboard/backend/app/controllers/__init__.py +0 -6
  36. dashboard/backend/app/controllers/run_controller.py +0 -86
  37. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  38. dashboard/backend/app/dependencies/__init__.py +0 -5
  39. dashboard/backend/app/dependencies/storage.py +0 -50
  40. dashboard/backend/app/repositories/__init__.py +0 -6
  41. dashboard/backend/app/repositories/run_repository.py +0 -80
  42. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  43. dashboard/backend/app/rest/__init__.py +0 -8
  44. dashboard/backend/app/rest/v1/__init__.py +0 -12
  45. dashboard/backend/app/rest/v1/health.py +0 -33
  46. dashboard/backend/app/rest/v1/runs.py +0 -133
  47. dashboard/backend/app/rest/v1/workflows.py +0 -41
  48. dashboard/backend/app/schemas/__init__.py +0 -23
  49. dashboard/backend/app/schemas/common.py +0 -16
  50. dashboard/backend/app/schemas/event.py +0 -24
  51. dashboard/backend/app/schemas/hook.py +0 -25
  52. dashboard/backend/app/schemas/run.py +0 -54
  53. dashboard/backend/app/schemas/step.py +0 -28
  54. dashboard/backend/app/schemas/workflow.py +0 -31
  55. dashboard/backend/app/server.py +0 -87
  56. dashboard/backend/app/services/__init__.py +0 -6
  57. dashboard/backend/app/services/run_service.py +0 -240
  58. dashboard/backend/app/services/workflow_service.py +0 -155
  59. dashboard/backend/main.py +0 -18
  60. docs/concepts/cancellation.mdx +0 -362
  61. docs/concepts/continue-as-new.mdx +0 -434
  62. docs/concepts/events.mdx +0 -266
  63. docs/concepts/fault-tolerance.mdx +0 -370
  64. docs/concepts/hooks.mdx +0 -552
  65. docs/concepts/limitations.mdx +0 -167
  66. docs/concepts/schedules.mdx +0 -775
  67. docs/concepts/sleep.mdx +0 -312
  68. docs/concepts/steps.mdx +0 -301
  69. docs/concepts/workflows.mdx +0 -255
  70. docs/guides/cli.mdx +0 -942
  71. docs/guides/configuration.mdx +0 -560
  72. docs/introduction.mdx +0 -155
  73. docs/quickstart.mdx +0 -279
  74. examples/__init__.py +0 -1
  75. examples/celery/__init__.py +0 -1
  76. examples/celery/durable/docker-compose.yml +0 -55
  77. examples/celery/durable/pyworkflow.config.yaml +0 -12
  78. examples/celery/durable/workflows/__init__.py +0 -122
  79. examples/celery/durable/workflows/basic.py +0 -87
  80. examples/celery/durable/workflows/batch_processing.py +0 -102
  81. examples/celery/durable/workflows/cancellation.py +0 -273
  82. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  83. examples/celery/durable/workflows/child_workflows.py +0 -202
  84. examples/celery/durable/workflows/continue_as_new.py +0 -260
  85. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  86. examples/celery/durable/workflows/hooks.py +0 -211
  87. examples/celery/durable/workflows/idempotency.py +0 -112
  88. examples/celery/durable/workflows/long_running.py +0 -99
  89. examples/celery/durable/workflows/retries.py +0 -101
  90. examples/celery/durable/workflows/schedules.py +0 -209
  91. examples/celery/transient/01_basic_workflow.py +0 -91
  92. examples/celery/transient/02_fault_tolerance.py +0 -257
  93. examples/celery/transient/__init__.py +0 -20
  94. examples/celery/transient/pyworkflow.config.yaml +0 -25
  95. examples/local/__init__.py +0 -1
  96. examples/local/durable/01_basic_workflow.py +0 -94
  97. examples/local/durable/02_file_storage.py +0 -132
  98. examples/local/durable/03_retries.py +0 -169
  99. examples/local/durable/04_long_running.py +0 -119
  100. examples/local/durable/05_event_log.py +0 -145
  101. examples/local/durable/06_idempotency.py +0 -148
  102. examples/local/durable/07_hooks.py +0 -334
  103. examples/local/durable/08_cancellation.py +0 -233
  104. examples/local/durable/09_child_workflows.py +0 -198
  105. examples/local/durable/10_child_workflow_patterns.py +0 -265
  106. examples/local/durable/11_continue_as_new.py +0 -249
  107. examples/local/durable/12_schedules.py +0 -198
  108. examples/local/durable/__init__.py +0 -1
  109. examples/local/transient/01_quick_tasks.py +0 -87
  110. examples/local/transient/02_retries.py +0 -130
  111. examples/local/transient/03_sleep.py +0 -141
  112. examples/local/transient/__init__.py +0 -1
  113. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  114. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  115. tests/examples/__init__.py +0 -0
  116. tests/integration/__init__.py +0 -0
  117. tests/integration/test_cancellation.py +0 -330
  118. tests/integration/test_child_workflows.py +0 -439
  119. tests/integration/test_continue_as_new.py +0 -428
  120. tests/integration/test_dynamodb_storage.py +0 -1146
  121. tests/integration/test_fault_tolerance.py +0 -369
  122. tests/integration/test_schedule_storage.py +0 -484
  123. tests/unit/__init__.py +0 -0
  124. tests/unit/backends/__init__.py +0 -1
  125. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  126. tests/unit/backends/test_postgres_storage.py +0 -1281
  127. tests/unit/backends/test_sqlite_storage.py +0 -1460
  128. tests/unit/conftest.py +0 -41
  129. tests/unit/test_cancellation.py +0 -364
  130. tests/unit/test_child_workflows.py +0 -680
  131. tests/unit/test_continue_as_new.py +0 -441
  132. tests/unit/test_event_limits.py +0 -316
  133. tests/unit/test_executor.py +0 -320
  134. tests/unit/test_fault_tolerance.py +0 -334
  135. tests/unit/test_hooks.py +0 -495
  136. tests/unit/test_registry.py +0 -261
  137. tests/unit/test_replay.py +0 -420
  138. tests/unit/test_schedule_schemas.py +0 -285
  139. tests/unit/test_schedule_utils.py +0 -286
  140. tests/unit/test_scheduled_workflow.py +0 -274
  141. tests/unit/test_step.py +0 -353
  142. tests/unit/test_workflow.py +0 -243
  143. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/WHEEL +0 -0
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/entry_points.txt +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/licenses/LICENSE +0 -0
@@ -1,133 +0,0 @@
1
- """Workflow run endpoints."""
2
-
3
- from datetime import datetime
4
-
5
- from fastapi import APIRouter, Depends, HTTPException, Query
6
-
7
- from app.controllers.run_controller import RunController
8
- from app.dependencies import get_storage
9
- from app.schemas.event import EventListResponse
10
- from app.schemas.run import RunDetailResponse, RunListResponse, StartRunRequest, StartRunResponse
11
- from pyworkflow.storage.base import StorageBackend
12
-
13
- router = APIRouter()
14
-
15
-
16
- @router.get("", response_model=RunListResponse)
17
- async def list_runs(
18
- query: str | None = Query(
19
- None, description="Search in workflow name and input kwargs (case-insensitive)"
20
- ),
21
- status: str | None = Query(
22
- None,
23
- description="Filter by status (pending, running, suspended, completed, failed, interrupted, cancelled)",
24
- ),
25
- start_time: datetime | None = Query(
26
- None, description="Filter runs started at or after this time (ISO 8601)"
27
- ),
28
- end_time: datetime | None = Query(
29
- None, description="Filter runs started before this time (ISO 8601)"
30
- ),
31
- limit: int = Query(100, ge=1, le=1000, description="Maximum results"),
32
- cursor: str | None = Query(None, description="Run ID to start after (for pagination)"),
33
- storage: StorageBackend = Depends(get_storage),
34
- ) -> RunListResponse:
35
- """List workflow runs with optional filtering and cursor-based pagination.
36
-
37
- Args:
38
- query: Case-insensitive search in workflow name and input kwargs.
39
- status: Filter by run status.
40
- start_time: Filter runs started at or after this time.
41
- end_time: Filter runs started before this time.
42
- limit: Maximum number of results (1-1000).
43
- cursor: Run ID to start after (for pagination).
44
- storage: Storage backend (injected).
45
-
46
- Returns:
47
- RunListResponse with matching runs and next_cursor.
48
- """
49
- controller = RunController(storage)
50
- return await controller.list_runs(
51
- query=query,
52
- status=status,
53
- start_time=start_time,
54
- end_time=end_time,
55
- limit=limit,
56
- cursor=cursor,
57
- )
58
-
59
-
60
- @router.post("", response_model=StartRunResponse, status_code=201)
61
- async def start_run(
62
- request: StartRunRequest,
63
- storage: StorageBackend = Depends(get_storage),
64
- ) -> StartRunResponse:
65
- """Start a new workflow run.
66
-
67
- Args:
68
- request: Start run request with workflow name and kwargs.
69
- storage: Storage backend (injected).
70
-
71
- Returns:
72
- StartRunResponse with run_id and workflow_name.
73
-
74
- Raises:
75
- HTTPException: 404 if workflow not found, 400 for validation errors.
76
- """
77
- controller = RunController(storage)
78
- try:
79
- return await controller.start_run(request)
80
- except ValueError as e:
81
- raise HTTPException(status_code=404, detail=str(e))
82
- except Exception as e:
83
- raise HTTPException(status_code=400, detail=str(e))
84
-
85
-
86
- @router.get("/{run_id}", response_model=RunDetailResponse)
87
- async def get_run(
88
- run_id: str,
89
- storage: StorageBackend = Depends(get_storage),
90
- ) -> RunDetailResponse:
91
- """Get detailed information about a workflow run.
92
-
93
- Args:
94
- run_id: The run ID.
95
- storage: Storage backend (injected).
96
-
97
- Returns:
98
- RunDetailResponse with run details.
99
-
100
- Raises:
101
- HTTPException: 404 if run not found.
102
- """
103
- controller = RunController(storage)
104
- run = await controller.get_run(run_id)
105
-
106
- if run is None:
107
- raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
108
-
109
- return run
110
-
111
-
112
- @router.get("/{run_id}/events", response_model=EventListResponse)
113
- async def get_run_events(
114
- run_id: str,
115
- storage: StorageBackend = Depends(get_storage),
116
- ) -> EventListResponse:
117
- """Get all events for a workflow run.
118
-
119
- Args:
120
- run_id: The run ID.
121
- storage: Storage backend (injected).
122
-
123
- Returns:
124
- EventListResponse with run events.
125
- """
126
- controller = RunController(storage)
127
-
128
- # Verify run exists
129
- run = await controller.get_run(run_id)
130
- if run is None:
131
- raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
132
-
133
- return await controller.get_events(run_id)
@@ -1,41 +0,0 @@
1
- """Workflow endpoints."""
2
-
3
- from fastapi import APIRouter, HTTPException
4
-
5
- from app.controllers.workflow_controller import WorkflowController
6
- from app.schemas.workflow import WorkflowListResponse, WorkflowResponse
7
-
8
- router = APIRouter()
9
-
10
-
11
- @router.get("", response_model=WorkflowListResponse)
12
- async def list_workflows() -> WorkflowListResponse:
13
- """List all registered workflows.
14
-
15
- Returns:
16
- WorkflowListResponse with all registered workflows.
17
- """
18
- controller = WorkflowController()
19
- return controller.list_workflows()
20
-
21
-
22
- @router.get("/{name}", response_model=WorkflowResponse)
23
- async def get_workflow(name: str) -> WorkflowResponse:
24
- """Get a specific workflow by name.
25
-
26
- Args:
27
- name: Workflow name.
28
-
29
- Returns:
30
- WorkflowResponse with workflow details.
31
-
32
- Raises:
33
- HTTPException: 404 if workflow not found.
34
- """
35
- controller = WorkflowController()
36
- workflow = controller.get_workflow(name)
37
-
38
- if workflow is None:
39
- raise HTTPException(status_code=404, detail=f"Workflow '{name}' not found")
40
-
41
- return workflow
@@ -1,23 +0,0 @@
1
- """Pydantic schemas for API request/response models."""
2
-
3
- from app.schemas.common import PaginatedResponse
4
- from app.schemas.event import EventListResponse, EventResponse
5
- from app.schemas.hook import HookListResponse, HookResponse
6
- from app.schemas.run import RunDetailResponse, RunListResponse, RunResponse
7
- from app.schemas.step import StepListResponse, StepResponse
8
- from app.schemas.workflow import WorkflowListResponse, WorkflowResponse
9
-
10
- __all__ = [
11
- "PaginatedResponse",
12
- "WorkflowResponse",
13
- "WorkflowListResponse",
14
- "RunResponse",
15
- "RunDetailResponse",
16
- "RunListResponse",
17
- "EventResponse",
18
- "EventListResponse",
19
- "StepResponse",
20
- "StepListResponse",
21
- "HookResponse",
22
- "HookListResponse",
23
- ]
@@ -1,16 +0,0 @@
1
- """Common schema types."""
2
-
3
- from typing import Generic, TypeVar
4
-
5
- from pydantic import BaseModel
6
-
7
- T = TypeVar("T")
8
-
9
-
10
- class PaginatedResponse(BaseModel, Generic[T]):
11
- """Base paginated response model."""
12
-
13
- items: list[T]
14
- count: int
15
- limit: int = 100
16
- offset: int = 0
@@ -1,24 +0,0 @@
1
- """Event response schemas."""
2
-
3
- from datetime import datetime
4
- from typing import Any
5
-
6
- from pydantic import BaseModel
7
-
8
-
9
- class EventResponse(BaseModel):
10
- """Response model for a workflow event."""
11
-
12
- event_id: str
13
- run_id: str
14
- type: str
15
- timestamp: datetime
16
- sequence: int | None = None
17
- data: dict[str, Any] = {}
18
-
19
-
20
- class EventListResponse(BaseModel):
21
- """Response model for listing events."""
22
-
23
- items: list[EventResponse]
24
- count: int
@@ -1,25 +0,0 @@
1
- """Hook response schemas."""
2
-
3
- from datetime import datetime
4
-
5
- from pydantic import BaseModel
6
-
7
-
8
- class HookResponse(BaseModel):
9
- """Response model for a hook."""
10
-
11
- hook_id: str
12
- run_id: str
13
- name: str | None = None
14
- status: str
15
- created_at: datetime
16
- received_at: datetime | None = None
17
- expires_at: datetime | None = None
18
- has_payload: bool = False
19
-
20
-
21
- class HookListResponse(BaseModel):
22
- """Response model for listing hooks."""
23
-
24
- items: list[HookResponse]
25
- count: int
@@ -1,54 +0,0 @@
1
- """Workflow run response schemas."""
2
-
3
- from datetime import datetime
4
- from typing import Any
5
-
6
- from pydantic import BaseModel
7
-
8
-
9
- class RunResponse(BaseModel):
10
- """Response model for a workflow run."""
11
-
12
- run_id: str
13
- workflow_name: str
14
- status: str
15
- created_at: datetime
16
- started_at: datetime | None = None
17
- completed_at: datetime | None = None
18
- duration_seconds: float | None = None
19
- error: str | None = None
20
- recovery_attempts: int = 0
21
-
22
-
23
- class RunDetailResponse(RunResponse):
24
- """Detailed response model for a workflow run."""
25
-
26
- input_args: Any | None = None
27
- input_kwargs: Any | None = None
28
- result: Any | None = None
29
- metadata: dict[str, Any] = {}
30
- max_duration: str | None = None
31
- max_recovery_attempts: int = 3
32
-
33
-
34
- class RunListResponse(BaseModel):
35
- """Response model for listing runs."""
36
-
37
- items: list[RunResponse]
38
- count: int
39
- limit: int = 100
40
- next_cursor: str | None = None
41
-
42
-
43
- class StartRunRequest(BaseModel):
44
- """Request model for starting a new workflow run."""
45
-
46
- workflow_name: str
47
- kwargs: dict[str, Any] = {}
48
-
49
-
50
- class StartRunResponse(BaseModel):
51
- """Response model for a newly started workflow run."""
52
-
53
- run_id: str
54
- workflow_name: str
@@ -1,28 +0,0 @@
1
- """Step execution response schemas."""
2
-
3
- from datetime import datetime
4
-
5
- from pydantic import BaseModel
6
-
7
-
8
- class StepResponse(BaseModel):
9
- """Response model for a step execution."""
10
-
11
- step_id: str
12
- run_id: str
13
- step_name: str
14
- status: str
15
- attempt: int = 1
16
- max_retries: int = 3
17
- created_at: datetime
18
- started_at: datetime | None = None
19
- completed_at: datetime | None = None
20
- duration_seconds: float | None = None
21
- error: str | None = None
22
-
23
-
24
- class StepListResponse(BaseModel):
25
- """Response model for listing steps."""
26
-
27
- items: list[StepResponse]
28
- count: int
@@ -1,31 +0,0 @@
1
- """Workflow-related response schemas."""
2
-
3
- from typing import Any
4
-
5
- from pydantic import BaseModel
6
-
7
-
8
- class WorkflowParameter(BaseModel):
9
- """Response model for a workflow parameter."""
10
-
11
- name: str
12
- type: str # "string", "number", "boolean", "object", "array", "any"
13
- required: bool
14
- default: Any | None = None
15
-
16
-
17
- class WorkflowResponse(BaseModel):
18
- """Response model for a registered workflow."""
19
-
20
- name: str
21
- description: str | None = None
22
- max_duration: str | None = None
23
- tags: list[str] = []
24
- parameters: list[WorkflowParameter] = []
25
-
26
-
27
- class WorkflowListResponse(BaseModel):
28
- """Response model for listing workflows."""
29
-
30
- items: list[WorkflowResponse]
31
- count: int
@@ -1,87 +0,0 @@
1
- """FastAPI application factory."""
2
-
3
- from collections.abc import AsyncGenerator
4
- from contextlib import asynccontextmanager
5
-
6
- from fastapi import FastAPI
7
- from fastapi.middleware.cors import CORSMiddleware
8
-
9
- from app.config import settings
10
- from app.rest import router as api_router
11
-
12
-
13
- def _initialize_pyworkflow() -> None:
14
- """Initialize pyworkflow configuration and discover workflows.
15
-
16
- Priority:
17
- 1. If pyworkflow_config_path is set, load from that path (includes discovery)
18
- 2. Otherwise, load from pyworkflow.config.yaml in cwd and discover workflows
19
- """
20
- import pyworkflow
21
-
22
- if settings.pyworkflow_config_path:
23
- # configure_from_yaml automatically discovers workflows
24
- pyworkflow.configure_from_yaml(settings.pyworkflow_config_path)
25
- else:
26
- # Load config without discovery, then discover from cwd config
27
- pyworkflow.get_config()
28
- # Discover workflows from pyworkflow.config.yaml in cwd
29
- pyworkflow.discover_workflows()
30
-
31
-
32
- @asynccontextmanager
33
- async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
34
- """Application lifespan handler.
35
-
36
- Startup: Initialize pyworkflow configuration
37
- Shutdown: (cleanup if needed in future)
38
- """
39
- # Startup
40
- _initialize_pyworkflow()
41
-
42
- # Reset cached storage instance to ensure fresh initialization
43
- from app.dependencies.storage import get_storage, reset_storage_cache
44
-
45
- reset_storage_cache()
46
-
47
- # Initialize and connect storage backend
48
- storage = await get_storage()
49
- if hasattr(storage, "connect"):
50
- await storage.connect()
51
- if hasattr(storage, "initialize"):
52
- await storage.initialize()
53
-
54
- yield
55
-
56
- # Shutdown - disconnect storage
57
- if hasattr(storage, "disconnect"):
58
- await storage.disconnect()
59
-
60
-
61
- def create_app() -> FastAPI:
62
- """Create and configure the FastAPI application."""
63
- app = FastAPI(
64
- title="PyWorkflow Dashboard API",
65
- description="REST API for monitoring PyWorkflow workflows",
66
- version="0.1.0",
67
- docs_url="/docs",
68
- redoc_url="/redoc",
69
- lifespan=lifespan,
70
- )
71
-
72
- # Configure CORS
73
- app.add_middleware(
74
- CORSMiddleware,
75
- allow_origins=settings.cors_origins,
76
- allow_credentials=True,
77
- allow_methods=["*"],
78
- allow_headers=["*"],
79
- )
80
-
81
- # Include API routes
82
- app.include_router(api_router)
83
-
84
- return app
85
-
86
-
87
- app = create_app()
@@ -1,6 +0,0 @@
1
- """Services layer."""
2
-
3
- from app.services.run_service import RunService
4
- from app.services.workflow_service import WorkflowService
5
-
6
- __all__ = ["WorkflowService", "RunService"]