pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/config.py +94 -17
  9. pyworkflow/context/__init__.py +13 -0
  10. pyworkflow/context/base.py +26 -0
  11. pyworkflow/context/local.py +80 -0
  12. pyworkflow/context/step_context.py +295 -0
  13. pyworkflow/core/registry.py +6 -1
  14. pyworkflow/core/step.py +141 -0
  15. pyworkflow/core/workflow.py +56 -0
  16. pyworkflow/engine/events.py +30 -0
  17. pyworkflow/engine/replay.py +39 -0
  18. pyworkflow/primitives/child_workflow.py +1 -1
  19. pyworkflow/runtime/local.py +1 -1
  20. pyworkflow/storage/__init__.py +14 -0
  21. pyworkflow/storage/base.py +35 -0
  22. pyworkflow/storage/cassandra.py +1747 -0
  23. pyworkflow/storage/config.py +69 -0
  24. pyworkflow/storage/dynamodb.py +31 -2
  25. pyworkflow/storage/file.py +28 -0
  26. pyworkflow/storage/memory.py +18 -0
  27. pyworkflow/storage/mysql.py +1159 -0
  28. pyworkflow/storage/postgres.py +27 -2
  29. pyworkflow/storage/schemas.py +4 -3
  30. pyworkflow/storage/sqlite.py +25 -2
  31. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/METADATA +7 -4
  32. pyworkflow_engine-0.1.10.dist-info/RECORD +91 -0
  33. pyworkflow_engine-0.1.10.dist-info/top_level.txt +1 -0
  34. dashboard/backend/app/__init__.py +0 -1
  35. dashboard/backend/app/config.py +0 -32
  36. dashboard/backend/app/controllers/__init__.py +0 -6
  37. dashboard/backend/app/controllers/run_controller.py +0 -86
  38. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  39. dashboard/backend/app/dependencies/__init__.py +0 -5
  40. dashboard/backend/app/dependencies/storage.py +0 -50
  41. dashboard/backend/app/repositories/__init__.py +0 -6
  42. dashboard/backend/app/repositories/run_repository.py +0 -80
  43. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  44. dashboard/backend/app/rest/__init__.py +0 -8
  45. dashboard/backend/app/rest/v1/__init__.py +0 -12
  46. dashboard/backend/app/rest/v1/health.py +0 -33
  47. dashboard/backend/app/rest/v1/runs.py +0 -133
  48. dashboard/backend/app/rest/v1/workflows.py +0 -41
  49. dashboard/backend/app/schemas/__init__.py +0 -23
  50. dashboard/backend/app/schemas/common.py +0 -16
  51. dashboard/backend/app/schemas/event.py +0 -24
  52. dashboard/backend/app/schemas/hook.py +0 -25
  53. dashboard/backend/app/schemas/run.py +0 -54
  54. dashboard/backend/app/schemas/step.py +0 -28
  55. dashboard/backend/app/schemas/workflow.py +0 -31
  56. dashboard/backend/app/server.py +0 -87
  57. dashboard/backend/app/services/__init__.py +0 -6
  58. dashboard/backend/app/services/run_service.py +0 -240
  59. dashboard/backend/app/services/workflow_service.py +0 -155
  60. dashboard/backend/main.py +0 -18
  61. docs/concepts/cancellation.mdx +0 -362
  62. docs/concepts/continue-as-new.mdx +0 -434
  63. docs/concepts/events.mdx +0 -266
  64. docs/concepts/fault-tolerance.mdx +0 -370
  65. docs/concepts/hooks.mdx +0 -552
  66. docs/concepts/limitations.mdx +0 -167
  67. docs/concepts/schedules.mdx +0 -775
  68. docs/concepts/sleep.mdx +0 -312
  69. docs/concepts/steps.mdx +0 -301
  70. docs/concepts/workflows.mdx +0 -255
  71. docs/guides/cli.mdx +0 -942
  72. docs/guides/configuration.mdx +0 -560
  73. docs/introduction.mdx +0 -155
  74. docs/quickstart.mdx +0 -279
  75. examples/__init__.py +0 -1
  76. examples/celery/__init__.py +0 -1
  77. examples/celery/durable/docker-compose.yml +0 -55
  78. examples/celery/durable/pyworkflow.config.yaml +0 -12
  79. examples/celery/durable/workflows/__init__.py +0 -122
  80. examples/celery/durable/workflows/basic.py +0 -87
  81. examples/celery/durable/workflows/batch_processing.py +0 -102
  82. examples/celery/durable/workflows/cancellation.py +0 -273
  83. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  84. examples/celery/durable/workflows/child_workflows.py +0 -202
  85. examples/celery/durable/workflows/continue_as_new.py +0 -260
  86. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  87. examples/celery/durable/workflows/hooks.py +0 -211
  88. examples/celery/durable/workflows/idempotency.py +0 -112
  89. examples/celery/durable/workflows/long_running.py +0 -99
  90. examples/celery/durable/workflows/retries.py +0 -101
  91. examples/celery/durable/workflows/schedules.py +0 -209
  92. examples/celery/transient/01_basic_workflow.py +0 -91
  93. examples/celery/transient/02_fault_tolerance.py +0 -257
  94. examples/celery/transient/__init__.py +0 -20
  95. examples/celery/transient/pyworkflow.config.yaml +0 -25
  96. examples/local/__init__.py +0 -1
  97. examples/local/durable/01_basic_workflow.py +0 -94
  98. examples/local/durable/02_file_storage.py +0 -132
  99. examples/local/durable/03_retries.py +0 -169
  100. examples/local/durable/04_long_running.py +0 -119
  101. examples/local/durable/05_event_log.py +0 -145
  102. examples/local/durable/06_idempotency.py +0 -148
  103. examples/local/durable/07_hooks.py +0 -334
  104. examples/local/durable/08_cancellation.py +0 -233
  105. examples/local/durable/09_child_workflows.py +0 -198
  106. examples/local/durable/10_child_workflow_patterns.py +0 -265
  107. examples/local/durable/11_continue_as_new.py +0 -249
  108. examples/local/durable/12_schedules.py +0 -198
  109. examples/local/durable/__init__.py +0 -1
  110. examples/local/transient/01_quick_tasks.py +0 -87
  111. examples/local/transient/02_retries.py +0 -130
  112. examples/local/transient/03_sleep.py +0 -141
  113. examples/local/transient/__init__.py +0 -1
  114. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  115. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  116. tests/examples/__init__.py +0 -0
  117. tests/integration/__init__.py +0 -0
  118. tests/integration/test_cancellation.py +0 -330
  119. tests/integration/test_child_workflows.py +0 -439
  120. tests/integration/test_continue_as_new.py +0 -428
  121. tests/integration/test_dynamodb_storage.py +0 -1146
  122. tests/integration/test_fault_tolerance.py +0 -369
  123. tests/integration/test_schedule_storage.py +0 -484
  124. tests/unit/__init__.py +0 -0
  125. tests/unit/backends/__init__.py +0 -1
  126. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  127. tests/unit/backends/test_postgres_storage.py +0 -1281
  128. tests/unit/backends/test_sqlite_storage.py +0 -1460
  129. tests/unit/conftest.py +0 -41
  130. tests/unit/test_cancellation.py +0 -364
  131. tests/unit/test_child_workflows.py +0 -680
  132. tests/unit/test_continue_as_new.py +0 -441
  133. tests/unit/test_event_limits.py +0 -316
  134. tests/unit/test_executor.py +0 -320
  135. tests/unit/test_fault_tolerance.py +0 -334
  136. tests/unit/test_hooks.py +0 -495
  137. tests/unit/test_registry.py +0 -261
  138. tests/unit/test_replay.py +0 -420
  139. tests/unit/test_schedule_schemas.py +0 -285
  140. tests/unit/test_schedule_utils.py +0 -286
  141. tests/unit/test_scheduled_workflow.py +0 -274
  142. tests/unit/test_step.py +0 -353
  143. tests/unit/test_workflow.py +0 -243
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/WHEEL +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/entry_points.txt +0 -0
  146. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,133 +0,0 @@
1
- """Workflow run endpoints."""
2
-
3
- from datetime import datetime
4
-
5
- from fastapi import APIRouter, Depends, HTTPException, Query
6
-
7
- from app.controllers.run_controller import RunController
8
- from app.dependencies import get_storage
9
- from app.schemas.event import EventListResponse
10
- from app.schemas.run import RunDetailResponse, RunListResponse, StartRunRequest, StartRunResponse
11
- from pyworkflow.storage.base import StorageBackend
12
-
13
- router = APIRouter()
14
-
15
-
16
- @router.get("", response_model=RunListResponse)
17
- async def list_runs(
18
- query: str | None = Query(
19
- None, description="Search in workflow name and input kwargs (case-insensitive)"
20
- ),
21
- status: str | None = Query(
22
- None,
23
- description="Filter by status (pending, running, suspended, completed, failed, interrupted, cancelled)",
24
- ),
25
- start_time: datetime | None = Query(
26
- None, description="Filter runs started at or after this time (ISO 8601)"
27
- ),
28
- end_time: datetime | None = Query(
29
- None, description="Filter runs started before this time (ISO 8601)"
30
- ),
31
- limit: int = Query(100, ge=1, le=1000, description="Maximum results"),
32
- cursor: str | None = Query(None, description="Run ID to start after (for pagination)"),
33
- storage: StorageBackend = Depends(get_storage),
34
- ) -> RunListResponse:
35
- """List workflow runs with optional filtering and cursor-based pagination.
36
-
37
- Args:
38
- query: Case-insensitive search in workflow name and input kwargs.
39
- status: Filter by run status.
40
- start_time: Filter runs started at or after this time.
41
- end_time: Filter runs started before this time.
42
- limit: Maximum number of results (1-1000).
43
- cursor: Run ID to start after (for pagination).
44
- storage: Storage backend (injected).
45
-
46
- Returns:
47
- RunListResponse with matching runs and next_cursor.
48
- """
49
- controller = RunController(storage)
50
- return await controller.list_runs(
51
- query=query,
52
- status=status,
53
- start_time=start_time,
54
- end_time=end_time,
55
- limit=limit,
56
- cursor=cursor,
57
- )
58
-
59
-
60
- @router.post("", response_model=StartRunResponse, status_code=201)
61
- async def start_run(
62
- request: StartRunRequest,
63
- storage: StorageBackend = Depends(get_storage),
64
- ) -> StartRunResponse:
65
- """Start a new workflow run.
66
-
67
- Args:
68
- request: Start run request with workflow name and kwargs.
69
- storage: Storage backend (injected).
70
-
71
- Returns:
72
- StartRunResponse with run_id and workflow_name.
73
-
74
- Raises:
75
- HTTPException: 404 if workflow not found, 400 for validation errors.
76
- """
77
- controller = RunController(storage)
78
- try:
79
- return await controller.start_run(request)
80
- except ValueError as e:
81
- raise HTTPException(status_code=404, detail=str(e))
82
- except Exception as e:
83
- raise HTTPException(status_code=400, detail=str(e))
84
-
85
-
86
- @router.get("/{run_id}", response_model=RunDetailResponse)
87
- async def get_run(
88
- run_id: str,
89
- storage: StorageBackend = Depends(get_storage),
90
- ) -> RunDetailResponse:
91
- """Get detailed information about a workflow run.
92
-
93
- Args:
94
- run_id: The run ID.
95
- storage: Storage backend (injected).
96
-
97
- Returns:
98
- RunDetailResponse with run details.
99
-
100
- Raises:
101
- HTTPException: 404 if run not found.
102
- """
103
- controller = RunController(storage)
104
- run = await controller.get_run(run_id)
105
-
106
- if run is None:
107
- raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
108
-
109
- return run
110
-
111
-
112
- @router.get("/{run_id}/events", response_model=EventListResponse)
113
- async def get_run_events(
114
- run_id: str,
115
- storage: StorageBackend = Depends(get_storage),
116
- ) -> EventListResponse:
117
- """Get all events for a workflow run.
118
-
119
- Args:
120
- run_id: The run ID.
121
- storage: Storage backend (injected).
122
-
123
- Returns:
124
- EventListResponse with run events.
125
- """
126
- controller = RunController(storage)
127
-
128
- # Verify run exists
129
- run = await controller.get_run(run_id)
130
- if run is None:
131
- raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
132
-
133
- return await controller.get_events(run_id)
@@ -1,41 +0,0 @@
1
- """Workflow endpoints."""
2
-
3
- from fastapi import APIRouter, HTTPException
4
-
5
- from app.controllers.workflow_controller import WorkflowController
6
- from app.schemas.workflow import WorkflowListResponse, WorkflowResponse
7
-
8
- router = APIRouter()
9
-
10
-
11
- @router.get("", response_model=WorkflowListResponse)
12
- async def list_workflows() -> WorkflowListResponse:
13
- """List all registered workflows.
14
-
15
- Returns:
16
- WorkflowListResponse with all registered workflows.
17
- """
18
- controller = WorkflowController()
19
- return controller.list_workflows()
20
-
21
-
22
- @router.get("/{name}", response_model=WorkflowResponse)
23
- async def get_workflow(name: str) -> WorkflowResponse:
24
- """Get a specific workflow by name.
25
-
26
- Args:
27
- name: Workflow name.
28
-
29
- Returns:
30
- WorkflowResponse with workflow details.
31
-
32
- Raises:
33
- HTTPException: 404 if workflow not found.
34
- """
35
- controller = WorkflowController()
36
- workflow = controller.get_workflow(name)
37
-
38
- if workflow is None:
39
- raise HTTPException(status_code=404, detail=f"Workflow '{name}' not found")
40
-
41
- return workflow
@@ -1,23 +0,0 @@
1
- """Pydantic schemas for API request/response models."""
2
-
3
- from app.schemas.common import PaginatedResponse
4
- from app.schemas.event import EventListResponse, EventResponse
5
- from app.schemas.hook import HookListResponse, HookResponse
6
- from app.schemas.run import RunDetailResponse, RunListResponse, RunResponse
7
- from app.schemas.step import StepListResponse, StepResponse
8
- from app.schemas.workflow import WorkflowListResponse, WorkflowResponse
9
-
10
- __all__ = [
11
- "PaginatedResponse",
12
- "WorkflowResponse",
13
- "WorkflowListResponse",
14
- "RunResponse",
15
- "RunDetailResponse",
16
- "RunListResponse",
17
- "EventResponse",
18
- "EventListResponse",
19
- "StepResponse",
20
- "StepListResponse",
21
- "HookResponse",
22
- "HookListResponse",
23
- ]
@@ -1,16 +0,0 @@
1
- """Common schema types."""
2
-
3
- from typing import Generic, TypeVar
4
-
5
- from pydantic import BaseModel
6
-
7
- T = TypeVar("T")
8
-
9
-
10
- class PaginatedResponse(BaseModel, Generic[T]):
11
- """Base paginated response model."""
12
-
13
- items: list[T]
14
- count: int
15
- limit: int = 100
16
- offset: int = 0
@@ -1,24 +0,0 @@
1
- """Event response schemas."""
2
-
3
- from datetime import datetime
4
- from typing import Any
5
-
6
- from pydantic import BaseModel
7
-
8
-
9
- class EventResponse(BaseModel):
10
- """Response model for a workflow event."""
11
-
12
- event_id: str
13
- run_id: str
14
- type: str
15
- timestamp: datetime
16
- sequence: int | None = None
17
- data: dict[str, Any] = {}
18
-
19
-
20
- class EventListResponse(BaseModel):
21
- """Response model for listing events."""
22
-
23
- items: list[EventResponse]
24
- count: int
@@ -1,25 +0,0 @@
1
- """Hook response schemas."""
2
-
3
- from datetime import datetime
4
-
5
- from pydantic import BaseModel
6
-
7
-
8
- class HookResponse(BaseModel):
9
- """Response model for a hook."""
10
-
11
- hook_id: str
12
- run_id: str
13
- name: str | None = None
14
- status: str
15
- created_at: datetime
16
- received_at: datetime | None = None
17
- expires_at: datetime | None = None
18
- has_payload: bool = False
19
-
20
-
21
- class HookListResponse(BaseModel):
22
- """Response model for listing hooks."""
23
-
24
- items: list[HookResponse]
25
- count: int
@@ -1,54 +0,0 @@
1
- """Workflow run response schemas."""
2
-
3
- from datetime import datetime
4
- from typing import Any
5
-
6
- from pydantic import BaseModel
7
-
8
-
9
- class RunResponse(BaseModel):
10
- """Response model for a workflow run."""
11
-
12
- run_id: str
13
- workflow_name: str
14
- status: str
15
- created_at: datetime
16
- started_at: datetime | None = None
17
- completed_at: datetime | None = None
18
- duration_seconds: float | None = None
19
- error: str | None = None
20
- recovery_attempts: int = 0
21
-
22
-
23
- class RunDetailResponse(RunResponse):
24
- """Detailed response model for a workflow run."""
25
-
26
- input_args: Any | None = None
27
- input_kwargs: Any | None = None
28
- result: Any | None = None
29
- metadata: dict[str, Any] = {}
30
- max_duration: str | None = None
31
- max_recovery_attempts: int = 3
32
-
33
-
34
- class RunListResponse(BaseModel):
35
- """Response model for listing runs."""
36
-
37
- items: list[RunResponse]
38
- count: int
39
- limit: int = 100
40
- next_cursor: str | None = None
41
-
42
-
43
- class StartRunRequest(BaseModel):
44
- """Request model for starting a new workflow run."""
45
-
46
- workflow_name: str
47
- kwargs: dict[str, Any] = {}
48
-
49
-
50
- class StartRunResponse(BaseModel):
51
- """Response model for a newly started workflow run."""
52
-
53
- run_id: str
54
- workflow_name: str
@@ -1,28 +0,0 @@
1
- """Step execution response schemas."""
2
-
3
- from datetime import datetime
4
-
5
- from pydantic import BaseModel
6
-
7
-
8
- class StepResponse(BaseModel):
9
- """Response model for a step execution."""
10
-
11
- step_id: str
12
- run_id: str
13
- step_name: str
14
- status: str
15
- attempt: int = 1
16
- max_retries: int = 3
17
- created_at: datetime
18
- started_at: datetime | None = None
19
- completed_at: datetime | None = None
20
- duration_seconds: float | None = None
21
- error: str | None = None
22
-
23
-
24
- class StepListResponse(BaseModel):
25
- """Response model for listing steps."""
26
-
27
- items: list[StepResponse]
28
- count: int
@@ -1,31 +0,0 @@
1
- """Workflow-related response schemas."""
2
-
3
- from typing import Any
4
-
5
- from pydantic import BaseModel
6
-
7
-
8
- class WorkflowParameter(BaseModel):
9
- """Response model for a workflow parameter."""
10
-
11
- name: str
12
- type: str # "string", "number", "boolean", "object", "array", "any"
13
- required: bool
14
- default: Any | None = None
15
-
16
-
17
- class WorkflowResponse(BaseModel):
18
- """Response model for a registered workflow."""
19
-
20
- name: str
21
- description: str | None = None
22
- max_duration: str | None = None
23
- tags: list[str] = []
24
- parameters: list[WorkflowParameter] = []
25
-
26
-
27
- class WorkflowListResponse(BaseModel):
28
- """Response model for listing workflows."""
29
-
30
- items: list[WorkflowResponse]
31
- count: int
@@ -1,87 +0,0 @@
1
- """FastAPI application factory."""
2
-
3
- from collections.abc import AsyncGenerator
4
- from contextlib import asynccontextmanager
5
-
6
- from fastapi import FastAPI
7
- from fastapi.middleware.cors import CORSMiddleware
8
-
9
- from app.config import settings
10
- from app.rest import router as api_router
11
-
12
-
13
- def _initialize_pyworkflow() -> None:
14
- """Initialize pyworkflow configuration and discover workflows.
15
-
16
- Priority:
17
- 1. If pyworkflow_config_path is set, load from that path (includes discovery)
18
- 2. Otherwise, load from pyworkflow.config.yaml in cwd and discover workflows
19
- """
20
- import pyworkflow
21
-
22
- if settings.pyworkflow_config_path:
23
- # configure_from_yaml automatically discovers workflows
24
- pyworkflow.configure_from_yaml(settings.pyworkflow_config_path)
25
- else:
26
- # Load config without discovery, then discover from cwd config
27
- pyworkflow.get_config()
28
- # Discover workflows from pyworkflow.config.yaml in cwd
29
- pyworkflow.discover_workflows()
30
-
31
-
32
- @asynccontextmanager
33
- async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]:
34
- """Application lifespan handler.
35
-
36
- Startup: Initialize pyworkflow configuration
37
- Shutdown: (cleanup if needed in future)
38
- """
39
- # Startup
40
- _initialize_pyworkflow()
41
-
42
- # Reset cached storage instance to ensure fresh initialization
43
- from app.dependencies.storage import get_storage, reset_storage_cache
44
-
45
- reset_storage_cache()
46
-
47
- # Initialize and connect storage backend
48
- storage = await get_storage()
49
- if hasattr(storage, "connect"):
50
- await storage.connect()
51
- if hasattr(storage, "initialize"):
52
- await storage.initialize()
53
-
54
- yield
55
-
56
- # Shutdown - disconnect storage
57
- if hasattr(storage, "disconnect"):
58
- await storage.disconnect()
59
-
60
-
61
- def create_app() -> FastAPI:
62
- """Create and configure the FastAPI application."""
63
- app = FastAPI(
64
- title="PyWorkflow Dashboard API",
65
- description="REST API for monitoring PyWorkflow workflows",
66
- version="0.1.0",
67
- docs_url="/docs",
68
- redoc_url="/redoc",
69
- lifespan=lifespan,
70
- )
71
-
72
- # Configure CORS
73
- app.add_middleware(
74
- CORSMiddleware,
75
- allow_origins=settings.cors_origins,
76
- allow_credentials=True,
77
- allow_methods=["*"],
78
- allow_headers=["*"],
79
- )
80
-
81
- # Include API routes
82
- app.include_router(api_router)
83
-
84
- return app
85
-
86
-
87
- app = create_app()
@@ -1,6 +0,0 @@
1
- """Services layer."""
2
-
3
- from app.services.run_service import RunService
4
- from app.services.workflow_service import WorkflowService
5
-
6
- __all__ = ["WorkflowService", "RunService"]