pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.10__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/config.py +94 -17
  9. pyworkflow/context/__init__.py +13 -0
  10. pyworkflow/context/base.py +26 -0
  11. pyworkflow/context/local.py +80 -0
  12. pyworkflow/context/step_context.py +295 -0
  13. pyworkflow/core/registry.py +6 -1
  14. pyworkflow/core/step.py +141 -0
  15. pyworkflow/core/workflow.py +56 -0
  16. pyworkflow/engine/events.py +30 -0
  17. pyworkflow/engine/replay.py +39 -0
  18. pyworkflow/primitives/child_workflow.py +1 -1
  19. pyworkflow/runtime/local.py +1 -1
  20. pyworkflow/storage/__init__.py +14 -0
  21. pyworkflow/storage/base.py +35 -0
  22. pyworkflow/storage/cassandra.py +1747 -0
  23. pyworkflow/storage/config.py +69 -0
  24. pyworkflow/storage/dynamodb.py +31 -2
  25. pyworkflow/storage/file.py +28 -0
  26. pyworkflow/storage/memory.py +18 -0
  27. pyworkflow/storage/mysql.py +1159 -0
  28. pyworkflow/storage/postgres.py +27 -2
  29. pyworkflow/storage/schemas.py +4 -3
  30. pyworkflow/storage/sqlite.py +25 -2
  31. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/METADATA +7 -4
  32. pyworkflow_engine-0.1.10.dist-info/RECORD +91 -0
  33. pyworkflow_engine-0.1.10.dist-info/top_level.txt +1 -0
  34. dashboard/backend/app/__init__.py +0 -1
  35. dashboard/backend/app/config.py +0 -32
  36. dashboard/backend/app/controllers/__init__.py +0 -6
  37. dashboard/backend/app/controllers/run_controller.py +0 -86
  38. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  39. dashboard/backend/app/dependencies/__init__.py +0 -5
  40. dashboard/backend/app/dependencies/storage.py +0 -50
  41. dashboard/backend/app/repositories/__init__.py +0 -6
  42. dashboard/backend/app/repositories/run_repository.py +0 -80
  43. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  44. dashboard/backend/app/rest/__init__.py +0 -8
  45. dashboard/backend/app/rest/v1/__init__.py +0 -12
  46. dashboard/backend/app/rest/v1/health.py +0 -33
  47. dashboard/backend/app/rest/v1/runs.py +0 -133
  48. dashboard/backend/app/rest/v1/workflows.py +0 -41
  49. dashboard/backend/app/schemas/__init__.py +0 -23
  50. dashboard/backend/app/schemas/common.py +0 -16
  51. dashboard/backend/app/schemas/event.py +0 -24
  52. dashboard/backend/app/schemas/hook.py +0 -25
  53. dashboard/backend/app/schemas/run.py +0 -54
  54. dashboard/backend/app/schemas/step.py +0 -28
  55. dashboard/backend/app/schemas/workflow.py +0 -31
  56. dashboard/backend/app/server.py +0 -87
  57. dashboard/backend/app/services/__init__.py +0 -6
  58. dashboard/backend/app/services/run_service.py +0 -240
  59. dashboard/backend/app/services/workflow_service.py +0 -155
  60. dashboard/backend/main.py +0 -18
  61. docs/concepts/cancellation.mdx +0 -362
  62. docs/concepts/continue-as-new.mdx +0 -434
  63. docs/concepts/events.mdx +0 -266
  64. docs/concepts/fault-tolerance.mdx +0 -370
  65. docs/concepts/hooks.mdx +0 -552
  66. docs/concepts/limitations.mdx +0 -167
  67. docs/concepts/schedules.mdx +0 -775
  68. docs/concepts/sleep.mdx +0 -312
  69. docs/concepts/steps.mdx +0 -301
  70. docs/concepts/workflows.mdx +0 -255
  71. docs/guides/cli.mdx +0 -942
  72. docs/guides/configuration.mdx +0 -560
  73. docs/introduction.mdx +0 -155
  74. docs/quickstart.mdx +0 -279
  75. examples/__init__.py +0 -1
  76. examples/celery/__init__.py +0 -1
  77. examples/celery/durable/docker-compose.yml +0 -55
  78. examples/celery/durable/pyworkflow.config.yaml +0 -12
  79. examples/celery/durable/workflows/__init__.py +0 -122
  80. examples/celery/durable/workflows/basic.py +0 -87
  81. examples/celery/durable/workflows/batch_processing.py +0 -102
  82. examples/celery/durable/workflows/cancellation.py +0 -273
  83. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  84. examples/celery/durable/workflows/child_workflows.py +0 -202
  85. examples/celery/durable/workflows/continue_as_new.py +0 -260
  86. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  87. examples/celery/durable/workflows/hooks.py +0 -211
  88. examples/celery/durable/workflows/idempotency.py +0 -112
  89. examples/celery/durable/workflows/long_running.py +0 -99
  90. examples/celery/durable/workflows/retries.py +0 -101
  91. examples/celery/durable/workflows/schedules.py +0 -209
  92. examples/celery/transient/01_basic_workflow.py +0 -91
  93. examples/celery/transient/02_fault_tolerance.py +0 -257
  94. examples/celery/transient/__init__.py +0 -20
  95. examples/celery/transient/pyworkflow.config.yaml +0 -25
  96. examples/local/__init__.py +0 -1
  97. examples/local/durable/01_basic_workflow.py +0 -94
  98. examples/local/durable/02_file_storage.py +0 -132
  99. examples/local/durable/03_retries.py +0 -169
  100. examples/local/durable/04_long_running.py +0 -119
  101. examples/local/durable/05_event_log.py +0 -145
  102. examples/local/durable/06_idempotency.py +0 -148
  103. examples/local/durable/07_hooks.py +0 -334
  104. examples/local/durable/08_cancellation.py +0 -233
  105. examples/local/durable/09_child_workflows.py +0 -198
  106. examples/local/durable/10_child_workflow_patterns.py +0 -265
  107. examples/local/durable/11_continue_as_new.py +0 -249
  108. examples/local/durable/12_schedules.py +0 -198
  109. examples/local/durable/__init__.py +0 -1
  110. examples/local/transient/01_quick_tasks.py +0 -87
  111. examples/local/transient/02_retries.py +0 -130
  112. examples/local/transient/03_sleep.py +0 -141
  113. examples/local/transient/__init__.py +0 -1
  114. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  115. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  116. tests/examples/__init__.py +0 -0
  117. tests/integration/__init__.py +0 -0
  118. tests/integration/test_cancellation.py +0 -330
  119. tests/integration/test_child_workflows.py +0 -439
  120. tests/integration/test_continue_as_new.py +0 -428
  121. tests/integration/test_dynamodb_storage.py +0 -1146
  122. tests/integration/test_fault_tolerance.py +0 -369
  123. tests/integration/test_schedule_storage.py +0 -484
  124. tests/unit/__init__.py +0 -0
  125. tests/unit/backends/__init__.py +0 -1
  126. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  127. tests/unit/backends/test_postgres_storage.py +0 -1281
  128. tests/unit/backends/test_sqlite_storage.py +0 -1460
  129. tests/unit/conftest.py +0 -41
  130. tests/unit/test_cancellation.py +0 -364
  131. tests/unit/test_child_workflows.py +0 -680
  132. tests/unit/test_continue_as_new.py +0 -441
  133. tests/unit/test_event_limits.py +0 -316
  134. tests/unit/test_executor.py +0 -320
  135. tests/unit/test_fault_tolerance.py +0 -334
  136. tests/unit/test_hooks.py +0 -495
  137. tests/unit/test_registry.py +0 -261
  138. tests/unit/test_replay.py +0 -420
  139. tests/unit/test_schedule_schemas.py +0 -285
  140. tests/unit/test_schedule_utils.py +0 -286
  141. tests/unit/test_scheduled_workflow.py +0 -274
  142. tests/unit/test_step.py +0 -353
  143. tests/unit/test_workflow.py +0 -243
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/WHEEL +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/entry_points.txt +0 -0
  146. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.10.dist-info}/licenses/LICENSE +0 -0
@@ -1,240 +0,0 @@
1
- """Service layer for workflow run operations."""
2
-
3
- import json
4
- from datetime import UTC, datetime
5
- from typing import Any
6
-
7
- import pyworkflow
8
- from app.repositories.run_repository import RunRepository
9
- from app.schemas.event import EventListResponse, EventResponse
10
- from app.schemas.run import (
11
- RunDetailResponse,
12
- RunListResponse,
13
- RunResponse,
14
- StartRunRequest,
15
- StartRunResponse,
16
- )
17
- from pyworkflow.storage.schemas import RunStatus, WorkflowRun
18
-
19
-
20
- class RunService:
21
- """Service for workflow run-related business logic."""
22
-
23
- def __init__(self, repository: RunRepository):
24
- """Initialize with run repository.
25
-
26
- Args:
27
- repository: RunRepository instance.
28
- """
29
- self.repository = repository
30
-
31
- async def list_runs(
32
- self,
33
- query: str | None = None,
34
- status: str | None = None,
35
- start_time: datetime | None = None,
36
- end_time: datetime | None = None,
37
- limit: int = 100,
38
- cursor: str | None = None,
39
- ) -> RunListResponse:
40
- """List workflow runs with optional filtering and cursor-based pagination.
41
-
42
- Args:
43
- query: Case-insensitive search in workflow name and input kwargs.
44
- status: Filter by status string.
45
- start_time: Filter runs started at or after this time.
46
- end_time: Filter runs started before this time.
47
- limit: Maximum number of results.
48
- cursor: Run ID to start after (for pagination).
49
-
50
- Returns:
51
- RunListResponse with list of runs and next_cursor.
52
- """
53
- status_enum = RunStatus(status) if status else None
54
-
55
- runs, next_cursor = await self.repository.list_runs(
56
- query=query,
57
- status=status_enum,
58
- start_time=start_time,
59
- end_time=end_time,
60
- limit=limit,
61
- cursor=cursor,
62
- )
63
-
64
- items = [self._run_to_response(run) for run in runs]
65
-
66
- return RunListResponse(
67
- items=items,
68
- count=len(items),
69
- limit=limit,
70
- next_cursor=next_cursor,
71
- )
72
-
73
- async def get_run(self, run_id: str) -> RunDetailResponse | None:
74
- """Get detailed information about a workflow run.
75
-
76
- Args:
77
- run_id: The run ID.
78
-
79
- Returns:
80
- RunDetailResponse if found, None otherwise.
81
- """
82
- run = await self.repository.get_run(run_id)
83
-
84
- if run is None:
85
- return None
86
-
87
- return self._run_to_detail_response(run)
88
-
89
- async def get_events(self, run_id: str) -> EventListResponse:
90
- """Get all events for a workflow run.
91
-
92
- Args:
93
- run_id: The run ID.
94
-
95
- Returns:
96
- EventListResponse with list of events.
97
- """
98
- events = await self.repository.get_events(run_id)
99
-
100
- items = [
101
- EventResponse(
102
- event_id=event.event_id,
103
- run_id=event.run_id,
104
- type=event.type.value,
105
- timestamp=event.timestamp,
106
- sequence=event.sequence,
107
- data=event.data,
108
- )
109
- for event in events
110
- ]
111
-
112
- return EventListResponse(
113
- items=items,
114
- count=len(items),
115
- )
116
-
117
- def _run_to_response(self, run: WorkflowRun) -> RunResponse:
118
- """Convert WorkflowRun to RunResponse.
119
-
120
- Args:
121
- run: WorkflowRun instance.
122
-
123
- Returns:
124
- RunResponse instance.
125
- """
126
- return RunResponse(
127
- run_id=run.run_id,
128
- workflow_name=run.workflow_name,
129
- status=run.status.value,
130
- created_at=run.created_at,
131
- started_at=run.started_at,
132
- completed_at=run.completed_at,
133
- duration_seconds=self._calculate_duration(run.started_at, run.completed_at),
134
- error=run.error,
135
- recovery_attempts=run.recovery_attempts,
136
- )
137
-
138
- def _run_to_detail_response(self, run: WorkflowRun) -> RunDetailResponse:
139
- """Convert WorkflowRun to RunDetailResponse.
140
-
141
- Args:
142
- run: WorkflowRun instance.
143
-
144
- Returns:
145
- RunDetailResponse instance.
146
- """
147
- # Parse JSON strings for input/result
148
- input_args = self._safe_json_parse(run.input_args)
149
- input_kwargs = self._safe_json_parse(run.input_kwargs)
150
- result = self._safe_json_parse(run.result) if run.result else None
151
-
152
- return RunDetailResponse(
153
- run_id=run.run_id,
154
- workflow_name=run.workflow_name,
155
- status=run.status.value,
156
- created_at=run.created_at,
157
- started_at=run.started_at,
158
- completed_at=run.completed_at,
159
- duration_seconds=self._calculate_duration(run.started_at, run.completed_at),
160
- error=run.error,
161
- recovery_attempts=run.recovery_attempts,
162
- input_args=input_args,
163
- input_kwargs=input_kwargs,
164
- result=result,
165
- metadata=run.metadata,
166
- max_duration=run.max_duration,
167
- max_recovery_attempts=run.max_recovery_attempts,
168
- )
169
-
170
- def _calculate_duration(
171
- self,
172
- started_at: datetime | None,
173
- completed_at: datetime | None,
174
- ) -> float | None:
175
- """Calculate duration in seconds.
176
-
177
- Args:
178
- started_at: Start timestamp.
179
- completed_at: Completion timestamp.
180
-
181
- Returns:
182
- Duration in seconds, or None if not calculable.
183
- """
184
- if started_at is None:
185
- return None
186
-
187
- end_time = completed_at or datetime.now(UTC)
188
-
189
- # Handle timezone-naive datetimes
190
- if started_at.tzinfo is None:
191
- started_at = started_at.replace(tzinfo=UTC)
192
- if end_time.tzinfo is None:
193
- end_time = end_time.replace(tzinfo=UTC)
194
-
195
- return (end_time - started_at).total_seconds()
196
-
197
- def _safe_json_parse(self, value: str | None) -> Any:
198
- """Safely parse a JSON string.
199
-
200
- Args:
201
- value: JSON string or None.
202
-
203
- Returns:
204
- Parsed value or the original string if parsing fails.
205
- """
206
- if value is None:
207
- return None
208
-
209
- try:
210
- return json.loads(value)
211
- except (json.JSONDecodeError, TypeError):
212
- return value
213
-
214
- async def start_run(self, request: StartRunRequest) -> StartRunResponse:
215
- """Start a new workflow run.
216
-
217
- Args:
218
- request: The start run request containing workflow name and kwargs.
219
-
220
- Returns:
221
- StartRunResponse with run_id and workflow_name.
222
-
223
- Raises:
224
- ValueError: If workflow not found.
225
- """
226
- # Get the workflow metadata
227
- workflow_meta = pyworkflow.get_workflow(request.workflow_name)
228
- if workflow_meta is None:
229
- raise ValueError(f"Workflow '{request.workflow_name}' not found")
230
-
231
- # Start the workflow using pyworkflow.start()
232
- run_id = await pyworkflow.start(
233
- workflow_meta.func,
234
- **request.kwargs,
235
- )
236
-
237
- return StartRunResponse(
238
- run_id=run_id,
239
- workflow_name=request.workflow_name,
240
- )
@@ -1,155 +0,0 @@
1
- """Service layer for workflow operations."""
2
-
3
- import inspect
4
- from typing import Any, get_origin, get_type_hints
5
-
6
- from app.repositories.workflow_repository import WorkflowRepository
7
- from app.schemas.workflow import (
8
- WorkflowListResponse,
9
- WorkflowParameter,
10
- WorkflowResponse,
11
- )
12
-
13
-
14
- def _get_type_name(type_hint: Any) -> str:
15
- """Convert a Python type hint to a simple type name for the frontend.
16
-
17
- Args:
18
- type_hint: The type hint to convert.
19
-
20
- Returns:
21
- A string representing the type ("string", "number", "boolean", "array", "object", "any").
22
- """
23
- if type_hint is Any or type_hint is inspect.Parameter.empty:
24
- return "any"
25
-
26
- # Handle None type
27
- if type_hint is type(None):
28
- return "any"
29
-
30
- # Get the origin for generic types (e.g., list[str] -> list)
31
- origin = get_origin(type_hint)
32
- if origin is not None:
33
- if origin is list:
34
- return "array"
35
- if origin is dict:
36
- return "object"
37
- # Union types (Optional, etc.)
38
- if origin is type(None):
39
- return "any"
40
-
41
- # Handle basic types
42
- if hasattr(type_hint, "__name__"):
43
- type_name = type_hint.__name__
44
- if type_name == "str":
45
- return "string"
46
- if type_name in ("int", "float"):
47
- return "number"
48
- if type_name == "bool":
49
- return "boolean"
50
- if type_name == "list":
51
- return "array"
52
- if type_name == "dict":
53
- return "object"
54
-
55
- return "any"
56
-
57
-
58
- def _extract_workflow_parameters(func: Any) -> list[WorkflowParameter]:
59
- """Extract parameter information from a workflow function.
60
-
61
- Args:
62
- func: The workflow function to inspect.
63
-
64
- Returns:
65
- List of WorkflowParameter objects.
66
- """
67
- sig = inspect.signature(func)
68
- params = []
69
-
70
- # Try to get type hints
71
- try:
72
- hints = get_type_hints(func)
73
- except Exception:
74
- hints = {}
75
-
76
- for param_name, param in sig.parameters.items():
77
- # Skip *args and **kwargs
78
- if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
79
- continue
80
-
81
- type_hint = hints.get(param_name, Any)
82
- has_default = param.default is not inspect.Parameter.empty
83
-
84
- # Serialize the default value
85
- default_value = None
86
- if has_default:
87
- default_value = param.default
88
-
89
- param_info = WorkflowParameter(
90
- name=param_name,
91
- type=_get_type_name(type_hint),
92
- required=not has_default,
93
- default=default_value,
94
- )
95
- params.append(param_info)
96
-
97
- return params
98
-
99
-
100
- class WorkflowService:
101
- """Service for workflow-related business logic."""
102
-
103
- def __init__(self, repository: WorkflowRepository):
104
- """Initialize with workflow repository.
105
-
106
- Args:
107
- repository: WorkflowRepository instance.
108
- """
109
- self.repository = repository
110
-
111
- def list_workflows(self) -> WorkflowListResponse:
112
- """Get all registered workflows.
113
-
114
- Returns:
115
- WorkflowListResponse with list of workflows.
116
- """
117
- workflows = self.repository.list_all()
118
-
119
- items = [
120
- WorkflowResponse(
121
- name=name,
122
- description=metadata.description,
123
- max_duration=metadata.max_duration,
124
- tags=metadata.tags or [],
125
- parameters=_extract_workflow_parameters(metadata.original_func),
126
- )
127
- for name, metadata in workflows.items()
128
- ]
129
-
130
- return WorkflowListResponse(
131
- items=items,
132
- count=len(items),
133
- )
134
-
135
- def get_workflow(self, name: str) -> WorkflowResponse | None:
136
- """Get a specific workflow by name.
137
-
138
- Args:
139
- name: Workflow name.
140
-
141
- Returns:
142
- WorkflowResponse if found, None otherwise.
143
- """
144
- metadata = self.repository.get_by_name(name)
145
-
146
- if metadata is None:
147
- return None
148
-
149
- return WorkflowResponse(
150
- name=metadata.name,
151
- description=metadata.description,
152
- max_duration=metadata.max_duration,
153
- tags=metadata.tags or [],
154
- parameters=_extract_workflow_parameters(metadata.original_func),
155
- )
dashboard/backend/main.py DELETED
@@ -1,18 +0,0 @@
1
- """CLI entry point for the dashboard backend server."""
2
-
3
- import uvicorn
4
- from app.config import settings
5
-
6
-
7
- def main():
8
- """Run the dashboard backend server."""
9
- uvicorn.run(
10
- "app.server:app",
11
- host=settings.host,
12
- port=settings.port,
13
- reload=settings.debug,
14
- )
15
-
16
-
17
- if __name__ == "__main__":
18
- main()