pyworkflow-engine 0.1.7__py3-none-any.whl → 0.1.9__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. pyworkflow/__init__.py +10 -1
  2. pyworkflow/celery/tasks.py +272 -24
  3. pyworkflow/cli/__init__.py +4 -1
  4. pyworkflow/cli/commands/runs.py +4 -4
  5. pyworkflow/cli/commands/setup.py +203 -4
  6. pyworkflow/cli/utils/config_generator.py +76 -3
  7. pyworkflow/cli/utils/docker_manager.py +232 -0
  8. pyworkflow/context/__init__.py +13 -0
  9. pyworkflow/context/base.py +26 -0
  10. pyworkflow/context/local.py +80 -0
  11. pyworkflow/context/step_context.py +295 -0
  12. pyworkflow/core/registry.py +6 -1
  13. pyworkflow/core/step.py +141 -0
  14. pyworkflow/core/workflow.py +56 -0
  15. pyworkflow/engine/events.py +30 -0
  16. pyworkflow/engine/replay.py +39 -0
  17. pyworkflow/primitives/child_workflow.py +1 -1
  18. pyworkflow/runtime/local.py +1 -1
  19. pyworkflow/storage/__init__.py +14 -0
  20. pyworkflow/storage/base.py +35 -0
  21. pyworkflow/storage/cassandra.py +1747 -0
  22. pyworkflow/storage/config.py +69 -0
  23. pyworkflow/storage/dynamodb.py +31 -2
  24. pyworkflow/storage/file.py +28 -0
  25. pyworkflow/storage/memory.py +18 -0
  26. pyworkflow/storage/mysql.py +1159 -0
  27. pyworkflow/storage/postgres.py +27 -2
  28. pyworkflow/storage/schemas.py +4 -3
  29. pyworkflow/storage/sqlite.py +25 -2
  30. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/METADATA +7 -4
  31. pyworkflow_engine-0.1.9.dist-info/RECORD +91 -0
  32. pyworkflow_engine-0.1.9.dist-info/top_level.txt +1 -0
  33. dashboard/backend/app/__init__.py +0 -1
  34. dashboard/backend/app/config.py +0 -32
  35. dashboard/backend/app/controllers/__init__.py +0 -6
  36. dashboard/backend/app/controllers/run_controller.py +0 -86
  37. dashboard/backend/app/controllers/workflow_controller.py +0 -33
  38. dashboard/backend/app/dependencies/__init__.py +0 -5
  39. dashboard/backend/app/dependencies/storage.py +0 -50
  40. dashboard/backend/app/repositories/__init__.py +0 -6
  41. dashboard/backend/app/repositories/run_repository.py +0 -80
  42. dashboard/backend/app/repositories/workflow_repository.py +0 -27
  43. dashboard/backend/app/rest/__init__.py +0 -8
  44. dashboard/backend/app/rest/v1/__init__.py +0 -12
  45. dashboard/backend/app/rest/v1/health.py +0 -33
  46. dashboard/backend/app/rest/v1/runs.py +0 -133
  47. dashboard/backend/app/rest/v1/workflows.py +0 -41
  48. dashboard/backend/app/schemas/__init__.py +0 -23
  49. dashboard/backend/app/schemas/common.py +0 -16
  50. dashboard/backend/app/schemas/event.py +0 -24
  51. dashboard/backend/app/schemas/hook.py +0 -25
  52. dashboard/backend/app/schemas/run.py +0 -54
  53. dashboard/backend/app/schemas/step.py +0 -28
  54. dashboard/backend/app/schemas/workflow.py +0 -31
  55. dashboard/backend/app/server.py +0 -87
  56. dashboard/backend/app/services/__init__.py +0 -6
  57. dashboard/backend/app/services/run_service.py +0 -240
  58. dashboard/backend/app/services/workflow_service.py +0 -155
  59. dashboard/backend/main.py +0 -18
  60. docs/concepts/cancellation.mdx +0 -362
  61. docs/concepts/continue-as-new.mdx +0 -434
  62. docs/concepts/events.mdx +0 -266
  63. docs/concepts/fault-tolerance.mdx +0 -370
  64. docs/concepts/hooks.mdx +0 -552
  65. docs/concepts/limitations.mdx +0 -167
  66. docs/concepts/schedules.mdx +0 -775
  67. docs/concepts/sleep.mdx +0 -312
  68. docs/concepts/steps.mdx +0 -301
  69. docs/concepts/workflows.mdx +0 -255
  70. docs/guides/cli.mdx +0 -942
  71. docs/guides/configuration.mdx +0 -560
  72. docs/introduction.mdx +0 -155
  73. docs/quickstart.mdx +0 -279
  74. examples/__init__.py +0 -1
  75. examples/celery/__init__.py +0 -1
  76. examples/celery/durable/docker-compose.yml +0 -55
  77. examples/celery/durable/pyworkflow.config.yaml +0 -12
  78. examples/celery/durable/workflows/__init__.py +0 -122
  79. examples/celery/durable/workflows/basic.py +0 -87
  80. examples/celery/durable/workflows/batch_processing.py +0 -102
  81. examples/celery/durable/workflows/cancellation.py +0 -273
  82. examples/celery/durable/workflows/child_workflow_patterns.py +0 -240
  83. examples/celery/durable/workflows/child_workflows.py +0 -202
  84. examples/celery/durable/workflows/continue_as_new.py +0 -260
  85. examples/celery/durable/workflows/fault_tolerance.py +0 -210
  86. examples/celery/durable/workflows/hooks.py +0 -211
  87. examples/celery/durable/workflows/idempotency.py +0 -112
  88. examples/celery/durable/workflows/long_running.py +0 -99
  89. examples/celery/durable/workflows/retries.py +0 -101
  90. examples/celery/durable/workflows/schedules.py +0 -209
  91. examples/celery/transient/01_basic_workflow.py +0 -91
  92. examples/celery/transient/02_fault_tolerance.py +0 -257
  93. examples/celery/transient/__init__.py +0 -20
  94. examples/celery/transient/pyworkflow.config.yaml +0 -25
  95. examples/local/__init__.py +0 -1
  96. examples/local/durable/01_basic_workflow.py +0 -94
  97. examples/local/durable/02_file_storage.py +0 -132
  98. examples/local/durable/03_retries.py +0 -169
  99. examples/local/durable/04_long_running.py +0 -119
  100. examples/local/durable/05_event_log.py +0 -145
  101. examples/local/durable/06_idempotency.py +0 -148
  102. examples/local/durable/07_hooks.py +0 -334
  103. examples/local/durable/08_cancellation.py +0 -233
  104. examples/local/durable/09_child_workflows.py +0 -198
  105. examples/local/durable/10_child_workflow_patterns.py +0 -265
  106. examples/local/durable/11_continue_as_new.py +0 -249
  107. examples/local/durable/12_schedules.py +0 -198
  108. examples/local/durable/__init__.py +0 -1
  109. examples/local/transient/01_quick_tasks.py +0 -87
  110. examples/local/transient/02_retries.py +0 -130
  111. examples/local/transient/03_sleep.py +0 -141
  112. examples/local/transient/__init__.py +0 -1
  113. pyworkflow_engine-0.1.7.dist-info/RECORD +0 -196
  114. pyworkflow_engine-0.1.7.dist-info/top_level.txt +0 -5
  115. tests/examples/__init__.py +0 -0
  116. tests/integration/__init__.py +0 -0
  117. tests/integration/test_cancellation.py +0 -330
  118. tests/integration/test_child_workflows.py +0 -439
  119. tests/integration/test_continue_as_new.py +0 -428
  120. tests/integration/test_dynamodb_storage.py +0 -1146
  121. tests/integration/test_fault_tolerance.py +0 -369
  122. tests/integration/test_schedule_storage.py +0 -484
  123. tests/unit/__init__.py +0 -0
  124. tests/unit/backends/__init__.py +0 -1
  125. tests/unit/backends/test_dynamodb_storage.py +0 -1554
  126. tests/unit/backends/test_postgres_storage.py +0 -1281
  127. tests/unit/backends/test_sqlite_storage.py +0 -1460
  128. tests/unit/conftest.py +0 -41
  129. tests/unit/test_cancellation.py +0 -364
  130. tests/unit/test_child_workflows.py +0 -680
  131. tests/unit/test_continue_as_new.py +0 -441
  132. tests/unit/test_event_limits.py +0 -316
  133. tests/unit/test_executor.py +0 -320
  134. tests/unit/test_fault_tolerance.py +0 -334
  135. tests/unit/test_hooks.py +0 -495
  136. tests/unit/test_registry.py +0 -261
  137. tests/unit/test_replay.py +0 -420
  138. tests/unit/test_schedule_schemas.py +0 -285
  139. tests/unit/test_schedule_utils.py +0 -286
  140. tests/unit/test_scheduled_workflow.py +0 -274
  141. tests/unit/test_step.py +0 -353
  142. tests/unit/test_workflow.py +0 -243
  143. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/WHEEL +0 -0
  144. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/entry_points.txt +0 -0
  145. {pyworkflow_engine-0.1.7.dist-info → pyworkflow_engine-0.1.9.dist-info}/licenses/LICENSE +0 -0
@@ -1,240 +0,0 @@
1
- """Service layer for workflow run operations."""
2
-
3
- import json
4
- from datetime import UTC, datetime
5
- from typing import Any
6
-
7
- import pyworkflow
8
- from app.repositories.run_repository import RunRepository
9
- from app.schemas.event import EventListResponse, EventResponse
10
- from app.schemas.run import (
11
- RunDetailResponse,
12
- RunListResponse,
13
- RunResponse,
14
- StartRunRequest,
15
- StartRunResponse,
16
- )
17
- from pyworkflow.storage.schemas import RunStatus, WorkflowRun
18
-
19
-
20
- class RunService:
21
- """Service for workflow run-related business logic."""
22
-
23
- def __init__(self, repository: RunRepository):
24
- """Initialize with run repository.
25
-
26
- Args:
27
- repository: RunRepository instance.
28
- """
29
- self.repository = repository
30
-
31
- async def list_runs(
32
- self,
33
- query: str | None = None,
34
- status: str | None = None,
35
- start_time: datetime | None = None,
36
- end_time: datetime | None = None,
37
- limit: int = 100,
38
- cursor: str | None = None,
39
- ) -> RunListResponse:
40
- """List workflow runs with optional filtering and cursor-based pagination.
41
-
42
- Args:
43
- query: Case-insensitive search in workflow name and input kwargs.
44
- status: Filter by status string.
45
- start_time: Filter runs started at or after this time.
46
- end_time: Filter runs started before this time.
47
- limit: Maximum number of results.
48
- cursor: Run ID to start after (for pagination).
49
-
50
- Returns:
51
- RunListResponse with list of runs and next_cursor.
52
- """
53
- status_enum = RunStatus(status) if status else None
54
-
55
- runs, next_cursor = await self.repository.list_runs(
56
- query=query,
57
- status=status_enum,
58
- start_time=start_time,
59
- end_time=end_time,
60
- limit=limit,
61
- cursor=cursor,
62
- )
63
-
64
- items = [self._run_to_response(run) for run in runs]
65
-
66
- return RunListResponse(
67
- items=items,
68
- count=len(items),
69
- limit=limit,
70
- next_cursor=next_cursor,
71
- )
72
-
73
- async def get_run(self, run_id: str) -> RunDetailResponse | None:
74
- """Get detailed information about a workflow run.
75
-
76
- Args:
77
- run_id: The run ID.
78
-
79
- Returns:
80
- RunDetailResponse if found, None otherwise.
81
- """
82
- run = await self.repository.get_run(run_id)
83
-
84
- if run is None:
85
- return None
86
-
87
- return self._run_to_detail_response(run)
88
-
89
- async def get_events(self, run_id: str) -> EventListResponse:
90
- """Get all events for a workflow run.
91
-
92
- Args:
93
- run_id: The run ID.
94
-
95
- Returns:
96
- EventListResponse with list of events.
97
- """
98
- events = await self.repository.get_events(run_id)
99
-
100
- items = [
101
- EventResponse(
102
- event_id=event.event_id,
103
- run_id=event.run_id,
104
- type=event.type.value,
105
- timestamp=event.timestamp,
106
- sequence=event.sequence,
107
- data=event.data,
108
- )
109
- for event in events
110
- ]
111
-
112
- return EventListResponse(
113
- items=items,
114
- count=len(items),
115
- )
116
-
117
- def _run_to_response(self, run: WorkflowRun) -> RunResponse:
118
- """Convert WorkflowRun to RunResponse.
119
-
120
- Args:
121
- run: WorkflowRun instance.
122
-
123
- Returns:
124
- RunResponse instance.
125
- """
126
- return RunResponse(
127
- run_id=run.run_id,
128
- workflow_name=run.workflow_name,
129
- status=run.status.value,
130
- created_at=run.created_at,
131
- started_at=run.started_at,
132
- completed_at=run.completed_at,
133
- duration_seconds=self._calculate_duration(run.started_at, run.completed_at),
134
- error=run.error,
135
- recovery_attempts=run.recovery_attempts,
136
- )
137
-
138
- def _run_to_detail_response(self, run: WorkflowRun) -> RunDetailResponse:
139
- """Convert WorkflowRun to RunDetailResponse.
140
-
141
- Args:
142
- run: WorkflowRun instance.
143
-
144
- Returns:
145
- RunDetailResponse instance.
146
- """
147
- # Parse JSON strings for input/result
148
- input_args = self._safe_json_parse(run.input_args)
149
- input_kwargs = self._safe_json_parse(run.input_kwargs)
150
- result = self._safe_json_parse(run.result) if run.result else None
151
-
152
- return RunDetailResponse(
153
- run_id=run.run_id,
154
- workflow_name=run.workflow_name,
155
- status=run.status.value,
156
- created_at=run.created_at,
157
- started_at=run.started_at,
158
- completed_at=run.completed_at,
159
- duration_seconds=self._calculate_duration(run.started_at, run.completed_at),
160
- error=run.error,
161
- recovery_attempts=run.recovery_attempts,
162
- input_args=input_args,
163
- input_kwargs=input_kwargs,
164
- result=result,
165
- metadata=run.metadata,
166
- max_duration=run.max_duration,
167
- max_recovery_attempts=run.max_recovery_attempts,
168
- )
169
-
170
- def _calculate_duration(
171
- self,
172
- started_at: datetime | None,
173
- completed_at: datetime | None,
174
- ) -> float | None:
175
- """Calculate duration in seconds.
176
-
177
- Args:
178
- started_at: Start timestamp.
179
- completed_at: Completion timestamp.
180
-
181
- Returns:
182
- Duration in seconds, or None if not calculable.
183
- """
184
- if started_at is None:
185
- return None
186
-
187
- end_time = completed_at or datetime.now(UTC)
188
-
189
- # Handle timezone-naive datetimes
190
- if started_at.tzinfo is None:
191
- started_at = started_at.replace(tzinfo=UTC)
192
- if end_time.tzinfo is None:
193
- end_time = end_time.replace(tzinfo=UTC)
194
-
195
- return (end_time - started_at).total_seconds()
196
-
197
- def _safe_json_parse(self, value: str | None) -> Any:
198
- """Safely parse a JSON string.
199
-
200
- Args:
201
- value: JSON string or None.
202
-
203
- Returns:
204
- Parsed value or the original string if parsing fails.
205
- """
206
- if value is None:
207
- return None
208
-
209
- try:
210
- return json.loads(value)
211
- except (json.JSONDecodeError, TypeError):
212
- return value
213
-
214
- async def start_run(self, request: StartRunRequest) -> StartRunResponse:
215
- """Start a new workflow run.
216
-
217
- Args:
218
- request: The start run request containing workflow name and kwargs.
219
-
220
- Returns:
221
- StartRunResponse with run_id and workflow_name.
222
-
223
- Raises:
224
- ValueError: If workflow not found.
225
- """
226
- # Get the workflow metadata
227
- workflow_meta = pyworkflow.get_workflow(request.workflow_name)
228
- if workflow_meta is None:
229
- raise ValueError(f"Workflow '{request.workflow_name}' not found")
230
-
231
- # Start the workflow using pyworkflow.start()
232
- run_id = await pyworkflow.start(
233
- workflow_meta.func,
234
- **request.kwargs,
235
- )
236
-
237
- return StartRunResponse(
238
- run_id=run_id,
239
- workflow_name=request.workflow_name,
240
- )
@@ -1,155 +0,0 @@
1
- """Service layer for workflow operations."""
2
-
3
- import inspect
4
- from typing import Any, get_origin, get_type_hints
5
-
6
- from app.repositories.workflow_repository import WorkflowRepository
7
- from app.schemas.workflow import (
8
- WorkflowListResponse,
9
- WorkflowParameter,
10
- WorkflowResponse,
11
- )
12
-
13
-
14
- def _get_type_name(type_hint: Any) -> str:
15
- """Convert a Python type hint to a simple type name for the frontend.
16
-
17
- Args:
18
- type_hint: The type hint to convert.
19
-
20
- Returns:
21
- A string representing the type ("string", "number", "boolean", "array", "object", "any").
22
- """
23
- if type_hint is Any or type_hint is inspect.Parameter.empty:
24
- return "any"
25
-
26
- # Handle None type
27
- if type_hint is type(None):
28
- return "any"
29
-
30
- # Get the origin for generic types (e.g., list[str] -> list)
31
- origin = get_origin(type_hint)
32
- if origin is not None:
33
- if origin is list:
34
- return "array"
35
- if origin is dict:
36
- return "object"
37
- # Union types (Optional, etc.)
38
- if origin is type(None):
39
- return "any"
40
-
41
- # Handle basic types
42
- if hasattr(type_hint, "__name__"):
43
- type_name = type_hint.__name__
44
- if type_name == "str":
45
- return "string"
46
- if type_name in ("int", "float"):
47
- return "number"
48
- if type_name == "bool":
49
- return "boolean"
50
- if type_name == "list":
51
- return "array"
52
- if type_name == "dict":
53
- return "object"
54
-
55
- return "any"
56
-
57
-
58
- def _extract_workflow_parameters(func: Any) -> list[WorkflowParameter]:
59
- """Extract parameter information from a workflow function.
60
-
61
- Args:
62
- func: The workflow function to inspect.
63
-
64
- Returns:
65
- List of WorkflowParameter objects.
66
- """
67
- sig = inspect.signature(func)
68
- params = []
69
-
70
- # Try to get type hints
71
- try:
72
- hints = get_type_hints(func)
73
- except Exception:
74
- hints = {}
75
-
76
- for param_name, param in sig.parameters.items():
77
- # Skip *args and **kwargs
78
- if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
79
- continue
80
-
81
- type_hint = hints.get(param_name, Any)
82
- has_default = param.default is not inspect.Parameter.empty
83
-
84
- # Serialize the default value
85
- default_value = None
86
- if has_default:
87
- default_value = param.default
88
-
89
- param_info = WorkflowParameter(
90
- name=param_name,
91
- type=_get_type_name(type_hint),
92
- required=not has_default,
93
- default=default_value,
94
- )
95
- params.append(param_info)
96
-
97
- return params
98
-
99
-
100
- class WorkflowService:
101
- """Service for workflow-related business logic."""
102
-
103
- def __init__(self, repository: WorkflowRepository):
104
- """Initialize with workflow repository.
105
-
106
- Args:
107
- repository: WorkflowRepository instance.
108
- """
109
- self.repository = repository
110
-
111
- def list_workflows(self) -> WorkflowListResponse:
112
- """Get all registered workflows.
113
-
114
- Returns:
115
- WorkflowListResponse with list of workflows.
116
- """
117
- workflows = self.repository.list_all()
118
-
119
- items = [
120
- WorkflowResponse(
121
- name=name,
122
- description=metadata.description,
123
- max_duration=metadata.max_duration,
124
- tags=metadata.tags or [],
125
- parameters=_extract_workflow_parameters(metadata.original_func),
126
- )
127
- for name, metadata in workflows.items()
128
- ]
129
-
130
- return WorkflowListResponse(
131
- items=items,
132
- count=len(items),
133
- )
134
-
135
- def get_workflow(self, name: str) -> WorkflowResponse | None:
136
- """Get a specific workflow by name.
137
-
138
- Args:
139
- name: Workflow name.
140
-
141
- Returns:
142
- WorkflowResponse if found, None otherwise.
143
- """
144
- metadata = self.repository.get_by_name(name)
145
-
146
- if metadata is None:
147
- return None
148
-
149
- return WorkflowResponse(
150
- name=metadata.name,
151
- description=metadata.description,
152
- max_duration=metadata.max_duration,
153
- tags=metadata.tags or [],
154
- parameters=_extract_workflow_parameters(metadata.original_func),
155
- )
dashboard/backend/main.py DELETED
@@ -1,18 +0,0 @@
1
- """CLI entry point for the dashboard backend server."""
2
-
3
- import uvicorn
4
- from app.config import settings
5
-
6
-
7
- def main():
8
- """Run the dashboard backend server."""
9
- uvicorn.run(
10
- "app.server:app",
11
- host=settings.host,
12
- port=settings.port,
13
- reload=settings.debug,
14
- )
15
-
16
-
17
- if __name__ == "__main__":
18
- main()