pyworkflow-engine 0.1.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dashboard/backend/app/__init__.py +1 -0
- dashboard/backend/app/config.py +32 -0
- dashboard/backend/app/controllers/__init__.py +6 -0
- dashboard/backend/app/controllers/run_controller.py +86 -0
- dashboard/backend/app/controllers/workflow_controller.py +33 -0
- dashboard/backend/app/dependencies/__init__.py +5 -0
- dashboard/backend/app/dependencies/storage.py +50 -0
- dashboard/backend/app/repositories/__init__.py +6 -0
- dashboard/backend/app/repositories/run_repository.py +80 -0
- dashboard/backend/app/repositories/workflow_repository.py +27 -0
- dashboard/backend/app/rest/__init__.py +8 -0
- dashboard/backend/app/rest/v1/__init__.py +12 -0
- dashboard/backend/app/rest/v1/health.py +33 -0
- dashboard/backend/app/rest/v1/runs.py +133 -0
- dashboard/backend/app/rest/v1/workflows.py +41 -0
- dashboard/backend/app/schemas/__init__.py +23 -0
- dashboard/backend/app/schemas/common.py +16 -0
- dashboard/backend/app/schemas/event.py +24 -0
- dashboard/backend/app/schemas/hook.py +25 -0
- dashboard/backend/app/schemas/run.py +54 -0
- dashboard/backend/app/schemas/step.py +28 -0
- dashboard/backend/app/schemas/workflow.py +31 -0
- dashboard/backend/app/server.py +87 -0
- dashboard/backend/app/services/__init__.py +6 -0
- dashboard/backend/app/services/run_service.py +240 -0
- dashboard/backend/app/services/workflow_service.py +155 -0
- dashboard/backend/main.py +18 -0
- docs/concepts/cancellation.mdx +362 -0
- docs/concepts/continue-as-new.mdx +434 -0
- docs/concepts/events.mdx +266 -0
- docs/concepts/fault-tolerance.mdx +370 -0
- docs/concepts/hooks.mdx +552 -0
- docs/concepts/limitations.mdx +167 -0
- docs/concepts/schedules.mdx +775 -0
- docs/concepts/sleep.mdx +312 -0
- docs/concepts/steps.mdx +301 -0
- docs/concepts/workflows.mdx +255 -0
- docs/guides/cli.mdx +942 -0
- docs/guides/configuration.mdx +560 -0
- docs/introduction.mdx +155 -0
- docs/quickstart.mdx +279 -0
- examples/__init__.py +1 -0
- examples/celery/__init__.py +1 -0
- examples/celery/durable/docker-compose.yml +55 -0
- examples/celery/durable/pyworkflow.config.yaml +12 -0
- examples/celery/durable/workflows/__init__.py +122 -0
- examples/celery/durable/workflows/basic.py +87 -0
- examples/celery/durable/workflows/batch_processing.py +102 -0
- examples/celery/durable/workflows/cancellation.py +273 -0
- examples/celery/durable/workflows/child_workflow_patterns.py +240 -0
- examples/celery/durable/workflows/child_workflows.py +202 -0
- examples/celery/durable/workflows/continue_as_new.py +260 -0
- examples/celery/durable/workflows/fault_tolerance.py +210 -0
- examples/celery/durable/workflows/hooks.py +211 -0
- examples/celery/durable/workflows/idempotency.py +112 -0
- examples/celery/durable/workflows/long_running.py +99 -0
- examples/celery/durable/workflows/retries.py +101 -0
- examples/celery/durable/workflows/schedules.py +209 -0
- examples/celery/transient/01_basic_workflow.py +91 -0
- examples/celery/transient/02_fault_tolerance.py +257 -0
- examples/celery/transient/__init__.py +20 -0
- examples/celery/transient/pyworkflow.config.yaml +25 -0
- examples/local/__init__.py +1 -0
- examples/local/durable/01_basic_workflow.py +94 -0
- examples/local/durable/02_file_storage.py +132 -0
- examples/local/durable/03_retries.py +169 -0
- examples/local/durable/04_long_running.py +119 -0
- examples/local/durable/05_event_log.py +145 -0
- examples/local/durable/06_idempotency.py +148 -0
- examples/local/durable/07_hooks.py +334 -0
- examples/local/durable/08_cancellation.py +233 -0
- examples/local/durable/09_child_workflows.py +198 -0
- examples/local/durable/10_child_workflow_patterns.py +265 -0
- examples/local/durable/11_continue_as_new.py +249 -0
- examples/local/durable/12_schedules.py +198 -0
- examples/local/durable/__init__.py +1 -0
- examples/local/transient/01_quick_tasks.py +87 -0
- examples/local/transient/02_retries.py +130 -0
- examples/local/transient/03_sleep.py +141 -0
- examples/local/transient/__init__.py +1 -0
- pyworkflow/__init__.py +256 -0
- pyworkflow/aws/__init__.py +68 -0
- pyworkflow/aws/context.py +234 -0
- pyworkflow/aws/handler.py +184 -0
- pyworkflow/aws/testing.py +310 -0
- pyworkflow/celery/__init__.py +41 -0
- pyworkflow/celery/app.py +198 -0
- pyworkflow/celery/scheduler.py +315 -0
- pyworkflow/celery/tasks.py +1746 -0
- pyworkflow/cli/__init__.py +132 -0
- pyworkflow/cli/__main__.py +6 -0
- pyworkflow/cli/commands/__init__.py +1 -0
- pyworkflow/cli/commands/hooks.py +640 -0
- pyworkflow/cli/commands/quickstart.py +495 -0
- pyworkflow/cli/commands/runs.py +773 -0
- pyworkflow/cli/commands/scheduler.py +130 -0
- pyworkflow/cli/commands/schedules.py +794 -0
- pyworkflow/cli/commands/setup.py +703 -0
- pyworkflow/cli/commands/worker.py +413 -0
- pyworkflow/cli/commands/workflows.py +1257 -0
- pyworkflow/cli/output/__init__.py +1 -0
- pyworkflow/cli/output/formatters.py +321 -0
- pyworkflow/cli/output/styles.py +121 -0
- pyworkflow/cli/utils/__init__.py +1 -0
- pyworkflow/cli/utils/async_helpers.py +30 -0
- pyworkflow/cli/utils/config.py +130 -0
- pyworkflow/cli/utils/config_generator.py +344 -0
- pyworkflow/cli/utils/discovery.py +53 -0
- pyworkflow/cli/utils/docker_manager.py +651 -0
- pyworkflow/cli/utils/interactive.py +364 -0
- pyworkflow/cli/utils/storage.py +115 -0
- pyworkflow/config.py +329 -0
- pyworkflow/context/__init__.py +63 -0
- pyworkflow/context/aws.py +230 -0
- pyworkflow/context/base.py +416 -0
- pyworkflow/context/local.py +930 -0
- pyworkflow/context/mock.py +381 -0
- pyworkflow/core/__init__.py +0 -0
- pyworkflow/core/exceptions.py +353 -0
- pyworkflow/core/registry.py +313 -0
- pyworkflow/core/scheduled.py +328 -0
- pyworkflow/core/step.py +494 -0
- pyworkflow/core/workflow.py +294 -0
- pyworkflow/discovery.py +248 -0
- pyworkflow/engine/__init__.py +0 -0
- pyworkflow/engine/events.py +879 -0
- pyworkflow/engine/executor.py +682 -0
- pyworkflow/engine/replay.py +273 -0
- pyworkflow/observability/__init__.py +19 -0
- pyworkflow/observability/logging.py +234 -0
- pyworkflow/primitives/__init__.py +33 -0
- pyworkflow/primitives/child_handle.py +174 -0
- pyworkflow/primitives/child_workflow.py +372 -0
- pyworkflow/primitives/continue_as_new.py +101 -0
- pyworkflow/primitives/define_hook.py +150 -0
- pyworkflow/primitives/hooks.py +97 -0
- pyworkflow/primitives/resume_hook.py +210 -0
- pyworkflow/primitives/schedule.py +545 -0
- pyworkflow/primitives/shield.py +96 -0
- pyworkflow/primitives/sleep.py +100 -0
- pyworkflow/runtime/__init__.py +21 -0
- pyworkflow/runtime/base.py +179 -0
- pyworkflow/runtime/celery.py +310 -0
- pyworkflow/runtime/factory.py +101 -0
- pyworkflow/runtime/local.py +706 -0
- pyworkflow/scheduler/__init__.py +9 -0
- pyworkflow/scheduler/local.py +248 -0
- pyworkflow/serialization/__init__.py +0 -0
- pyworkflow/serialization/decoder.py +146 -0
- pyworkflow/serialization/encoder.py +162 -0
- pyworkflow/storage/__init__.py +54 -0
- pyworkflow/storage/base.py +612 -0
- pyworkflow/storage/config.py +185 -0
- pyworkflow/storage/dynamodb.py +1315 -0
- pyworkflow/storage/file.py +827 -0
- pyworkflow/storage/memory.py +549 -0
- pyworkflow/storage/postgres.py +1161 -0
- pyworkflow/storage/schemas.py +486 -0
- pyworkflow/storage/sqlite.py +1136 -0
- pyworkflow/utils/__init__.py +0 -0
- pyworkflow/utils/duration.py +177 -0
- pyworkflow/utils/schedule.py +391 -0
- pyworkflow_engine-0.1.7.dist-info/METADATA +687 -0
- pyworkflow_engine-0.1.7.dist-info/RECORD +196 -0
- pyworkflow_engine-0.1.7.dist-info/WHEEL +5 -0
- pyworkflow_engine-0.1.7.dist-info/entry_points.txt +2 -0
- pyworkflow_engine-0.1.7.dist-info/licenses/LICENSE +21 -0
- pyworkflow_engine-0.1.7.dist-info/top_level.txt +5 -0
- tests/examples/__init__.py +0 -0
- tests/integration/__init__.py +0 -0
- tests/integration/test_cancellation.py +330 -0
- tests/integration/test_child_workflows.py +439 -0
- tests/integration/test_continue_as_new.py +428 -0
- tests/integration/test_dynamodb_storage.py +1146 -0
- tests/integration/test_fault_tolerance.py +369 -0
- tests/integration/test_schedule_storage.py +484 -0
- tests/unit/__init__.py +0 -0
- tests/unit/backends/__init__.py +1 -0
- tests/unit/backends/test_dynamodb_storage.py +1554 -0
- tests/unit/backends/test_postgres_storage.py +1281 -0
- tests/unit/backends/test_sqlite_storage.py +1460 -0
- tests/unit/conftest.py +41 -0
- tests/unit/test_cancellation.py +364 -0
- tests/unit/test_child_workflows.py +680 -0
- tests/unit/test_continue_as_new.py +441 -0
- tests/unit/test_event_limits.py +316 -0
- tests/unit/test_executor.py +320 -0
- tests/unit/test_fault_tolerance.py +334 -0
- tests/unit/test_hooks.py +495 -0
- tests/unit/test_registry.py +261 -0
- tests/unit/test_replay.py +420 -0
- tests/unit/test_schedule_schemas.py +285 -0
- tests/unit/test_schedule_utils.py +286 -0
- tests/unit/test_scheduled_workflow.py +274 -0
- tests/unit/test_step.py +353 -0
- tests/unit/test_workflow.py +243 -0
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Interactive prompt utilities using InquirerPy.
|
|
3
|
+
|
|
4
|
+
This module provides reusable prompt functions for interactive CLI workflows,
|
|
5
|
+
all styled with PyWorkflow branding.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from collections.abc import Callable
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
|
|
11
|
+
import click
|
|
12
|
+
from InquirerPy import inquirer
|
|
13
|
+
from InquirerPy.base.control import Choice
|
|
14
|
+
|
|
15
|
+
from pyworkflow.cli.output.styles import PYWORKFLOW_STYLE
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def confirm(message: str, default: bool = True) -> bool:
|
|
19
|
+
"""
|
|
20
|
+
Display a yes/no confirmation prompt.
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
message: Question to ask the user
|
|
24
|
+
default: Default answer if user just presses Enter
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
True if user confirms, False otherwise
|
|
28
|
+
|
|
29
|
+
Raises:
|
|
30
|
+
click.Abort: If user presses Ctrl+C
|
|
31
|
+
|
|
32
|
+
Example:
|
|
33
|
+
>>> if confirm("Continue with setup?"):
|
|
34
|
+
... print("Continuing...")
|
|
35
|
+
"""
|
|
36
|
+
try:
|
|
37
|
+
return inquirer.confirm(
|
|
38
|
+
message=message,
|
|
39
|
+
default=default,
|
|
40
|
+
style=PYWORKFLOW_STYLE,
|
|
41
|
+
).execute()
|
|
42
|
+
except KeyboardInterrupt:
|
|
43
|
+
raise click.Abort()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def select(
|
|
47
|
+
message: str,
|
|
48
|
+
choices: list[str] | list[dict[str, str]],
|
|
49
|
+
default: str | None = None,
|
|
50
|
+
) -> str:
|
|
51
|
+
"""
|
|
52
|
+
Display a single-selection list.
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
message: Question to ask the user
|
|
56
|
+
choices: List of options. Can be:
|
|
57
|
+
- List of strings: ["option1", "option2"]
|
|
58
|
+
- List of dicts: [{"name": "Display Text", "value": "value1"}]
|
|
59
|
+
default: Default selected value
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
Selected value
|
|
63
|
+
|
|
64
|
+
Raises:
|
|
65
|
+
click.Abort: If user presses Ctrl+C
|
|
66
|
+
|
|
67
|
+
Example:
|
|
68
|
+
>>> storage = select(
|
|
69
|
+
... "Choose storage:",
|
|
70
|
+
... choices=[
|
|
71
|
+
... {"name": "SQLite (recommended)", "value": "sqlite"},
|
|
72
|
+
... {"name": "File storage", "value": "file"}
|
|
73
|
+
... ]
|
|
74
|
+
... )
|
|
75
|
+
"""
|
|
76
|
+
try:
|
|
77
|
+
# Convert string list to Choice objects
|
|
78
|
+
choice_objects: list[Choice] = []
|
|
79
|
+
if choices and isinstance(choices[0], str):
|
|
80
|
+
for c in choices:
|
|
81
|
+
if isinstance(c, str):
|
|
82
|
+
choice_objects.append(Choice(value=c, name=c))
|
|
83
|
+
else:
|
|
84
|
+
# Dict format: {"name": "...", "value": "..."}
|
|
85
|
+
for c in choices:
|
|
86
|
+
if isinstance(c, dict):
|
|
87
|
+
choice_objects.append(Choice(value=c.get("value", c["name"]), name=c["name"]))
|
|
88
|
+
|
|
89
|
+
return inquirer.select(
|
|
90
|
+
message=message,
|
|
91
|
+
choices=choice_objects,
|
|
92
|
+
default=default,
|
|
93
|
+
style=PYWORKFLOW_STYLE,
|
|
94
|
+
).execute()
|
|
95
|
+
except KeyboardInterrupt:
|
|
96
|
+
raise click.Abort()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def input_text(
|
|
100
|
+
message: str,
|
|
101
|
+
default: str = "",
|
|
102
|
+
validate: Callable[[str], bool | str] | None = None,
|
|
103
|
+
) -> str:
|
|
104
|
+
"""
|
|
105
|
+
Display a text input prompt.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
message: Question or prompt text
|
|
109
|
+
default: Default value if user just presses Enter
|
|
110
|
+
validate: Optional validation function. Should return:
|
|
111
|
+
- True if valid
|
|
112
|
+
- False or error message string if invalid
|
|
113
|
+
|
|
114
|
+
Returns:
|
|
115
|
+
User's input text
|
|
116
|
+
|
|
117
|
+
Raises:
|
|
118
|
+
click.Abort: If user presses Ctrl+C
|
|
119
|
+
|
|
120
|
+
Example:
|
|
121
|
+
>>> def validate_module(value: str) -> bool | str:
|
|
122
|
+
... if not value:
|
|
123
|
+
... return True # Empty is OK
|
|
124
|
+
... if "." not in value:
|
|
125
|
+
... return "Module path should contain dots (e.g., myapp.workflows)"
|
|
126
|
+
... return True
|
|
127
|
+
>>>
|
|
128
|
+
>>> module = input_text(
|
|
129
|
+
... "Workflow module path:",
|
|
130
|
+
... default="myapp.workflows",
|
|
131
|
+
... validate=validate_module
|
|
132
|
+
... )
|
|
133
|
+
"""
|
|
134
|
+
try:
|
|
135
|
+
return inquirer.text(
|
|
136
|
+
message=message,
|
|
137
|
+
default=default,
|
|
138
|
+
validate=validate, # type: ignore[arg-type]
|
|
139
|
+
style=PYWORKFLOW_STYLE,
|
|
140
|
+
).execute()
|
|
141
|
+
except KeyboardInterrupt:
|
|
142
|
+
raise click.Abort()
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def filepath(
|
|
146
|
+
message: str,
|
|
147
|
+
default: str = "",
|
|
148
|
+
only_directories: bool = False,
|
|
149
|
+
) -> str:
|
|
150
|
+
"""
|
|
151
|
+
Display a filepath input prompt with tab completion.
|
|
152
|
+
|
|
153
|
+
Args:
|
|
154
|
+
message: Question or prompt text
|
|
155
|
+
default: Default path value
|
|
156
|
+
only_directories: If True, only allow directory paths
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
Selected or entered file path
|
|
160
|
+
|
|
161
|
+
Raises:
|
|
162
|
+
click.Abort: If user presses Ctrl+C
|
|
163
|
+
|
|
164
|
+
Example:
|
|
165
|
+
>>> db_path = filepath(
|
|
166
|
+
... "SQLite database path:",
|
|
167
|
+
... default="./pyworkflow_data/pyworkflow.db"
|
|
168
|
+
... )
|
|
169
|
+
>>>
|
|
170
|
+
>>> data_dir = filepath(
|
|
171
|
+
... "Data directory:",
|
|
172
|
+
... default="./pyworkflow_data",
|
|
173
|
+
... only_directories=True
|
|
174
|
+
... )
|
|
175
|
+
"""
|
|
176
|
+
try:
|
|
177
|
+
result = inquirer.filepath(
|
|
178
|
+
message=message,
|
|
179
|
+
default=default,
|
|
180
|
+
only_directories=only_directories,
|
|
181
|
+
style=PYWORKFLOW_STYLE,
|
|
182
|
+
).execute()
|
|
183
|
+
|
|
184
|
+
# Convert to absolute path and resolve
|
|
185
|
+
return str(Path(result).expanduser())
|
|
186
|
+
except KeyboardInterrupt:
|
|
187
|
+
raise click.Abort()
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def multiselect(
|
|
191
|
+
message: str,
|
|
192
|
+
choices: list[str] | list[dict[str, str]],
|
|
193
|
+
default: list[str] | None = None,
|
|
194
|
+
) -> list[str]:
|
|
195
|
+
"""
|
|
196
|
+
Display a multi-selection checkbox list.
|
|
197
|
+
|
|
198
|
+
Args:
|
|
199
|
+
message: Question to ask the user
|
|
200
|
+
choices: List of options. Can be:
|
|
201
|
+
- List of strings: ["option1", "option2"]
|
|
202
|
+
- List of dicts: [{"name": "Display Text", "value": "value1"}]
|
|
203
|
+
default: List of default selected values
|
|
204
|
+
|
|
205
|
+
Returns:
|
|
206
|
+
List of selected values
|
|
207
|
+
|
|
208
|
+
Raises:
|
|
209
|
+
click.Abort: If user presses Ctrl+C
|
|
210
|
+
|
|
211
|
+
Example:
|
|
212
|
+
>>> features = multiselect(
|
|
213
|
+
... "Select features to enable:",
|
|
214
|
+
... choices=[
|
|
215
|
+
... {"name": "Dashboard", "value": "dashboard"},
|
|
216
|
+
... {"name": "Monitoring", "value": "monitoring"},
|
|
217
|
+
... {"name": "Metrics", "value": "metrics"}
|
|
218
|
+
... ],
|
|
219
|
+
... default=["dashboard"]
|
|
220
|
+
... )
|
|
221
|
+
"""
|
|
222
|
+
try:
|
|
223
|
+
# Convert string list to Choice objects
|
|
224
|
+
choice_objects: list[Choice] = []
|
|
225
|
+
if choices and isinstance(choices[0], str):
|
|
226
|
+
for c in choices:
|
|
227
|
+
if isinstance(c, str):
|
|
228
|
+
choice_objects.append(Choice(value=c, name=c, enabled=(c in (default or []))))
|
|
229
|
+
else:
|
|
230
|
+
# Dict format: {"name": "...", "value": "..."}
|
|
231
|
+
for c in choices:
|
|
232
|
+
if isinstance(c, dict):
|
|
233
|
+
value = c.get("value", c["name"])
|
|
234
|
+
choice_objects.append(
|
|
235
|
+
Choice(
|
|
236
|
+
value=value,
|
|
237
|
+
name=c["name"],
|
|
238
|
+
enabled=(value in (default or [])),
|
|
239
|
+
)
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
return inquirer.checkbox(
|
|
243
|
+
message=message,
|
|
244
|
+
choices=choice_objects,
|
|
245
|
+
style=PYWORKFLOW_STYLE,
|
|
246
|
+
).execute()
|
|
247
|
+
except KeyboardInterrupt:
|
|
248
|
+
raise click.Abort()
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def password(message: str, validate: Callable[[str], bool | str] | None = None) -> str:
|
|
252
|
+
"""
|
|
253
|
+
Display a password input prompt (hidden text).
|
|
254
|
+
|
|
255
|
+
Args:
|
|
256
|
+
message: Question or prompt text
|
|
257
|
+
validate: Optional validation function
|
|
258
|
+
|
|
259
|
+
Returns:
|
|
260
|
+
Entered password
|
|
261
|
+
|
|
262
|
+
Raises:
|
|
263
|
+
click.Abort: If user presses Ctrl+C
|
|
264
|
+
|
|
265
|
+
Example:
|
|
266
|
+
>>> def validate_password(value: str) -> bool | str:
|
|
267
|
+
... if len(value) < 8:
|
|
268
|
+
... return "Password must be at least 8 characters"
|
|
269
|
+
... return True
|
|
270
|
+
>>>
|
|
271
|
+
>>> pwd = password("Enter password:", validate=validate_password)
|
|
272
|
+
"""
|
|
273
|
+
try:
|
|
274
|
+
return inquirer.secret(
|
|
275
|
+
message=message,
|
|
276
|
+
validate=validate, # type: ignore[arg-type]
|
|
277
|
+
style=PYWORKFLOW_STYLE,
|
|
278
|
+
).execute()
|
|
279
|
+
except KeyboardInterrupt:
|
|
280
|
+
raise click.Abort()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# Validation helper functions
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def validate_module_path(value: str) -> bool | str:
|
|
287
|
+
"""
|
|
288
|
+
Validate Python module path.
|
|
289
|
+
|
|
290
|
+
Args:
|
|
291
|
+
value: Module path string (e.g., "myapp.workflows")
|
|
292
|
+
|
|
293
|
+
Returns:
|
|
294
|
+
True if valid, error message string if invalid
|
|
295
|
+
|
|
296
|
+
Example:
|
|
297
|
+
>>> validate_module_path("myapp.workflows")
|
|
298
|
+
True
|
|
299
|
+
>>> validate_module_path("invalid module")
|
|
300
|
+
'Module path cannot contain spaces'
|
|
301
|
+
"""
|
|
302
|
+
if not value:
|
|
303
|
+
return True # Empty is allowed
|
|
304
|
+
|
|
305
|
+
if " " in value:
|
|
306
|
+
return "Module path cannot contain spaces"
|
|
307
|
+
|
|
308
|
+
if not all(part.isidentifier() for part in value.split(".")):
|
|
309
|
+
return "Module path must be valid Python identifiers separated by dots"
|
|
310
|
+
|
|
311
|
+
return True
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
def validate_nonempty(value: str) -> bool | str:
|
|
315
|
+
"""
|
|
316
|
+
Validate that input is not empty.
|
|
317
|
+
|
|
318
|
+
Args:
|
|
319
|
+
value: Input string
|
|
320
|
+
|
|
321
|
+
Returns:
|
|
322
|
+
True if not empty, error message if empty
|
|
323
|
+
"""
|
|
324
|
+
if not value or not value.strip():
|
|
325
|
+
return "This field cannot be empty"
|
|
326
|
+
return True
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def validate_port(value: str) -> bool | str:
|
|
330
|
+
"""
|
|
331
|
+
Validate port number.
|
|
332
|
+
|
|
333
|
+
Args:
|
|
334
|
+
value: Port number as string
|
|
335
|
+
|
|
336
|
+
Returns:
|
|
337
|
+
True if valid port (1-65535), error message otherwise
|
|
338
|
+
"""
|
|
339
|
+
try:
|
|
340
|
+
port = int(value)
|
|
341
|
+
if 1 <= port <= 65535:
|
|
342
|
+
return True
|
|
343
|
+
return "Port must be between 1 and 65535"
|
|
344
|
+
except ValueError:
|
|
345
|
+
return "Port must be a number"
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def validate_url(value: str) -> bool | str:
|
|
349
|
+
"""
|
|
350
|
+
Validate URL format.
|
|
351
|
+
|
|
352
|
+
Args:
|
|
353
|
+
value: URL string
|
|
354
|
+
|
|
355
|
+
Returns:
|
|
356
|
+
True if valid URL format, error message otherwise
|
|
357
|
+
"""
|
|
358
|
+
if not value:
|
|
359
|
+
return "URL cannot be empty"
|
|
360
|
+
|
|
361
|
+
if not (value.startswith("http://") or value.startswith("https://")):
|
|
362
|
+
return "URL must start with http:// or https://"
|
|
363
|
+
|
|
364
|
+
return True
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Storage backend factory utilities."""
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from loguru import logger
|
|
6
|
+
|
|
7
|
+
from pyworkflow import StorageBackend
|
|
8
|
+
from pyworkflow.storage.config import config_to_storage
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def create_storage(
|
|
12
|
+
backend_type: str | None = None,
|
|
13
|
+
path: str | None = None,
|
|
14
|
+
config: dict[str, Any] | None = None,
|
|
15
|
+
) -> StorageBackend:
|
|
16
|
+
"""
|
|
17
|
+
Create storage backend from configuration.
|
|
18
|
+
|
|
19
|
+
Configuration priority:
|
|
20
|
+
1. CLI flags (backend_type, path arguments)
|
|
21
|
+
2. Environment variables (handled by Click)
|
|
22
|
+
3. Config file (config dict)
|
|
23
|
+
4. Default (file backend with ./workflow_data)
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
backend_type: Storage backend type ("file", "memory", "redis", "sqlite", "dynamodb")
|
|
27
|
+
path: Storage path (for file/sqlite backends)
|
|
28
|
+
config: Configuration dict from pyworkflow.toml
|
|
29
|
+
|
|
30
|
+
Returns:
|
|
31
|
+
Configured StorageBackend instance
|
|
32
|
+
|
|
33
|
+
Raises:
|
|
34
|
+
ValueError: If backend type is unsupported
|
|
35
|
+
|
|
36
|
+
Examples:
|
|
37
|
+
# File storage with explicit path
|
|
38
|
+
storage = create_storage(backend_type="file", path="./data")
|
|
39
|
+
|
|
40
|
+
# From config
|
|
41
|
+
config = {"storage": {"type": "file", "base_path": "./workflow_data"}}
|
|
42
|
+
storage = create_storage(config=config)
|
|
43
|
+
|
|
44
|
+
# Default (file storage)
|
|
45
|
+
storage = create_storage()
|
|
46
|
+
"""
|
|
47
|
+
# Resolve backend type with priority: CLI flag > config file > default
|
|
48
|
+
backend = backend_type
|
|
49
|
+
|
|
50
|
+
if not backend and config:
|
|
51
|
+
backend = config.get("storage", {}).get("type")
|
|
52
|
+
|
|
53
|
+
if not backend:
|
|
54
|
+
backend = "file" # Default
|
|
55
|
+
|
|
56
|
+
logger.debug(f"Creating storage backend: {backend}")
|
|
57
|
+
|
|
58
|
+
# Resolve storage path with priority: CLI flag > config file > default
|
|
59
|
+
storage_path = path
|
|
60
|
+
if not storage_path and config:
|
|
61
|
+
storage_path = config.get("storage", {}).get("base_path")
|
|
62
|
+
|
|
63
|
+
# Build unified config dict
|
|
64
|
+
storage_config: dict[str, Any] = {"type": backend}
|
|
65
|
+
if storage_path:
|
|
66
|
+
storage_config["base_path"] = storage_path
|
|
67
|
+
|
|
68
|
+
# Extract redis config if present
|
|
69
|
+
if backend == "redis" and config:
|
|
70
|
+
storage_section = config.get("storage", {})
|
|
71
|
+
if "host" in storage_section:
|
|
72
|
+
storage_config["host"] = storage_section["host"]
|
|
73
|
+
if "port" in storage_section:
|
|
74
|
+
storage_config["port"] = storage_section["port"]
|
|
75
|
+
if "db" in storage_section:
|
|
76
|
+
storage_config["db"] = storage_section["db"]
|
|
77
|
+
|
|
78
|
+
# Extract postgres config if present
|
|
79
|
+
if backend == "postgres" and config:
|
|
80
|
+
storage_section = config.get("storage", {})
|
|
81
|
+
if "dsn" in storage_section:
|
|
82
|
+
storage_config["dsn"] = storage_section["dsn"]
|
|
83
|
+
else:
|
|
84
|
+
if "host" in storage_section:
|
|
85
|
+
storage_config["host"] = storage_section["host"]
|
|
86
|
+
if "port" in storage_section:
|
|
87
|
+
storage_config["port"] = storage_section["port"]
|
|
88
|
+
if "user" in storage_section:
|
|
89
|
+
storage_config["user"] = storage_section["user"]
|
|
90
|
+
if "password" in storage_section:
|
|
91
|
+
storage_config["password"] = storage_section["password"]
|
|
92
|
+
if "database" in storage_section:
|
|
93
|
+
storage_config["database"] = storage_section["database"]
|
|
94
|
+
|
|
95
|
+
# Extract dynamodb config if present
|
|
96
|
+
if backend == "dynamodb" and config:
|
|
97
|
+
storage_section = config.get("storage", {})
|
|
98
|
+
if "table_name" in storage_section:
|
|
99
|
+
storage_config["table_name"] = storage_section["table_name"]
|
|
100
|
+
if "region" in storage_section:
|
|
101
|
+
storage_config["region"] = storage_section["region"]
|
|
102
|
+
if "endpoint_url" in storage_section:
|
|
103
|
+
storage_config["endpoint_url"] = storage_section["endpoint_url"]
|
|
104
|
+
|
|
105
|
+
# Use unified config_to_storage
|
|
106
|
+
storage = config_to_storage(storage_config)
|
|
107
|
+
|
|
108
|
+
# Log which backend was created
|
|
109
|
+
backend_name = storage.__class__.__name__
|
|
110
|
+
if hasattr(storage, "base_path"):
|
|
111
|
+
logger.info(f"Using {backend_name} with path: {storage.base_path}")
|
|
112
|
+
else:
|
|
113
|
+
logger.info(f"Using {backend_name}")
|
|
114
|
+
|
|
115
|
+
return storage
|