prefect-client 2.19.2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +8 -56
- prefect/_internal/compatibility/deprecated.py +6 -115
- prefect/_internal/compatibility/experimental.py +4 -79
- prefect/_internal/concurrency/api.py +0 -34
- prefect/_internal/concurrency/calls.py +0 -6
- prefect/_internal/concurrency/cancellation.py +0 -3
- prefect/_internal/concurrency/event_loop.py +0 -20
- prefect/_internal/concurrency/inspection.py +3 -3
- prefect/_internal/concurrency/threads.py +35 -0
- prefect/_internal/concurrency/waiters.py +0 -28
- prefect/_internal/pydantic/__init__.py +0 -45
- prefect/_internal/pydantic/v1_schema.py +21 -22
- prefect/_internal/pydantic/v2_schema.py +0 -2
- prefect/_internal/pydantic/v2_validated_func.py +18 -23
- prefect/_internal/schemas/bases.py +44 -177
- prefect/_internal/schemas/fields.py +1 -43
- prefect/_internal/schemas/validators.py +60 -158
- prefect/artifacts.py +161 -14
- prefect/automations.py +39 -4
- prefect/blocks/abstract.py +1 -1
- prefect/blocks/core.py +268 -148
- prefect/blocks/fields.py +2 -57
- prefect/blocks/kubernetes.py +8 -12
- prefect/blocks/notifications.py +40 -20
- prefect/blocks/system.py +22 -11
- prefect/blocks/webhook.py +2 -9
- prefect/client/base.py +4 -4
- prefect/client/cloud.py +8 -13
- prefect/client/orchestration.py +347 -341
- prefect/client/schemas/actions.py +92 -86
- prefect/client/schemas/filters.py +20 -40
- prefect/client/schemas/objects.py +151 -145
- prefect/client/schemas/responses.py +16 -24
- prefect/client/schemas/schedules.py +47 -35
- prefect/client/subscriptions.py +2 -2
- prefect/client/utilities.py +5 -2
- prefect/concurrency/asyncio.py +3 -1
- prefect/concurrency/events.py +1 -1
- prefect/concurrency/services.py +6 -3
- prefect/context.py +195 -27
- prefect/deployments/__init__.py +5 -6
- prefect/deployments/base.py +7 -5
- prefect/deployments/flow_runs.py +185 -0
- prefect/deployments/runner.py +50 -45
- prefect/deployments/schedules.py +28 -23
- prefect/deployments/steps/__init__.py +0 -1
- prefect/deployments/steps/core.py +1 -0
- prefect/deployments/steps/pull.py +7 -21
- prefect/engine.py +12 -2422
- prefect/events/actions.py +17 -23
- prefect/events/cli/automations.py +19 -6
- prefect/events/clients.py +14 -37
- prefect/events/filters.py +14 -18
- prefect/events/related.py +2 -2
- prefect/events/schemas/__init__.py +0 -5
- prefect/events/schemas/automations.py +55 -46
- prefect/events/schemas/deployment_triggers.py +7 -197
- prefect/events/schemas/events.py +34 -65
- prefect/events/schemas/labelling.py +10 -14
- prefect/events/utilities.py +2 -3
- prefect/events/worker.py +2 -3
- prefect/filesystems.py +6 -517
- prefect/{new_flow_engine.py → flow_engine.py} +313 -72
- prefect/flow_runs.py +377 -5
- prefect/flows.py +307 -166
- prefect/futures.py +186 -345
- prefect/infrastructure/__init__.py +0 -27
- prefect/infrastructure/provisioners/__init__.py +5 -3
- prefect/infrastructure/provisioners/cloud_run.py +11 -6
- prefect/infrastructure/provisioners/container_instance.py +11 -7
- prefect/infrastructure/provisioners/ecs.py +6 -4
- prefect/infrastructure/provisioners/modal.py +8 -5
- prefect/input/actions.py +2 -4
- prefect/input/run_input.py +5 -7
- prefect/logging/formatters.py +0 -2
- prefect/logging/handlers.py +3 -11
- prefect/logging/loggers.py +2 -2
- prefect/manifests.py +2 -1
- prefect/records/__init__.py +1 -0
- prefect/records/result_store.py +42 -0
- prefect/records/store.py +9 -0
- prefect/results.py +43 -39
- prefect/runner/runner.py +19 -15
- prefect/runner/server.py +6 -10
- prefect/runner/storage.py +3 -8
- prefect/runner/submit.py +2 -2
- prefect/runner/utils.py +2 -2
- prefect/serializers.py +24 -35
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
- prefect/settings.py +70 -133
- prefect/states.py +17 -47
- prefect/task_engine.py +697 -58
- prefect/task_runners.py +269 -301
- prefect/task_server.py +53 -34
- prefect/tasks.py +327 -337
- prefect/transactions.py +220 -0
- prefect/types/__init__.py +61 -82
- prefect/utilities/asyncutils.py +195 -136
- prefect/utilities/callables.py +311 -43
- prefect/utilities/collections.py +23 -38
- prefect/utilities/dispatch.py +11 -3
- prefect/utilities/dockerutils.py +4 -0
- prefect/utilities/engine.py +140 -20
- prefect/utilities/importtools.py +97 -27
- prefect/utilities/pydantic.py +128 -38
- prefect/utilities/schema_tools/hydration.py +5 -1
- prefect/utilities/templating.py +12 -2
- prefect/variables.py +78 -61
- prefect/workers/__init__.py +0 -1
- prefect/workers/base.py +15 -17
- prefect/workers/process.py +3 -8
- prefect/workers/server.py +2 -2
- {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/METADATA +22 -21
- prefect_client-3.0.0rc1.dist-info/RECORD +176 -0
- prefect/_internal/pydantic/_base_model.py +0 -51
- prefect/_internal/pydantic/_compat.py +0 -82
- prefect/_internal/pydantic/_flags.py +0 -20
- prefect/_internal/pydantic/_types.py +0 -8
- prefect/_internal/pydantic/utilities/__init__.py +0 -0
- prefect/_internal/pydantic/utilities/config_dict.py +0 -72
- prefect/_internal/pydantic/utilities/field_validator.py +0 -150
- prefect/_internal/pydantic/utilities/model_construct.py +0 -56
- prefect/_internal/pydantic/utilities/model_copy.py +0 -55
- prefect/_internal/pydantic/utilities/model_dump.py +0 -136
- prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
- prefect/_internal/pydantic/utilities/model_fields.py +0 -50
- prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
- prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
- prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
- prefect/_internal/pydantic/utilities/model_validate.py +0 -75
- prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
- prefect/_internal/pydantic/utilities/model_validator.py +0 -87
- prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
- prefect/_vendor/__init__.py +0 -0
- prefect/_vendor/fastapi/__init__.py +0 -25
- prefect/_vendor/fastapi/applications.py +0 -946
- prefect/_vendor/fastapi/background.py +0 -3
- prefect/_vendor/fastapi/concurrency.py +0 -44
- prefect/_vendor/fastapi/datastructures.py +0 -58
- prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
- prefect/_vendor/fastapi/dependencies/models.py +0 -64
- prefect/_vendor/fastapi/dependencies/utils.py +0 -877
- prefect/_vendor/fastapi/encoders.py +0 -177
- prefect/_vendor/fastapi/exception_handlers.py +0 -40
- prefect/_vendor/fastapi/exceptions.py +0 -46
- prefect/_vendor/fastapi/logger.py +0 -3
- prefect/_vendor/fastapi/middleware/__init__.py +0 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
- prefect/_vendor/fastapi/middleware/cors.py +0 -3
- prefect/_vendor/fastapi/middleware/gzip.py +0 -3
- prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
- prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
- prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
- prefect/_vendor/fastapi/openapi/__init__.py +0 -0
- prefect/_vendor/fastapi/openapi/constants.py +0 -2
- prefect/_vendor/fastapi/openapi/docs.py +0 -203
- prefect/_vendor/fastapi/openapi/models.py +0 -480
- prefect/_vendor/fastapi/openapi/utils.py +0 -485
- prefect/_vendor/fastapi/param_functions.py +0 -340
- prefect/_vendor/fastapi/params.py +0 -453
- prefect/_vendor/fastapi/requests.py +0 -4
- prefect/_vendor/fastapi/responses.py +0 -40
- prefect/_vendor/fastapi/routing.py +0 -1331
- prefect/_vendor/fastapi/security/__init__.py +0 -15
- prefect/_vendor/fastapi/security/api_key.py +0 -98
- prefect/_vendor/fastapi/security/base.py +0 -6
- prefect/_vendor/fastapi/security/http.py +0 -172
- prefect/_vendor/fastapi/security/oauth2.py +0 -227
- prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
- prefect/_vendor/fastapi/security/utils.py +0 -10
- prefect/_vendor/fastapi/staticfiles.py +0 -1
- prefect/_vendor/fastapi/templating.py +0 -3
- prefect/_vendor/fastapi/testclient.py +0 -1
- prefect/_vendor/fastapi/types.py +0 -3
- prefect/_vendor/fastapi/utils.py +0 -235
- prefect/_vendor/fastapi/websockets.py +0 -7
- prefect/_vendor/starlette/__init__.py +0 -1
- prefect/_vendor/starlette/_compat.py +0 -28
- prefect/_vendor/starlette/_exception_handler.py +0 -80
- prefect/_vendor/starlette/_utils.py +0 -88
- prefect/_vendor/starlette/applications.py +0 -261
- prefect/_vendor/starlette/authentication.py +0 -159
- prefect/_vendor/starlette/background.py +0 -43
- prefect/_vendor/starlette/concurrency.py +0 -59
- prefect/_vendor/starlette/config.py +0 -151
- prefect/_vendor/starlette/convertors.py +0 -87
- prefect/_vendor/starlette/datastructures.py +0 -707
- prefect/_vendor/starlette/endpoints.py +0 -130
- prefect/_vendor/starlette/exceptions.py +0 -60
- prefect/_vendor/starlette/formparsers.py +0 -276
- prefect/_vendor/starlette/middleware/__init__.py +0 -17
- prefect/_vendor/starlette/middleware/authentication.py +0 -52
- prefect/_vendor/starlette/middleware/base.py +0 -220
- prefect/_vendor/starlette/middleware/cors.py +0 -176
- prefect/_vendor/starlette/middleware/errors.py +0 -265
- prefect/_vendor/starlette/middleware/exceptions.py +0 -74
- prefect/_vendor/starlette/middleware/gzip.py +0 -113
- prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
- prefect/_vendor/starlette/middleware/sessions.py +0 -82
- prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
- prefect/_vendor/starlette/middleware/wsgi.py +0 -147
- prefect/_vendor/starlette/requests.py +0 -328
- prefect/_vendor/starlette/responses.py +0 -347
- prefect/_vendor/starlette/routing.py +0 -933
- prefect/_vendor/starlette/schemas.py +0 -154
- prefect/_vendor/starlette/staticfiles.py +0 -248
- prefect/_vendor/starlette/status.py +0 -199
- prefect/_vendor/starlette/templating.py +0 -231
- prefect/_vendor/starlette/testclient.py +0 -804
- prefect/_vendor/starlette/types.py +0 -30
- prefect/_vendor/starlette/websockets.py +0 -193
- prefect/agent.py +0 -698
- prefect/deployments/deployments.py +0 -1042
- prefect/deprecated/__init__.py +0 -0
- prefect/deprecated/data_documents.py +0 -350
- prefect/deprecated/packaging/__init__.py +0 -12
- prefect/deprecated/packaging/base.py +0 -96
- prefect/deprecated/packaging/docker.py +0 -146
- prefect/deprecated/packaging/file.py +0 -92
- prefect/deprecated/packaging/orion.py +0 -80
- prefect/deprecated/packaging/serializers.py +0 -171
- prefect/events/instrument.py +0 -135
- prefect/infrastructure/base.py +0 -323
- prefect/infrastructure/container.py +0 -818
- prefect/infrastructure/kubernetes.py +0 -920
- prefect/infrastructure/process.py +0 -289
- prefect/new_task_engine.py +0 -423
- prefect/pydantic/__init__.py +0 -76
- prefect/pydantic/main.py +0 -39
- prefect/software/__init__.py +0 -2
- prefect/software/base.py +0 -50
- prefect/software/conda.py +0 -199
- prefect/software/pip.py +0 -122
- prefect/software/python.py +0 -52
- prefect/workers/block.py +0 -218
- prefect_client-2.19.2.dist-info/RECORD +0 -292
- {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/LICENSE +0 -0
- {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/WHEEL +0 -0
- {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -5,9 +5,14 @@ Module containing the base workflow class and decorator - for most use cases, us
|
|
5
5
|
# This file requires type-checking with pyright because mypy does not yet support PEP612
|
6
6
|
# See https://github.com/python/mypy/issues/8645
|
7
7
|
|
8
|
+
import ast
|
8
9
|
import datetime
|
10
|
+
import importlib.util
|
9
11
|
import inspect
|
12
|
+
import json
|
10
13
|
import os
|
14
|
+
import re
|
15
|
+
import sys
|
11
16
|
import tempfile
|
12
17
|
import warnings
|
13
18
|
from functools import partial, update_wrapper
|
@@ -34,50 +39,38 @@ from typing import (
|
|
34
39
|
)
|
35
40
|
from uuid import UUID
|
36
41
|
|
42
|
+
import pydantic
|
43
|
+
from fastapi.encoders import jsonable_encoder
|
44
|
+
from pydantic.v1 import BaseModel as V1BaseModel
|
45
|
+
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
46
|
+
from pydantic.v1.errors import ConfigError # TODO
|
37
47
|
from rich.console import Console
|
38
48
|
from typing_extensions import Literal, ParamSpec, Self
|
39
49
|
|
40
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
41
|
-
|
42
|
-
if HAS_PYDANTIC_V2:
|
43
|
-
import pydantic.v1 as pydantic
|
44
|
-
from pydantic import ValidationError as V2ValidationError
|
45
|
-
from pydantic.v1 import BaseModel as V1BaseModel
|
46
|
-
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
47
|
-
|
48
|
-
from ._internal.pydantic.v2_schema import is_v2_type
|
49
|
-
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
50
|
-
from ._internal.pydantic.v2_validated_func import (
|
51
|
-
V2ValidatedFunction as ValidatedFunction,
|
52
|
-
)
|
53
|
-
|
54
|
-
else:
|
55
|
-
import pydantic
|
56
|
-
from pydantic.decorator import ValidatedFunction
|
57
|
-
|
58
|
-
V2ValidationError = None
|
59
|
-
|
60
|
-
from prefect._vendor.fastapi.encoders import jsonable_encoder
|
61
|
-
|
62
50
|
from prefect._internal.compatibility.deprecated import deprecated_parameter
|
63
51
|
from prefect._internal.concurrency.api import create_call, from_async
|
64
|
-
from prefect.
|
52
|
+
from prefect.blocks.core import Block
|
65
53
|
from prefect.client.orchestration import get_client
|
54
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
66
55
|
from prefect.client.schemas.objects import Flow as FlowSchema
|
67
|
-
from prefect.client.schemas.objects import FlowRun
|
56
|
+
from prefect.client.schemas.objects import FlowRun
|
68
57
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
58
|
+
from prefect.client.utilities import client_injector
|
69
59
|
from prefect.context import PrefectObjectRegistry, registry_from_script
|
70
60
|
from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
|
61
|
+
from prefect.deployments.steps.core import run_steps
|
71
62
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
72
63
|
from prefect.exceptions import (
|
64
|
+
InvalidNameError,
|
73
65
|
MissingFlowError,
|
74
66
|
ObjectNotFound,
|
75
67
|
ParameterTypeError,
|
76
68
|
UnspecifiedFlowError,
|
77
69
|
)
|
78
|
-
from prefect.filesystems import ReadableDeploymentStorage
|
70
|
+
from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
|
79
71
|
from prefect.futures import PrefectFuture
|
80
72
|
from prefect.logging import get_logger
|
73
|
+
from prefect.logging.loggers import flow_run_logger
|
81
74
|
from prefect.results import ResultSerializer, ResultStorage
|
82
75
|
from prefect.runner.storage import (
|
83
76
|
BlockStorageAdapter,
|
@@ -86,16 +79,20 @@ from prefect.runner.storage import (
|
|
86
79
|
)
|
87
80
|
from prefect.settings import (
|
88
81
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
89
|
-
PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE,
|
90
82
|
PREFECT_FLOW_DEFAULT_RETRIES,
|
91
83
|
PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS,
|
92
84
|
PREFECT_UI_URL,
|
93
85
|
PREFECT_UNIT_TEST_MODE,
|
94
86
|
)
|
95
87
|
from prefect.states import State
|
96
|
-
from prefect.task_runners import
|
88
|
+
from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
|
89
|
+
from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
|
97
90
|
from prefect.utilities.annotations import NotSet
|
98
|
-
from prefect.utilities.asyncutils import
|
91
|
+
from prefect.utilities.asyncutils import (
|
92
|
+
is_async_fn,
|
93
|
+
run_sync_in_worker_thread,
|
94
|
+
sync_compatible,
|
95
|
+
)
|
99
96
|
from prefect.utilities.callables import (
|
100
97
|
get_call_parameters,
|
101
98
|
parameter_schema,
|
@@ -103,18 +100,14 @@ from prefect.utilities.callables import (
|
|
103
100
|
raise_for_reserved_arguments,
|
104
101
|
)
|
105
102
|
from prefect.utilities.collections import listrepr
|
103
|
+
from prefect.utilities.filesystem import relative_path_to_current_platform
|
106
104
|
from prefect.utilities.hashing import file_hash
|
107
105
|
from prefect.utilities.importtools import import_object
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
VisualizationUnsupportedError,
|
114
|
-
build_task_dependencies,
|
115
|
-
get_task_viz_tracker,
|
116
|
-
track_viz_task,
|
117
|
-
visualize_task_dependencies,
|
106
|
+
|
107
|
+
from ._internal.pydantic.v2_schema import is_v2_type
|
108
|
+
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
109
|
+
from ._internal.pydantic.v2_validated_func import (
|
110
|
+
V2ValidatedFunction as ValidatedFunction,
|
118
111
|
)
|
119
112
|
|
120
113
|
T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
|
@@ -125,7 +118,9 @@ F = TypeVar("F", bound="Flow") # The type of the flow
|
|
125
118
|
logger = get_logger("flows")
|
126
119
|
|
127
120
|
if TYPE_CHECKING:
|
121
|
+
from prefect.client.orchestration import PrefectClient
|
128
122
|
from prefect.deployments.runner import FlexibleScheduleList, RunnerDeployment
|
123
|
+
from prefect.flows import FlowRun
|
129
124
|
|
130
125
|
|
131
126
|
@PrefectObjectRegistry.register_instances
|
@@ -196,9 +191,9 @@ class Flow(Generic[P, R]):
|
|
196
191
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
197
192
|
retries: Optional[int] = None,
|
198
193
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
199
|
-
task_runner: Union[Type[
|
200
|
-
description: str = None,
|
201
|
-
timeout_seconds: Union[int, float] = None,
|
194
|
+
task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
|
195
|
+
description: Optional[str] = None,
|
196
|
+
timeout_seconds: Union[int, float, None] = None,
|
202
197
|
validate_parameters: bool = True,
|
203
198
|
persist_result: Optional[bool] = None,
|
204
199
|
result_storage: Optional[ResultStorage] = None,
|
@@ -245,8 +240,6 @@ class Flow(Generic[P, R]):
|
|
245
240
|
]
|
246
241
|
for hooks, hook_name in zip(hook_categories, hook_names):
|
247
242
|
if hooks is not None:
|
248
|
-
if not hooks:
|
249
|
-
raise ValueError(f"Empty list passed for '{hook_name}'")
|
250
243
|
try:
|
251
244
|
hooks = list(hooks)
|
252
245
|
except TypeError:
|
@@ -273,7 +266,7 @@ class Flow(Generic[P, R]):
|
|
273
266
|
|
274
267
|
# Validate name if given
|
275
268
|
if name:
|
276
|
-
|
269
|
+
_raise_on_name_with_banned_characters(name)
|
277
270
|
|
278
271
|
self.name = name or fn.__name__.replace("_", "-")
|
279
272
|
|
@@ -285,7 +278,8 @@ class Flow(Generic[P, R]):
|
|
285
278
|
)
|
286
279
|
self.flow_run_name = flow_run_name
|
287
280
|
|
288
|
-
|
281
|
+
default_task_runner = ThreadPoolTaskRunner()
|
282
|
+
task_runner = task_runner or default_task_runner
|
289
283
|
self.task_runner = (
|
290
284
|
task_runner() if isinstance(task_runner, type) else task_runner
|
291
285
|
)
|
@@ -333,7 +327,7 @@ class Flow(Generic[P, R]):
|
|
333
327
|
# is not picklable in some environments
|
334
328
|
try:
|
335
329
|
ValidatedFunction(self.fn, config={"arbitrary_types_allowed": True})
|
336
|
-
except
|
330
|
+
except ConfigError as exc:
|
337
331
|
raise ValueError(
|
338
332
|
"Flow function is not compatible with `validate_parameters`. "
|
339
333
|
"Disable validation or change the argument names."
|
@@ -343,11 +337,11 @@ class Flow(Generic[P, R]):
|
|
343
337
|
self.result_storage = result_storage
|
344
338
|
self.result_serializer = result_serializer
|
345
339
|
self.cache_result_in_memory = cache_result_in_memory
|
346
|
-
self.
|
347
|
-
self.
|
348
|
-
self.
|
349
|
-
self.
|
350
|
-
self.
|
340
|
+
self.on_completion_hooks = on_completion or []
|
341
|
+
self.on_failure_hooks = on_failure or []
|
342
|
+
self.on_cancellation_hooks = on_cancellation or []
|
343
|
+
self.on_crashed_hooks = on_crashed or []
|
344
|
+
self.on_running_hooks = on_running or []
|
351
345
|
|
352
346
|
# Used for flows loaded from remote storage
|
353
347
|
self._storage: Optional[RunnerStorage] = None
|
@@ -363,20 +357,20 @@ class Flow(Generic[P, R]):
|
|
363
357
|
def with_options(
|
364
358
|
self,
|
365
359
|
*,
|
366
|
-
name: str = None,
|
367
|
-
version: str = None,
|
360
|
+
name: Optional[str] = None,
|
361
|
+
version: Optional[str] = None,
|
368
362
|
retries: Optional[int] = None,
|
369
363
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
370
|
-
description: str = None,
|
364
|
+
description: Optional[str] = None,
|
371
365
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
372
|
-
task_runner: Union[Type[
|
373
|
-
timeout_seconds: Union[int, float] = None,
|
374
|
-
validate_parameters: bool = None,
|
375
|
-
persist_result: Optional[bool] = NotSet,
|
376
|
-
result_storage: Optional[ResultStorage] = NotSet,
|
377
|
-
result_serializer: Optional[ResultSerializer] = NotSet,
|
378
|
-
cache_result_in_memory: bool = None,
|
379
|
-
log_prints: Optional[bool] = NotSet,
|
366
|
+
task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
|
367
|
+
timeout_seconds: Union[int, float, None] = None,
|
368
|
+
validate_parameters: Optional[bool] = None,
|
369
|
+
persist_result: Optional[bool] = NotSet, # type: ignore
|
370
|
+
result_storage: Optional[ResultStorage] = NotSet, # type: ignore
|
371
|
+
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
372
|
+
cache_result_in_memory: Optional[bool] = None,
|
373
|
+
log_prints: Optional[bool] = NotSet, # type: ignore
|
380
374
|
on_completion: Optional[
|
381
375
|
List[Callable[[FlowSchema, FlowRun, State], None]]
|
382
376
|
] = None,
|
@@ -432,15 +426,14 @@ class Flow(Generic[P, R]):
|
|
432
426
|
Create a new flow from an existing flow, update the task runner, and call
|
433
427
|
it without an intermediate variable:
|
434
428
|
|
435
|
-
>>> from prefect.task_runners import
|
429
|
+
>>> from prefect.task_runners import ThreadPoolTaskRunner
|
436
430
|
>>>
|
437
431
|
>>> @flow
|
438
432
|
>>> def my_flow(x, y):
|
439
433
|
>>> return x + y
|
440
434
|
>>>
|
441
|
-
>>> state = my_flow.with_options(task_runner=
|
435
|
+
>>> state = my_flow.with_options(task_runner=ThreadPoolTaskRunner)(1, 3)
|
442
436
|
>>> assert state.result() == 4
|
443
|
-
|
444
437
|
"""
|
445
438
|
new_flow = Flow(
|
446
439
|
fn=self.fn,
|
@@ -480,11 +473,11 @@ class Flow(Generic[P, R]):
|
|
480
473
|
else self.cache_result_in_memory
|
481
474
|
),
|
482
475
|
log_prints=log_prints if log_prints is not NotSet else self.log_prints,
|
483
|
-
on_completion=on_completion or self.
|
484
|
-
on_failure=on_failure or self.
|
485
|
-
on_cancellation=on_cancellation or self.
|
486
|
-
on_crashed=on_crashed or self.
|
487
|
-
on_running=on_running or self.
|
476
|
+
on_completion=on_completion or self.on_completion_hooks,
|
477
|
+
on_failure=on_failure or self.on_failure_hooks,
|
478
|
+
on_cancellation=on_cancellation or self.on_cancellation_hooks,
|
479
|
+
on_crashed=on_crashed or self.on_crashed_hooks,
|
480
|
+
on_running=on_running or self.on_running_hooks,
|
488
481
|
)
|
489
482
|
new_flow._storage = self._storage
|
490
483
|
new_flow._entrypoint = self._entrypoint
|
@@ -503,49 +496,52 @@ class Flow(Generic[P, R]):
|
|
503
496
|
"""
|
504
497
|
args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
|
505
498
|
|
506
|
-
|
499
|
+
with warnings.catch_warnings():
|
500
|
+
warnings.filterwarnings(
|
501
|
+
"ignore", category=pydantic.warnings.PydanticDeprecatedSince20
|
502
|
+
)
|
507
503
|
has_v1_models = any(isinstance(o, V1BaseModel) for o in args) or any(
|
508
504
|
isinstance(o, V1BaseModel) for o in kwargs.values()
|
509
505
|
)
|
510
|
-
has_v2_types = any(is_v2_type(o) for o in args) or any(
|
511
|
-
is_v2_type(o) for o in kwargs.values()
|
512
|
-
)
|
513
506
|
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
)
|
507
|
+
has_v2_types = any(is_v2_type(o) for o in args) or any(
|
508
|
+
is_v2_type(o) for o in kwargs.values()
|
509
|
+
)
|
518
510
|
|
519
|
-
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
else:
|
524
|
-
validated_fn = V2ValidatedFunction(
|
525
|
-
self.fn, config={"arbitrary_types_allowed": True}
|
526
|
-
)
|
511
|
+
if has_v1_models and has_v2_types:
|
512
|
+
raise ParameterTypeError(
|
513
|
+
"Cannot mix Pydantic v1 and v2 types as arguments to a flow."
|
514
|
+
)
|
527
515
|
|
528
|
-
|
529
|
-
validated_fn =
|
516
|
+
if has_v1_models:
|
517
|
+
validated_fn = V1ValidatedFunction(
|
530
518
|
self.fn, config={"arbitrary_types_allowed": True}
|
531
519
|
)
|
520
|
+
else:
|
521
|
+
validated_fn = V2ValidatedFunction(
|
522
|
+
self.fn, config=pydantic.ConfigDict(arbitrary_types_allowed=True)
|
523
|
+
)
|
532
524
|
|
533
525
|
try:
|
534
|
-
|
526
|
+
with warnings.catch_warnings():
|
527
|
+
warnings.filterwarnings(
|
528
|
+
"ignore", category=pydantic.warnings.PydanticDeprecatedSince20
|
529
|
+
)
|
530
|
+
model = validated_fn.init_model_instance(*args, **kwargs)
|
535
531
|
except pydantic.ValidationError as exc:
|
536
532
|
# We capture the pydantic exception and raise our own because the pydantic
|
537
533
|
# exception is not picklable when using a cythonized pydantic installation
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
534
|
+
logger.error(
|
535
|
+
f"Parameter validation failed for flow {self.name!r}: {exc.errors()}"
|
536
|
+
f"\nParameters: {parameters}"
|
537
|
+
)
|
542
538
|
raise ParameterTypeError.from_validation_error(exc) from None
|
543
539
|
|
544
540
|
# Get the updated parameter dict with cast values from the model
|
545
541
|
cast_parameters = {
|
546
542
|
k: v
|
547
|
-
for k, v in model.
|
548
|
-
if k in model.
|
543
|
+
for k, v in dict(model).items()
|
544
|
+
if k in model.model_fields_set or model.model_fields[k].default_factory
|
549
545
|
}
|
550
546
|
return cast_parameters
|
551
547
|
|
@@ -605,7 +601,7 @@ class Flow(Generic[P, R]):
|
|
605
601
|
description: Optional[str] = None,
|
606
602
|
tags: Optional[List[str]] = None,
|
607
603
|
version: Optional[str] = None,
|
608
|
-
enforce_parameter_schema: bool =
|
604
|
+
enforce_parameter_schema: bool = True,
|
609
605
|
work_pool_name: Optional[str] = None,
|
610
606
|
work_queue_name: Optional[str] = None,
|
611
607
|
job_variables: Optional[Dict[str, Any]] = None,
|
@@ -667,7 +663,8 @@ class Flow(Generic[P, R]):
|
|
667
663
|
from prefect.deployments.runner import RunnerDeployment
|
668
664
|
|
669
665
|
if not name.endswith(".py"):
|
670
|
-
|
666
|
+
_raise_on_name_with_banned_characters(name)
|
667
|
+
|
671
668
|
if self._storage and self._entrypoint:
|
672
669
|
return await RunnerDeployment.from_storage(
|
673
670
|
storage=self._storage,
|
@@ -713,6 +710,36 @@ class Flow(Generic[P, R]):
|
|
713
710
|
entrypoint_type=entrypoint_type,
|
714
711
|
)
|
715
712
|
|
713
|
+
def on_completion(
|
714
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
715
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
716
|
+
self.on_completion_hooks.append(fn)
|
717
|
+
return fn
|
718
|
+
|
719
|
+
def on_cancellation(
|
720
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
721
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
722
|
+
self.on_cancellation_hooks.append(fn)
|
723
|
+
return fn
|
724
|
+
|
725
|
+
def on_crashed(
|
726
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
727
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
728
|
+
self.on_crashed_hooks.append(fn)
|
729
|
+
return fn
|
730
|
+
|
731
|
+
def on_running(
|
732
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
733
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
734
|
+
self.on_running_hooks.append(fn)
|
735
|
+
return fn
|
736
|
+
|
737
|
+
def on_failure(
|
738
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
739
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
740
|
+
self.on_failure_hooks.append(fn)
|
741
|
+
return fn
|
742
|
+
|
716
743
|
@sync_compatible
|
717
744
|
async def serve(
|
718
745
|
self,
|
@@ -736,7 +763,7 @@ class Flow(Generic[P, R]):
|
|
736
763
|
description: Optional[str] = None,
|
737
764
|
tags: Optional[List[str]] = None,
|
738
765
|
version: Optional[str] = None,
|
739
|
-
enforce_parameter_schema: bool =
|
766
|
+
enforce_parameter_schema: bool = True,
|
740
767
|
pause_on_shutdown: bool = True,
|
741
768
|
print_starting_message: bool = True,
|
742
769
|
limit: Optional[int] = None,
|
@@ -945,7 +972,7 @@ class Flow(Generic[P, R]):
|
|
945
972
|
cron: Optional[str] = None,
|
946
973
|
rrule: Optional[str] = None,
|
947
974
|
paused: Optional[bool] = None,
|
948
|
-
schedules: Optional[List[
|
975
|
+
schedules: Optional[List[DeploymentScheduleCreate]] = None,
|
949
976
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
950
977
|
is_schedule_active: Optional[bool] = None,
|
951
978
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
@@ -953,7 +980,7 @@ class Flow(Generic[P, R]):
|
|
953
980
|
description: Optional[str] = None,
|
954
981
|
tags: Optional[List[str]] = None,
|
955
982
|
version: Optional[str] = None,
|
956
|
-
enforce_parameter_schema: bool =
|
983
|
+
enforce_parameter_schema: bool = True,
|
957
984
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
958
985
|
print_next_steps: bool = True,
|
959
986
|
ignore_warnings: bool = False,
|
@@ -1198,7 +1225,10 @@ class Flow(Generic[P, R]):
|
|
1198
1225
|
>>> with tags("db", "blue"):
|
1199
1226
|
>>> my_flow("foo")
|
1200
1227
|
"""
|
1201
|
-
from prefect.
|
1228
|
+
from prefect.utilities.visualization import (
|
1229
|
+
get_task_viz_tracker,
|
1230
|
+
track_viz_task,
|
1231
|
+
)
|
1202
1232
|
|
1203
1233
|
# Convert the call args/kwargs to a parameter dict
|
1204
1234
|
parameters = get_call_parameters(self.fn, args, kwargs)
|
@@ -1211,72 +1241,19 @@ class Flow(Generic[P, R]):
|
|
1211
1241
|
# we can add support for exploring subflows for tasks in the future.
|
1212
1242
|
return track_viz_task(self.isasync, self.name, parameters)
|
1213
1243
|
|
1214
|
-
|
1215
|
-
from prefect.new_flow_engine import run_flow, run_flow_sync
|
1244
|
+
from prefect.flow_engine import run_flow, run_flow_sync
|
1216
1245
|
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1220
|
-
wait_for=wait_for,
|
1221
|
-
return_type=return_type,
|
1222
|
-
)
|
1223
|
-
if self.isasync:
|
1224
|
-
# this returns an awaitable coroutine
|
1225
|
-
return run_flow(**run_kwargs)
|
1226
|
-
else:
|
1227
|
-
return run_flow_sync(**run_kwargs)
|
1228
|
-
|
1229
|
-
return enter_flow_run_engine_from_flow_call(
|
1230
|
-
self,
|
1231
|
-
parameters,
|
1246
|
+
run_kwargs = dict(
|
1247
|
+
flow=self,
|
1248
|
+
parameters=parameters,
|
1232
1249
|
wait_for=wait_for,
|
1233
1250
|
return_type=return_type,
|
1234
1251
|
)
|
1235
|
-
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
...
|
1241
|
-
|
1242
|
-
@overload
|
1243
|
-
def _run(
|
1244
|
-
self: "Flow[P, Coroutine[Any, Any, T]]", *args: P.args, **kwargs: P.kwargs
|
1245
|
-
) -> Awaitable[T]:
|
1246
|
-
...
|
1247
|
-
|
1248
|
-
@overload
|
1249
|
-
def _run(self: "Flow[P, T]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
|
1250
|
-
...
|
1251
|
-
|
1252
|
-
def _run(
|
1253
|
-
self,
|
1254
|
-
*args: "P.args",
|
1255
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1256
|
-
**kwargs: "P.kwargs",
|
1257
|
-
):
|
1258
|
-
"""
|
1259
|
-
Run the flow and return its final state.
|
1260
|
-
|
1261
|
-
Examples:
|
1262
|
-
|
1263
|
-
Run a flow and get the returned result
|
1264
|
-
|
1265
|
-
>>> state = my_flow._run("marvin")
|
1266
|
-
>>> state.result()
|
1267
|
-
"goodbye marvin"
|
1268
|
-
"""
|
1269
|
-
from prefect.engine import enter_flow_run_engine_from_flow_call
|
1270
|
-
|
1271
|
-
# Convert the call args/kwargs to a parameter dict
|
1272
|
-
parameters = get_call_parameters(self.fn, args, kwargs)
|
1273
|
-
|
1274
|
-
return enter_flow_run_engine_from_flow_call(
|
1275
|
-
self,
|
1276
|
-
parameters,
|
1277
|
-
wait_for=wait_for,
|
1278
|
-
return_type="state",
|
1279
|
-
)
|
1252
|
+
if self.isasync:
|
1253
|
+
# this returns an awaitable coroutine
|
1254
|
+
return run_flow(**run_kwargs)
|
1255
|
+
else:
|
1256
|
+
return run_flow_sync(**run_kwargs)
|
1280
1257
|
|
1281
1258
|
@sync_compatible
|
1282
1259
|
async def visualize(self, *args, **kwargs):
|
@@ -1289,6 +1266,16 @@ class Flow(Generic[P, R]):
|
|
1289
1266
|
- GraphvizExecutableNotFoundError: If the `dot` executable isn't found.
|
1290
1267
|
- FlowVisualizationError: If the flow can't be visualized for any other reason.
|
1291
1268
|
"""
|
1269
|
+
from prefect.utilities.visualization import (
|
1270
|
+
FlowVisualizationError,
|
1271
|
+
GraphvizExecutableNotFoundError,
|
1272
|
+
GraphvizImportError,
|
1273
|
+
TaskVizTracker,
|
1274
|
+
VisualizationUnsupportedError,
|
1275
|
+
build_task_dependencies,
|
1276
|
+
visualize_task_dependencies,
|
1277
|
+
)
|
1278
|
+
|
1292
1279
|
if not PREFECT_UNIT_TEST_MODE:
|
1293
1280
|
warnings.warn(
|
1294
1281
|
"`flow.visualize()` will execute code inside of your flow that is not"
|
@@ -1341,7 +1328,7 @@ def flow(
|
|
1341
1328
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1342
1329
|
retries: Optional[int] = None,
|
1343
1330
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
1344
|
-
task_runner:
|
1331
|
+
task_runner: Optional[TaskRunner] = None,
|
1345
1332
|
description: str = None,
|
1346
1333
|
timeout_seconds: Union[int, float] = None,
|
1347
1334
|
validate_parameters: bool = True,
|
@@ -1373,7 +1360,7 @@ def flow(
|
|
1373
1360
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1374
1361
|
retries: int = None,
|
1375
1362
|
retry_delay_seconds: Union[int, float] = None,
|
1376
|
-
task_runner:
|
1363
|
+
task_runner: Optional[TaskRunner] = None,
|
1377
1364
|
description: str = None,
|
1378
1365
|
timeout_seconds: Union[int, float] = None,
|
1379
1366
|
validate_parameters: bool = True,
|
@@ -1551,6 +1538,23 @@ def flow(
|
|
1551
1538
|
)
|
1552
1539
|
|
1553
1540
|
|
1541
|
+
def _raise_on_name_with_banned_characters(name: str) -> str:
|
1542
|
+
"""
|
1543
|
+
Raise an InvalidNameError if the given name contains any invalid
|
1544
|
+
characters.
|
1545
|
+
"""
|
1546
|
+
if name is None:
|
1547
|
+
return name
|
1548
|
+
|
1549
|
+
if not re.match(WITHOUT_BANNED_CHARACTERS, name):
|
1550
|
+
raise InvalidNameError(
|
1551
|
+
f"Name {name!r} contains an invalid character. "
|
1552
|
+
f"Must not contain any of: {BANNED_CHARACTERS}."
|
1553
|
+
)
|
1554
|
+
|
1555
|
+
return name
|
1556
|
+
|
1557
|
+
|
1554
1558
|
# Add from_source so it is available on the flow function we all know and love
|
1555
1559
|
flow.from_source = Flow.from_source
|
1556
1560
|
|
@@ -1640,7 +1644,9 @@ def load_flow_from_script(path: str, flow_name: str = None) -> Flow:
|
|
1640
1644
|
)
|
1641
1645
|
|
1642
1646
|
|
1643
|
-
def load_flow_from_entrypoint(
|
1647
|
+
def load_flow_from_entrypoint(
|
1648
|
+
entrypoint: str,
|
1649
|
+
) -> Flow:
|
1644
1650
|
"""
|
1645
1651
|
Extract a flow object from a script at an entrypoint by running all of the code in the file.
|
1646
1652
|
|
@@ -1793,3 +1799,138 @@ async def serve(
|
|
1793
1799
|
)
|
1794
1800
|
|
1795
1801
|
await runner.start()
|
1802
|
+
|
1803
|
+
|
1804
|
+
@client_injector
|
1805
|
+
async def load_flow_from_flow_run(
|
1806
|
+
client: "PrefectClient",
|
1807
|
+
flow_run: "FlowRun",
|
1808
|
+
ignore_storage: bool = False,
|
1809
|
+
storage_base_path: Optional[str] = None,
|
1810
|
+
) -> "Flow":
|
1811
|
+
"""
|
1812
|
+
Load a flow from the location/script provided in a deployment's storage document.
|
1813
|
+
|
1814
|
+
If `ignore_storage=True` is provided, no pull from remote storage occurs. This flag
|
1815
|
+
is largely for testing, and assumes the flow is already available locally.
|
1816
|
+
"""
|
1817
|
+
deployment = await client.read_deployment(flow_run.deployment_id)
|
1818
|
+
|
1819
|
+
if deployment.entrypoint is None:
|
1820
|
+
raise ValueError(
|
1821
|
+
f"Deployment {deployment.id} does not have an entrypoint and can not be run."
|
1822
|
+
)
|
1823
|
+
|
1824
|
+
run_logger = flow_run_logger(flow_run)
|
1825
|
+
|
1826
|
+
runner_storage_base_path = storage_base_path or os.environ.get(
|
1827
|
+
"PREFECT__STORAGE_BASE_PATH"
|
1828
|
+
)
|
1829
|
+
|
1830
|
+
# If there's no colon, assume it's a module path
|
1831
|
+
if ":" not in deployment.entrypoint:
|
1832
|
+
run_logger.debug(
|
1833
|
+
f"Importing flow code from module path {deployment.entrypoint}"
|
1834
|
+
)
|
1835
|
+
flow = await run_sync_in_worker_thread(
|
1836
|
+
load_flow_from_entrypoint, deployment.entrypoint
|
1837
|
+
)
|
1838
|
+
return flow
|
1839
|
+
|
1840
|
+
if not ignore_storage and not deployment.pull_steps:
|
1841
|
+
sys.path.insert(0, ".")
|
1842
|
+
if deployment.storage_document_id:
|
1843
|
+
storage_document = await client.read_block_document(
|
1844
|
+
deployment.storage_document_id
|
1845
|
+
)
|
1846
|
+
storage_block = Block._from_block_document(storage_document)
|
1847
|
+
else:
|
1848
|
+
basepath = deployment.path or Path(deployment.manifest_path).parent
|
1849
|
+
if runner_storage_base_path:
|
1850
|
+
basepath = str(basepath).replace(
|
1851
|
+
"$STORAGE_BASE_PATH", runner_storage_base_path
|
1852
|
+
)
|
1853
|
+
storage_block = LocalFileSystem(basepath=basepath)
|
1854
|
+
|
1855
|
+
from_path = (
|
1856
|
+
str(deployment.path).replace("$STORAGE_BASE_PATH", runner_storage_base_path)
|
1857
|
+
if runner_storage_base_path and deployment.path
|
1858
|
+
else deployment.path
|
1859
|
+
)
|
1860
|
+
run_logger.info(f"Downloading flow code from storage at {from_path!r}")
|
1861
|
+
await storage_block.get_directory(from_path=from_path, local_path=".")
|
1862
|
+
|
1863
|
+
if deployment.pull_steps:
|
1864
|
+
run_logger.debug(f"Running {len(deployment.pull_steps)} deployment pull steps")
|
1865
|
+
output = await run_steps(deployment.pull_steps)
|
1866
|
+
if output.get("directory"):
|
1867
|
+
run_logger.debug(f"Changing working directory to {output['directory']!r}")
|
1868
|
+
os.chdir(output["directory"])
|
1869
|
+
|
1870
|
+
import_path = relative_path_to_current_platform(deployment.entrypoint)
|
1871
|
+
# for backwards compat
|
1872
|
+
if deployment.manifest_path:
|
1873
|
+
with open(deployment.manifest_path, "r") as f:
|
1874
|
+
import_path = json.load(f)["import_path"]
|
1875
|
+
import_path = (
|
1876
|
+
Path(deployment.manifest_path).parent / import_path
|
1877
|
+
).absolute()
|
1878
|
+
run_logger.debug(f"Importing flow code from '{import_path}'")
|
1879
|
+
|
1880
|
+
flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, str(import_path))
|
1881
|
+
|
1882
|
+
return flow
|
1883
|
+
|
1884
|
+
|
1885
|
+
def load_flow_argument_from_entrypoint(
|
1886
|
+
entrypoint: str, arg: str = "name"
|
1887
|
+
) -> Optional[str]:
|
1888
|
+
"""
|
1889
|
+
Extract a flow argument from an entrypoint string.
|
1890
|
+
|
1891
|
+
Loads the source code of the entrypoint and extracts the flow argument from the
|
1892
|
+
`flow` decorator.
|
1893
|
+
|
1894
|
+
Args:
|
1895
|
+
entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
|
1896
|
+
to a flow function
|
1897
|
+
|
1898
|
+
Returns:
|
1899
|
+
The flow argument value
|
1900
|
+
"""
|
1901
|
+
if ":" in entrypoint:
|
1902
|
+
# split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
|
1903
|
+
path, func_name = entrypoint.rsplit(":", maxsplit=1)
|
1904
|
+
source_code = Path(path).read_text()
|
1905
|
+
else:
|
1906
|
+
path, func_name = entrypoint.rsplit(".", maxsplit=1)
|
1907
|
+
spec = importlib.util.find_spec(path)
|
1908
|
+
if not spec or not spec.origin:
|
1909
|
+
raise ValueError(f"Could not find module {path!r}")
|
1910
|
+
source_code = Path(spec.origin).read_text()
|
1911
|
+
parsed_code = ast.parse(source_code)
|
1912
|
+
func_def = next(
|
1913
|
+
(
|
1914
|
+
node
|
1915
|
+
for node in ast.walk(parsed_code)
|
1916
|
+
if isinstance(node, ast.FunctionDef) and node.name == func_name
|
1917
|
+
),
|
1918
|
+
None,
|
1919
|
+
)
|
1920
|
+
if not func_def:
|
1921
|
+
raise ValueError(f"Could not find flow {func_name!r} in {path!r}")
|
1922
|
+
for decorator in func_def.decorator_list:
|
1923
|
+
if (
|
1924
|
+
isinstance(decorator, ast.Call)
|
1925
|
+
and getattr(decorator.func, "id", "") == "flow"
|
1926
|
+
):
|
1927
|
+
for keyword in decorator.keywords:
|
1928
|
+
if keyword.arg == arg:
|
1929
|
+
return (
|
1930
|
+
keyword.value.value
|
1931
|
+
) # Return the string value of the argument
|
1932
|
+
|
1933
|
+
if arg == "name":
|
1934
|
+
return func_name.replace(
|
1935
|
+
"_", "-"
|
1936
|
+
) # If no matching decorator or keyword argument is found
|