prefect-client 2.19.3__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +8 -56
- prefect/_internal/compatibility/deprecated.py +6 -115
- prefect/_internal/compatibility/experimental.py +4 -79
- prefect/_internal/concurrency/api.py +0 -34
- prefect/_internal/concurrency/calls.py +0 -6
- prefect/_internal/concurrency/cancellation.py +0 -3
- prefect/_internal/concurrency/event_loop.py +0 -20
- prefect/_internal/concurrency/inspection.py +3 -3
- prefect/_internal/concurrency/threads.py +35 -0
- prefect/_internal/concurrency/waiters.py +0 -28
- prefect/_internal/pydantic/__init__.py +0 -45
- prefect/_internal/pydantic/v1_schema.py +21 -22
- prefect/_internal/pydantic/v2_schema.py +0 -2
- prefect/_internal/pydantic/v2_validated_func.py +18 -23
- prefect/_internal/schemas/bases.py +44 -177
- prefect/_internal/schemas/fields.py +1 -43
- prefect/_internal/schemas/validators.py +60 -158
- prefect/artifacts.py +161 -14
- prefect/automations.py +39 -4
- prefect/blocks/abstract.py +1 -1
- prefect/blocks/core.py +268 -148
- prefect/blocks/fields.py +2 -57
- prefect/blocks/kubernetes.py +8 -12
- prefect/blocks/notifications.py +40 -20
- prefect/blocks/system.py +22 -11
- prefect/blocks/webhook.py +2 -9
- prefect/client/base.py +4 -4
- prefect/client/cloud.py +8 -13
- prefect/client/orchestration.py +347 -341
- prefect/client/schemas/actions.py +92 -86
- prefect/client/schemas/filters.py +20 -40
- prefect/client/schemas/objects.py +147 -145
- prefect/client/schemas/responses.py +16 -24
- prefect/client/schemas/schedules.py +47 -35
- prefect/client/subscriptions.py +2 -2
- prefect/client/utilities.py +5 -2
- prefect/concurrency/asyncio.py +3 -1
- prefect/concurrency/events.py +1 -1
- prefect/concurrency/services.py +6 -3
- prefect/context.py +195 -27
- prefect/deployments/__init__.py +5 -6
- prefect/deployments/base.py +7 -5
- prefect/deployments/flow_runs.py +185 -0
- prefect/deployments/runner.py +50 -45
- prefect/deployments/schedules.py +28 -23
- prefect/deployments/steps/__init__.py +0 -1
- prefect/deployments/steps/core.py +1 -0
- prefect/deployments/steps/pull.py +7 -21
- prefect/engine.py +12 -2422
- prefect/events/actions.py +17 -23
- prefect/events/cli/automations.py +19 -6
- prefect/events/clients.py +14 -37
- prefect/events/filters.py +14 -18
- prefect/events/related.py +2 -2
- prefect/events/schemas/__init__.py +0 -5
- prefect/events/schemas/automations.py +55 -46
- prefect/events/schemas/deployment_triggers.py +7 -197
- prefect/events/schemas/events.py +34 -65
- prefect/events/schemas/labelling.py +10 -14
- prefect/events/utilities.py +2 -3
- prefect/events/worker.py +2 -3
- prefect/filesystems.py +6 -517
- prefect/{new_flow_engine.py → flow_engine.py} +313 -72
- prefect/flow_runs.py +377 -5
- prefect/flows.py +248 -165
- prefect/futures.py +186 -345
- prefect/infrastructure/__init__.py +0 -27
- prefect/infrastructure/provisioners/__init__.py +5 -3
- prefect/infrastructure/provisioners/cloud_run.py +11 -6
- prefect/infrastructure/provisioners/container_instance.py +11 -7
- prefect/infrastructure/provisioners/ecs.py +6 -4
- prefect/infrastructure/provisioners/modal.py +8 -5
- prefect/input/actions.py +2 -4
- prefect/input/run_input.py +5 -7
- prefect/logging/formatters.py +0 -2
- prefect/logging/handlers.py +3 -11
- prefect/logging/loggers.py +2 -2
- prefect/manifests.py +2 -1
- prefect/records/__init__.py +1 -0
- prefect/records/result_store.py +42 -0
- prefect/records/store.py +9 -0
- prefect/results.py +43 -39
- prefect/runner/runner.py +9 -9
- prefect/runner/server.py +6 -10
- prefect/runner/storage.py +3 -8
- prefect/runner/submit.py +2 -2
- prefect/runner/utils.py +2 -2
- prefect/serializers.py +24 -35
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
- prefect/settings.py +70 -133
- prefect/states.py +17 -47
- prefect/task_engine.py +697 -58
- prefect/task_runners.py +269 -301
- prefect/task_server.py +53 -34
- prefect/tasks.py +327 -337
- prefect/transactions.py +220 -0
- prefect/types/__init__.py +61 -82
- prefect/utilities/asyncutils.py +195 -136
- prefect/utilities/callables.py +121 -41
- prefect/utilities/collections.py +23 -38
- prefect/utilities/dispatch.py +11 -3
- prefect/utilities/dockerutils.py +4 -0
- prefect/utilities/engine.py +140 -20
- prefect/utilities/importtools.py +26 -27
- prefect/utilities/pydantic.py +128 -38
- prefect/utilities/schema_tools/hydration.py +5 -1
- prefect/utilities/templating.py +12 -2
- prefect/variables.py +78 -61
- prefect/workers/__init__.py +0 -1
- prefect/workers/base.py +15 -17
- prefect/workers/process.py +3 -8
- prefect/workers/server.py +2 -2
- {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/METADATA +22 -21
- prefect_client-3.0.0rc1.dist-info/RECORD +176 -0
- prefect/_internal/pydantic/_base_model.py +0 -51
- prefect/_internal/pydantic/_compat.py +0 -82
- prefect/_internal/pydantic/_flags.py +0 -20
- prefect/_internal/pydantic/_types.py +0 -8
- prefect/_internal/pydantic/utilities/__init__.py +0 -0
- prefect/_internal/pydantic/utilities/config_dict.py +0 -72
- prefect/_internal/pydantic/utilities/field_validator.py +0 -150
- prefect/_internal/pydantic/utilities/model_construct.py +0 -56
- prefect/_internal/pydantic/utilities/model_copy.py +0 -55
- prefect/_internal/pydantic/utilities/model_dump.py +0 -136
- prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
- prefect/_internal/pydantic/utilities/model_fields.py +0 -50
- prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
- prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
- prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
- prefect/_internal/pydantic/utilities/model_validate.py +0 -75
- prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
- prefect/_internal/pydantic/utilities/model_validator.py +0 -87
- prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
- prefect/_vendor/__init__.py +0 -0
- prefect/_vendor/fastapi/__init__.py +0 -25
- prefect/_vendor/fastapi/applications.py +0 -946
- prefect/_vendor/fastapi/background.py +0 -3
- prefect/_vendor/fastapi/concurrency.py +0 -44
- prefect/_vendor/fastapi/datastructures.py +0 -58
- prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
- prefect/_vendor/fastapi/dependencies/models.py +0 -64
- prefect/_vendor/fastapi/dependencies/utils.py +0 -877
- prefect/_vendor/fastapi/encoders.py +0 -177
- prefect/_vendor/fastapi/exception_handlers.py +0 -40
- prefect/_vendor/fastapi/exceptions.py +0 -46
- prefect/_vendor/fastapi/logger.py +0 -3
- prefect/_vendor/fastapi/middleware/__init__.py +0 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
- prefect/_vendor/fastapi/middleware/cors.py +0 -3
- prefect/_vendor/fastapi/middleware/gzip.py +0 -3
- prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
- prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
- prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
- prefect/_vendor/fastapi/openapi/__init__.py +0 -0
- prefect/_vendor/fastapi/openapi/constants.py +0 -2
- prefect/_vendor/fastapi/openapi/docs.py +0 -203
- prefect/_vendor/fastapi/openapi/models.py +0 -480
- prefect/_vendor/fastapi/openapi/utils.py +0 -485
- prefect/_vendor/fastapi/param_functions.py +0 -340
- prefect/_vendor/fastapi/params.py +0 -453
- prefect/_vendor/fastapi/requests.py +0 -4
- prefect/_vendor/fastapi/responses.py +0 -40
- prefect/_vendor/fastapi/routing.py +0 -1331
- prefect/_vendor/fastapi/security/__init__.py +0 -15
- prefect/_vendor/fastapi/security/api_key.py +0 -98
- prefect/_vendor/fastapi/security/base.py +0 -6
- prefect/_vendor/fastapi/security/http.py +0 -172
- prefect/_vendor/fastapi/security/oauth2.py +0 -227
- prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
- prefect/_vendor/fastapi/security/utils.py +0 -10
- prefect/_vendor/fastapi/staticfiles.py +0 -1
- prefect/_vendor/fastapi/templating.py +0 -3
- prefect/_vendor/fastapi/testclient.py +0 -1
- prefect/_vendor/fastapi/types.py +0 -3
- prefect/_vendor/fastapi/utils.py +0 -235
- prefect/_vendor/fastapi/websockets.py +0 -7
- prefect/_vendor/starlette/__init__.py +0 -1
- prefect/_vendor/starlette/_compat.py +0 -28
- prefect/_vendor/starlette/_exception_handler.py +0 -80
- prefect/_vendor/starlette/_utils.py +0 -88
- prefect/_vendor/starlette/applications.py +0 -261
- prefect/_vendor/starlette/authentication.py +0 -159
- prefect/_vendor/starlette/background.py +0 -43
- prefect/_vendor/starlette/concurrency.py +0 -59
- prefect/_vendor/starlette/config.py +0 -151
- prefect/_vendor/starlette/convertors.py +0 -87
- prefect/_vendor/starlette/datastructures.py +0 -707
- prefect/_vendor/starlette/endpoints.py +0 -130
- prefect/_vendor/starlette/exceptions.py +0 -60
- prefect/_vendor/starlette/formparsers.py +0 -276
- prefect/_vendor/starlette/middleware/__init__.py +0 -17
- prefect/_vendor/starlette/middleware/authentication.py +0 -52
- prefect/_vendor/starlette/middleware/base.py +0 -220
- prefect/_vendor/starlette/middleware/cors.py +0 -176
- prefect/_vendor/starlette/middleware/errors.py +0 -265
- prefect/_vendor/starlette/middleware/exceptions.py +0 -74
- prefect/_vendor/starlette/middleware/gzip.py +0 -113
- prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
- prefect/_vendor/starlette/middleware/sessions.py +0 -82
- prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
- prefect/_vendor/starlette/middleware/wsgi.py +0 -147
- prefect/_vendor/starlette/requests.py +0 -328
- prefect/_vendor/starlette/responses.py +0 -347
- prefect/_vendor/starlette/routing.py +0 -933
- prefect/_vendor/starlette/schemas.py +0 -154
- prefect/_vendor/starlette/staticfiles.py +0 -248
- prefect/_vendor/starlette/status.py +0 -199
- prefect/_vendor/starlette/templating.py +0 -231
- prefect/_vendor/starlette/testclient.py +0 -804
- prefect/_vendor/starlette/types.py +0 -30
- prefect/_vendor/starlette/websockets.py +0 -193
- prefect/agent.py +0 -698
- prefect/deployments/deployments.py +0 -1042
- prefect/deprecated/__init__.py +0 -0
- prefect/deprecated/data_documents.py +0 -350
- prefect/deprecated/packaging/__init__.py +0 -12
- prefect/deprecated/packaging/base.py +0 -96
- prefect/deprecated/packaging/docker.py +0 -146
- prefect/deprecated/packaging/file.py +0 -92
- prefect/deprecated/packaging/orion.py +0 -80
- prefect/deprecated/packaging/serializers.py +0 -171
- prefect/events/instrument.py +0 -135
- prefect/infrastructure/base.py +0 -323
- prefect/infrastructure/container.py +0 -818
- prefect/infrastructure/kubernetes.py +0 -920
- prefect/infrastructure/process.py +0 -289
- prefect/new_task_engine.py +0 -423
- prefect/pydantic/__init__.py +0 -76
- prefect/pydantic/main.py +0 -39
- prefect/software/__init__.py +0 -2
- prefect/software/base.py +0 -50
- prefect/software/conda.py +0 -199
- prefect/software/pip.py +0 -122
- prefect/software/python.py +0 -52
- prefect/workers/block.py +0 -218
- prefect_client-2.19.3.dist-info/RECORD +0 -292
- {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/LICENSE +0 -0
- {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/WHEEL +0 -0
- {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -9,7 +9,10 @@ import ast
|
|
9
9
|
import datetime
|
10
10
|
import importlib.util
|
11
11
|
import inspect
|
12
|
+
import json
|
12
13
|
import os
|
14
|
+
import re
|
15
|
+
import sys
|
13
16
|
import tempfile
|
14
17
|
import warnings
|
15
18
|
from functools import partial, update_wrapper
|
@@ -36,50 +39,38 @@ from typing import (
|
|
36
39
|
)
|
37
40
|
from uuid import UUID
|
38
41
|
|
42
|
+
import pydantic
|
43
|
+
from fastapi.encoders import jsonable_encoder
|
44
|
+
from pydantic.v1 import BaseModel as V1BaseModel
|
45
|
+
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
46
|
+
from pydantic.v1.errors import ConfigError # TODO
|
39
47
|
from rich.console import Console
|
40
48
|
from typing_extensions import Literal, ParamSpec, Self
|
41
49
|
|
42
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
43
|
-
|
44
|
-
if HAS_PYDANTIC_V2:
|
45
|
-
import pydantic.v1 as pydantic
|
46
|
-
from pydantic import ValidationError as V2ValidationError
|
47
|
-
from pydantic.v1 import BaseModel as V1BaseModel
|
48
|
-
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
49
|
-
|
50
|
-
from ._internal.pydantic.v2_schema import is_v2_type
|
51
|
-
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
52
|
-
from ._internal.pydantic.v2_validated_func import (
|
53
|
-
V2ValidatedFunction as ValidatedFunction,
|
54
|
-
)
|
55
|
-
|
56
|
-
else:
|
57
|
-
import pydantic
|
58
|
-
from pydantic.decorator import ValidatedFunction
|
59
|
-
|
60
|
-
V2ValidationError = None
|
61
|
-
|
62
|
-
from prefect._vendor.fastapi.encoders import jsonable_encoder
|
63
|
-
|
64
50
|
from prefect._internal.compatibility.deprecated import deprecated_parameter
|
65
51
|
from prefect._internal.concurrency.api import create_call, from_async
|
66
|
-
from prefect.
|
52
|
+
from prefect.blocks.core import Block
|
67
53
|
from prefect.client.orchestration import get_client
|
54
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
68
55
|
from prefect.client.schemas.objects import Flow as FlowSchema
|
69
|
-
from prefect.client.schemas.objects import FlowRun
|
56
|
+
from prefect.client.schemas.objects import FlowRun
|
70
57
|
from prefect.client.schemas.schedules import SCHEDULE_TYPES
|
58
|
+
from prefect.client.utilities import client_injector
|
71
59
|
from prefect.context import PrefectObjectRegistry, registry_from_script
|
72
60
|
from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
|
61
|
+
from prefect.deployments.steps.core import run_steps
|
73
62
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
74
63
|
from prefect.exceptions import (
|
64
|
+
InvalidNameError,
|
75
65
|
MissingFlowError,
|
76
66
|
ObjectNotFound,
|
77
67
|
ParameterTypeError,
|
78
68
|
UnspecifiedFlowError,
|
79
69
|
)
|
80
|
-
from prefect.filesystems import ReadableDeploymentStorage
|
70
|
+
from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
|
81
71
|
from prefect.futures import PrefectFuture
|
82
72
|
from prefect.logging import get_logger
|
73
|
+
from prefect.logging.loggers import flow_run_logger
|
83
74
|
from prefect.results import ResultSerializer, ResultStorage
|
84
75
|
from prefect.runner.storage import (
|
85
76
|
BlockStorageAdapter,
|
@@ -88,16 +79,20 @@ from prefect.runner.storage import (
|
|
88
79
|
)
|
89
80
|
from prefect.settings import (
|
90
81
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
91
|
-
PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE,
|
92
82
|
PREFECT_FLOW_DEFAULT_RETRIES,
|
93
83
|
PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS,
|
94
84
|
PREFECT_UI_URL,
|
95
85
|
PREFECT_UNIT_TEST_MODE,
|
96
86
|
)
|
97
87
|
from prefect.states import State
|
98
|
-
from prefect.task_runners import
|
88
|
+
from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
|
89
|
+
from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
|
99
90
|
from prefect.utilities.annotations import NotSet
|
100
|
-
from prefect.utilities.asyncutils import
|
91
|
+
from prefect.utilities.asyncutils import (
|
92
|
+
is_async_fn,
|
93
|
+
run_sync_in_worker_thread,
|
94
|
+
sync_compatible,
|
95
|
+
)
|
101
96
|
from prefect.utilities.callables import (
|
102
97
|
get_call_parameters,
|
103
98
|
parameter_schema,
|
@@ -105,18 +100,14 @@ from prefect.utilities.callables import (
|
|
105
100
|
raise_for_reserved_arguments,
|
106
101
|
)
|
107
102
|
from prefect.utilities.collections import listrepr
|
103
|
+
from prefect.utilities.filesystem import relative_path_to_current_platform
|
108
104
|
from prefect.utilities.hashing import file_hash
|
109
105
|
from prefect.utilities.importtools import import_object
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
VisualizationUnsupportedError,
|
116
|
-
build_task_dependencies,
|
117
|
-
get_task_viz_tracker,
|
118
|
-
track_viz_task,
|
119
|
-
visualize_task_dependencies,
|
106
|
+
|
107
|
+
from ._internal.pydantic.v2_schema import is_v2_type
|
108
|
+
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
109
|
+
from ._internal.pydantic.v2_validated_func import (
|
110
|
+
V2ValidatedFunction as ValidatedFunction,
|
120
111
|
)
|
121
112
|
|
122
113
|
T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
|
@@ -127,7 +118,9 @@ F = TypeVar("F", bound="Flow") # The type of the flow
|
|
127
118
|
logger = get_logger("flows")
|
128
119
|
|
129
120
|
if TYPE_CHECKING:
|
121
|
+
from prefect.client.orchestration import PrefectClient
|
130
122
|
from prefect.deployments.runner import FlexibleScheduleList, RunnerDeployment
|
123
|
+
from prefect.flows import FlowRun
|
131
124
|
|
132
125
|
|
133
126
|
@PrefectObjectRegistry.register_instances
|
@@ -198,9 +191,9 @@ class Flow(Generic[P, R]):
|
|
198
191
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
199
192
|
retries: Optional[int] = None,
|
200
193
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
201
|
-
task_runner: Union[Type[
|
202
|
-
description: str = None,
|
203
|
-
timeout_seconds: Union[int, float] = None,
|
194
|
+
task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
|
195
|
+
description: Optional[str] = None,
|
196
|
+
timeout_seconds: Union[int, float, None] = None,
|
204
197
|
validate_parameters: bool = True,
|
205
198
|
persist_result: Optional[bool] = None,
|
206
199
|
result_storage: Optional[ResultStorage] = None,
|
@@ -247,8 +240,6 @@ class Flow(Generic[P, R]):
|
|
247
240
|
]
|
248
241
|
for hooks, hook_name in zip(hook_categories, hook_names):
|
249
242
|
if hooks is not None:
|
250
|
-
if not hooks:
|
251
|
-
raise ValueError(f"Empty list passed for '{hook_name}'")
|
252
243
|
try:
|
253
244
|
hooks = list(hooks)
|
254
245
|
except TypeError:
|
@@ -275,7 +266,7 @@ class Flow(Generic[P, R]):
|
|
275
266
|
|
276
267
|
# Validate name if given
|
277
268
|
if name:
|
278
|
-
|
269
|
+
_raise_on_name_with_banned_characters(name)
|
279
270
|
|
280
271
|
self.name = name or fn.__name__.replace("_", "-")
|
281
272
|
|
@@ -287,7 +278,8 @@ class Flow(Generic[P, R]):
|
|
287
278
|
)
|
288
279
|
self.flow_run_name = flow_run_name
|
289
280
|
|
290
|
-
|
281
|
+
default_task_runner = ThreadPoolTaskRunner()
|
282
|
+
task_runner = task_runner or default_task_runner
|
291
283
|
self.task_runner = (
|
292
284
|
task_runner() if isinstance(task_runner, type) else task_runner
|
293
285
|
)
|
@@ -335,7 +327,7 @@ class Flow(Generic[P, R]):
|
|
335
327
|
# is not picklable in some environments
|
336
328
|
try:
|
337
329
|
ValidatedFunction(self.fn, config={"arbitrary_types_allowed": True})
|
338
|
-
except
|
330
|
+
except ConfigError as exc:
|
339
331
|
raise ValueError(
|
340
332
|
"Flow function is not compatible with `validate_parameters`. "
|
341
333
|
"Disable validation or change the argument names."
|
@@ -345,11 +337,11 @@ class Flow(Generic[P, R]):
|
|
345
337
|
self.result_storage = result_storage
|
346
338
|
self.result_serializer = result_serializer
|
347
339
|
self.cache_result_in_memory = cache_result_in_memory
|
348
|
-
self.
|
349
|
-
self.
|
350
|
-
self.
|
351
|
-
self.
|
352
|
-
self.
|
340
|
+
self.on_completion_hooks = on_completion or []
|
341
|
+
self.on_failure_hooks = on_failure or []
|
342
|
+
self.on_cancellation_hooks = on_cancellation or []
|
343
|
+
self.on_crashed_hooks = on_crashed or []
|
344
|
+
self.on_running_hooks = on_running or []
|
353
345
|
|
354
346
|
# Used for flows loaded from remote storage
|
355
347
|
self._storage: Optional[RunnerStorage] = None
|
@@ -365,20 +357,20 @@ class Flow(Generic[P, R]):
|
|
365
357
|
def with_options(
|
366
358
|
self,
|
367
359
|
*,
|
368
|
-
name: str = None,
|
369
|
-
version: str = None,
|
360
|
+
name: Optional[str] = None,
|
361
|
+
version: Optional[str] = None,
|
370
362
|
retries: Optional[int] = None,
|
371
363
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
372
|
-
description: str = None,
|
364
|
+
description: Optional[str] = None,
|
373
365
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
374
|
-
task_runner: Union[Type[
|
375
|
-
timeout_seconds: Union[int, float] = None,
|
376
|
-
validate_parameters: bool = None,
|
377
|
-
persist_result: Optional[bool] = NotSet,
|
378
|
-
result_storage: Optional[ResultStorage] = NotSet,
|
379
|
-
result_serializer: Optional[ResultSerializer] = NotSet,
|
380
|
-
cache_result_in_memory: bool = None,
|
381
|
-
log_prints: Optional[bool] = NotSet,
|
366
|
+
task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
|
367
|
+
timeout_seconds: Union[int, float, None] = None,
|
368
|
+
validate_parameters: Optional[bool] = None,
|
369
|
+
persist_result: Optional[bool] = NotSet, # type: ignore
|
370
|
+
result_storage: Optional[ResultStorage] = NotSet, # type: ignore
|
371
|
+
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
372
|
+
cache_result_in_memory: Optional[bool] = None,
|
373
|
+
log_prints: Optional[bool] = NotSet, # type: ignore
|
382
374
|
on_completion: Optional[
|
383
375
|
List[Callable[[FlowSchema, FlowRun, State], None]]
|
384
376
|
] = None,
|
@@ -434,15 +426,14 @@ class Flow(Generic[P, R]):
|
|
434
426
|
Create a new flow from an existing flow, update the task runner, and call
|
435
427
|
it without an intermediate variable:
|
436
428
|
|
437
|
-
>>> from prefect.task_runners import
|
429
|
+
>>> from prefect.task_runners import ThreadPoolTaskRunner
|
438
430
|
>>>
|
439
431
|
>>> @flow
|
440
432
|
>>> def my_flow(x, y):
|
441
433
|
>>> return x + y
|
442
434
|
>>>
|
443
|
-
>>> state = my_flow.with_options(task_runner=
|
435
|
+
>>> state = my_flow.with_options(task_runner=ThreadPoolTaskRunner)(1, 3)
|
444
436
|
>>> assert state.result() == 4
|
445
|
-
|
446
437
|
"""
|
447
438
|
new_flow = Flow(
|
448
439
|
fn=self.fn,
|
@@ -482,11 +473,11 @@ class Flow(Generic[P, R]):
|
|
482
473
|
else self.cache_result_in_memory
|
483
474
|
),
|
484
475
|
log_prints=log_prints if log_prints is not NotSet else self.log_prints,
|
485
|
-
on_completion=on_completion or self.
|
486
|
-
on_failure=on_failure or self.
|
487
|
-
on_cancellation=on_cancellation or self.
|
488
|
-
on_crashed=on_crashed or self.
|
489
|
-
on_running=on_running or self.
|
476
|
+
on_completion=on_completion or self.on_completion_hooks,
|
477
|
+
on_failure=on_failure or self.on_failure_hooks,
|
478
|
+
on_cancellation=on_cancellation or self.on_cancellation_hooks,
|
479
|
+
on_crashed=on_crashed or self.on_crashed_hooks,
|
480
|
+
on_running=on_running or self.on_running_hooks,
|
490
481
|
)
|
491
482
|
new_flow._storage = self._storage
|
492
483
|
new_flow._entrypoint = self._entrypoint
|
@@ -505,49 +496,52 @@ class Flow(Generic[P, R]):
|
|
505
496
|
"""
|
506
497
|
args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
|
507
498
|
|
508
|
-
|
499
|
+
with warnings.catch_warnings():
|
500
|
+
warnings.filterwarnings(
|
501
|
+
"ignore", category=pydantic.warnings.PydanticDeprecatedSince20
|
502
|
+
)
|
509
503
|
has_v1_models = any(isinstance(o, V1BaseModel) for o in args) or any(
|
510
504
|
isinstance(o, V1BaseModel) for o in kwargs.values()
|
511
505
|
)
|
512
|
-
has_v2_types = any(is_v2_type(o) for o in args) or any(
|
513
|
-
is_v2_type(o) for o in kwargs.values()
|
514
|
-
)
|
515
506
|
|
516
|
-
|
517
|
-
|
518
|
-
|
519
|
-
)
|
507
|
+
has_v2_types = any(is_v2_type(o) for o in args) or any(
|
508
|
+
is_v2_type(o) for o in kwargs.values()
|
509
|
+
)
|
520
510
|
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
else:
|
526
|
-
validated_fn = V2ValidatedFunction(
|
527
|
-
self.fn, config={"arbitrary_types_allowed": True}
|
528
|
-
)
|
511
|
+
if has_v1_models and has_v2_types:
|
512
|
+
raise ParameterTypeError(
|
513
|
+
"Cannot mix Pydantic v1 and v2 types as arguments to a flow."
|
514
|
+
)
|
529
515
|
|
530
|
-
|
531
|
-
validated_fn =
|
516
|
+
if has_v1_models:
|
517
|
+
validated_fn = V1ValidatedFunction(
|
532
518
|
self.fn, config={"arbitrary_types_allowed": True}
|
533
519
|
)
|
520
|
+
else:
|
521
|
+
validated_fn = V2ValidatedFunction(
|
522
|
+
self.fn, config=pydantic.ConfigDict(arbitrary_types_allowed=True)
|
523
|
+
)
|
534
524
|
|
535
525
|
try:
|
536
|
-
|
526
|
+
with warnings.catch_warnings():
|
527
|
+
warnings.filterwarnings(
|
528
|
+
"ignore", category=pydantic.warnings.PydanticDeprecatedSince20
|
529
|
+
)
|
530
|
+
model = validated_fn.init_model_instance(*args, **kwargs)
|
537
531
|
except pydantic.ValidationError as exc:
|
538
532
|
# We capture the pydantic exception and raise our own because the pydantic
|
539
533
|
# exception is not picklable when using a cythonized pydantic installation
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
534
|
+
logger.error(
|
535
|
+
f"Parameter validation failed for flow {self.name!r}: {exc.errors()}"
|
536
|
+
f"\nParameters: {parameters}"
|
537
|
+
)
|
544
538
|
raise ParameterTypeError.from_validation_error(exc) from None
|
545
539
|
|
546
540
|
# Get the updated parameter dict with cast values from the model
|
547
541
|
cast_parameters = {
|
548
542
|
k: v
|
549
|
-
for k, v in model.
|
550
|
-
if k in model.
|
543
|
+
for k, v in dict(model).items()
|
544
|
+
if k in model.model_fields_set or model.model_fields[k].default_factory
|
551
545
|
}
|
552
546
|
return cast_parameters
|
553
547
|
|
@@ -607,7 +601,7 @@ class Flow(Generic[P, R]):
|
|
607
601
|
description: Optional[str] = None,
|
608
602
|
tags: Optional[List[str]] = None,
|
609
603
|
version: Optional[str] = None,
|
610
|
-
enforce_parameter_schema: bool =
|
604
|
+
enforce_parameter_schema: bool = True,
|
611
605
|
work_pool_name: Optional[str] = None,
|
612
606
|
work_queue_name: Optional[str] = None,
|
613
607
|
job_variables: Optional[Dict[str, Any]] = None,
|
@@ -669,7 +663,8 @@ class Flow(Generic[P, R]):
|
|
669
663
|
from prefect.deployments.runner import RunnerDeployment
|
670
664
|
|
671
665
|
if not name.endswith(".py"):
|
672
|
-
|
666
|
+
_raise_on_name_with_banned_characters(name)
|
667
|
+
|
673
668
|
if self._storage and self._entrypoint:
|
674
669
|
return await RunnerDeployment.from_storage(
|
675
670
|
storage=self._storage,
|
@@ -715,6 +710,36 @@ class Flow(Generic[P, R]):
|
|
715
710
|
entrypoint_type=entrypoint_type,
|
716
711
|
)
|
717
712
|
|
713
|
+
def on_completion(
|
714
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
715
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
716
|
+
self.on_completion_hooks.append(fn)
|
717
|
+
return fn
|
718
|
+
|
719
|
+
def on_cancellation(
|
720
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
721
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
722
|
+
self.on_cancellation_hooks.append(fn)
|
723
|
+
return fn
|
724
|
+
|
725
|
+
def on_crashed(
|
726
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
727
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
728
|
+
self.on_crashed_hooks.append(fn)
|
729
|
+
return fn
|
730
|
+
|
731
|
+
def on_running(
|
732
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
733
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
734
|
+
self.on_running_hooks.append(fn)
|
735
|
+
return fn
|
736
|
+
|
737
|
+
def on_failure(
|
738
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
739
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
740
|
+
self.on_failure_hooks.append(fn)
|
741
|
+
return fn
|
742
|
+
|
718
743
|
@sync_compatible
|
719
744
|
async def serve(
|
720
745
|
self,
|
@@ -738,7 +763,7 @@ class Flow(Generic[P, R]):
|
|
738
763
|
description: Optional[str] = None,
|
739
764
|
tags: Optional[List[str]] = None,
|
740
765
|
version: Optional[str] = None,
|
741
|
-
enforce_parameter_schema: bool =
|
766
|
+
enforce_parameter_schema: bool = True,
|
742
767
|
pause_on_shutdown: bool = True,
|
743
768
|
print_starting_message: bool = True,
|
744
769
|
limit: Optional[int] = None,
|
@@ -947,7 +972,7 @@ class Flow(Generic[P, R]):
|
|
947
972
|
cron: Optional[str] = None,
|
948
973
|
rrule: Optional[str] = None,
|
949
974
|
paused: Optional[bool] = None,
|
950
|
-
schedules: Optional[List[
|
975
|
+
schedules: Optional[List[DeploymentScheduleCreate]] = None,
|
951
976
|
schedule: Optional[SCHEDULE_TYPES] = None,
|
952
977
|
is_schedule_active: Optional[bool] = None,
|
953
978
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
@@ -955,7 +980,7 @@ class Flow(Generic[P, R]):
|
|
955
980
|
description: Optional[str] = None,
|
956
981
|
tags: Optional[List[str]] = None,
|
957
982
|
version: Optional[str] = None,
|
958
|
-
enforce_parameter_schema: bool =
|
983
|
+
enforce_parameter_schema: bool = True,
|
959
984
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
960
985
|
print_next_steps: bool = True,
|
961
986
|
ignore_warnings: bool = False,
|
@@ -1200,7 +1225,10 @@ class Flow(Generic[P, R]):
|
|
1200
1225
|
>>> with tags("db", "blue"):
|
1201
1226
|
>>> my_flow("foo")
|
1202
1227
|
"""
|
1203
|
-
from prefect.
|
1228
|
+
from prefect.utilities.visualization import (
|
1229
|
+
get_task_viz_tracker,
|
1230
|
+
track_viz_task,
|
1231
|
+
)
|
1204
1232
|
|
1205
1233
|
# Convert the call args/kwargs to a parameter dict
|
1206
1234
|
parameters = get_call_parameters(self.fn, args, kwargs)
|
@@ -1213,72 +1241,19 @@ class Flow(Generic[P, R]):
|
|
1213
1241
|
# we can add support for exploring subflows for tasks in the future.
|
1214
1242
|
return track_viz_task(self.isasync, self.name, parameters)
|
1215
1243
|
|
1216
|
-
|
1217
|
-
from prefect.new_flow_engine import run_flow, run_flow_sync
|
1244
|
+
from prefect.flow_engine import run_flow, run_flow_sync
|
1218
1245
|
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
wait_for=wait_for,
|
1223
|
-
return_type=return_type,
|
1224
|
-
)
|
1225
|
-
if self.isasync:
|
1226
|
-
# this returns an awaitable coroutine
|
1227
|
-
return run_flow(**run_kwargs)
|
1228
|
-
else:
|
1229
|
-
return run_flow_sync(**run_kwargs)
|
1230
|
-
|
1231
|
-
return enter_flow_run_engine_from_flow_call(
|
1232
|
-
self,
|
1233
|
-
parameters,
|
1246
|
+
run_kwargs = dict(
|
1247
|
+
flow=self,
|
1248
|
+
parameters=parameters,
|
1234
1249
|
wait_for=wait_for,
|
1235
1250
|
return_type=return_type,
|
1236
1251
|
)
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
...
|
1243
|
-
|
1244
|
-
@overload
|
1245
|
-
def _run(
|
1246
|
-
self: "Flow[P, Coroutine[Any, Any, T]]", *args: P.args, **kwargs: P.kwargs
|
1247
|
-
) -> Awaitable[T]:
|
1248
|
-
...
|
1249
|
-
|
1250
|
-
@overload
|
1251
|
-
def _run(self: "Flow[P, T]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
|
1252
|
-
...
|
1253
|
-
|
1254
|
-
def _run(
|
1255
|
-
self,
|
1256
|
-
*args: "P.args",
|
1257
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1258
|
-
**kwargs: "P.kwargs",
|
1259
|
-
):
|
1260
|
-
"""
|
1261
|
-
Run the flow and return its final state.
|
1262
|
-
|
1263
|
-
Examples:
|
1264
|
-
|
1265
|
-
Run a flow and get the returned result
|
1266
|
-
|
1267
|
-
>>> state = my_flow._run("marvin")
|
1268
|
-
>>> state.result()
|
1269
|
-
"goodbye marvin"
|
1270
|
-
"""
|
1271
|
-
from prefect.engine import enter_flow_run_engine_from_flow_call
|
1272
|
-
|
1273
|
-
# Convert the call args/kwargs to a parameter dict
|
1274
|
-
parameters = get_call_parameters(self.fn, args, kwargs)
|
1275
|
-
|
1276
|
-
return enter_flow_run_engine_from_flow_call(
|
1277
|
-
self,
|
1278
|
-
parameters,
|
1279
|
-
wait_for=wait_for,
|
1280
|
-
return_type="state",
|
1281
|
-
)
|
1252
|
+
if self.isasync:
|
1253
|
+
# this returns an awaitable coroutine
|
1254
|
+
return run_flow(**run_kwargs)
|
1255
|
+
else:
|
1256
|
+
return run_flow_sync(**run_kwargs)
|
1282
1257
|
|
1283
1258
|
@sync_compatible
|
1284
1259
|
async def visualize(self, *args, **kwargs):
|
@@ -1291,6 +1266,16 @@ class Flow(Generic[P, R]):
|
|
1291
1266
|
- GraphvizExecutableNotFoundError: If the `dot` executable isn't found.
|
1292
1267
|
- FlowVisualizationError: If the flow can't be visualized for any other reason.
|
1293
1268
|
"""
|
1269
|
+
from prefect.utilities.visualization import (
|
1270
|
+
FlowVisualizationError,
|
1271
|
+
GraphvizExecutableNotFoundError,
|
1272
|
+
GraphvizImportError,
|
1273
|
+
TaskVizTracker,
|
1274
|
+
VisualizationUnsupportedError,
|
1275
|
+
build_task_dependencies,
|
1276
|
+
visualize_task_dependencies,
|
1277
|
+
)
|
1278
|
+
|
1294
1279
|
if not PREFECT_UNIT_TEST_MODE:
|
1295
1280
|
warnings.warn(
|
1296
1281
|
"`flow.visualize()` will execute code inside of your flow that is not"
|
@@ -1343,7 +1328,7 @@ def flow(
|
|
1343
1328
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1344
1329
|
retries: Optional[int] = None,
|
1345
1330
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
1346
|
-
task_runner:
|
1331
|
+
task_runner: Optional[TaskRunner] = None,
|
1347
1332
|
description: str = None,
|
1348
1333
|
timeout_seconds: Union[int, float] = None,
|
1349
1334
|
validate_parameters: bool = True,
|
@@ -1375,7 +1360,7 @@ def flow(
|
|
1375
1360
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1376
1361
|
retries: int = None,
|
1377
1362
|
retry_delay_seconds: Union[int, float] = None,
|
1378
|
-
task_runner:
|
1363
|
+
task_runner: Optional[TaskRunner] = None,
|
1379
1364
|
description: str = None,
|
1380
1365
|
timeout_seconds: Union[int, float] = None,
|
1381
1366
|
validate_parameters: bool = True,
|
@@ -1553,6 +1538,23 @@ def flow(
|
|
1553
1538
|
)
|
1554
1539
|
|
1555
1540
|
|
1541
|
+
def _raise_on_name_with_banned_characters(name: str) -> str:
|
1542
|
+
"""
|
1543
|
+
Raise an InvalidNameError if the given name contains any invalid
|
1544
|
+
characters.
|
1545
|
+
"""
|
1546
|
+
if name is None:
|
1547
|
+
return name
|
1548
|
+
|
1549
|
+
if not re.match(WITHOUT_BANNED_CHARACTERS, name):
|
1550
|
+
raise InvalidNameError(
|
1551
|
+
f"Name {name!r} contains an invalid character. "
|
1552
|
+
f"Must not contain any of: {BANNED_CHARACTERS}."
|
1553
|
+
)
|
1554
|
+
|
1555
|
+
return name
|
1556
|
+
|
1557
|
+
|
1556
1558
|
# Add from_source so it is available on the flow function we all know and love
|
1557
1559
|
flow.from_source = Flow.from_source
|
1558
1560
|
|
@@ -1799,6 +1801,87 @@ async def serve(
|
|
1799
1801
|
await runner.start()
|
1800
1802
|
|
1801
1803
|
|
1804
|
+
@client_injector
|
1805
|
+
async def load_flow_from_flow_run(
|
1806
|
+
client: "PrefectClient",
|
1807
|
+
flow_run: "FlowRun",
|
1808
|
+
ignore_storage: bool = False,
|
1809
|
+
storage_base_path: Optional[str] = None,
|
1810
|
+
) -> "Flow":
|
1811
|
+
"""
|
1812
|
+
Load a flow from the location/script provided in a deployment's storage document.
|
1813
|
+
|
1814
|
+
If `ignore_storage=True` is provided, no pull from remote storage occurs. This flag
|
1815
|
+
is largely for testing, and assumes the flow is already available locally.
|
1816
|
+
"""
|
1817
|
+
deployment = await client.read_deployment(flow_run.deployment_id)
|
1818
|
+
|
1819
|
+
if deployment.entrypoint is None:
|
1820
|
+
raise ValueError(
|
1821
|
+
f"Deployment {deployment.id} does not have an entrypoint and can not be run."
|
1822
|
+
)
|
1823
|
+
|
1824
|
+
run_logger = flow_run_logger(flow_run)
|
1825
|
+
|
1826
|
+
runner_storage_base_path = storage_base_path or os.environ.get(
|
1827
|
+
"PREFECT__STORAGE_BASE_PATH"
|
1828
|
+
)
|
1829
|
+
|
1830
|
+
# If there's no colon, assume it's a module path
|
1831
|
+
if ":" not in deployment.entrypoint:
|
1832
|
+
run_logger.debug(
|
1833
|
+
f"Importing flow code from module path {deployment.entrypoint}"
|
1834
|
+
)
|
1835
|
+
flow = await run_sync_in_worker_thread(
|
1836
|
+
load_flow_from_entrypoint, deployment.entrypoint
|
1837
|
+
)
|
1838
|
+
return flow
|
1839
|
+
|
1840
|
+
if not ignore_storage and not deployment.pull_steps:
|
1841
|
+
sys.path.insert(0, ".")
|
1842
|
+
if deployment.storage_document_id:
|
1843
|
+
storage_document = await client.read_block_document(
|
1844
|
+
deployment.storage_document_id
|
1845
|
+
)
|
1846
|
+
storage_block = Block._from_block_document(storage_document)
|
1847
|
+
else:
|
1848
|
+
basepath = deployment.path or Path(deployment.manifest_path).parent
|
1849
|
+
if runner_storage_base_path:
|
1850
|
+
basepath = str(basepath).replace(
|
1851
|
+
"$STORAGE_BASE_PATH", runner_storage_base_path
|
1852
|
+
)
|
1853
|
+
storage_block = LocalFileSystem(basepath=basepath)
|
1854
|
+
|
1855
|
+
from_path = (
|
1856
|
+
str(deployment.path).replace("$STORAGE_BASE_PATH", runner_storage_base_path)
|
1857
|
+
if runner_storage_base_path and deployment.path
|
1858
|
+
else deployment.path
|
1859
|
+
)
|
1860
|
+
run_logger.info(f"Downloading flow code from storage at {from_path!r}")
|
1861
|
+
await storage_block.get_directory(from_path=from_path, local_path=".")
|
1862
|
+
|
1863
|
+
if deployment.pull_steps:
|
1864
|
+
run_logger.debug(f"Running {len(deployment.pull_steps)} deployment pull steps")
|
1865
|
+
output = await run_steps(deployment.pull_steps)
|
1866
|
+
if output.get("directory"):
|
1867
|
+
run_logger.debug(f"Changing working directory to {output['directory']!r}")
|
1868
|
+
os.chdir(output["directory"])
|
1869
|
+
|
1870
|
+
import_path = relative_path_to_current_platform(deployment.entrypoint)
|
1871
|
+
# for backwards compat
|
1872
|
+
if deployment.manifest_path:
|
1873
|
+
with open(deployment.manifest_path, "r") as f:
|
1874
|
+
import_path = json.load(f)["import_path"]
|
1875
|
+
import_path = (
|
1876
|
+
Path(deployment.manifest_path).parent / import_path
|
1877
|
+
).absolute()
|
1878
|
+
run_logger.debug(f"Importing flow code from '{import_path}'")
|
1879
|
+
|
1880
|
+
flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, str(import_path))
|
1881
|
+
|
1882
|
+
return flow
|
1883
|
+
|
1884
|
+
|
1802
1885
|
def load_flow_argument_from_entrypoint(
|
1803
1886
|
entrypoint: str, arg: str = "name"
|
1804
1887
|
) -> Optional[str]:
|