prefect-client 2.20.4__py3-none-any.whl → 3.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- prefect/__init__.py +74 -110
- prefect/_internal/compatibility/deprecated.py +6 -115
- prefect/_internal/compatibility/experimental.py +4 -79
- prefect/_internal/compatibility/migration.py +166 -0
- prefect/_internal/concurrency/__init__.py +2 -2
- prefect/_internal/concurrency/api.py +1 -35
- prefect/_internal/concurrency/calls.py +0 -6
- prefect/_internal/concurrency/cancellation.py +0 -3
- prefect/_internal/concurrency/event_loop.py +0 -20
- prefect/_internal/concurrency/inspection.py +3 -3
- prefect/_internal/concurrency/primitives.py +1 -0
- prefect/_internal/concurrency/services.py +23 -0
- prefect/_internal/concurrency/threads.py +35 -0
- prefect/_internal/concurrency/waiters.py +0 -28
- prefect/_internal/integrations.py +7 -0
- prefect/_internal/pydantic/__init__.py +0 -45
- prefect/_internal/pydantic/annotations/pendulum.py +2 -2
- prefect/_internal/pydantic/v1_schema.py +21 -22
- prefect/_internal/pydantic/v2_schema.py +0 -2
- prefect/_internal/pydantic/v2_validated_func.py +18 -23
- prefect/_internal/pytz.py +1 -1
- prefect/_internal/retries.py +61 -0
- prefect/_internal/schemas/bases.py +45 -177
- prefect/_internal/schemas/fields.py +1 -43
- prefect/_internal/schemas/validators.py +47 -233
- prefect/agent.py +3 -695
- prefect/artifacts.py +173 -14
- prefect/automations.py +39 -4
- prefect/blocks/abstract.py +1 -1
- prefect/blocks/core.py +405 -153
- prefect/blocks/fields.py +2 -57
- prefect/blocks/notifications.py +43 -28
- prefect/blocks/redis.py +168 -0
- prefect/blocks/system.py +67 -20
- prefect/blocks/webhook.py +2 -9
- prefect/cache_policies.py +239 -0
- prefect/client/__init__.py +4 -0
- prefect/client/base.py +33 -27
- prefect/client/cloud.py +65 -20
- prefect/client/collections.py +1 -1
- prefect/client/orchestration.py +650 -442
- prefect/client/schemas/actions.py +115 -100
- prefect/client/schemas/filters.py +46 -52
- prefect/client/schemas/objects.py +228 -178
- prefect/client/schemas/responses.py +18 -36
- prefect/client/schemas/schedules.py +55 -36
- prefect/client/schemas/sorting.py +2 -0
- prefect/client/subscriptions.py +8 -7
- prefect/client/types/flexible_schedule_list.py +11 -0
- prefect/client/utilities.py +9 -6
- prefect/concurrency/asyncio.py +60 -11
- prefect/concurrency/context.py +24 -0
- prefect/concurrency/events.py +2 -2
- prefect/concurrency/services.py +46 -16
- prefect/concurrency/sync.py +51 -7
- prefect/concurrency/v1/asyncio.py +143 -0
- prefect/concurrency/v1/context.py +27 -0
- prefect/concurrency/v1/events.py +61 -0
- prefect/concurrency/v1/services.py +116 -0
- prefect/concurrency/v1/sync.py +92 -0
- prefect/context.py +246 -149
- prefect/deployments/__init__.py +33 -18
- prefect/deployments/base.py +10 -15
- prefect/deployments/deployments.py +2 -1048
- prefect/deployments/flow_runs.py +178 -0
- prefect/deployments/runner.py +72 -173
- prefect/deployments/schedules.py +31 -25
- prefect/deployments/steps/__init__.py +0 -1
- prefect/deployments/steps/core.py +7 -0
- prefect/deployments/steps/pull.py +15 -21
- prefect/deployments/steps/utility.py +2 -1
- prefect/docker/__init__.py +20 -0
- prefect/docker/docker_image.py +82 -0
- prefect/engine.py +15 -2475
- prefect/events/actions.py +17 -23
- prefect/events/cli/automations.py +20 -7
- prefect/events/clients.py +142 -80
- prefect/events/filters.py +14 -18
- prefect/events/related.py +74 -75
- prefect/events/schemas/__init__.py +0 -5
- prefect/events/schemas/automations.py +55 -46
- prefect/events/schemas/deployment_triggers.py +7 -197
- prefect/events/schemas/events.py +46 -65
- prefect/events/schemas/labelling.py +10 -14
- prefect/events/utilities.py +4 -5
- prefect/events/worker.py +23 -8
- prefect/exceptions.py +15 -0
- prefect/filesystems.py +30 -529
- prefect/flow_engine.py +827 -0
- prefect/flow_runs.py +379 -7
- prefect/flows.py +470 -360
- prefect/futures.py +382 -331
- prefect/infrastructure/__init__.py +5 -26
- prefect/infrastructure/base.py +3 -320
- prefect/infrastructure/provisioners/__init__.py +5 -3
- prefect/infrastructure/provisioners/cloud_run.py +13 -8
- prefect/infrastructure/provisioners/container_instance.py +14 -9
- prefect/infrastructure/provisioners/ecs.py +10 -8
- prefect/infrastructure/provisioners/modal.py +8 -5
- prefect/input/__init__.py +4 -0
- prefect/input/actions.py +2 -4
- prefect/input/run_input.py +9 -9
- prefect/logging/formatters.py +2 -4
- prefect/logging/handlers.py +9 -14
- prefect/logging/loggers.py +5 -5
- prefect/main.py +72 -0
- prefect/plugins.py +2 -64
- prefect/profiles.toml +16 -2
- prefect/records/__init__.py +1 -0
- prefect/records/base.py +223 -0
- prefect/records/filesystem.py +207 -0
- prefect/records/memory.py +178 -0
- prefect/records/result_store.py +64 -0
- prefect/results.py +577 -504
- prefect/runner/runner.py +117 -47
- prefect/runner/server.py +32 -34
- prefect/runner/storage.py +3 -12
- prefect/runner/submit.py +2 -10
- prefect/runner/utils.py +2 -2
- prefect/runtime/__init__.py +1 -0
- prefect/runtime/deployment.py +1 -0
- prefect/runtime/flow_run.py +40 -5
- prefect/runtime/task_run.py +1 -0
- prefect/serializers.py +28 -39
- prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
- prefect/settings.py +209 -332
- prefect/states.py +160 -63
- prefect/task_engine.py +1478 -57
- prefect/task_runners.py +383 -287
- prefect/task_runs.py +240 -0
- prefect/task_worker.py +463 -0
- prefect/tasks.py +684 -374
- prefect/transactions.py +410 -0
- prefect/types/__init__.py +72 -86
- prefect/types/entrypoint.py +13 -0
- prefect/utilities/annotations.py +4 -3
- prefect/utilities/asyncutils.py +227 -148
- prefect/utilities/callables.py +137 -45
- prefect/utilities/collections.py +134 -86
- prefect/utilities/dispatch.py +27 -14
- prefect/utilities/dockerutils.py +11 -4
- prefect/utilities/engine.py +186 -32
- prefect/utilities/filesystem.py +4 -5
- prefect/utilities/importtools.py +26 -27
- prefect/utilities/pydantic.py +128 -38
- prefect/utilities/schema_tools/hydration.py +18 -1
- prefect/utilities/schema_tools/validation.py +30 -0
- prefect/utilities/services.py +35 -9
- prefect/utilities/templating.py +12 -2
- prefect/utilities/timeout.py +20 -5
- prefect/utilities/urls.py +195 -0
- prefect/utilities/visualization.py +1 -0
- prefect/variables.py +78 -59
- prefect/workers/__init__.py +0 -1
- prefect/workers/base.py +237 -244
- prefect/workers/block.py +5 -226
- prefect/workers/cloud.py +6 -0
- prefect/workers/process.py +265 -12
- prefect/workers/server.py +29 -11
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/METADATA +28 -24
- prefect_client-3.0.0.dist-info/RECORD +201 -0
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/WHEEL +1 -1
- prefect/_internal/pydantic/_base_model.py +0 -51
- prefect/_internal/pydantic/_compat.py +0 -82
- prefect/_internal/pydantic/_flags.py +0 -20
- prefect/_internal/pydantic/_types.py +0 -8
- prefect/_internal/pydantic/utilities/config_dict.py +0 -72
- prefect/_internal/pydantic/utilities/field_validator.py +0 -150
- prefect/_internal/pydantic/utilities/model_construct.py +0 -56
- prefect/_internal/pydantic/utilities/model_copy.py +0 -55
- prefect/_internal/pydantic/utilities/model_dump.py +0 -136
- prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
- prefect/_internal/pydantic/utilities/model_fields.py +0 -50
- prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
- prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
- prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
- prefect/_internal/pydantic/utilities/model_validate.py +0 -75
- prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
- prefect/_internal/pydantic/utilities/model_validator.py +0 -87
- prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
- prefect/_vendor/fastapi/__init__.py +0 -25
- prefect/_vendor/fastapi/applications.py +0 -946
- prefect/_vendor/fastapi/background.py +0 -3
- prefect/_vendor/fastapi/concurrency.py +0 -44
- prefect/_vendor/fastapi/datastructures.py +0 -58
- prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
- prefect/_vendor/fastapi/dependencies/models.py +0 -64
- prefect/_vendor/fastapi/dependencies/utils.py +0 -877
- prefect/_vendor/fastapi/encoders.py +0 -177
- prefect/_vendor/fastapi/exception_handlers.py +0 -40
- prefect/_vendor/fastapi/exceptions.py +0 -46
- prefect/_vendor/fastapi/logger.py +0 -3
- prefect/_vendor/fastapi/middleware/__init__.py +0 -1
- prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
- prefect/_vendor/fastapi/middleware/cors.py +0 -3
- prefect/_vendor/fastapi/middleware/gzip.py +0 -3
- prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
- prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
- prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
- prefect/_vendor/fastapi/openapi/__init__.py +0 -0
- prefect/_vendor/fastapi/openapi/constants.py +0 -2
- prefect/_vendor/fastapi/openapi/docs.py +0 -203
- prefect/_vendor/fastapi/openapi/models.py +0 -480
- prefect/_vendor/fastapi/openapi/utils.py +0 -485
- prefect/_vendor/fastapi/param_functions.py +0 -340
- prefect/_vendor/fastapi/params.py +0 -453
- prefect/_vendor/fastapi/py.typed +0 -0
- prefect/_vendor/fastapi/requests.py +0 -4
- prefect/_vendor/fastapi/responses.py +0 -40
- prefect/_vendor/fastapi/routing.py +0 -1331
- prefect/_vendor/fastapi/security/__init__.py +0 -15
- prefect/_vendor/fastapi/security/api_key.py +0 -98
- prefect/_vendor/fastapi/security/base.py +0 -6
- prefect/_vendor/fastapi/security/http.py +0 -172
- prefect/_vendor/fastapi/security/oauth2.py +0 -227
- prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
- prefect/_vendor/fastapi/security/utils.py +0 -10
- prefect/_vendor/fastapi/staticfiles.py +0 -1
- prefect/_vendor/fastapi/templating.py +0 -3
- prefect/_vendor/fastapi/testclient.py +0 -1
- prefect/_vendor/fastapi/types.py +0 -3
- prefect/_vendor/fastapi/utils.py +0 -235
- prefect/_vendor/fastapi/websockets.py +0 -7
- prefect/_vendor/starlette/__init__.py +0 -1
- prefect/_vendor/starlette/_compat.py +0 -28
- prefect/_vendor/starlette/_exception_handler.py +0 -80
- prefect/_vendor/starlette/_utils.py +0 -88
- prefect/_vendor/starlette/applications.py +0 -261
- prefect/_vendor/starlette/authentication.py +0 -159
- prefect/_vendor/starlette/background.py +0 -43
- prefect/_vendor/starlette/concurrency.py +0 -59
- prefect/_vendor/starlette/config.py +0 -151
- prefect/_vendor/starlette/convertors.py +0 -87
- prefect/_vendor/starlette/datastructures.py +0 -707
- prefect/_vendor/starlette/endpoints.py +0 -130
- prefect/_vendor/starlette/exceptions.py +0 -60
- prefect/_vendor/starlette/formparsers.py +0 -276
- prefect/_vendor/starlette/middleware/__init__.py +0 -17
- prefect/_vendor/starlette/middleware/authentication.py +0 -52
- prefect/_vendor/starlette/middleware/base.py +0 -220
- prefect/_vendor/starlette/middleware/cors.py +0 -176
- prefect/_vendor/starlette/middleware/errors.py +0 -265
- prefect/_vendor/starlette/middleware/exceptions.py +0 -74
- prefect/_vendor/starlette/middleware/gzip.py +0 -113
- prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
- prefect/_vendor/starlette/middleware/sessions.py +0 -82
- prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
- prefect/_vendor/starlette/middleware/wsgi.py +0 -147
- prefect/_vendor/starlette/py.typed +0 -0
- prefect/_vendor/starlette/requests.py +0 -328
- prefect/_vendor/starlette/responses.py +0 -347
- prefect/_vendor/starlette/routing.py +0 -933
- prefect/_vendor/starlette/schemas.py +0 -154
- prefect/_vendor/starlette/staticfiles.py +0 -248
- prefect/_vendor/starlette/status.py +0 -199
- prefect/_vendor/starlette/templating.py +0 -231
- prefect/_vendor/starlette/testclient.py +0 -804
- prefect/_vendor/starlette/types.py +0 -30
- prefect/_vendor/starlette/websockets.py +0 -193
- prefect/blocks/kubernetes.py +0 -119
- prefect/deprecated/__init__.py +0 -0
- prefect/deprecated/data_documents.py +0 -350
- prefect/deprecated/packaging/__init__.py +0 -12
- prefect/deprecated/packaging/base.py +0 -96
- prefect/deprecated/packaging/docker.py +0 -146
- prefect/deprecated/packaging/file.py +0 -92
- prefect/deprecated/packaging/orion.py +0 -80
- prefect/deprecated/packaging/serializers.py +0 -171
- prefect/events/instrument.py +0 -135
- prefect/infrastructure/container.py +0 -824
- prefect/infrastructure/kubernetes.py +0 -920
- prefect/infrastructure/process.py +0 -289
- prefect/manifests.py +0 -20
- prefect/new_flow_engine.py +0 -449
- prefect/new_task_engine.py +0 -423
- prefect/pydantic/__init__.py +0 -76
- prefect/pydantic/main.py +0 -39
- prefect/software/__init__.py +0 -2
- prefect/software/base.py +0 -50
- prefect/software/conda.py +0 -199
- prefect/software/pip.py +0 -122
- prefect/software/python.py +0 -52
- prefect/task_server.py +0 -322
- prefect_client-2.20.4.dist-info/RECORD +0 -294
- /prefect/{_internal/pydantic/utilities → client/types}/__init__.py +0 -0
- /prefect/{_vendor → concurrency/v1}/__init__.py +0 -0
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/LICENSE +0 -0
- {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/top_level.txt +0 -0
prefect/flows.py
CHANGED
@@ -4,21 +4,22 @@ Module containing the base workflow class and decorator - for most use cases, us
|
|
4
4
|
|
5
5
|
# This file requires type-checking with pyright because mypy does not yet support PEP612
|
6
6
|
# See https://github.com/python/mypy/issues/8645
|
7
|
-
|
8
7
|
import ast
|
8
|
+
import asyncio
|
9
9
|
import datetime
|
10
10
|
import importlib.util
|
11
11
|
import inspect
|
12
12
|
import os
|
13
|
+
import re
|
14
|
+
import sys
|
13
15
|
import tempfile
|
14
16
|
import warnings
|
17
|
+
from copy import copy
|
15
18
|
from functools import partial, update_wrapper
|
16
19
|
from pathlib import Path
|
17
|
-
from tempfile import NamedTemporaryFile
|
18
20
|
from typing import (
|
19
21
|
TYPE_CHECKING,
|
20
22
|
Any,
|
21
|
-
AnyStr,
|
22
23
|
Awaitable,
|
23
24
|
Callable,
|
24
25
|
Coroutine,
|
@@ -38,89 +39,68 @@ from typing import (
|
|
38
39
|
)
|
39
40
|
from uuid import UUID
|
40
41
|
|
42
|
+
import pydantic
|
43
|
+
from fastapi.encoders import jsonable_encoder
|
44
|
+
from pydantic.v1 import BaseModel as V1BaseModel
|
45
|
+
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
46
|
+
from pydantic.v1.errors import ConfigError # TODO
|
41
47
|
from rich.console import Console
|
42
48
|
from typing_extensions import Literal, ParamSpec, Self
|
43
49
|
|
44
|
-
from prefect._internal.pydantic import HAS_PYDANTIC_V2
|
45
|
-
|
46
|
-
if HAS_PYDANTIC_V2:
|
47
|
-
import pydantic.v1 as pydantic
|
48
|
-
from pydantic import ValidationError as V2ValidationError
|
49
|
-
from pydantic.v1 import BaseModel as V1BaseModel
|
50
|
-
from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
|
51
|
-
|
52
|
-
from ._internal.pydantic.v2_schema import is_v2_type
|
53
|
-
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
54
|
-
from ._internal.pydantic.v2_validated_func import (
|
55
|
-
V2ValidatedFunction as ValidatedFunction,
|
56
|
-
)
|
57
|
-
|
58
|
-
else:
|
59
|
-
import pydantic
|
60
|
-
from pydantic.decorator import ValidatedFunction
|
61
|
-
|
62
|
-
V2ValidationError = None
|
63
|
-
|
64
|
-
from prefect._vendor.fastapi.encoders import jsonable_encoder
|
65
|
-
|
66
|
-
from prefect._internal.compatibility.deprecated import deprecated_parameter
|
67
50
|
from prefect._internal.concurrency.api import create_call, from_async
|
68
|
-
from prefect.
|
51
|
+
from prefect.blocks.core import Block
|
69
52
|
from prefect.client.orchestration import get_client
|
53
|
+
from prefect.client.schemas.actions import DeploymentScheduleCreate
|
70
54
|
from prefect.client.schemas.objects import Flow as FlowSchema
|
71
|
-
from prefect.client.schemas.objects import FlowRun
|
72
|
-
from prefect.client.
|
73
|
-
from prefect.
|
74
|
-
from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
|
55
|
+
from prefect.client.schemas.objects import FlowRun
|
56
|
+
from prefect.client.utilities import client_injector
|
57
|
+
from prefect.docker.docker_image import DockerImage
|
75
58
|
from prefect.events import DeploymentTriggerTypes, TriggerTypes
|
76
59
|
from prefect.exceptions import (
|
60
|
+
InvalidNameError,
|
77
61
|
MissingFlowError,
|
78
62
|
ObjectNotFound,
|
79
63
|
ParameterTypeError,
|
80
64
|
ScriptError,
|
65
|
+
TerminationSignal,
|
81
66
|
UnspecifiedFlowError,
|
82
67
|
)
|
83
|
-
from prefect.filesystems import ReadableDeploymentStorage
|
68
|
+
from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
|
84
69
|
from prefect.futures import PrefectFuture
|
85
70
|
from prefect.logging import get_logger
|
71
|
+
from prefect.logging.loggers import flow_run_logger
|
86
72
|
from prefect.results import ResultSerializer, ResultStorage
|
87
|
-
from prefect.runner.storage import (
|
88
|
-
BlockStorageAdapter,
|
89
|
-
LocalStorage,
|
90
|
-
RunnerStorage,
|
91
|
-
create_storage_from_source,
|
92
|
-
)
|
93
73
|
from prefect.settings import (
|
94
74
|
PREFECT_DEFAULT_WORK_POOL_NAME,
|
95
|
-
PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE,
|
96
75
|
PREFECT_FLOW_DEFAULT_RETRIES,
|
97
76
|
PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS,
|
98
77
|
PREFECT_UI_URL,
|
99
78
|
PREFECT_UNIT_TEST_MODE,
|
100
79
|
)
|
101
80
|
from prefect.states import State
|
102
|
-
from prefect.task_runners import
|
81
|
+
from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
|
82
|
+
from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
|
83
|
+
from prefect.types.entrypoint import EntrypointType
|
103
84
|
from prefect.utilities.annotations import NotSet
|
104
|
-
from prefect.utilities.asyncutils import
|
85
|
+
from prefect.utilities.asyncutils import (
|
86
|
+
run_sync_in_worker_thread,
|
87
|
+
sync_compatible,
|
88
|
+
)
|
105
89
|
from prefect.utilities.callables import (
|
106
90
|
get_call_parameters,
|
107
91
|
parameter_schema,
|
108
92
|
parameters_to_args_kwargs,
|
109
93
|
raise_for_reserved_arguments,
|
110
94
|
)
|
111
|
-
from prefect.utilities.collections import listrepr
|
95
|
+
from prefect.utilities.collections import listrepr, visit_collection
|
96
|
+
from prefect.utilities.filesystem import relative_path_to_current_platform
|
112
97
|
from prefect.utilities.hashing import file_hash
|
113
98
|
from prefect.utilities.importtools import import_object, safe_load_namespace
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
VisualizationUnsupportedError,
|
120
|
-
build_task_dependencies,
|
121
|
-
get_task_viz_tracker,
|
122
|
-
track_viz_task,
|
123
|
-
visualize_task_dependencies,
|
99
|
+
|
100
|
+
from ._internal.pydantic.v2_schema import is_v2_type
|
101
|
+
from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
|
102
|
+
from ._internal.pydantic.v2_validated_func import (
|
103
|
+
V2ValidatedFunction as ValidatedFunction,
|
124
104
|
)
|
125
105
|
|
126
106
|
T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
|
@@ -131,10 +111,13 @@ F = TypeVar("F", bound="Flow") # The type of the flow
|
|
131
111
|
logger = get_logger("flows")
|
132
112
|
|
133
113
|
if TYPE_CHECKING:
|
134
|
-
from prefect.
|
114
|
+
from prefect.client.orchestration import PrefectClient
|
115
|
+
from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
|
116
|
+
from prefect.deployments.runner import RunnerDeployment
|
117
|
+
from prefect.flows import FlowRun
|
118
|
+
from prefect.runner.storage import RunnerStorage
|
135
119
|
|
136
120
|
|
137
|
-
@PrefectObjectRegistry.register_instances
|
138
121
|
class Flow(Generic[P, R]):
|
139
122
|
"""
|
140
123
|
A Prefect workflow definition.
|
@@ -157,7 +140,7 @@ class Flow(Generic[P, R]):
|
|
157
140
|
be provided as a string template with the flow's parameters as variables,
|
158
141
|
or a function that returns a string.
|
159
142
|
task_runner: An optional task runner to use for task execution within the flow;
|
160
|
-
if not provided, a `
|
143
|
+
if not provided, a `ThreadPoolTaskRunner` will be used.
|
161
144
|
description: An optional string description for the flow; if not provided, the
|
162
145
|
description will be pulled from the docstring for the decorated function.
|
163
146
|
timeout_seconds: An optional number of seconds indicating a maximum runtime for
|
@@ -202,12 +185,12 @@ class Flow(Generic[P, R]):
|
|
202
185
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
203
186
|
retries: Optional[int] = None,
|
204
187
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
205
|
-
task_runner: Union[Type[
|
206
|
-
description: str = None,
|
207
|
-
timeout_seconds: Union[int, float] = None,
|
188
|
+
task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
|
189
|
+
description: Optional[str] = None,
|
190
|
+
timeout_seconds: Union[int, float, None] = None,
|
208
191
|
validate_parameters: bool = True,
|
209
192
|
persist_result: Optional[bool] = None,
|
210
|
-
result_storage: Optional[ResultStorage] = None,
|
193
|
+
result_storage: Optional[Union[ResultStorage, str]] = None,
|
211
194
|
result_serializer: Optional[ResultSerializer] = None,
|
212
195
|
cache_result_in_memory: bool = True,
|
213
196
|
log_prints: Optional[bool] = None,
|
@@ -251,8 +234,6 @@ class Flow(Generic[P, R]):
|
|
251
234
|
]
|
252
235
|
for hooks, hook_name in zip(hook_categories, hook_names):
|
253
236
|
if hooks is not None:
|
254
|
-
if not hooks:
|
255
|
-
raise ValueError(f"Empty list passed for '{hook_name}'")
|
256
237
|
try:
|
257
238
|
hooks = list(hooks)
|
258
239
|
except TypeError:
|
@@ -279,7 +260,7 @@ class Flow(Generic[P, R]):
|
|
279
260
|
|
280
261
|
# Validate name if given
|
281
262
|
if name:
|
282
|
-
|
263
|
+
_raise_on_name_with_banned_characters(name)
|
283
264
|
|
284
265
|
self.name = name or fn.__name__.replace("_", "-")
|
285
266
|
|
@@ -291,7 +272,8 @@ class Flow(Generic[P, R]):
|
|
291
272
|
)
|
292
273
|
self.flow_run_name = flow_run_name
|
293
274
|
|
294
|
-
|
275
|
+
default_task_runner = ThreadPoolTaskRunner()
|
276
|
+
task_runner = task_runner or default_task_runner
|
295
277
|
self.task_runner = (
|
296
278
|
task_runner() if isinstance(task_runner, type) else task_runner
|
297
279
|
)
|
@@ -301,7 +283,18 @@ class Flow(Generic[P, R]):
|
|
301
283
|
self.description = description or inspect.getdoc(fn)
|
302
284
|
update_wrapper(self, fn)
|
303
285
|
self.fn = fn
|
304
|
-
|
286
|
+
|
287
|
+
# the flow is considered async if its function is async or an async
|
288
|
+
# generator
|
289
|
+
self.isasync = inspect.iscoroutinefunction(
|
290
|
+
self.fn
|
291
|
+
) or inspect.isasyncgenfunction(self.fn)
|
292
|
+
|
293
|
+
# the flow is considered a generator if its function is a generator or
|
294
|
+
# an async generator
|
295
|
+
self.isgenerator = inspect.isgeneratorfunction(
|
296
|
+
self.fn
|
297
|
+
) or inspect.isasyncgenfunction(self.fn)
|
305
298
|
|
306
299
|
raise_for_reserved_arguments(self.fn, ["return_state", "wait_for"])
|
307
300
|
|
@@ -339,24 +332,35 @@ class Flow(Generic[P, R]):
|
|
339
332
|
# is not picklable in some environments
|
340
333
|
try:
|
341
334
|
ValidatedFunction(self.fn, config={"arbitrary_types_allowed": True})
|
342
|
-
except
|
335
|
+
except ConfigError as exc:
|
343
336
|
raise ValueError(
|
344
337
|
"Flow function is not compatible with `validate_parameters`. "
|
345
338
|
"Disable validation or change the argument names."
|
346
339
|
) from exc
|
347
340
|
|
341
|
+
# result persistence settings
|
342
|
+
if persist_result is None:
|
343
|
+
if result_storage is not None or result_serializer is not None:
|
344
|
+
persist_result = True
|
345
|
+
|
348
346
|
self.persist_result = persist_result
|
347
|
+
if result_storage and not isinstance(result_storage, str):
|
348
|
+
if getattr(result_storage, "_block_document_id", None) is None:
|
349
|
+
raise TypeError(
|
350
|
+
"Result storage configuration must be persisted server-side."
|
351
|
+
" Please call `.save()` on your block before passing it in."
|
352
|
+
)
|
349
353
|
self.result_storage = result_storage
|
350
354
|
self.result_serializer = result_serializer
|
351
355
|
self.cache_result_in_memory = cache_result_in_memory
|
352
|
-
self.
|
353
|
-
self.
|
354
|
-
self.
|
355
|
-
self.
|
356
|
-
self.
|
356
|
+
self.on_completion_hooks = on_completion or []
|
357
|
+
self.on_failure_hooks = on_failure or []
|
358
|
+
self.on_cancellation_hooks = on_cancellation or []
|
359
|
+
self.on_crashed_hooks = on_crashed or []
|
360
|
+
self.on_running_hooks = on_running or []
|
357
361
|
|
358
362
|
# Used for flows loaded from remote storage
|
359
|
-
self._storage: Optional[RunnerStorage] = None
|
363
|
+
self._storage: Optional["RunnerStorage"] = None
|
360
364
|
self._entrypoint: Optional[str] = None
|
361
365
|
|
362
366
|
module = fn.__module__
|
@@ -366,23 +370,45 @@ class Flow(Generic[P, R]):
|
|
366
370
|
|
367
371
|
self._entrypoint = f"{module}:{fn.__name__}"
|
368
372
|
|
373
|
+
@property
|
374
|
+
def ismethod(self) -> bool:
|
375
|
+
return hasattr(self.fn, "__prefect_self__")
|
376
|
+
|
377
|
+
def __get__(self, instance, owner):
|
378
|
+
"""
|
379
|
+
Implement the descriptor protocol so that the flow can be used as an instance method.
|
380
|
+
When an instance method is loaded, this method is called with the "self" instance as
|
381
|
+
an argument. We return a copy of the flow with that instance bound to the flow's function.
|
382
|
+
"""
|
383
|
+
|
384
|
+
# if no instance is provided, it's being accessed on the class
|
385
|
+
if instance is None:
|
386
|
+
return self
|
387
|
+
|
388
|
+
# if the flow is being accessed on an instance, bind the instance to the __prefect_self__ attribute
|
389
|
+
# of the flow's function. This will allow it to be automatically added to the flow's parameters
|
390
|
+
else:
|
391
|
+
bound_flow = copy(self)
|
392
|
+
bound_flow.fn.__prefect_self__ = instance
|
393
|
+
return bound_flow
|
394
|
+
|
369
395
|
def with_options(
|
370
396
|
self,
|
371
397
|
*,
|
372
|
-
name: str = None,
|
373
|
-
version: str = None,
|
398
|
+
name: Optional[str] = None,
|
399
|
+
version: Optional[str] = None,
|
374
400
|
retries: Optional[int] = None,
|
375
401
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
376
|
-
description: str = None,
|
402
|
+
description: Optional[str] = None,
|
377
403
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
378
|
-
task_runner: Union[Type[
|
379
|
-
timeout_seconds: Union[int, float] = None,
|
380
|
-
validate_parameters: bool = None,
|
381
|
-
persist_result: Optional[bool] = NotSet,
|
382
|
-
result_storage: Optional[ResultStorage] = NotSet,
|
383
|
-
result_serializer: Optional[ResultSerializer] = NotSet,
|
384
|
-
cache_result_in_memory: bool = None,
|
385
|
-
log_prints: Optional[bool] = NotSet,
|
404
|
+
task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
|
405
|
+
timeout_seconds: Union[int, float, None] = None,
|
406
|
+
validate_parameters: Optional[bool] = None,
|
407
|
+
persist_result: Optional[bool] = NotSet, # type: ignore
|
408
|
+
result_storage: Optional[ResultStorage] = NotSet, # type: ignore
|
409
|
+
result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
|
410
|
+
cache_result_in_memory: Optional[bool] = None,
|
411
|
+
log_prints: Optional[bool] = NotSet, # type: ignore
|
386
412
|
on_completion: Optional[
|
387
413
|
List[Callable[[FlowSchema, FlowRun, State], None]]
|
388
414
|
] = None,
|
@@ -438,15 +464,14 @@ class Flow(Generic[P, R]):
|
|
438
464
|
Create a new flow from an existing flow, update the task runner, and call
|
439
465
|
it without an intermediate variable:
|
440
466
|
|
441
|
-
>>> from prefect.task_runners import
|
467
|
+
>>> from prefect.task_runners import ThreadPoolTaskRunner
|
442
468
|
>>>
|
443
469
|
>>> @flow
|
444
470
|
>>> def my_flow(x, y):
|
445
471
|
>>> return x + y
|
446
472
|
>>>
|
447
|
-
>>> state = my_flow.with_options(task_runner=
|
473
|
+
>>> state = my_flow.with_options(task_runner=ThreadPoolTaskRunner)(1, 3)
|
448
474
|
>>> assert state.result() == 4
|
449
|
-
|
450
475
|
"""
|
451
476
|
new_flow = Flow(
|
452
477
|
fn=self.fn,
|
@@ -486,11 +511,11 @@ class Flow(Generic[P, R]):
|
|
486
511
|
else self.cache_result_in_memory
|
487
512
|
),
|
488
513
|
log_prints=log_prints if log_prints is not NotSet else self.log_prints,
|
489
|
-
on_completion=on_completion or self.
|
490
|
-
on_failure=on_failure or self.
|
491
|
-
on_cancellation=on_cancellation or self.
|
492
|
-
on_crashed=on_crashed or self.
|
493
|
-
on_running=on_running or self.
|
514
|
+
on_completion=on_completion or self.on_completion_hooks,
|
515
|
+
on_failure=on_failure or self.on_failure_hooks,
|
516
|
+
on_cancellation=on_cancellation or self.on_cancellation_hooks,
|
517
|
+
on_crashed=on_crashed or self.on_crashed_hooks,
|
518
|
+
on_running=on_running or self.on_running_hooks,
|
494
519
|
)
|
495
520
|
new_flow._storage = self._storage
|
496
521
|
new_flow._entrypoint = self._entrypoint
|
@@ -507,51 +532,69 @@ class Flow(Generic[P, R]):
|
|
507
532
|
Raises:
|
508
533
|
ParameterTypeError: if the provided parameters are not valid
|
509
534
|
"""
|
535
|
+
|
536
|
+
def resolve_block_reference(data: Any) -> Any:
|
537
|
+
if isinstance(data, dict) and "$ref" in data:
|
538
|
+
return Block.load_from_ref(data["$ref"])
|
539
|
+
return data
|
540
|
+
|
541
|
+
try:
|
542
|
+
parameters = visit_collection(
|
543
|
+
parameters, resolve_block_reference, return_data=True
|
544
|
+
)
|
545
|
+
except (ValueError, RuntimeError) as exc:
|
546
|
+
raise ParameterTypeError(
|
547
|
+
"Failed to resolve block references in parameters."
|
548
|
+
) from exc
|
549
|
+
|
510
550
|
args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
|
511
551
|
|
512
|
-
|
552
|
+
with warnings.catch_warnings():
|
553
|
+
warnings.filterwarnings(
|
554
|
+
"ignore", category=pydantic.warnings.PydanticDeprecatedSince20
|
555
|
+
)
|
513
556
|
has_v1_models = any(isinstance(o, V1BaseModel) for o in args) or any(
|
514
557
|
isinstance(o, V1BaseModel) for o in kwargs.values()
|
515
558
|
)
|
516
|
-
has_v2_types = any(is_v2_type(o) for o in args) or any(
|
517
|
-
is_v2_type(o) for o in kwargs.values()
|
518
|
-
)
|
519
559
|
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
)
|
560
|
+
has_v2_types = any(is_v2_type(o) for o in args) or any(
|
561
|
+
is_v2_type(o) for o in kwargs.values()
|
562
|
+
)
|
524
563
|
|
525
|
-
|
526
|
-
|
527
|
-
|
528
|
-
|
529
|
-
else:
|
530
|
-
validated_fn = V2ValidatedFunction(
|
531
|
-
self.fn, config={"arbitrary_types_allowed": True}
|
532
|
-
)
|
564
|
+
if has_v1_models and has_v2_types:
|
565
|
+
raise ParameterTypeError(
|
566
|
+
"Cannot mix Pydantic v1 and v2 types as arguments to a flow."
|
567
|
+
)
|
533
568
|
|
534
|
-
|
535
|
-
validated_fn =
|
569
|
+
if has_v1_models:
|
570
|
+
validated_fn = V1ValidatedFunction(
|
536
571
|
self.fn, config={"arbitrary_types_allowed": True}
|
537
572
|
)
|
573
|
+
else:
|
574
|
+
validated_fn = V2ValidatedFunction(
|
575
|
+
self.fn, config=pydantic.ConfigDict(arbitrary_types_allowed=True)
|
576
|
+
)
|
538
577
|
|
539
578
|
try:
|
540
|
-
|
579
|
+
with warnings.catch_warnings():
|
580
|
+
warnings.filterwarnings(
|
581
|
+
"ignore", category=pydantic.warnings.PydanticDeprecatedSince20
|
582
|
+
)
|
583
|
+
model = validated_fn.init_model_instance(*args, **kwargs)
|
541
584
|
except pydantic.ValidationError as exc:
|
542
585
|
# We capture the pydantic exception and raise our own because the pydantic
|
543
586
|
# exception is not picklable when using a cythonized pydantic installation
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
587
|
+
logger.error(
|
588
|
+
f"Parameter validation failed for flow {self.name!r}: {exc.errors()}"
|
589
|
+
f"\nParameters: {parameters}"
|
590
|
+
)
|
548
591
|
raise ParameterTypeError.from_validation_error(exc) from None
|
549
592
|
|
550
593
|
# Get the updated parameter dict with cast values from the model
|
551
594
|
cast_parameters = {
|
552
595
|
k: v
|
553
|
-
for k, v in model.
|
554
|
-
if k in model.
|
596
|
+
for k, v in dict(model).items()
|
597
|
+
if k in model.model_fields_set or model.model_fields[k].default_factory
|
555
598
|
}
|
556
599
|
return cast_parameters
|
557
600
|
|
@@ -565,30 +608,26 @@ class Flow(Generic[P, R]):
|
|
565
608
|
"""
|
566
609
|
serialized_parameters = {}
|
567
610
|
for key, value in parameters.items():
|
611
|
+
# do not serialize the bound self object
|
612
|
+
if self.ismethod and value is self.fn.__prefect_self__:
|
613
|
+
continue
|
614
|
+
if isinstance(value, (PrefectFuture, State)):
|
615
|
+
# Don't call jsonable_encoder() on a PrefectFuture or State to
|
616
|
+
# avoid triggering a __getitem__ call
|
617
|
+
serialized_parameters[key] = f"<{type(value).__name__}>"
|
618
|
+
continue
|
568
619
|
try:
|
569
620
|
serialized_parameters[key] = jsonable_encoder(value)
|
570
621
|
except (TypeError, ValueError):
|
571
622
|
logger.debug(
|
572
|
-
f"Parameter {key!r} for flow {self.name!r} is
|
573
|
-
f"
|
623
|
+
f"Parameter {key!r} for flow {self.name!r} is unserializable. "
|
624
|
+
f"Type {type(value).__name__!r} and will not be stored "
|
574
625
|
"in the backend."
|
575
626
|
)
|
576
627
|
serialized_parameters[key] = f"<{type(value).__name__}>"
|
577
628
|
return serialized_parameters
|
578
629
|
|
579
630
|
@sync_compatible
|
580
|
-
@deprecated_parameter(
|
581
|
-
"schedule",
|
582
|
-
start_date="Mar 2024",
|
583
|
-
when=lambda p: p is not None,
|
584
|
-
help="Use `schedules` instead.",
|
585
|
-
)
|
586
|
-
@deprecated_parameter(
|
587
|
-
"is_schedule_active",
|
588
|
-
start_date="Mar 2024",
|
589
|
-
when=lambda p: p is not None,
|
590
|
-
help="Use `paused` instead.",
|
591
|
-
)
|
592
631
|
async def to_deployment(
|
593
632
|
self,
|
594
633
|
name: str,
|
@@ -603,15 +642,14 @@ class Flow(Generic[P, R]):
|
|
603
642
|
cron: Optional[Union[Iterable[str], str]] = None,
|
604
643
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
605
644
|
paused: Optional[bool] = None,
|
606
|
-
schedules: Optional[
|
607
|
-
|
608
|
-
is_schedule_active: Optional[bool] = None,
|
645
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
646
|
+
concurrency_limit: Optional[int] = None,
|
609
647
|
parameters: Optional[dict] = None,
|
610
648
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
611
649
|
description: Optional[str] = None,
|
612
650
|
tags: Optional[List[str]] = None,
|
613
651
|
version: Optional[str] = None,
|
614
|
-
enforce_parameter_schema: bool =
|
652
|
+
enforce_parameter_schema: bool = True,
|
615
653
|
work_pool_name: Optional[str] = None,
|
616
654
|
work_queue_name: Optional[str] = None,
|
617
655
|
job_variables: Optional[Dict[str, Any]] = None,
|
@@ -629,10 +667,7 @@ class Flow(Generic[P, R]):
|
|
629
667
|
paused: Whether or not to set this deployment as paused.
|
630
668
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
631
669
|
Used to define multiple schedules or additional scheduling options such as `timezone`.
|
632
|
-
|
633
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
634
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
635
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
670
|
+
concurrency_limit: The maximum number of runs of this deployment that can run at the same time.
|
636
671
|
parameters: A dictionary of default parameter values to pass to runs of this deployment.
|
637
672
|
triggers: A list of triggers that will kick off runs of this deployment.
|
638
673
|
description: A description for the created deployment. Defaults to the flow's
|
@@ -673,7 +708,8 @@ class Flow(Generic[P, R]):
|
|
673
708
|
from prefect.deployments.runner import RunnerDeployment
|
674
709
|
|
675
710
|
if not name.endswith(".py"):
|
676
|
-
|
711
|
+
_raise_on_name_with_banned_characters(name)
|
712
|
+
|
677
713
|
if self._storage and self._entrypoint:
|
678
714
|
return await RunnerDeployment.from_storage(
|
679
715
|
storage=self._storage,
|
@@ -684,8 +720,7 @@ class Flow(Generic[P, R]):
|
|
684
720
|
rrule=rrule,
|
685
721
|
paused=paused,
|
686
722
|
schedules=schedules,
|
687
|
-
|
688
|
-
is_schedule_active=is_schedule_active,
|
723
|
+
concurrency_limit=concurrency_limit,
|
689
724
|
tags=tags,
|
690
725
|
triggers=triggers,
|
691
726
|
parameters=parameters or {},
|
@@ -695,7 +730,7 @@ class Flow(Generic[P, R]):
|
|
695
730
|
work_pool_name=work_pool_name,
|
696
731
|
work_queue_name=work_queue_name,
|
697
732
|
job_variables=job_variables,
|
698
|
-
)
|
733
|
+
) # type: ignore # TODO: remove sync_compatible
|
699
734
|
else:
|
700
735
|
return RunnerDeployment.from_flow(
|
701
736
|
self,
|
@@ -705,8 +740,7 @@ class Flow(Generic[P, R]):
|
|
705
740
|
rrule=rrule,
|
706
741
|
paused=paused,
|
707
742
|
schedules=schedules,
|
708
|
-
|
709
|
-
is_schedule_active=is_schedule_active,
|
743
|
+
concurrency_limit=concurrency_limit,
|
710
744
|
tags=tags,
|
711
745
|
triggers=triggers,
|
712
746
|
parameters=parameters or {},
|
@@ -719,8 +753,37 @@ class Flow(Generic[P, R]):
|
|
719
753
|
entrypoint_type=entrypoint_type,
|
720
754
|
)
|
721
755
|
|
722
|
-
|
723
|
-
|
756
|
+
def on_completion(
|
757
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
758
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
759
|
+
self.on_completion_hooks.append(fn)
|
760
|
+
return fn
|
761
|
+
|
762
|
+
def on_cancellation(
|
763
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
764
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
765
|
+
self.on_cancellation_hooks.append(fn)
|
766
|
+
return fn
|
767
|
+
|
768
|
+
def on_crashed(
|
769
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
770
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
771
|
+
self.on_crashed_hooks.append(fn)
|
772
|
+
return fn
|
773
|
+
|
774
|
+
def on_running(
|
775
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
776
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
777
|
+
self.on_running_hooks.append(fn)
|
778
|
+
return fn
|
779
|
+
|
780
|
+
def on_failure(
|
781
|
+
self, fn: Callable[["Flow", FlowRun, State], None]
|
782
|
+
) -> Callable[["Flow", FlowRun, State], None]:
|
783
|
+
self.on_failure_hooks.append(fn)
|
784
|
+
return fn
|
785
|
+
|
786
|
+
def serve(
|
724
787
|
self,
|
725
788
|
name: Optional[str] = None,
|
726
789
|
interval: Optional[
|
@@ -734,15 +797,13 @@ class Flow(Generic[P, R]):
|
|
734
797
|
cron: Optional[Union[Iterable[str], str]] = None,
|
735
798
|
rrule: Optional[Union[Iterable[str], str]] = None,
|
736
799
|
paused: Optional[bool] = None,
|
737
|
-
schedules: Optional[
|
738
|
-
schedule: Optional[SCHEDULE_TYPES] = None,
|
739
|
-
is_schedule_active: Optional[bool] = None,
|
800
|
+
schedules: Optional["FlexibleScheduleList"] = None,
|
740
801
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
741
802
|
parameters: Optional[dict] = None,
|
742
803
|
description: Optional[str] = None,
|
743
804
|
tags: Optional[List[str]] = None,
|
744
805
|
version: Optional[str] = None,
|
745
|
-
enforce_parameter_schema: bool =
|
806
|
+
enforce_parameter_schema: bool = True,
|
746
807
|
pause_on_shutdown: bool = True,
|
747
808
|
print_starting_message: bool = True,
|
748
809
|
limit: Optional[int] = None,
|
@@ -766,11 +827,6 @@ class Flow(Generic[P, R]):
|
|
766
827
|
paused: Whether or not to set this deployment as paused.
|
767
828
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
768
829
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
769
|
-
schedule: A schedule object defining when to execute runs of this deployment. Used to
|
770
|
-
define additional scheduling options such as `timezone`.
|
771
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
772
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
773
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
774
830
|
parameters: A dictionary of default parameter values to pass to runs of this deployment.
|
775
831
|
description: A description for the created deployment. Defaults to the flow's
|
776
832
|
description if not provided.
|
@@ -825,7 +881,7 @@ class Flow(Generic[P, R]):
|
|
825
881
|
name = Path(name).stem
|
826
882
|
|
827
883
|
runner = Runner(name=name, pause_on_shutdown=pause_on_shutdown, limit=limit)
|
828
|
-
deployment_id =
|
884
|
+
deployment_id = runner.add_flow(
|
829
885
|
self,
|
830
886
|
name=name,
|
831
887
|
triggers=triggers,
|
@@ -834,8 +890,6 @@ class Flow(Generic[P, R]):
|
|
834
890
|
rrule=rrule,
|
835
891
|
paused=paused,
|
836
892
|
schedules=schedules,
|
837
|
-
schedule=schedule,
|
838
|
-
is_schedule_active=is_schedule_active,
|
839
893
|
parameters=parameters,
|
840
894
|
description=description,
|
841
895
|
tags=tags,
|
@@ -858,15 +912,32 @@ class Flow(Generic[P, R]):
|
|
858
912
|
|
859
913
|
console = Console()
|
860
914
|
console.print(help_message, soft_wrap=True)
|
861
|
-
|
915
|
+
|
916
|
+
try:
|
917
|
+
loop = asyncio.get_running_loop()
|
918
|
+
except RuntimeError as exc:
|
919
|
+
if "no running event loop" in str(exc):
|
920
|
+
loop = None
|
921
|
+
else:
|
922
|
+
raise
|
923
|
+
|
924
|
+
try:
|
925
|
+
if loop is not None:
|
926
|
+
loop.run_until_complete(runner.start(webserver=webserver))
|
927
|
+
else:
|
928
|
+
asyncio.run(runner.start(webserver=webserver))
|
929
|
+
except (KeyboardInterrupt, TerminationSignal) as exc:
|
930
|
+
logger.info(f"Received {type(exc).__name__}, shutting down...")
|
931
|
+
if loop is not None:
|
932
|
+
loop.stop()
|
862
933
|
|
863
934
|
@classmethod
|
864
935
|
@sync_compatible
|
865
936
|
async def from_source(
|
866
|
-
cls: Type[
|
867
|
-
source: Union[str, RunnerStorage, ReadableDeploymentStorage],
|
937
|
+
cls: Type["Flow[P, R]"],
|
938
|
+
source: Union[str, "RunnerStorage", ReadableDeploymentStorage],
|
868
939
|
entrypoint: str,
|
869
|
-
) ->
|
940
|
+
) -> "Flow[P, R]":
|
870
941
|
"""
|
871
942
|
Loads a flow from a remote source.
|
872
943
|
|
@@ -912,8 +983,41 @@ class Flow(Generic[P, R]):
|
|
912
983
|
|
913
984
|
my_flow()
|
914
985
|
```
|
986
|
+
|
987
|
+
Load a flow from a local directory:
|
988
|
+
|
989
|
+
``` python
|
990
|
+
# from_local_source.py
|
991
|
+
|
992
|
+
from pathlib import Path
|
993
|
+
from prefect import flow
|
994
|
+
|
995
|
+
@flow(log_prints=True)
|
996
|
+
def my_flow(name: str = "world"):
|
997
|
+
print(f"Hello {name}! I'm a flow from a Python script!")
|
998
|
+
|
999
|
+
if __name__ == "__main__":
|
1000
|
+
my_flow.from_source(
|
1001
|
+
source=str(Path(__file__).parent),
|
1002
|
+
entrypoint="from_local_source.py:my_flow",
|
1003
|
+
).deploy(
|
1004
|
+
name="my-deployment",
|
1005
|
+
parameters=dict(name="Marvin"),
|
1006
|
+
work_pool_name="local",
|
1007
|
+
)
|
1008
|
+
```
|
915
1009
|
"""
|
916
|
-
|
1010
|
+
|
1011
|
+
from prefect.runner.storage import (
|
1012
|
+
BlockStorageAdapter,
|
1013
|
+
LocalStorage,
|
1014
|
+
RunnerStorage,
|
1015
|
+
create_storage_from_source,
|
1016
|
+
)
|
1017
|
+
|
1018
|
+
if isinstance(source, (Path, str)):
|
1019
|
+
if isinstance(source, Path):
|
1020
|
+
source = str(source)
|
917
1021
|
storage = create_storage_from_source(source)
|
918
1022
|
elif isinstance(source, RunnerStorage):
|
919
1023
|
storage = source
|
@@ -924,14 +1028,15 @@ class Flow(Generic[P, R]):
|
|
924
1028
|
f"Unsupported source type {type(source).__name__!r}. Please provide a"
|
925
1029
|
" URL to remote storage or a storage object."
|
926
1030
|
)
|
927
|
-
|
928
1031
|
with tempfile.TemporaryDirectory() as tmpdir:
|
929
1032
|
if not isinstance(storage, LocalStorage):
|
930
1033
|
storage.set_base_path(Path(tmpdir))
|
931
1034
|
await storage.pull_code()
|
1035
|
+
storage.set_base_path(Path(tmpdir))
|
1036
|
+
await storage.pull_code()
|
932
1037
|
|
933
1038
|
full_entrypoint = str(storage.destination / entrypoint)
|
934
|
-
flow:
|
1039
|
+
flow: Flow = await from_async.wait_for_call_in_new_thread(
|
935
1040
|
create_call(load_flow_from_entrypoint, full_entrypoint)
|
936
1041
|
)
|
937
1042
|
flow._storage = storage
|
@@ -944,7 +1049,7 @@ class Flow(Generic[P, R]):
|
|
944
1049
|
self,
|
945
1050
|
name: str,
|
946
1051
|
work_pool_name: Optional[str] = None,
|
947
|
-
image: Optional[Union[str,
|
1052
|
+
image: Optional[Union[str, DockerImage]] = None,
|
948
1053
|
build: bool = True,
|
949
1054
|
push: bool = True,
|
950
1055
|
work_queue_name: Optional[str] = None,
|
@@ -953,15 +1058,14 @@ class Flow(Generic[P, R]):
|
|
953
1058
|
cron: Optional[str] = None,
|
954
1059
|
rrule: Optional[str] = None,
|
955
1060
|
paused: Optional[bool] = None,
|
956
|
-
schedules: Optional[List[
|
957
|
-
|
958
|
-
is_schedule_active: Optional[bool] = None,
|
1061
|
+
schedules: Optional[List[DeploymentScheduleCreate]] = None,
|
1062
|
+
concurrency_limit: Optional[int] = None,
|
959
1063
|
triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
|
960
1064
|
parameters: Optional[dict] = None,
|
961
1065
|
description: Optional[str] = None,
|
962
1066
|
tags: Optional[List[str]] = None,
|
963
1067
|
version: Optional[str] = None,
|
964
|
-
enforce_parameter_schema: bool =
|
1068
|
+
enforce_parameter_schema: bool = True,
|
965
1069
|
entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
|
966
1070
|
print_next_steps: bool = True,
|
967
1071
|
ignore_warnings: bool = False,
|
@@ -980,7 +1084,7 @@ class Flow(Generic[P, R]):
|
|
980
1084
|
work_pool_name: The name of the work pool to use for this deployment. Defaults to
|
981
1085
|
the value of `PREFECT_DEFAULT_WORK_POOL_NAME`.
|
982
1086
|
image: The name of the Docker image to build, including the registry and
|
983
|
-
repository. Pass a
|
1087
|
+
repository. Pass a DockerImage instance to customize the Dockerfile used
|
984
1088
|
and build arguments.
|
985
1089
|
build: Whether or not to build a new image for the flow. If False, the provided
|
986
1090
|
image will be used as-is and pulled at runtime.
|
@@ -1002,11 +1106,7 @@ class Flow(Generic[P, R]):
|
|
1002
1106
|
paused: Whether or not to set this deployment as paused.
|
1003
1107
|
schedules: A list of schedule objects defining when to execute runs of this deployment.
|
1004
1108
|
Used to define multiple schedules or additional scheduling options like `timezone`.
|
1005
|
-
|
1006
|
-
define additional scheduling options like `timezone`.
|
1007
|
-
is_schedule_active: Whether or not to set the schedule for this deployment as active. If
|
1008
|
-
not provided when creating a deployment, the schedule will be set as active. If not
|
1009
|
-
provided when updating a deployment, the schedule's activation will not be changed.
|
1109
|
+
concurrency_limit: The maximum number of runs that can be executed concurrently.
|
1010
1110
|
parameters: A dictionary of default parameter values to pass to runs of this deployment.
|
1011
1111
|
description: A description for the created deployment. Defaults to the flow's
|
1012
1112
|
description if not provided.
|
@@ -1058,7 +1158,13 @@ class Flow(Generic[P, R]):
|
|
1058
1158
|
)
|
1059
1159
|
```
|
1060
1160
|
"""
|
1061
|
-
|
1161
|
+
if not (
|
1162
|
+
work_pool_name := work_pool_name or PREFECT_DEFAULT_WORK_POOL_NAME.value()
|
1163
|
+
):
|
1164
|
+
raise ValueError(
|
1165
|
+
"No work pool name provided. Please provide a `work_pool_name` or set the"
|
1166
|
+
" `PREFECT_DEFAULT_WORK_POOL_NAME` environment variable."
|
1167
|
+
)
|
1062
1168
|
|
1063
1169
|
try:
|
1064
1170
|
async with get_client() as client:
|
@@ -1075,9 +1181,8 @@ class Flow(Generic[P, R]):
|
|
1075
1181
|
cron=cron,
|
1076
1182
|
rrule=rrule,
|
1077
1183
|
schedules=schedules,
|
1184
|
+
concurrency_limit=concurrency_limit,
|
1078
1185
|
paused=paused,
|
1079
|
-
schedule=schedule,
|
1080
|
-
is_schedule_active=is_schedule_active,
|
1081
1186
|
triggers=triggers,
|
1082
1187
|
parameters=parameters,
|
1083
1188
|
description=description,
|
@@ -1089,6 +1194,8 @@ class Flow(Generic[P, R]):
|
|
1089
1194
|
entrypoint_type=entrypoint_type,
|
1090
1195
|
)
|
1091
1196
|
|
1197
|
+
from prefect.deployments.runner import deploy
|
1198
|
+
|
1092
1199
|
deployment_ids = await deploy(
|
1093
1200
|
deployment,
|
1094
1201
|
work_pool_name=work_pool_name,
|
@@ -1206,7 +1313,10 @@ class Flow(Generic[P, R]):
|
|
1206
1313
|
>>> with tags("db", "blue"):
|
1207
1314
|
>>> my_flow("foo")
|
1208
1315
|
"""
|
1209
|
-
from prefect.
|
1316
|
+
from prefect.utilities.visualization import (
|
1317
|
+
get_task_viz_tracker,
|
1318
|
+
track_viz_task,
|
1319
|
+
)
|
1210
1320
|
|
1211
1321
|
# Convert the call args/kwargs to a parameter dict
|
1212
1322
|
parameters = get_call_parameters(self.fn, args, kwargs)
|
@@ -1219,73 +1329,15 @@ class Flow(Generic[P, R]):
|
|
1219
1329
|
# we can add support for exploring subflows for tasks in the future.
|
1220
1330
|
return track_viz_task(self.isasync, self.name, parameters)
|
1221
1331
|
|
1222
|
-
|
1223
|
-
from prefect.new_flow_engine import run_flow, run_flow_sync
|
1224
|
-
|
1225
|
-
run_kwargs = dict(
|
1226
|
-
flow=self,
|
1227
|
-
parameters=parameters,
|
1228
|
-
wait_for=wait_for,
|
1229
|
-
return_type=return_type,
|
1230
|
-
)
|
1231
|
-
if self.isasync:
|
1232
|
-
# this returns an awaitable coroutine
|
1233
|
-
return run_flow(**run_kwargs)
|
1234
|
-
else:
|
1235
|
-
return run_flow_sync(**run_kwargs)
|
1332
|
+
from prefect.flow_engine import run_flow
|
1236
1333
|
|
1237
|
-
return
|
1238
|
-
self,
|
1239
|
-
parameters,
|
1334
|
+
return run_flow(
|
1335
|
+
flow=self,
|
1336
|
+
parameters=parameters,
|
1240
1337
|
wait_for=wait_for,
|
1241
1338
|
return_type=return_type,
|
1242
1339
|
)
|
1243
1340
|
|
1244
|
-
@overload
|
1245
|
-
def _run(self: "Flow[P, NoReturn]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
|
1246
|
-
# `NoReturn` matches if a type can't be inferred for the function which stops a
|
1247
|
-
# sync function from matching the `Coroutine` overload
|
1248
|
-
...
|
1249
|
-
|
1250
|
-
@overload
|
1251
|
-
def _run(
|
1252
|
-
self: "Flow[P, Coroutine[Any, Any, T]]", *args: P.args, **kwargs: P.kwargs
|
1253
|
-
) -> Awaitable[T]:
|
1254
|
-
...
|
1255
|
-
|
1256
|
-
@overload
|
1257
|
-
def _run(self: "Flow[P, T]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
|
1258
|
-
...
|
1259
|
-
|
1260
|
-
def _run(
|
1261
|
-
self,
|
1262
|
-
*args: "P.args",
|
1263
|
-
wait_for: Optional[Iterable[PrefectFuture]] = None,
|
1264
|
-
**kwargs: "P.kwargs",
|
1265
|
-
):
|
1266
|
-
"""
|
1267
|
-
Run the flow and return its final state.
|
1268
|
-
|
1269
|
-
Examples:
|
1270
|
-
|
1271
|
-
Run a flow and get the returned result
|
1272
|
-
|
1273
|
-
>>> state = my_flow._run("marvin")
|
1274
|
-
>>> state.result()
|
1275
|
-
"goodbye marvin"
|
1276
|
-
"""
|
1277
|
-
from prefect.engine import enter_flow_run_engine_from_flow_call
|
1278
|
-
|
1279
|
-
# Convert the call args/kwargs to a parameter dict
|
1280
|
-
parameters = get_call_parameters(self.fn, args, kwargs)
|
1281
|
-
|
1282
|
-
return enter_flow_run_engine_from_flow_call(
|
1283
|
-
self,
|
1284
|
-
parameters,
|
1285
|
-
wait_for=wait_for,
|
1286
|
-
return_type="state",
|
1287
|
-
)
|
1288
|
-
|
1289
1341
|
@sync_compatible
|
1290
1342
|
async def visualize(self, *args, **kwargs):
|
1291
1343
|
"""
|
@@ -1297,6 +1349,16 @@ class Flow(Generic[P, R]):
|
|
1297
1349
|
- GraphvizExecutableNotFoundError: If the `dot` executable isn't found.
|
1298
1350
|
- FlowVisualizationError: If the flow can't be visualized for any other reason.
|
1299
1351
|
"""
|
1352
|
+
from prefect.utilities.visualization import (
|
1353
|
+
FlowVisualizationError,
|
1354
|
+
GraphvizExecutableNotFoundError,
|
1355
|
+
GraphvizImportError,
|
1356
|
+
TaskVizTracker,
|
1357
|
+
VisualizationUnsupportedError,
|
1358
|
+
build_task_dependencies,
|
1359
|
+
visualize_task_dependencies,
|
1360
|
+
)
|
1361
|
+
|
1300
1362
|
if not PREFECT_UNIT_TEST_MODE:
|
1301
1363
|
warnings.warn(
|
1302
1364
|
"`flow.visualize()` will execute code inside of your flow that is not"
|
@@ -1349,9 +1411,9 @@ def flow(
|
|
1349
1411
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1350
1412
|
retries: Optional[int] = None,
|
1351
1413
|
retry_delay_seconds: Optional[Union[int, float]] = None,
|
1352
|
-
task_runner:
|
1353
|
-
description: str = None,
|
1354
|
-
timeout_seconds: Union[int, float] = None,
|
1414
|
+
task_runner: Optional[TaskRunner] = None,
|
1415
|
+
description: Optional[str] = None,
|
1416
|
+
timeout_seconds: Union[int, float, None] = None,
|
1355
1417
|
validate_parameters: bool = True,
|
1356
1418
|
persist_result: Optional[bool] = None,
|
1357
1419
|
result_storage: Optional[ResultStorage] = None,
|
@@ -1379,11 +1441,11 @@ def flow(
|
|
1379
1441
|
name: Optional[str] = None,
|
1380
1442
|
version: Optional[str] = None,
|
1381
1443
|
flow_run_name: Optional[Union[Callable[[], str], str]] = None,
|
1382
|
-
retries: int = None,
|
1383
|
-
retry_delay_seconds: Union[int, float] = None,
|
1384
|
-
task_runner:
|
1385
|
-
description: str = None,
|
1386
|
-
timeout_seconds: Union[int, float] = None,
|
1444
|
+
retries: Optional[int] = None,
|
1445
|
+
retry_delay_seconds: Union[int, float, None] = None,
|
1446
|
+
task_runner: Optional[TaskRunner] = None,
|
1447
|
+
description: Optional[str] = None,
|
1448
|
+
timeout_seconds: Union[int, float, None] = None,
|
1387
1449
|
validate_parameters: bool = True,
|
1388
1450
|
persist_result: Optional[bool] = None,
|
1389
1451
|
result_storage: Optional[ResultStorage] = None,
|
@@ -1506,6 +1568,9 @@ def flow(
|
|
1506
1568
|
>>> pass
|
1507
1569
|
"""
|
1508
1570
|
if __fn:
|
1571
|
+
if isinstance(__fn, (classmethod, staticmethod)):
|
1572
|
+
method_decorator = type(__fn).__name__
|
1573
|
+
raise TypeError(f"@{method_decorator} should be applied on top of @flow")
|
1509
1574
|
return cast(
|
1510
1575
|
Flow[P, R],
|
1511
1576
|
Flow(
|
@@ -1559,12 +1624,31 @@ def flow(
|
|
1559
1624
|
)
|
1560
1625
|
|
1561
1626
|
|
1627
|
+
def _raise_on_name_with_banned_characters(name: str) -> str:
|
1628
|
+
"""
|
1629
|
+
Raise an InvalidNameError if the given name contains any invalid
|
1630
|
+
characters.
|
1631
|
+
"""
|
1632
|
+
if name is None:
|
1633
|
+
return name
|
1634
|
+
|
1635
|
+
if not re.match(WITHOUT_BANNED_CHARACTERS, name):
|
1636
|
+
raise InvalidNameError(
|
1637
|
+
f"Name {name!r} contains an invalid character. "
|
1638
|
+
f"Must not contain any of: {BANNED_CHARACTERS}."
|
1639
|
+
)
|
1640
|
+
|
1641
|
+
return name
|
1642
|
+
|
1643
|
+
|
1562
1644
|
# Add from_source so it is available on the flow function we all know and love
|
1563
1645
|
flow.from_source = Flow.from_source
|
1564
1646
|
|
1565
1647
|
|
1566
1648
|
def select_flow(
|
1567
|
-
flows: Iterable[Flow],
|
1649
|
+
flows: Iterable[Flow],
|
1650
|
+
flow_name: Optional[str] = None,
|
1651
|
+
from_message: Optional[str] = None,
|
1568
1652
|
) -> Flow:
|
1569
1653
|
"""
|
1570
1654
|
Select the only flow in an iterable or a flow specified by name.
|
@@ -1578,74 +1662,33 @@ def select_flow(
|
|
1578
1662
|
UnspecifiedFlowError: If multiple flows exist but no flow name was provided
|
1579
1663
|
"""
|
1580
1664
|
# Convert to flows by name
|
1581
|
-
|
1665
|
+
flows_dict = {f.name: f for f in flows}
|
1582
1666
|
|
1583
1667
|
# Add a leading space if given, otherwise use an empty string
|
1584
1668
|
from_message = (" " + from_message) if from_message else ""
|
1585
|
-
if not
|
1669
|
+
if not Optional:
|
1586
1670
|
raise MissingFlowError(f"No flows found{from_message}.")
|
1587
1671
|
|
1588
|
-
elif flow_name and flow_name not in
|
1672
|
+
elif flow_name and flow_name not in flows_dict:
|
1589
1673
|
raise MissingFlowError(
|
1590
1674
|
f"Flow {flow_name!r} not found{from_message}. "
|
1591
|
-
f"Found the following flows: {listrepr(
|
1675
|
+
f"Found the following flows: {listrepr(flows_dict.keys())}. "
|
1592
1676
|
"Check to make sure that your flow function is decorated with `@flow`."
|
1593
1677
|
)
|
1594
1678
|
|
1595
|
-
elif not flow_name and len(
|
1679
|
+
elif not flow_name and len(flows_dict) > 1:
|
1596
1680
|
raise UnspecifiedFlowError(
|
1597
1681
|
(
|
1598
|
-
f"Found {len(
|
1599
|
-
f" {listrepr(sorted(
|
1682
|
+
f"Found {len(flows_dict)} flows{from_message}:"
|
1683
|
+
f" {listrepr(sorted(flows_dict.keys()))}. Specify a flow name to select a"
|
1600
1684
|
" flow."
|
1601
1685
|
),
|
1602
1686
|
)
|
1603
1687
|
|
1604
1688
|
if flow_name:
|
1605
|
-
return
|
1689
|
+
return flows_dict[flow_name]
|
1606
1690
|
else:
|
1607
|
-
return list(
|
1608
|
-
|
1609
|
-
|
1610
|
-
def load_flows_from_script(path: str) -> List[Flow]:
|
1611
|
-
"""
|
1612
|
-
Load all flow objects from the given python script. All of the code in the file
|
1613
|
-
will be executed.
|
1614
|
-
|
1615
|
-
Returns:
|
1616
|
-
A list of flows
|
1617
|
-
|
1618
|
-
Raises:
|
1619
|
-
FlowScriptError: If an exception is encountered while running the script
|
1620
|
-
"""
|
1621
|
-
return registry_from_script(path).get_instances(Flow)
|
1622
|
-
|
1623
|
-
|
1624
|
-
def load_flow_from_script(path: str, flow_name: str = None) -> Flow:
|
1625
|
-
"""
|
1626
|
-
Extract a flow object from a script by running all of the code in the file.
|
1627
|
-
|
1628
|
-
If the script has multiple flows in it, a flow name must be provided to specify
|
1629
|
-
the flow to return.
|
1630
|
-
|
1631
|
-
Args:
|
1632
|
-
path: A path to a Python script containing flows
|
1633
|
-
flow_name: An optional flow name to look for in the script
|
1634
|
-
|
1635
|
-
Returns:
|
1636
|
-
The flow object from the script
|
1637
|
-
|
1638
|
-
Raises:
|
1639
|
-
FlowScriptError: If an exception is encountered while running the script
|
1640
|
-
MissingFlowError: If no flows exist in the iterable
|
1641
|
-
MissingFlowError: If a flow name is provided and that flow does not exist
|
1642
|
-
UnspecifiedFlowError: If multiple flows exist but no flow name was provided
|
1643
|
-
"""
|
1644
|
-
return select_flow(
|
1645
|
-
load_flows_from_script(path),
|
1646
|
-
flow_name=flow_name,
|
1647
|
-
from_message=f"in script '{path}'",
|
1648
|
-
)
|
1691
|
+
return list(flows_dict.values())[0]
|
1649
1692
|
|
1650
1693
|
|
1651
1694
|
def load_flow_from_entrypoint(
|
@@ -1658,9 +1701,8 @@ def load_flow_from_entrypoint(
|
|
1658
1701
|
Args:
|
1659
1702
|
entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
|
1660
1703
|
to a flow function
|
1661
|
-
use_placeholder_flow:
|
1662
|
-
cannot be loaded
|
1663
|
-
exception will be raised.
|
1704
|
+
use_placeholder_flow: if True, use a placeholder Flow object if the actual flow object
|
1705
|
+
cannot be loaded from the entrypoint (e.g. dependencies are missing)
|
1664
1706
|
|
1665
1707
|
Returns:
|
1666
1708
|
The flow object from the script
|
@@ -1669,67 +1711,38 @@ def load_flow_from_entrypoint(
|
|
1669
1711
|
FlowScriptError: If an exception is encountered while running the script
|
1670
1712
|
MissingFlowError: If the flow function specified in the entrypoint does not exist
|
1671
1713
|
"""
|
1672
|
-
with PrefectObjectRegistry(
|
1673
|
-
block_code_execution=True,
|
1674
|
-
capture_failures=True,
|
1675
|
-
):
|
1676
|
-
if ":" in entrypoint:
|
1677
|
-
# split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
|
1678
|
-
path, func_name = entrypoint.rsplit(":", maxsplit=1)
|
1679
1714
|
|
1680
|
-
|
1681
|
-
|
1682
|
-
|
1683
|
-
|
1684
|
-
|
1685
|
-
|
1686
|
-
|
1687
|
-
|
1688
|
-
|
1689
|
-
|
1690
|
-
|
1691
|
-
|
1692
|
-
|
1693
|
-
|
1694
|
-
|
1695
|
-
|
1715
|
+
if ":" in entrypoint:
|
1716
|
+
# split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
|
1717
|
+
path, func_name = entrypoint.rsplit(":", maxsplit=1)
|
1718
|
+
else:
|
1719
|
+
path, func_name = entrypoint.rsplit(".", maxsplit=1)
|
1720
|
+
try:
|
1721
|
+
flow = import_object(entrypoint)
|
1722
|
+
except AttributeError as exc:
|
1723
|
+
raise MissingFlowError(
|
1724
|
+
f"Flow function with name {func_name!r} not found in {path!r}. "
|
1725
|
+
) from exc
|
1726
|
+
except ScriptError:
|
1727
|
+
# If the flow has dependencies that are not installed in the current
|
1728
|
+
# environment, fallback to loading the flow via AST parsing.
|
1729
|
+
if use_placeholder_flow:
|
1730
|
+
flow = safe_load_flow_from_entrypoint(entrypoint)
|
1731
|
+
if flow is None:
|
1696
1732
|
raise
|
1733
|
+
else:
|
1734
|
+
raise
|
1697
1735
|
|
1698
|
-
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
|
1703
|
-
|
1704
|
-
return flow
|
1705
|
-
|
1706
|
-
|
1707
|
-
def load_flow_from_text(script_contents: AnyStr, flow_name: str):
|
1708
|
-
"""
|
1709
|
-
Load a flow from a text script.
|
1736
|
+
if not isinstance(flow, Flow):
|
1737
|
+
raise MissingFlowError(
|
1738
|
+
f"Function with name {func_name!r} is not a flow. Make sure that it is "
|
1739
|
+
"decorated with '@flow'."
|
1740
|
+
)
|
1710
1741
|
|
1711
|
-
The script will be written to a temporary local file path so errors can refer
|
1712
|
-
to line numbers and contextual tracebacks can be provided.
|
1713
|
-
"""
|
1714
|
-
with NamedTemporaryFile(
|
1715
|
-
mode="wt" if isinstance(script_contents, str) else "wb",
|
1716
|
-
prefix=f"flow-script-{flow_name}",
|
1717
|
-
suffix=".py",
|
1718
|
-
delete=False,
|
1719
|
-
) as tmpfile:
|
1720
|
-
tmpfile.write(script_contents)
|
1721
|
-
tmpfile.flush()
|
1722
|
-
try:
|
1723
|
-
flow = load_flow_from_script(tmpfile.name, flow_name=flow_name)
|
1724
|
-
finally:
|
1725
|
-
# windows compat
|
1726
|
-
tmpfile.close()
|
1727
|
-
os.remove(tmpfile.name)
|
1728
1742
|
return flow
|
1729
1743
|
|
1730
1744
|
|
1731
|
-
|
1732
|
-
async def serve(
|
1745
|
+
def serve(
|
1733
1746
|
*args: "RunnerDeployment",
|
1734
1747
|
pause_on_shutdown: bool = True,
|
1735
1748
|
print_starting_message: bool = True,
|
@@ -1785,7 +1798,7 @@ async def serve(
|
|
1785
1798
|
|
1786
1799
|
runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
|
1787
1800
|
for deployment in args:
|
1788
|
-
|
1801
|
+
runner.add_deployment(deployment)
|
1789
1802
|
|
1790
1803
|
if print_starting_message:
|
1791
1804
|
help_message_top = (
|
@@ -1816,7 +1829,104 @@ async def serve(
|
|
1816
1829
|
Group(help_message_top, table, help_message_bottom), soft_wrap=True
|
1817
1830
|
)
|
1818
1831
|
|
1819
|
-
|
1832
|
+
try:
|
1833
|
+
loop = asyncio.get_running_loop()
|
1834
|
+
except RuntimeError as exc:
|
1835
|
+
if "no running event loop" in str(exc):
|
1836
|
+
loop = None
|
1837
|
+
else:
|
1838
|
+
raise
|
1839
|
+
|
1840
|
+
if loop is not None:
|
1841
|
+
loop.run_until_complete(runner.start())
|
1842
|
+
else:
|
1843
|
+
asyncio.run(runner.start())
|
1844
|
+
|
1845
|
+
|
1846
|
+
@client_injector
|
1847
|
+
async def load_flow_from_flow_run(
|
1848
|
+
client: "PrefectClient",
|
1849
|
+
flow_run: "FlowRun",
|
1850
|
+
ignore_storage: bool = False,
|
1851
|
+
storage_base_path: Optional[str] = None,
|
1852
|
+
use_placeholder_flow: bool = True,
|
1853
|
+
) -> Flow:
|
1854
|
+
"""
|
1855
|
+
Load a flow from the location/script provided in a deployment's storage document.
|
1856
|
+
|
1857
|
+
If `ignore_storage=True` is provided, no pull from remote storage occurs. This flag
|
1858
|
+
is largely for testing, and assumes the flow is already available locally.
|
1859
|
+
"""
|
1860
|
+
deployment = await client.read_deployment(flow_run.deployment_id)
|
1861
|
+
|
1862
|
+
if deployment.entrypoint is None:
|
1863
|
+
raise ValueError(
|
1864
|
+
f"Deployment {deployment.id} does not have an entrypoint and can not be run."
|
1865
|
+
)
|
1866
|
+
|
1867
|
+
run_logger = flow_run_logger(flow_run)
|
1868
|
+
|
1869
|
+
runner_storage_base_path = storage_base_path or os.environ.get(
|
1870
|
+
"PREFECT__STORAGE_BASE_PATH"
|
1871
|
+
)
|
1872
|
+
|
1873
|
+
# If there's no colon, assume it's a module path
|
1874
|
+
if ":" not in deployment.entrypoint:
|
1875
|
+
run_logger.debug(
|
1876
|
+
f"Importing flow code from module path {deployment.entrypoint}"
|
1877
|
+
)
|
1878
|
+
flow = await run_sync_in_worker_thread(
|
1879
|
+
load_flow_from_entrypoint,
|
1880
|
+
deployment.entrypoint,
|
1881
|
+
use_placeholder_flow=use_placeholder_flow,
|
1882
|
+
)
|
1883
|
+
return flow
|
1884
|
+
|
1885
|
+
if not ignore_storage and not deployment.pull_steps:
|
1886
|
+
sys.path.insert(0, ".")
|
1887
|
+
if deployment.storage_document_id:
|
1888
|
+
storage_document = await client.read_block_document(
|
1889
|
+
deployment.storage_document_id
|
1890
|
+
)
|
1891
|
+
storage_block = Block._from_block_document(storage_document)
|
1892
|
+
else:
|
1893
|
+
basepath = deployment.path
|
1894
|
+
if runner_storage_base_path:
|
1895
|
+
basepath = str(basepath).replace(
|
1896
|
+
"$STORAGE_BASE_PATH", runner_storage_base_path
|
1897
|
+
)
|
1898
|
+
storage_block = LocalFileSystem(basepath=basepath)
|
1899
|
+
|
1900
|
+
from_path = (
|
1901
|
+
str(deployment.path).replace("$STORAGE_BASE_PATH", runner_storage_base_path)
|
1902
|
+
if runner_storage_base_path and deployment.path
|
1903
|
+
else deployment.path
|
1904
|
+
)
|
1905
|
+
run_logger.info(f"Downloading flow code from storage at {from_path!r}")
|
1906
|
+
await storage_block.get_directory(from_path=from_path, local_path=".")
|
1907
|
+
|
1908
|
+
if deployment.pull_steps:
|
1909
|
+
run_logger.debug(
|
1910
|
+
f"Running {len(deployment.pull_steps)} deployment pull step(s)"
|
1911
|
+
)
|
1912
|
+
|
1913
|
+
from prefect.deployments.steps.core import run_steps
|
1914
|
+
|
1915
|
+
output = await run_steps(deployment.pull_steps)
|
1916
|
+
if output.get("directory"):
|
1917
|
+
run_logger.debug(f"Changing working directory to {output['directory']!r}")
|
1918
|
+
os.chdir(output["directory"])
|
1919
|
+
|
1920
|
+
import_path = relative_path_to_current_platform(deployment.entrypoint)
|
1921
|
+
run_logger.debug(f"Importing flow code from '{import_path}'")
|
1922
|
+
|
1923
|
+
flow = await run_sync_in_worker_thread(
|
1924
|
+
load_flow_from_entrypoint,
|
1925
|
+
str(import_path),
|
1926
|
+
use_placeholder_flow=use_placeholder_flow,
|
1927
|
+
)
|
1928
|
+
|
1929
|
+
return flow
|
1820
1930
|
|
1821
1931
|
|
1822
1932
|
def load_placeholder_flow(entrypoint: str, raises: Exception):
|
@@ -1997,7 +2107,7 @@ def _sanitize_and_load_flow(
|
|
1997
2107
|
|
1998
2108
|
def load_flow_arguments_from_entrypoint(
|
1999
2109
|
entrypoint: str, arguments: Optional[Union[List[str], Set[str]]] = None
|
2000
|
-
) ->
|
2110
|
+
) -> dict[str, Any]:
|
2001
2111
|
"""
|
2002
2112
|
Extract flow arguments from an entrypoint string.
|
2003
2113
|
|