prefect-client 2.19.2__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. prefect/__init__.py +8 -56
  2. prefect/_internal/compatibility/deprecated.py +6 -115
  3. prefect/_internal/compatibility/experimental.py +4 -79
  4. prefect/_internal/concurrency/api.py +0 -34
  5. prefect/_internal/concurrency/calls.py +0 -6
  6. prefect/_internal/concurrency/cancellation.py +0 -3
  7. prefect/_internal/concurrency/event_loop.py +0 -20
  8. prefect/_internal/concurrency/inspection.py +3 -3
  9. prefect/_internal/concurrency/threads.py +35 -0
  10. prefect/_internal/concurrency/waiters.py +0 -28
  11. prefect/_internal/pydantic/__init__.py +0 -45
  12. prefect/_internal/pydantic/v1_schema.py +21 -22
  13. prefect/_internal/pydantic/v2_schema.py +0 -2
  14. prefect/_internal/pydantic/v2_validated_func.py +18 -23
  15. prefect/_internal/schemas/bases.py +44 -177
  16. prefect/_internal/schemas/fields.py +1 -43
  17. prefect/_internal/schemas/validators.py +60 -158
  18. prefect/artifacts.py +161 -14
  19. prefect/automations.py +39 -4
  20. prefect/blocks/abstract.py +1 -1
  21. prefect/blocks/core.py +268 -148
  22. prefect/blocks/fields.py +2 -57
  23. prefect/blocks/kubernetes.py +8 -12
  24. prefect/blocks/notifications.py +40 -20
  25. prefect/blocks/system.py +22 -11
  26. prefect/blocks/webhook.py +2 -9
  27. prefect/client/base.py +4 -4
  28. prefect/client/cloud.py +8 -13
  29. prefect/client/orchestration.py +347 -341
  30. prefect/client/schemas/actions.py +92 -86
  31. prefect/client/schemas/filters.py +20 -40
  32. prefect/client/schemas/objects.py +151 -145
  33. prefect/client/schemas/responses.py +16 -24
  34. prefect/client/schemas/schedules.py +47 -35
  35. prefect/client/subscriptions.py +2 -2
  36. prefect/client/utilities.py +5 -2
  37. prefect/concurrency/asyncio.py +3 -1
  38. prefect/concurrency/events.py +1 -1
  39. prefect/concurrency/services.py +6 -3
  40. prefect/context.py +195 -27
  41. prefect/deployments/__init__.py +5 -6
  42. prefect/deployments/base.py +7 -5
  43. prefect/deployments/flow_runs.py +185 -0
  44. prefect/deployments/runner.py +50 -45
  45. prefect/deployments/schedules.py +28 -23
  46. prefect/deployments/steps/__init__.py +0 -1
  47. prefect/deployments/steps/core.py +1 -0
  48. prefect/deployments/steps/pull.py +7 -21
  49. prefect/engine.py +12 -2422
  50. prefect/events/actions.py +17 -23
  51. prefect/events/cli/automations.py +19 -6
  52. prefect/events/clients.py +14 -37
  53. prefect/events/filters.py +14 -18
  54. prefect/events/related.py +2 -2
  55. prefect/events/schemas/__init__.py +0 -5
  56. prefect/events/schemas/automations.py +55 -46
  57. prefect/events/schemas/deployment_triggers.py +7 -197
  58. prefect/events/schemas/events.py +34 -65
  59. prefect/events/schemas/labelling.py +10 -14
  60. prefect/events/utilities.py +2 -3
  61. prefect/events/worker.py +2 -3
  62. prefect/filesystems.py +6 -517
  63. prefect/{new_flow_engine.py → flow_engine.py} +313 -72
  64. prefect/flow_runs.py +377 -5
  65. prefect/flows.py +307 -166
  66. prefect/futures.py +186 -345
  67. prefect/infrastructure/__init__.py +0 -27
  68. prefect/infrastructure/provisioners/__init__.py +5 -3
  69. prefect/infrastructure/provisioners/cloud_run.py +11 -6
  70. prefect/infrastructure/provisioners/container_instance.py +11 -7
  71. prefect/infrastructure/provisioners/ecs.py +6 -4
  72. prefect/infrastructure/provisioners/modal.py +8 -5
  73. prefect/input/actions.py +2 -4
  74. prefect/input/run_input.py +5 -7
  75. prefect/logging/formatters.py +0 -2
  76. prefect/logging/handlers.py +3 -11
  77. prefect/logging/loggers.py +2 -2
  78. prefect/manifests.py +2 -1
  79. prefect/records/__init__.py +1 -0
  80. prefect/records/result_store.py +42 -0
  81. prefect/records/store.py +9 -0
  82. prefect/results.py +43 -39
  83. prefect/runner/runner.py +19 -15
  84. prefect/runner/server.py +6 -10
  85. prefect/runner/storage.py +3 -8
  86. prefect/runner/submit.py +2 -2
  87. prefect/runner/utils.py +2 -2
  88. prefect/serializers.py +24 -35
  89. prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
  90. prefect/settings.py +70 -133
  91. prefect/states.py +17 -47
  92. prefect/task_engine.py +697 -58
  93. prefect/task_runners.py +269 -301
  94. prefect/task_server.py +53 -34
  95. prefect/tasks.py +327 -337
  96. prefect/transactions.py +220 -0
  97. prefect/types/__init__.py +61 -82
  98. prefect/utilities/asyncutils.py +195 -136
  99. prefect/utilities/callables.py +311 -43
  100. prefect/utilities/collections.py +23 -38
  101. prefect/utilities/dispatch.py +11 -3
  102. prefect/utilities/dockerutils.py +4 -0
  103. prefect/utilities/engine.py +140 -20
  104. prefect/utilities/importtools.py +97 -27
  105. prefect/utilities/pydantic.py +128 -38
  106. prefect/utilities/schema_tools/hydration.py +5 -1
  107. prefect/utilities/templating.py +12 -2
  108. prefect/variables.py +78 -61
  109. prefect/workers/__init__.py +0 -1
  110. prefect/workers/base.py +15 -17
  111. prefect/workers/process.py +3 -8
  112. prefect/workers/server.py +2 -2
  113. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/METADATA +22 -21
  114. prefect_client-3.0.0rc1.dist-info/RECORD +176 -0
  115. prefect/_internal/pydantic/_base_model.py +0 -51
  116. prefect/_internal/pydantic/_compat.py +0 -82
  117. prefect/_internal/pydantic/_flags.py +0 -20
  118. prefect/_internal/pydantic/_types.py +0 -8
  119. prefect/_internal/pydantic/utilities/__init__.py +0 -0
  120. prefect/_internal/pydantic/utilities/config_dict.py +0 -72
  121. prefect/_internal/pydantic/utilities/field_validator.py +0 -150
  122. prefect/_internal/pydantic/utilities/model_construct.py +0 -56
  123. prefect/_internal/pydantic/utilities/model_copy.py +0 -55
  124. prefect/_internal/pydantic/utilities/model_dump.py +0 -136
  125. prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
  126. prefect/_internal/pydantic/utilities/model_fields.py +0 -50
  127. prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
  128. prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
  129. prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
  130. prefect/_internal/pydantic/utilities/model_validate.py +0 -75
  131. prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
  132. prefect/_internal/pydantic/utilities/model_validator.py +0 -87
  133. prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
  134. prefect/_vendor/__init__.py +0 -0
  135. prefect/_vendor/fastapi/__init__.py +0 -25
  136. prefect/_vendor/fastapi/applications.py +0 -946
  137. prefect/_vendor/fastapi/background.py +0 -3
  138. prefect/_vendor/fastapi/concurrency.py +0 -44
  139. prefect/_vendor/fastapi/datastructures.py +0 -58
  140. prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
  141. prefect/_vendor/fastapi/dependencies/models.py +0 -64
  142. prefect/_vendor/fastapi/dependencies/utils.py +0 -877
  143. prefect/_vendor/fastapi/encoders.py +0 -177
  144. prefect/_vendor/fastapi/exception_handlers.py +0 -40
  145. prefect/_vendor/fastapi/exceptions.py +0 -46
  146. prefect/_vendor/fastapi/logger.py +0 -3
  147. prefect/_vendor/fastapi/middleware/__init__.py +0 -1
  148. prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
  149. prefect/_vendor/fastapi/middleware/cors.py +0 -3
  150. prefect/_vendor/fastapi/middleware/gzip.py +0 -3
  151. prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
  152. prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
  153. prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
  154. prefect/_vendor/fastapi/openapi/__init__.py +0 -0
  155. prefect/_vendor/fastapi/openapi/constants.py +0 -2
  156. prefect/_vendor/fastapi/openapi/docs.py +0 -203
  157. prefect/_vendor/fastapi/openapi/models.py +0 -480
  158. prefect/_vendor/fastapi/openapi/utils.py +0 -485
  159. prefect/_vendor/fastapi/param_functions.py +0 -340
  160. prefect/_vendor/fastapi/params.py +0 -453
  161. prefect/_vendor/fastapi/requests.py +0 -4
  162. prefect/_vendor/fastapi/responses.py +0 -40
  163. prefect/_vendor/fastapi/routing.py +0 -1331
  164. prefect/_vendor/fastapi/security/__init__.py +0 -15
  165. prefect/_vendor/fastapi/security/api_key.py +0 -98
  166. prefect/_vendor/fastapi/security/base.py +0 -6
  167. prefect/_vendor/fastapi/security/http.py +0 -172
  168. prefect/_vendor/fastapi/security/oauth2.py +0 -227
  169. prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
  170. prefect/_vendor/fastapi/security/utils.py +0 -10
  171. prefect/_vendor/fastapi/staticfiles.py +0 -1
  172. prefect/_vendor/fastapi/templating.py +0 -3
  173. prefect/_vendor/fastapi/testclient.py +0 -1
  174. prefect/_vendor/fastapi/types.py +0 -3
  175. prefect/_vendor/fastapi/utils.py +0 -235
  176. prefect/_vendor/fastapi/websockets.py +0 -7
  177. prefect/_vendor/starlette/__init__.py +0 -1
  178. prefect/_vendor/starlette/_compat.py +0 -28
  179. prefect/_vendor/starlette/_exception_handler.py +0 -80
  180. prefect/_vendor/starlette/_utils.py +0 -88
  181. prefect/_vendor/starlette/applications.py +0 -261
  182. prefect/_vendor/starlette/authentication.py +0 -159
  183. prefect/_vendor/starlette/background.py +0 -43
  184. prefect/_vendor/starlette/concurrency.py +0 -59
  185. prefect/_vendor/starlette/config.py +0 -151
  186. prefect/_vendor/starlette/convertors.py +0 -87
  187. prefect/_vendor/starlette/datastructures.py +0 -707
  188. prefect/_vendor/starlette/endpoints.py +0 -130
  189. prefect/_vendor/starlette/exceptions.py +0 -60
  190. prefect/_vendor/starlette/formparsers.py +0 -276
  191. prefect/_vendor/starlette/middleware/__init__.py +0 -17
  192. prefect/_vendor/starlette/middleware/authentication.py +0 -52
  193. prefect/_vendor/starlette/middleware/base.py +0 -220
  194. prefect/_vendor/starlette/middleware/cors.py +0 -176
  195. prefect/_vendor/starlette/middleware/errors.py +0 -265
  196. prefect/_vendor/starlette/middleware/exceptions.py +0 -74
  197. prefect/_vendor/starlette/middleware/gzip.py +0 -113
  198. prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
  199. prefect/_vendor/starlette/middleware/sessions.py +0 -82
  200. prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
  201. prefect/_vendor/starlette/middleware/wsgi.py +0 -147
  202. prefect/_vendor/starlette/requests.py +0 -328
  203. prefect/_vendor/starlette/responses.py +0 -347
  204. prefect/_vendor/starlette/routing.py +0 -933
  205. prefect/_vendor/starlette/schemas.py +0 -154
  206. prefect/_vendor/starlette/staticfiles.py +0 -248
  207. prefect/_vendor/starlette/status.py +0 -199
  208. prefect/_vendor/starlette/templating.py +0 -231
  209. prefect/_vendor/starlette/testclient.py +0 -804
  210. prefect/_vendor/starlette/types.py +0 -30
  211. prefect/_vendor/starlette/websockets.py +0 -193
  212. prefect/agent.py +0 -698
  213. prefect/deployments/deployments.py +0 -1042
  214. prefect/deprecated/__init__.py +0 -0
  215. prefect/deprecated/data_documents.py +0 -350
  216. prefect/deprecated/packaging/__init__.py +0 -12
  217. prefect/deprecated/packaging/base.py +0 -96
  218. prefect/deprecated/packaging/docker.py +0 -146
  219. prefect/deprecated/packaging/file.py +0 -92
  220. prefect/deprecated/packaging/orion.py +0 -80
  221. prefect/deprecated/packaging/serializers.py +0 -171
  222. prefect/events/instrument.py +0 -135
  223. prefect/infrastructure/base.py +0 -323
  224. prefect/infrastructure/container.py +0 -818
  225. prefect/infrastructure/kubernetes.py +0 -920
  226. prefect/infrastructure/process.py +0 -289
  227. prefect/new_task_engine.py +0 -423
  228. prefect/pydantic/__init__.py +0 -76
  229. prefect/pydantic/main.py +0 -39
  230. prefect/software/__init__.py +0 -2
  231. prefect/software/base.py +0 -50
  232. prefect/software/conda.py +0 -199
  233. prefect/software/pip.py +0 -122
  234. prefect/software/python.py +0 -52
  235. prefect/workers/block.py +0 -218
  236. prefect_client-2.19.2.dist-info/RECORD +0 -292
  237. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/LICENSE +0 -0
  238. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/WHEEL +0 -0
  239. {prefect_client-2.19.2.dist-info → prefect_client-3.0.0rc1.dist-info}/top_level.txt +0 -0
prefect/flows.py CHANGED
@@ -5,9 +5,14 @@ Module containing the base workflow class and decorator - for most use cases, us
5
5
  # This file requires type-checking with pyright because mypy does not yet support PEP612
6
6
  # See https://github.com/python/mypy/issues/8645
7
7
 
8
+ import ast
8
9
  import datetime
10
+ import importlib.util
9
11
  import inspect
12
+ import json
10
13
  import os
14
+ import re
15
+ import sys
11
16
  import tempfile
12
17
  import warnings
13
18
  from functools import partial, update_wrapper
@@ -34,50 +39,38 @@ from typing import (
34
39
  )
35
40
  from uuid import UUID
36
41
 
42
+ import pydantic
43
+ from fastapi.encoders import jsonable_encoder
44
+ from pydantic.v1 import BaseModel as V1BaseModel
45
+ from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
46
+ from pydantic.v1.errors import ConfigError # TODO
37
47
  from rich.console import Console
38
48
  from typing_extensions import Literal, ParamSpec, Self
39
49
 
40
- from prefect._internal.pydantic import HAS_PYDANTIC_V2
41
-
42
- if HAS_PYDANTIC_V2:
43
- import pydantic.v1 as pydantic
44
- from pydantic import ValidationError as V2ValidationError
45
- from pydantic.v1 import BaseModel as V1BaseModel
46
- from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
47
-
48
- from ._internal.pydantic.v2_schema import is_v2_type
49
- from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
50
- from ._internal.pydantic.v2_validated_func import (
51
- V2ValidatedFunction as ValidatedFunction,
52
- )
53
-
54
- else:
55
- import pydantic
56
- from pydantic.decorator import ValidatedFunction
57
-
58
- V2ValidationError = None
59
-
60
- from prefect._vendor.fastapi.encoders import jsonable_encoder
61
-
62
50
  from prefect._internal.compatibility.deprecated import deprecated_parameter
63
51
  from prefect._internal.concurrency.api import create_call, from_async
64
- from prefect._internal.schemas.validators import raise_on_name_with_banned_characters
52
+ from prefect.blocks.core import Block
65
53
  from prefect.client.orchestration import get_client
54
+ from prefect.client.schemas.actions import DeploymentScheduleCreate
66
55
  from prefect.client.schemas.objects import Flow as FlowSchema
67
- from prefect.client.schemas.objects import FlowRun, MinimalDeploymentSchedule
56
+ from prefect.client.schemas.objects import FlowRun
68
57
  from prefect.client.schemas.schedules import SCHEDULE_TYPES
58
+ from prefect.client.utilities import client_injector
69
59
  from prefect.context import PrefectObjectRegistry, registry_from_script
70
60
  from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
61
+ from prefect.deployments.steps.core import run_steps
71
62
  from prefect.events import DeploymentTriggerTypes, TriggerTypes
72
63
  from prefect.exceptions import (
64
+ InvalidNameError,
73
65
  MissingFlowError,
74
66
  ObjectNotFound,
75
67
  ParameterTypeError,
76
68
  UnspecifiedFlowError,
77
69
  )
78
- from prefect.filesystems import ReadableDeploymentStorage
70
+ from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
79
71
  from prefect.futures import PrefectFuture
80
72
  from prefect.logging import get_logger
73
+ from prefect.logging.loggers import flow_run_logger
81
74
  from prefect.results import ResultSerializer, ResultStorage
82
75
  from prefect.runner.storage import (
83
76
  BlockStorageAdapter,
@@ -86,16 +79,20 @@ from prefect.runner.storage import (
86
79
  )
87
80
  from prefect.settings import (
88
81
  PREFECT_DEFAULT_WORK_POOL_NAME,
89
- PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE,
90
82
  PREFECT_FLOW_DEFAULT_RETRIES,
91
83
  PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS,
92
84
  PREFECT_UI_URL,
93
85
  PREFECT_UNIT_TEST_MODE,
94
86
  )
95
87
  from prefect.states import State
96
- from prefect.task_runners import BaseTaskRunner, ConcurrentTaskRunner
88
+ from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
89
+ from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
97
90
  from prefect.utilities.annotations import NotSet
98
- from prefect.utilities.asyncutils import is_async_fn, sync_compatible
91
+ from prefect.utilities.asyncutils import (
92
+ is_async_fn,
93
+ run_sync_in_worker_thread,
94
+ sync_compatible,
95
+ )
99
96
  from prefect.utilities.callables import (
100
97
  get_call_parameters,
101
98
  parameter_schema,
@@ -103,18 +100,14 @@ from prefect.utilities.callables import (
103
100
  raise_for_reserved_arguments,
104
101
  )
105
102
  from prefect.utilities.collections import listrepr
103
+ from prefect.utilities.filesystem import relative_path_to_current_platform
106
104
  from prefect.utilities.hashing import file_hash
107
105
  from prefect.utilities.importtools import import_object
108
- from prefect.utilities.visualization import (
109
- FlowVisualizationError,
110
- GraphvizExecutableNotFoundError,
111
- GraphvizImportError,
112
- TaskVizTracker,
113
- VisualizationUnsupportedError,
114
- build_task_dependencies,
115
- get_task_viz_tracker,
116
- track_viz_task,
117
- visualize_task_dependencies,
106
+
107
+ from ._internal.pydantic.v2_schema import is_v2_type
108
+ from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
109
+ from ._internal.pydantic.v2_validated_func import (
110
+ V2ValidatedFunction as ValidatedFunction,
118
111
  )
119
112
 
120
113
  T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
@@ -125,7 +118,9 @@ F = TypeVar("F", bound="Flow") # The type of the flow
125
118
  logger = get_logger("flows")
126
119
 
127
120
  if TYPE_CHECKING:
121
+ from prefect.client.orchestration import PrefectClient
128
122
  from prefect.deployments.runner import FlexibleScheduleList, RunnerDeployment
123
+ from prefect.flows import FlowRun
129
124
 
130
125
 
131
126
  @PrefectObjectRegistry.register_instances
@@ -196,9 +191,9 @@ class Flow(Generic[P, R]):
196
191
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
197
192
  retries: Optional[int] = None,
198
193
  retry_delay_seconds: Optional[Union[int, float]] = None,
199
- task_runner: Union[Type[BaseTaskRunner], BaseTaskRunner] = ConcurrentTaskRunner,
200
- description: str = None,
201
- timeout_seconds: Union[int, float] = None,
194
+ task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
195
+ description: Optional[str] = None,
196
+ timeout_seconds: Union[int, float, None] = None,
202
197
  validate_parameters: bool = True,
203
198
  persist_result: Optional[bool] = None,
204
199
  result_storage: Optional[ResultStorage] = None,
@@ -245,8 +240,6 @@ class Flow(Generic[P, R]):
245
240
  ]
246
241
  for hooks, hook_name in zip(hook_categories, hook_names):
247
242
  if hooks is not None:
248
- if not hooks:
249
- raise ValueError(f"Empty list passed for '{hook_name}'")
250
243
  try:
251
244
  hooks = list(hooks)
252
245
  except TypeError:
@@ -273,7 +266,7 @@ class Flow(Generic[P, R]):
273
266
 
274
267
  # Validate name if given
275
268
  if name:
276
- raise_on_name_with_banned_characters(name)
269
+ _raise_on_name_with_banned_characters(name)
277
270
 
278
271
  self.name = name or fn.__name__.replace("_", "-")
279
272
 
@@ -285,7 +278,8 @@ class Flow(Generic[P, R]):
285
278
  )
286
279
  self.flow_run_name = flow_run_name
287
280
 
288
- task_runner = task_runner or ConcurrentTaskRunner()
281
+ default_task_runner = ThreadPoolTaskRunner()
282
+ task_runner = task_runner or default_task_runner
289
283
  self.task_runner = (
290
284
  task_runner() if isinstance(task_runner, type) else task_runner
291
285
  )
@@ -333,7 +327,7 @@ class Flow(Generic[P, R]):
333
327
  # is not picklable in some environments
334
328
  try:
335
329
  ValidatedFunction(self.fn, config={"arbitrary_types_allowed": True})
336
- except pydantic.ConfigError as exc:
330
+ except ConfigError as exc:
337
331
  raise ValueError(
338
332
  "Flow function is not compatible with `validate_parameters`. "
339
333
  "Disable validation or change the argument names."
@@ -343,11 +337,11 @@ class Flow(Generic[P, R]):
343
337
  self.result_storage = result_storage
344
338
  self.result_serializer = result_serializer
345
339
  self.cache_result_in_memory = cache_result_in_memory
346
- self.on_completion = on_completion
347
- self.on_failure = on_failure
348
- self.on_cancellation = on_cancellation
349
- self.on_crashed = on_crashed
350
- self.on_running = on_running
340
+ self.on_completion_hooks = on_completion or []
341
+ self.on_failure_hooks = on_failure or []
342
+ self.on_cancellation_hooks = on_cancellation or []
343
+ self.on_crashed_hooks = on_crashed or []
344
+ self.on_running_hooks = on_running or []
351
345
 
352
346
  # Used for flows loaded from remote storage
353
347
  self._storage: Optional[RunnerStorage] = None
@@ -363,20 +357,20 @@ class Flow(Generic[P, R]):
363
357
  def with_options(
364
358
  self,
365
359
  *,
366
- name: str = None,
367
- version: str = None,
360
+ name: Optional[str] = None,
361
+ version: Optional[str] = None,
368
362
  retries: Optional[int] = None,
369
363
  retry_delay_seconds: Optional[Union[int, float]] = None,
370
- description: str = None,
364
+ description: Optional[str] = None,
371
365
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
372
- task_runner: Union[Type[BaseTaskRunner], BaseTaskRunner] = None,
373
- timeout_seconds: Union[int, float] = None,
374
- validate_parameters: bool = None,
375
- persist_result: Optional[bool] = NotSet,
376
- result_storage: Optional[ResultStorage] = NotSet,
377
- result_serializer: Optional[ResultSerializer] = NotSet,
378
- cache_result_in_memory: bool = None,
379
- log_prints: Optional[bool] = NotSet,
366
+ task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
367
+ timeout_seconds: Union[int, float, None] = None,
368
+ validate_parameters: Optional[bool] = None,
369
+ persist_result: Optional[bool] = NotSet, # type: ignore
370
+ result_storage: Optional[ResultStorage] = NotSet, # type: ignore
371
+ result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
372
+ cache_result_in_memory: Optional[bool] = None,
373
+ log_prints: Optional[bool] = NotSet, # type: ignore
380
374
  on_completion: Optional[
381
375
  List[Callable[[FlowSchema, FlowRun, State], None]]
382
376
  ] = None,
@@ -432,15 +426,14 @@ class Flow(Generic[P, R]):
432
426
  Create a new flow from an existing flow, update the task runner, and call
433
427
  it without an intermediate variable:
434
428
 
435
- >>> from prefect.task_runners import SequentialTaskRunner
429
+ >>> from prefect.task_runners import ThreadPoolTaskRunner
436
430
  >>>
437
431
  >>> @flow
438
432
  >>> def my_flow(x, y):
439
433
  >>> return x + y
440
434
  >>>
441
- >>> state = my_flow.with_options(task_runner=SequentialTaskRunner)(1, 3)
435
+ >>> state = my_flow.with_options(task_runner=ThreadPoolTaskRunner)(1, 3)
442
436
  >>> assert state.result() == 4
443
-
444
437
  """
445
438
  new_flow = Flow(
446
439
  fn=self.fn,
@@ -480,11 +473,11 @@ class Flow(Generic[P, R]):
480
473
  else self.cache_result_in_memory
481
474
  ),
482
475
  log_prints=log_prints if log_prints is not NotSet else self.log_prints,
483
- on_completion=on_completion or self.on_completion,
484
- on_failure=on_failure or self.on_failure,
485
- on_cancellation=on_cancellation or self.on_cancellation,
486
- on_crashed=on_crashed or self.on_crashed,
487
- on_running=on_running or self.on_running,
476
+ on_completion=on_completion or self.on_completion_hooks,
477
+ on_failure=on_failure or self.on_failure_hooks,
478
+ on_cancellation=on_cancellation or self.on_cancellation_hooks,
479
+ on_crashed=on_crashed or self.on_crashed_hooks,
480
+ on_running=on_running or self.on_running_hooks,
488
481
  )
489
482
  new_flow._storage = self._storage
490
483
  new_flow._entrypoint = self._entrypoint
@@ -503,49 +496,52 @@ class Flow(Generic[P, R]):
503
496
  """
504
497
  args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
505
498
 
506
- if HAS_PYDANTIC_V2:
499
+ with warnings.catch_warnings():
500
+ warnings.filterwarnings(
501
+ "ignore", category=pydantic.warnings.PydanticDeprecatedSince20
502
+ )
507
503
  has_v1_models = any(isinstance(o, V1BaseModel) for o in args) or any(
508
504
  isinstance(o, V1BaseModel) for o in kwargs.values()
509
505
  )
510
- has_v2_types = any(is_v2_type(o) for o in args) or any(
511
- is_v2_type(o) for o in kwargs.values()
512
- )
513
506
 
514
- if has_v1_models and has_v2_types:
515
- raise ParameterTypeError(
516
- "Cannot mix Pydantic v1 and v2 types as arguments to a flow."
517
- )
507
+ has_v2_types = any(is_v2_type(o) for o in args) or any(
508
+ is_v2_type(o) for o in kwargs.values()
509
+ )
518
510
 
519
- if has_v1_models:
520
- validated_fn = V1ValidatedFunction(
521
- self.fn, config={"arbitrary_types_allowed": True}
522
- )
523
- else:
524
- validated_fn = V2ValidatedFunction(
525
- self.fn, config={"arbitrary_types_allowed": True}
526
- )
511
+ if has_v1_models and has_v2_types:
512
+ raise ParameterTypeError(
513
+ "Cannot mix Pydantic v1 and v2 types as arguments to a flow."
514
+ )
527
515
 
528
- else:
529
- validated_fn = ValidatedFunction(
516
+ if has_v1_models:
517
+ validated_fn = V1ValidatedFunction(
530
518
  self.fn, config={"arbitrary_types_allowed": True}
531
519
  )
520
+ else:
521
+ validated_fn = V2ValidatedFunction(
522
+ self.fn, config=pydantic.ConfigDict(arbitrary_types_allowed=True)
523
+ )
532
524
 
533
525
  try:
534
- model = validated_fn.init_model_instance(*args, **kwargs)
526
+ with warnings.catch_warnings():
527
+ warnings.filterwarnings(
528
+ "ignore", category=pydantic.warnings.PydanticDeprecatedSince20
529
+ )
530
+ model = validated_fn.init_model_instance(*args, **kwargs)
535
531
  except pydantic.ValidationError as exc:
536
532
  # We capture the pydantic exception and raise our own because the pydantic
537
533
  # exception is not picklable when using a cythonized pydantic installation
538
- raise ParameterTypeError.from_validation_error(exc) from None
539
- except V2ValidationError as exc:
540
- # We capture the pydantic exception and raise our own because the pydantic
541
- # exception is not picklable when using a cythonized pydantic installation
534
+ logger.error(
535
+ f"Parameter validation failed for flow {self.name!r}: {exc.errors()}"
536
+ f"\nParameters: {parameters}"
537
+ )
542
538
  raise ParameterTypeError.from_validation_error(exc) from None
543
539
 
544
540
  # Get the updated parameter dict with cast values from the model
545
541
  cast_parameters = {
546
542
  k: v
547
- for k, v in model._iter()
548
- if k in model.__fields_set__ or model.__fields__[k].default_factory
543
+ for k, v in dict(model).items()
544
+ if k in model.model_fields_set or model.model_fields[k].default_factory
549
545
  }
550
546
  return cast_parameters
551
547
 
@@ -605,7 +601,7 @@ class Flow(Generic[P, R]):
605
601
  description: Optional[str] = None,
606
602
  tags: Optional[List[str]] = None,
607
603
  version: Optional[str] = None,
608
- enforce_parameter_schema: bool = False,
604
+ enforce_parameter_schema: bool = True,
609
605
  work_pool_name: Optional[str] = None,
610
606
  work_queue_name: Optional[str] = None,
611
607
  job_variables: Optional[Dict[str, Any]] = None,
@@ -667,7 +663,8 @@ class Flow(Generic[P, R]):
667
663
  from prefect.deployments.runner import RunnerDeployment
668
664
 
669
665
  if not name.endswith(".py"):
670
- raise_on_name_with_banned_characters(name)
666
+ _raise_on_name_with_banned_characters(name)
667
+
671
668
  if self._storage and self._entrypoint:
672
669
  return await RunnerDeployment.from_storage(
673
670
  storage=self._storage,
@@ -713,6 +710,36 @@ class Flow(Generic[P, R]):
713
710
  entrypoint_type=entrypoint_type,
714
711
  )
715
712
 
713
+ def on_completion(
714
+ self, fn: Callable[["Flow", FlowRun, State], None]
715
+ ) -> Callable[["Flow", FlowRun, State], None]:
716
+ self.on_completion_hooks.append(fn)
717
+ return fn
718
+
719
+ def on_cancellation(
720
+ self, fn: Callable[["Flow", FlowRun, State], None]
721
+ ) -> Callable[["Flow", FlowRun, State], None]:
722
+ self.on_cancellation_hooks.append(fn)
723
+ return fn
724
+
725
+ def on_crashed(
726
+ self, fn: Callable[["Flow", FlowRun, State], None]
727
+ ) -> Callable[["Flow", FlowRun, State], None]:
728
+ self.on_crashed_hooks.append(fn)
729
+ return fn
730
+
731
+ def on_running(
732
+ self, fn: Callable[["Flow", FlowRun, State], None]
733
+ ) -> Callable[["Flow", FlowRun, State], None]:
734
+ self.on_running_hooks.append(fn)
735
+ return fn
736
+
737
+ def on_failure(
738
+ self, fn: Callable[["Flow", FlowRun, State], None]
739
+ ) -> Callable[["Flow", FlowRun, State], None]:
740
+ self.on_failure_hooks.append(fn)
741
+ return fn
742
+
716
743
  @sync_compatible
717
744
  async def serve(
718
745
  self,
@@ -736,7 +763,7 @@ class Flow(Generic[P, R]):
736
763
  description: Optional[str] = None,
737
764
  tags: Optional[List[str]] = None,
738
765
  version: Optional[str] = None,
739
- enforce_parameter_schema: bool = False,
766
+ enforce_parameter_schema: bool = True,
740
767
  pause_on_shutdown: bool = True,
741
768
  print_starting_message: bool = True,
742
769
  limit: Optional[int] = None,
@@ -945,7 +972,7 @@ class Flow(Generic[P, R]):
945
972
  cron: Optional[str] = None,
946
973
  rrule: Optional[str] = None,
947
974
  paused: Optional[bool] = None,
948
- schedules: Optional[List[MinimalDeploymentSchedule]] = None,
975
+ schedules: Optional[List[DeploymentScheduleCreate]] = None,
949
976
  schedule: Optional[SCHEDULE_TYPES] = None,
950
977
  is_schedule_active: Optional[bool] = None,
951
978
  triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
@@ -953,7 +980,7 @@ class Flow(Generic[P, R]):
953
980
  description: Optional[str] = None,
954
981
  tags: Optional[List[str]] = None,
955
982
  version: Optional[str] = None,
956
- enforce_parameter_schema: bool = False,
983
+ enforce_parameter_schema: bool = True,
957
984
  entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
958
985
  print_next_steps: bool = True,
959
986
  ignore_warnings: bool = False,
@@ -1198,7 +1225,10 @@ class Flow(Generic[P, R]):
1198
1225
  >>> with tags("db", "blue"):
1199
1226
  >>> my_flow("foo")
1200
1227
  """
1201
- from prefect.engine import enter_flow_run_engine_from_flow_call
1228
+ from prefect.utilities.visualization import (
1229
+ get_task_viz_tracker,
1230
+ track_viz_task,
1231
+ )
1202
1232
 
1203
1233
  # Convert the call args/kwargs to a parameter dict
1204
1234
  parameters = get_call_parameters(self.fn, args, kwargs)
@@ -1211,72 +1241,19 @@ class Flow(Generic[P, R]):
1211
1241
  # we can add support for exploring subflows for tasks in the future.
1212
1242
  return track_viz_task(self.isasync, self.name, parameters)
1213
1243
 
1214
- if PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE.value():
1215
- from prefect.new_flow_engine import run_flow, run_flow_sync
1244
+ from prefect.flow_engine import run_flow, run_flow_sync
1216
1245
 
1217
- run_kwargs = dict(
1218
- flow=self,
1219
- parameters=parameters,
1220
- wait_for=wait_for,
1221
- return_type=return_type,
1222
- )
1223
- if self.isasync:
1224
- # this returns an awaitable coroutine
1225
- return run_flow(**run_kwargs)
1226
- else:
1227
- return run_flow_sync(**run_kwargs)
1228
-
1229
- return enter_flow_run_engine_from_flow_call(
1230
- self,
1231
- parameters,
1246
+ run_kwargs = dict(
1247
+ flow=self,
1248
+ parameters=parameters,
1232
1249
  wait_for=wait_for,
1233
1250
  return_type=return_type,
1234
1251
  )
1235
-
1236
- @overload
1237
- def _run(self: "Flow[P, NoReturn]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
1238
- # `NoReturn` matches if a type can't be inferred for the function which stops a
1239
- # sync function from matching the `Coroutine` overload
1240
- ...
1241
-
1242
- @overload
1243
- def _run(
1244
- self: "Flow[P, Coroutine[Any, Any, T]]", *args: P.args, **kwargs: P.kwargs
1245
- ) -> Awaitable[T]:
1246
- ...
1247
-
1248
- @overload
1249
- def _run(self: "Flow[P, T]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
1250
- ...
1251
-
1252
- def _run(
1253
- self,
1254
- *args: "P.args",
1255
- wait_for: Optional[Iterable[PrefectFuture]] = None,
1256
- **kwargs: "P.kwargs",
1257
- ):
1258
- """
1259
- Run the flow and return its final state.
1260
-
1261
- Examples:
1262
-
1263
- Run a flow and get the returned result
1264
-
1265
- >>> state = my_flow._run("marvin")
1266
- >>> state.result()
1267
- "goodbye marvin"
1268
- """
1269
- from prefect.engine import enter_flow_run_engine_from_flow_call
1270
-
1271
- # Convert the call args/kwargs to a parameter dict
1272
- parameters = get_call_parameters(self.fn, args, kwargs)
1273
-
1274
- return enter_flow_run_engine_from_flow_call(
1275
- self,
1276
- parameters,
1277
- wait_for=wait_for,
1278
- return_type="state",
1279
- )
1252
+ if self.isasync:
1253
+ # this returns an awaitable coroutine
1254
+ return run_flow(**run_kwargs)
1255
+ else:
1256
+ return run_flow_sync(**run_kwargs)
1280
1257
 
1281
1258
  @sync_compatible
1282
1259
  async def visualize(self, *args, **kwargs):
@@ -1289,6 +1266,16 @@ class Flow(Generic[P, R]):
1289
1266
  - GraphvizExecutableNotFoundError: If the `dot` executable isn't found.
1290
1267
  - FlowVisualizationError: If the flow can't be visualized for any other reason.
1291
1268
  """
1269
+ from prefect.utilities.visualization import (
1270
+ FlowVisualizationError,
1271
+ GraphvizExecutableNotFoundError,
1272
+ GraphvizImportError,
1273
+ TaskVizTracker,
1274
+ VisualizationUnsupportedError,
1275
+ build_task_dependencies,
1276
+ visualize_task_dependencies,
1277
+ )
1278
+
1292
1279
  if not PREFECT_UNIT_TEST_MODE:
1293
1280
  warnings.warn(
1294
1281
  "`flow.visualize()` will execute code inside of your flow that is not"
@@ -1341,7 +1328,7 @@ def flow(
1341
1328
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
1342
1329
  retries: Optional[int] = None,
1343
1330
  retry_delay_seconds: Optional[Union[int, float]] = None,
1344
- task_runner: BaseTaskRunner = ConcurrentTaskRunner,
1331
+ task_runner: Optional[TaskRunner] = None,
1345
1332
  description: str = None,
1346
1333
  timeout_seconds: Union[int, float] = None,
1347
1334
  validate_parameters: bool = True,
@@ -1373,7 +1360,7 @@ def flow(
1373
1360
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
1374
1361
  retries: int = None,
1375
1362
  retry_delay_seconds: Union[int, float] = None,
1376
- task_runner: BaseTaskRunner = ConcurrentTaskRunner,
1363
+ task_runner: Optional[TaskRunner] = None,
1377
1364
  description: str = None,
1378
1365
  timeout_seconds: Union[int, float] = None,
1379
1366
  validate_parameters: bool = True,
@@ -1551,6 +1538,23 @@ def flow(
1551
1538
  )
1552
1539
 
1553
1540
 
1541
+ def _raise_on_name_with_banned_characters(name: str) -> str:
1542
+ """
1543
+ Raise an InvalidNameError if the given name contains any invalid
1544
+ characters.
1545
+ """
1546
+ if name is None:
1547
+ return name
1548
+
1549
+ if not re.match(WITHOUT_BANNED_CHARACTERS, name):
1550
+ raise InvalidNameError(
1551
+ f"Name {name!r} contains an invalid character. "
1552
+ f"Must not contain any of: {BANNED_CHARACTERS}."
1553
+ )
1554
+
1555
+ return name
1556
+
1557
+
1554
1558
  # Add from_source so it is available on the flow function we all know and love
1555
1559
  flow.from_source = Flow.from_source
1556
1560
 
@@ -1640,7 +1644,9 @@ def load_flow_from_script(path: str, flow_name: str = None) -> Flow:
1640
1644
  )
1641
1645
 
1642
1646
 
1643
- def load_flow_from_entrypoint(entrypoint: str) -> Flow:
1647
+ def load_flow_from_entrypoint(
1648
+ entrypoint: str,
1649
+ ) -> Flow:
1644
1650
  """
1645
1651
  Extract a flow object from a script at an entrypoint by running all of the code in the file.
1646
1652
 
@@ -1793,3 +1799,138 @@ async def serve(
1793
1799
  )
1794
1800
 
1795
1801
  await runner.start()
1802
+
1803
+
1804
+ @client_injector
1805
+ async def load_flow_from_flow_run(
1806
+ client: "PrefectClient",
1807
+ flow_run: "FlowRun",
1808
+ ignore_storage: bool = False,
1809
+ storage_base_path: Optional[str] = None,
1810
+ ) -> "Flow":
1811
+ """
1812
+ Load a flow from the location/script provided in a deployment's storage document.
1813
+
1814
+ If `ignore_storage=True` is provided, no pull from remote storage occurs. This flag
1815
+ is largely for testing, and assumes the flow is already available locally.
1816
+ """
1817
+ deployment = await client.read_deployment(flow_run.deployment_id)
1818
+
1819
+ if deployment.entrypoint is None:
1820
+ raise ValueError(
1821
+ f"Deployment {deployment.id} does not have an entrypoint and can not be run."
1822
+ )
1823
+
1824
+ run_logger = flow_run_logger(flow_run)
1825
+
1826
+ runner_storage_base_path = storage_base_path or os.environ.get(
1827
+ "PREFECT__STORAGE_BASE_PATH"
1828
+ )
1829
+
1830
+ # If there's no colon, assume it's a module path
1831
+ if ":" not in deployment.entrypoint:
1832
+ run_logger.debug(
1833
+ f"Importing flow code from module path {deployment.entrypoint}"
1834
+ )
1835
+ flow = await run_sync_in_worker_thread(
1836
+ load_flow_from_entrypoint, deployment.entrypoint
1837
+ )
1838
+ return flow
1839
+
1840
+ if not ignore_storage and not deployment.pull_steps:
1841
+ sys.path.insert(0, ".")
1842
+ if deployment.storage_document_id:
1843
+ storage_document = await client.read_block_document(
1844
+ deployment.storage_document_id
1845
+ )
1846
+ storage_block = Block._from_block_document(storage_document)
1847
+ else:
1848
+ basepath = deployment.path or Path(deployment.manifest_path).parent
1849
+ if runner_storage_base_path:
1850
+ basepath = str(basepath).replace(
1851
+ "$STORAGE_BASE_PATH", runner_storage_base_path
1852
+ )
1853
+ storage_block = LocalFileSystem(basepath=basepath)
1854
+
1855
+ from_path = (
1856
+ str(deployment.path).replace("$STORAGE_BASE_PATH", runner_storage_base_path)
1857
+ if runner_storage_base_path and deployment.path
1858
+ else deployment.path
1859
+ )
1860
+ run_logger.info(f"Downloading flow code from storage at {from_path!r}")
1861
+ await storage_block.get_directory(from_path=from_path, local_path=".")
1862
+
1863
+ if deployment.pull_steps:
1864
+ run_logger.debug(f"Running {len(deployment.pull_steps)} deployment pull steps")
1865
+ output = await run_steps(deployment.pull_steps)
1866
+ if output.get("directory"):
1867
+ run_logger.debug(f"Changing working directory to {output['directory']!r}")
1868
+ os.chdir(output["directory"])
1869
+
1870
+ import_path = relative_path_to_current_platform(deployment.entrypoint)
1871
+ # for backwards compat
1872
+ if deployment.manifest_path:
1873
+ with open(deployment.manifest_path, "r") as f:
1874
+ import_path = json.load(f)["import_path"]
1875
+ import_path = (
1876
+ Path(deployment.manifest_path).parent / import_path
1877
+ ).absolute()
1878
+ run_logger.debug(f"Importing flow code from '{import_path}'")
1879
+
1880
+ flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, str(import_path))
1881
+
1882
+ return flow
1883
+
1884
+
1885
+ def load_flow_argument_from_entrypoint(
1886
+ entrypoint: str, arg: str = "name"
1887
+ ) -> Optional[str]:
1888
+ """
1889
+ Extract a flow argument from an entrypoint string.
1890
+
1891
+ Loads the source code of the entrypoint and extracts the flow argument from the
1892
+ `flow` decorator.
1893
+
1894
+ Args:
1895
+ entrypoint: a string in the format `<path_to_script>:<flow_func_name>` or a module path
1896
+ to a flow function
1897
+
1898
+ Returns:
1899
+ The flow argument value
1900
+ """
1901
+ if ":" in entrypoint:
1902
+ # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
1903
+ path, func_name = entrypoint.rsplit(":", maxsplit=1)
1904
+ source_code = Path(path).read_text()
1905
+ else:
1906
+ path, func_name = entrypoint.rsplit(".", maxsplit=1)
1907
+ spec = importlib.util.find_spec(path)
1908
+ if not spec or not spec.origin:
1909
+ raise ValueError(f"Could not find module {path!r}")
1910
+ source_code = Path(spec.origin).read_text()
1911
+ parsed_code = ast.parse(source_code)
1912
+ func_def = next(
1913
+ (
1914
+ node
1915
+ for node in ast.walk(parsed_code)
1916
+ if isinstance(node, ast.FunctionDef) and node.name == func_name
1917
+ ),
1918
+ None,
1919
+ )
1920
+ if not func_def:
1921
+ raise ValueError(f"Could not find flow {func_name!r} in {path!r}")
1922
+ for decorator in func_def.decorator_list:
1923
+ if (
1924
+ isinstance(decorator, ast.Call)
1925
+ and getattr(decorator.func, "id", "") == "flow"
1926
+ ):
1927
+ for keyword in decorator.keywords:
1928
+ if keyword.arg == arg:
1929
+ return (
1930
+ keyword.value.value
1931
+ ) # Return the string value of the argument
1932
+
1933
+ if arg == "name":
1934
+ return func_name.replace(
1935
+ "_", "-"
1936
+ ) # If no matching decorator or keyword argument is found