prefect-client 2.19.3__py3-none-any.whl → 3.0.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. prefect/__init__.py +8 -56
  2. prefect/_internal/compatibility/deprecated.py +6 -115
  3. prefect/_internal/compatibility/experimental.py +4 -79
  4. prefect/_internal/concurrency/api.py +0 -34
  5. prefect/_internal/concurrency/calls.py +0 -6
  6. prefect/_internal/concurrency/cancellation.py +0 -3
  7. prefect/_internal/concurrency/event_loop.py +0 -20
  8. prefect/_internal/concurrency/inspection.py +3 -3
  9. prefect/_internal/concurrency/threads.py +35 -0
  10. prefect/_internal/concurrency/waiters.py +0 -28
  11. prefect/_internal/pydantic/__init__.py +0 -45
  12. prefect/_internal/pydantic/v1_schema.py +21 -22
  13. prefect/_internal/pydantic/v2_schema.py +0 -2
  14. prefect/_internal/pydantic/v2_validated_func.py +18 -23
  15. prefect/_internal/schemas/bases.py +44 -177
  16. prefect/_internal/schemas/fields.py +1 -43
  17. prefect/_internal/schemas/validators.py +60 -158
  18. prefect/artifacts.py +161 -14
  19. prefect/automations.py +39 -4
  20. prefect/blocks/abstract.py +1 -1
  21. prefect/blocks/core.py +268 -148
  22. prefect/blocks/fields.py +2 -57
  23. prefect/blocks/kubernetes.py +8 -12
  24. prefect/blocks/notifications.py +40 -20
  25. prefect/blocks/system.py +22 -11
  26. prefect/blocks/webhook.py +2 -9
  27. prefect/client/base.py +4 -4
  28. prefect/client/cloud.py +8 -13
  29. prefect/client/orchestration.py +347 -341
  30. prefect/client/schemas/actions.py +92 -86
  31. prefect/client/schemas/filters.py +20 -40
  32. prefect/client/schemas/objects.py +147 -145
  33. prefect/client/schemas/responses.py +16 -24
  34. prefect/client/schemas/schedules.py +47 -35
  35. prefect/client/subscriptions.py +2 -2
  36. prefect/client/utilities.py +5 -2
  37. prefect/concurrency/asyncio.py +3 -1
  38. prefect/concurrency/events.py +1 -1
  39. prefect/concurrency/services.py +6 -3
  40. prefect/context.py +195 -27
  41. prefect/deployments/__init__.py +5 -6
  42. prefect/deployments/base.py +7 -5
  43. prefect/deployments/flow_runs.py +185 -0
  44. prefect/deployments/runner.py +50 -45
  45. prefect/deployments/schedules.py +28 -23
  46. prefect/deployments/steps/__init__.py +0 -1
  47. prefect/deployments/steps/core.py +1 -0
  48. prefect/deployments/steps/pull.py +7 -21
  49. prefect/engine.py +12 -2422
  50. prefect/events/actions.py +17 -23
  51. prefect/events/cli/automations.py +19 -6
  52. prefect/events/clients.py +14 -37
  53. prefect/events/filters.py +14 -18
  54. prefect/events/related.py +2 -2
  55. prefect/events/schemas/__init__.py +0 -5
  56. prefect/events/schemas/automations.py +55 -46
  57. prefect/events/schemas/deployment_triggers.py +7 -197
  58. prefect/events/schemas/events.py +34 -65
  59. prefect/events/schemas/labelling.py +10 -14
  60. prefect/events/utilities.py +2 -3
  61. prefect/events/worker.py +2 -3
  62. prefect/filesystems.py +6 -517
  63. prefect/{new_flow_engine.py → flow_engine.py} +313 -72
  64. prefect/flow_runs.py +377 -5
  65. prefect/flows.py +248 -165
  66. prefect/futures.py +186 -345
  67. prefect/infrastructure/__init__.py +0 -27
  68. prefect/infrastructure/provisioners/__init__.py +5 -3
  69. prefect/infrastructure/provisioners/cloud_run.py +11 -6
  70. prefect/infrastructure/provisioners/container_instance.py +11 -7
  71. prefect/infrastructure/provisioners/ecs.py +6 -4
  72. prefect/infrastructure/provisioners/modal.py +8 -5
  73. prefect/input/actions.py +2 -4
  74. prefect/input/run_input.py +5 -7
  75. prefect/logging/formatters.py +0 -2
  76. prefect/logging/handlers.py +3 -11
  77. prefect/logging/loggers.py +2 -2
  78. prefect/manifests.py +2 -1
  79. prefect/records/__init__.py +1 -0
  80. prefect/records/result_store.py +42 -0
  81. prefect/records/store.py +9 -0
  82. prefect/results.py +43 -39
  83. prefect/runner/runner.py +9 -9
  84. prefect/runner/server.py +6 -10
  85. prefect/runner/storage.py +3 -8
  86. prefect/runner/submit.py +2 -2
  87. prefect/runner/utils.py +2 -2
  88. prefect/serializers.py +24 -35
  89. prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
  90. prefect/settings.py +70 -133
  91. prefect/states.py +17 -47
  92. prefect/task_engine.py +697 -58
  93. prefect/task_runners.py +269 -301
  94. prefect/task_server.py +53 -34
  95. prefect/tasks.py +327 -337
  96. prefect/transactions.py +220 -0
  97. prefect/types/__init__.py +61 -82
  98. prefect/utilities/asyncutils.py +195 -136
  99. prefect/utilities/callables.py +121 -41
  100. prefect/utilities/collections.py +23 -38
  101. prefect/utilities/dispatch.py +11 -3
  102. prefect/utilities/dockerutils.py +4 -0
  103. prefect/utilities/engine.py +140 -20
  104. prefect/utilities/importtools.py +26 -27
  105. prefect/utilities/pydantic.py +128 -38
  106. prefect/utilities/schema_tools/hydration.py +5 -1
  107. prefect/utilities/templating.py +12 -2
  108. prefect/variables.py +78 -61
  109. prefect/workers/__init__.py +0 -1
  110. prefect/workers/base.py +15 -17
  111. prefect/workers/process.py +3 -8
  112. prefect/workers/server.py +2 -2
  113. {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/METADATA +22 -21
  114. prefect_client-3.0.0rc1.dist-info/RECORD +176 -0
  115. prefect/_internal/pydantic/_base_model.py +0 -51
  116. prefect/_internal/pydantic/_compat.py +0 -82
  117. prefect/_internal/pydantic/_flags.py +0 -20
  118. prefect/_internal/pydantic/_types.py +0 -8
  119. prefect/_internal/pydantic/utilities/__init__.py +0 -0
  120. prefect/_internal/pydantic/utilities/config_dict.py +0 -72
  121. prefect/_internal/pydantic/utilities/field_validator.py +0 -150
  122. prefect/_internal/pydantic/utilities/model_construct.py +0 -56
  123. prefect/_internal/pydantic/utilities/model_copy.py +0 -55
  124. prefect/_internal/pydantic/utilities/model_dump.py +0 -136
  125. prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
  126. prefect/_internal/pydantic/utilities/model_fields.py +0 -50
  127. prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
  128. prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
  129. prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
  130. prefect/_internal/pydantic/utilities/model_validate.py +0 -75
  131. prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
  132. prefect/_internal/pydantic/utilities/model_validator.py +0 -87
  133. prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
  134. prefect/_vendor/__init__.py +0 -0
  135. prefect/_vendor/fastapi/__init__.py +0 -25
  136. prefect/_vendor/fastapi/applications.py +0 -946
  137. prefect/_vendor/fastapi/background.py +0 -3
  138. prefect/_vendor/fastapi/concurrency.py +0 -44
  139. prefect/_vendor/fastapi/datastructures.py +0 -58
  140. prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
  141. prefect/_vendor/fastapi/dependencies/models.py +0 -64
  142. prefect/_vendor/fastapi/dependencies/utils.py +0 -877
  143. prefect/_vendor/fastapi/encoders.py +0 -177
  144. prefect/_vendor/fastapi/exception_handlers.py +0 -40
  145. prefect/_vendor/fastapi/exceptions.py +0 -46
  146. prefect/_vendor/fastapi/logger.py +0 -3
  147. prefect/_vendor/fastapi/middleware/__init__.py +0 -1
  148. prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
  149. prefect/_vendor/fastapi/middleware/cors.py +0 -3
  150. prefect/_vendor/fastapi/middleware/gzip.py +0 -3
  151. prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
  152. prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
  153. prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
  154. prefect/_vendor/fastapi/openapi/__init__.py +0 -0
  155. prefect/_vendor/fastapi/openapi/constants.py +0 -2
  156. prefect/_vendor/fastapi/openapi/docs.py +0 -203
  157. prefect/_vendor/fastapi/openapi/models.py +0 -480
  158. prefect/_vendor/fastapi/openapi/utils.py +0 -485
  159. prefect/_vendor/fastapi/param_functions.py +0 -340
  160. prefect/_vendor/fastapi/params.py +0 -453
  161. prefect/_vendor/fastapi/requests.py +0 -4
  162. prefect/_vendor/fastapi/responses.py +0 -40
  163. prefect/_vendor/fastapi/routing.py +0 -1331
  164. prefect/_vendor/fastapi/security/__init__.py +0 -15
  165. prefect/_vendor/fastapi/security/api_key.py +0 -98
  166. prefect/_vendor/fastapi/security/base.py +0 -6
  167. prefect/_vendor/fastapi/security/http.py +0 -172
  168. prefect/_vendor/fastapi/security/oauth2.py +0 -227
  169. prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
  170. prefect/_vendor/fastapi/security/utils.py +0 -10
  171. prefect/_vendor/fastapi/staticfiles.py +0 -1
  172. prefect/_vendor/fastapi/templating.py +0 -3
  173. prefect/_vendor/fastapi/testclient.py +0 -1
  174. prefect/_vendor/fastapi/types.py +0 -3
  175. prefect/_vendor/fastapi/utils.py +0 -235
  176. prefect/_vendor/fastapi/websockets.py +0 -7
  177. prefect/_vendor/starlette/__init__.py +0 -1
  178. prefect/_vendor/starlette/_compat.py +0 -28
  179. prefect/_vendor/starlette/_exception_handler.py +0 -80
  180. prefect/_vendor/starlette/_utils.py +0 -88
  181. prefect/_vendor/starlette/applications.py +0 -261
  182. prefect/_vendor/starlette/authentication.py +0 -159
  183. prefect/_vendor/starlette/background.py +0 -43
  184. prefect/_vendor/starlette/concurrency.py +0 -59
  185. prefect/_vendor/starlette/config.py +0 -151
  186. prefect/_vendor/starlette/convertors.py +0 -87
  187. prefect/_vendor/starlette/datastructures.py +0 -707
  188. prefect/_vendor/starlette/endpoints.py +0 -130
  189. prefect/_vendor/starlette/exceptions.py +0 -60
  190. prefect/_vendor/starlette/formparsers.py +0 -276
  191. prefect/_vendor/starlette/middleware/__init__.py +0 -17
  192. prefect/_vendor/starlette/middleware/authentication.py +0 -52
  193. prefect/_vendor/starlette/middleware/base.py +0 -220
  194. prefect/_vendor/starlette/middleware/cors.py +0 -176
  195. prefect/_vendor/starlette/middleware/errors.py +0 -265
  196. prefect/_vendor/starlette/middleware/exceptions.py +0 -74
  197. prefect/_vendor/starlette/middleware/gzip.py +0 -113
  198. prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
  199. prefect/_vendor/starlette/middleware/sessions.py +0 -82
  200. prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
  201. prefect/_vendor/starlette/middleware/wsgi.py +0 -147
  202. prefect/_vendor/starlette/requests.py +0 -328
  203. prefect/_vendor/starlette/responses.py +0 -347
  204. prefect/_vendor/starlette/routing.py +0 -933
  205. prefect/_vendor/starlette/schemas.py +0 -154
  206. prefect/_vendor/starlette/staticfiles.py +0 -248
  207. prefect/_vendor/starlette/status.py +0 -199
  208. prefect/_vendor/starlette/templating.py +0 -231
  209. prefect/_vendor/starlette/testclient.py +0 -804
  210. prefect/_vendor/starlette/types.py +0 -30
  211. prefect/_vendor/starlette/websockets.py +0 -193
  212. prefect/agent.py +0 -698
  213. prefect/deployments/deployments.py +0 -1042
  214. prefect/deprecated/__init__.py +0 -0
  215. prefect/deprecated/data_documents.py +0 -350
  216. prefect/deprecated/packaging/__init__.py +0 -12
  217. prefect/deprecated/packaging/base.py +0 -96
  218. prefect/deprecated/packaging/docker.py +0 -146
  219. prefect/deprecated/packaging/file.py +0 -92
  220. prefect/deprecated/packaging/orion.py +0 -80
  221. prefect/deprecated/packaging/serializers.py +0 -171
  222. prefect/events/instrument.py +0 -135
  223. prefect/infrastructure/base.py +0 -323
  224. prefect/infrastructure/container.py +0 -818
  225. prefect/infrastructure/kubernetes.py +0 -920
  226. prefect/infrastructure/process.py +0 -289
  227. prefect/new_task_engine.py +0 -423
  228. prefect/pydantic/__init__.py +0 -76
  229. prefect/pydantic/main.py +0 -39
  230. prefect/software/__init__.py +0 -2
  231. prefect/software/base.py +0 -50
  232. prefect/software/conda.py +0 -199
  233. prefect/software/pip.py +0 -122
  234. prefect/software/python.py +0 -52
  235. prefect/workers/block.py +0 -218
  236. prefect_client-2.19.3.dist-info/RECORD +0 -292
  237. {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/LICENSE +0 -0
  238. {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/WHEEL +0 -0
  239. {prefect_client-2.19.3.dist-info → prefect_client-3.0.0rc1.dist-info}/top_level.txt +0 -0
prefect/flows.py CHANGED
@@ -9,7 +9,10 @@ import ast
9
9
  import datetime
10
10
  import importlib.util
11
11
  import inspect
12
+ import json
12
13
  import os
14
+ import re
15
+ import sys
13
16
  import tempfile
14
17
  import warnings
15
18
  from functools import partial, update_wrapper
@@ -36,50 +39,38 @@ from typing import (
36
39
  )
37
40
  from uuid import UUID
38
41
 
42
+ import pydantic
43
+ from fastapi.encoders import jsonable_encoder
44
+ from pydantic.v1 import BaseModel as V1BaseModel
45
+ from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
46
+ from pydantic.v1.errors import ConfigError # TODO
39
47
  from rich.console import Console
40
48
  from typing_extensions import Literal, ParamSpec, Self
41
49
 
42
- from prefect._internal.pydantic import HAS_PYDANTIC_V2
43
-
44
- if HAS_PYDANTIC_V2:
45
- import pydantic.v1 as pydantic
46
- from pydantic import ValidationError as V2ValidationError
47
- from pydantic.v1 import BaseModel as V1BaseModel
48
- from pydantic.v1.decorator import ValidatedFunction as V1ValidatedFunction
49
-
50
- from ._internal.pydantic.v2_schema import is_v2_type
51
- from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
52
- from ._internal.pydantic.v2_validated_func import (
53
- V2ValidatedFunction as ValidatedFunction,
54
- )
55
-
56
- else:
57
- import pydantic
58
- from pydantic.decorator import ValidatedFunction
59
-
60
- V2ValidationError = None
61
-
62
- from prefect._vendor.fastapi.encoders import jsonable_encoder
63
-
64
50
  from prefect._internal.compatibility.deprecated import deprecated_parameter
65
51
  from prefect._internal.concurrency.api import create_call, from_async
66
- from prefect._internal.schemas.validators import raise_on_name_with_banned_characters
52
+ from prefect.blocks.core import Block
67
53
  from prefect.client.orchestration import get_client
54
+ from prefect.client.schemas.actions import DeploymentScheduleCreate
68
55
  from prefect.client.schemas.objects import Flow as FlowSchema
69
- from prefect.client.schemas.objects import FlowRun, MinimalDeploymentSchedule
56
+ from prefect.client.schemas.objects import FlowRun
70
57
  from prefect.client.schemas.schedules import SCHEDULE_TYPES
58
+ from prefect.client.utilities import client_injector
71
59
  from prefect.context import PrefectObjectRegistry, registry_from_script
72
60
  from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
61
+ from prefect.deployments.steps.core import run_steps
73
62
  from prefect.events import DeploymentTriggerTypes, TriggerTypes
74
63
  from prefect.exceptions import (
64
+ InvalidNameError,
75
65
  MissingFlowError,
76
66
  ObjectNotFound,
77
67
  ParameterTypeError,
78
68
  UnspecifiedFlowError,
79
69
  )
80
- from prefect.filesystems import ReadableDeploymentStorage
70
+ from prefect.filesystems import LocalFileSystem, ReadableDeploymentStorage
81
71
  from prefect.futures import PrefectFuture
82
72
  from prefect.logging import get_logger
73
+ from prefect.logging.loggers import flow_run_logger
83
74
  from prefect.results import ResultSerializer, ResultStorage
84
75
  from prefect.runner.storage import (
85
76
  BlockStorageAdapter,
@@ -88,16 +79,20 @@ from prefect.runner.storage import (
88
79
  )
89
80
  from prefect.settings import (
90
81
  PREFECT_DEFAULT_WORK_POOL_NAME,
91
- PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE,
92
82
  PREFECT_FLOW_DEFAULT_RETRIES,
93
83
  PREFECT_FLOW_DEFAULT_RETRY_DELAY_SECONDS,
94
84
  PREFECT_UI_URL,
95
85
  PREFECT_UNIT_TEST_MODE,
96
86
  )
97
87
  from prefect.states import State
98
- from prefect.task_runners import BaseTaskRunner, ConcurrentTaskRunner
88
+ from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
89
+ from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
99
90
  from prefect.utilities.annotations import NotSet
100
- from prefect.utilities.asyncutils import is_async_fn, sync_compatible
91
+ from prefect.utilities.asyncutils import (
92
+ is_async_fn,
93
+ run_sync_in_worker_thread,
94
+ sync_compatible,
95
+ )
101
96
  from prefect.utilities.callables import (
102
97
  get_call_parameters,
103
98
  parameter_schema,
@@ -105,18 +100,14 @@ from prefect.utilities.callables import (
105
100
  raise_for_reserved_arguments,
106
101
  )
107
102
  from prefect.utilities.collections import listrepr
103
+ from prefect.utilities.filesystem import relative_path_to_current_platform
108
104
  from prefect.utilities.hashing import file_hash
109
105
  from prefect.utilities.importtools import import_object
110
- from prefect.utilities.visualization import (
111
- FlowVisualizationError,
112
- GraphvizExecutableNotFoundError,
113
- GraphvizImportError,
114
- TaskVizTracker,
115
- VisualizationUnsupportedError,
116
- build_task_dependencies,
117
- get_task_viz_tracker,
118
- track_viz_task,
119
- visualize_task_dependencies,
106
+
107
+ from ._internal.pydantic.v2_schema import is_v2_type
108
+ from ._internal.pydantic.v2_validated_func import V2ValidatedFunction
109
+ from ._internal.pydantic.v2_validated_func import (
110
+ V2ValidatedFunction as ValidatedFunction,
120
111
  )
121
112
 
122
113
  T = TypeVar("T") # Generic type var for capturing the inner return type of async funcs
@@ -127,7 +118,9 @@ F = TypeVar("F", bound="Flow") # The type of the flow
127
118
  logger = get_logger("flows")
128
119
 
129
120
  if TYPE_CHECKING:
121
+ from prefect.client.orchestration import PrefectClient
130
122
  from prefect.deployments.runner import FlexibleScheduleList, RunnerDeployment
123
+ from prefect.flows import FlowRun
131
124
 
132
125
 
133
126
  @PrefectObjectRegistry.register_instances
@@ -198,9 +191,9 @@ class Flow(Generic[P, R]):
198
191
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
199
192
  retries: Optional[int] = None,
200
193
  retry_delay_seconds: Optional[Union[int, float]] = None,
201
- task_runner: Union[Type[BaseTaskRunner], BaseTaskRunner] = ConcurrentTaskRunner,
202
- description: str = None,
203
- timeout_seconds: Union[int, float] = None,
194
+ task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
195
+ description: Optional[str] = None,
196
+ timeout_seconds: Union[int, float, None] = None,
204
197
  validate_parameters: bool = True,
205
198
  persist_result: Optional[bool] = None,
206
199
  result_storage: Optional[ResultStorage] = None,
@@ -247,8 +240,6 @@ class Flow(Generic[P, R]):
247
240
  ]
248
241
  for hooks, hook_name in zip(hook_categories, hook_names):
249
242
  if hooks is not None:
250
- if not hooks:
251
- raise ValueError(f"Empty list passed for '{hook_name}'")
252
243
  try:
253
244
  hooks = list(hooks)
254
245
  except TypeError:
@@ -275,7 +266,7 @@ class Flow(Generic[P, R]):
275
266
 
276
267
  # Validate name if given
277
268
  if name:
278
- raise_on_name_with_banned_characters(name)
269
+ _raise_on_name_with_banned_characters(name)
279
270
 
280
271
  self.name = name or fn.__name__.replace("_", "-")
281
272
 
@@ -287,7 +278,8 @@ class Flow(Generic[P, R]):
287
278
  )
288
279
  self.flow_run_name = flow_run_name
289
280
 
290
- task_runner = task_runner or ConcurrentTaskRunner()
281
+ default_task_runner = ThreadPoolTaskRunner()
282
+ task_runner = task_runner or default_task_runner
291
283
  self.task_runner = (
292
284
  task_runner() if isinstance(task_runner, type) else task_runner
293
285
  )
@@ -335,7 +327,7 @@ class Flow(Generic[P, R]):
335
327
  # is not picklable in some environments
336
328
  try:
337
329
  ValidatedFunction(self.fn, config={"arbitrary_types_allowed": True})
338
- except pydantic.ConfigError as exc:
330
+ except ConfigError as exc:
339
331
  raise ValueError(
340
332
  "Flow function is not compatible with `validate_parameters`. "
341
333
  "Disable validation or change the argument names."
@@ -345,11 +337,11 @@ class Flow(Generic[P, R]):
345
337
  self.result_storage = result_storage
346
338
  self.result_serializer = result_serializer
347
339
  self.cache_result_in_memory = cache_result_in_memory
348
- self.on_completion = on_completion
349
- self.on_failure = on_failure
350
- self.on_cancellation = on_cancellation
351
- self.on_crashed = on_crashed
352
- self.on_running = on_running
340
+ self.on_completion_hooks = on_completion or []
341
+ self.on_failure_hooks = on_failure or []
342
+ self.on_cancellation_hooks = on_cancellation or []
343
+ self.on_crashed_hooks = on_crashed or []
344
+ self.on_running_hooks = on_running or []
353
345
 
354
346
  # Used for flows loaded from remote storage
355
347
  self._storage: Optional[RunnerStorage] = None
@@ -365,20 +357,20 @@ class Flow(Generic[P, R]):
365
357
  def with_options(
366
358
  self,
367
359
  *,
368
- name: str = None,
369
- version: str = None,
360
+ name: Optional[str] = None,
361
+ version: Optional[str] = None,
370
362
  retries: Optional[int] = None,
371
363
  retry_delay_seconds: Optional[Union[int, float]] = None,
372
- description: str = None,
364
+ description: Optional[str] = None,
373
365
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
374
- task_runner: Union[Type[BaseTaskRunner], BaseTaskRunner] = None,
375
- timeout_seconds: Union[int, float] = None,
376
- validate_parameters: bool = None,
377
- persist_result: Optional[bool] = NotSet,
378
- result_storage: Optional[ResultStorage] = NotSet,
379
- result_serializer: Optional[ResultSerializer] = NotSet,
380
- cache_result_in_memory: bool = None,
381
- log_prints: Optional[bool] = NotSet,
366
+ task_runner: Union[Type[TaskRunner], TaskRunner, None] = None,
367
+ timeout_seconds: Union[int, float, None] = None,
368
+ validate_parameters: Optional[bool] = None,
369
+ persist_result: Optional[bool] = NotSet, # type: ignore
370
+ result_storage: Optional[ResultStorage] = NotSet, # type: ignore
371
+ result_serializer: Optional[ResultSerializer] = NotSet, # type: ignore
372
+ cache_result_in_memory: Optional[bool] = None,
373
+ log_prints: Optional[bool] = NotSet, # type: ignore
382
374
  on_completion: Optional[
383
375
  List[Callable[[FlowSchema, FlowRun, State], None]]
384
376
  ] = None,
@@ -434,15 +426,14 @@ class Flow(Generic[P, R]):
434
426
  Create a new flow from an existing flow, update the task runner, and call
435
427
  it without an intermediate variable:
436
428
 
437
- >>> from prefect.task_runners import SequentialTaskRunner
429
+ >>> from prefect.task_runners import ThreadPoolTaskRunner
438
430
  >>>
439
431
  >>> @flow
440
432
  >>> def my_flow(x, y):
441
433
  >>> return x + y
442
434
  >>>
443
- >>> state = my_flow.with_options(task_runner=SequentialTaskRunner)(1, 3)
435
+ >>> state = my_flow.with_options(task_runner=ThreadPoolTaskRunner)(1, 3)
444
436
  >>> assert state.result() == 4
445
-
446
437
  """
447
438
  new_flow = Flow(
448
439
  fn=self.fn,
@@ -482,11 +473,11 @@ class Flow(Generic[P, R]):
482
473
  else self.cache_result_in_memory
483
474
  ),
484
475
  log_prints=log_prints if log_prints is not NotSet else self.log_prints,
485
- on_completion=on_completion or self.on_completion,
486
- on_failure=on_failure or self.on_failure,
487
- on_cancellation=on_cancellation or self.on_cancellation,
488
- on_crashed=on_crashed or self.on_crashed,
489
- on_running=on_running or self.on_running,
476
+ on_completion=on_completion or self.on_completion_hooks,
477
+ on_failure=on_failure or self.on_failure_hooks,
478
+ on_cancellation=on_cancellation or self.on_cancellation_hooks,
479
+ on_crashed=on_crashed or self.on_crashed_hooks,
480
+ on_running=on_running or self.on_running_hooks,
490
481
  )
491
482
  new_flow._storage = self._storage
492
483
  new_flow._entrypoint = self._entrypoint
@@ -505,49 +496,52 @@ class Flow(Generic[P, R]):
505
496
  """
506
497
  args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
507
498
 
508
- if HAS_PYDANTIC_V2:
499
+ with warnings.catch_warnings():
500
+ warnings.filterwarnings(
501
+ "ignore", category=pydantic.warnings.PydanticDeprecatedSince20
502
+ )
509
503
  has_v1_models = any(isinstance(o, V1BaseModel) for o in args) or any(
510
504
  isinstance(o, V1BaseModel) for o in kwargs.values()
511
505
  )
512
- has_v2_types = any(is_v2_type(o) for o in args) or any(
513
- is_v2_type(o) for o in kwargs.values()
514
- )
515
506
 
516
- if has_v1_models and has_v2_types:
517
- raise ParameterTypeError(
518
- "Cannot mix Pydantic v1 and v2 types as arguments to a flow."
519
- )
507
+ has_v2_types = any(is_v2_type(o) for o in args) or any(
508
+ is_v2_type(o) for o in kwargs.values()
509
+ )
520
510
 
521
- if has_v1_models:
522
- validated_fn = V1ValidatedFunction(
523
- self.fn, config={"arbitrary_types_allowed": True}
524
- )
525
- else:
526
- validated_fn = V2ValidatedFunction(
527
- self.fn, config={"arbitrary_types_allowed": True}
528
- )
511
+ if has_v1_models and has_v2_types:
512
+ raise ParameterTypeError(
513
+ "Cannot mix Pydantic v1 and v2 types as arguments to a flow."
514
+ )
529
515
 
530
- else:
531
- validated_fn = ValidatedFunction(
516
+ if has_v1_models:
517
+ validated_fn = V1ValidatedFunction(
532
518
  self.fn, config={"arbitrary_types_allowed": True}
533
519
  )
520
+ else:
521
+ validated_fn = V2ValidatedFunction(
522
+ self.fn, config=pydantic.ConfigDict(arbitrary_types_allowed=True)
523
+ )
534
524
 
535
525
  try:
536
- model = validated_fn.init_model_instance(*args, **kwargs)
526
+ with warnings.catch_warnings():
527
+ warnings.filterwarnings(
528
+ "ignore", category=pydantic.warnings.PydanticDeprecatedSince20
529
+ )
530
+ model = validated_fn.init_model_instance(*args, **kwargs)
537
531
  except pydantic.ValidationError as exc:
538
532
  # We capture the pydantic exception and raise our own because the pydantic
539
533
  # exception is not picklable when using a cythonized pydantic installation
540
- raise ParameterTypeError.from_validation_error(exc) from None
541
- except V2ValidationError as exc:
542
- # We capture the pydantic exception and raise our own because the pydantic
543
- # exception is not picklable when using a cythonized pydantic installation
534
+ logger.error(
535
+ f"Parameter validation failed for flow {self.name!r}: {exc.errors()}"
536
+ f"\nParameters: {parameters}"
537
+ )
544
538
  raise ParameterTypeError.from_validation_error(exc) from None
545
539
 
546
540
  # Get the updated parameter dict with cast values from the model
547
541
  cast_parameters = {
548
542
  k: v
549
- for k, v in model._iter()
550
- if k in model.__fields_set__ or model.__fields__[k].default_factory
543
+ for k, v in dict(model).items()
544
+ if k in model.model_fields_set or model.model_fields[k].default_factory
551
545
  }
552
546
  return cast_parameters
553
547
 
@@ -607,7 +601,7 @@ class Flow(Generic[P, R]):
607
601
  description: Optional[str] = None,
608
602
  tags: Optional[List[str]] = None,
609
603
  version: Optional[str] = None,
610
- enforce_parameter_schema: bool = False,
604
+ enforce_parameter_schema: bool = True,
611
605
  work_pool_name: Optional[str] = None,
612
606
  work_queue_name: Optional[str] = None,
613
607
  job_variables: Optional[Dict[str, Any]] = None,
@@ -669,7 +663,8 @@ class Flow(Generic[P, R]):
669
663
  from prefect.deployments.runner import RunnerDeployment
670
664
 
671
665
  if not name.endswith(".py"):
672
- raise_on_name_with_banned_characters(name)
666
+ _raise_on_name_with_banned_characters(name)
667
+
673
668
  if self._storage and self._entrypoint:
674
669
  return await RunnerDeployment.from_storage(
675
670
  storage=self._storage,
@@ -715,6 +710,36 @@ class Flow(Generic[P, R]):
715
710
  entrypoint_type=entrypoint_type,
716
711
  )
717
712
 
713
+ def on_completion(
714
+ self, fn: Callable[["Flow", FlowRun, State], None]
715
+ ) -> Callable[["Flow", FlowRun, State], None]:
716
+ self.on_completion_hooks.append(fn)
717
+ return fn
718
+
719
+ def on_cancellation(
720
+ self, fn: Callable[["Flow", FlowRun, State], None]
721
+ ) -> Callable[["Flow", FlowRun, State], None]:
722
+ self.on_cancellation_hooks.append(fn)
723
+ return fn
724
+
725
+ def on_crashed(
726
+ self, fn: Callable[["Flow", FlowRun, State], None]
727
+ ) -> Callable[["Flow", FlowRun, State], None]:
728
+ self.on_crashed_hooks.append(fn)
729
+ return fn
730
+
731
+ def on_running(
732
+ self, fn: Callable[["Flow", FlowRun, State], None]
733
+ ) -> Callable[["Flow", FlowRun, State], None]:
734
+ self.on_running_hooks.append(fn)
735
+ return fn
736
+
737
+ def on_failure(
738
+ self, fn: Callable[["Flow", FlowRun, State], None]
739
+ ) -> Callable[["Flow", FlowRun, State], None]:
740
+ self.on_failure_hooks.append(fn)
741
+ return fn
742
+
718
743
  @sync_compatible
719
744
  async def serve(
720
745
  self,
@@ -738,7 +763,7 @@ class Flow(Generic[P, R]):
738
763
  description: Optional[str] = None,
739
764
  tags: Optional[List[str]] = None,
740
765
  version: Optional[str] = None,
741
- enforce_parameter_schema: bool = False,
766
+ enforce_parameter_schema: bool = True,
742
767
  pause_on_shutdown: bool = True,
743
768
  print_starting_message: bool = True,
744
769
  limit: Optional[int] = None,
@@ -947,7 +972,7 @@ class Flow(Generic[P, R]):
947
972
  cron: Optional[str] = None,
948
973
  rrule: Optional[str] = None,
949
974
  paused: Optional[bool] = None,
950
- schedules: Optional[List[MinimalDeploymentSchedule]] = None,
975
+ schedules: Optional[List[DeploymentScheduleCreate]] = None,
951
976
  schedule: Optional[SCHEDULE_TYPES] = None,
952
977
  is_schedule_active: Optional[bool] = None,
953
978
  triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
@@ -955,7 +980,7 @@ class Flow(Generic[P, R]):
955
980
  description: Optional[str] = None,
956
981
  tags: Optional[List[str]] = None,
957
982
  version: Optional[str] = None,
958
- enforce_parameter_schema: bool = False,
983
+ enforce_parameter_schema: bool = True,
959
984
  entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
960
985
  print_next_steps: bool = True,
961
986
  ignore_warnings: bool = False,
@@ -1200,7 +1225,10 @@ class Flow(Generic[P, R]):
1200
1225
  >>> with tags("db", "blue"):
1201
1226
  >>> my_flow("foo")
1202
1227
  """
1203
- from prefect.engine import enter_flow_run_engine_from_flow_call
1228
+ from prefect.utilities.visualization import (
1229
+ get_task_viz_tracker,
1230
+ track_viz_task,
1231
+ )
1204
1232
 
1205
1233
  # Convert the call args/kwargs to a parameter dict
1206
1234
  parameters = get_call_parameters(self.fn, args, kwargs)
@@ -1213,72 +1241,19 @@ class Flow(Generic[P, R]):
1213
1241
  # we can add support for exploring subflows for tasks in the future.
1214
1242
  return track_viz_task(self.isasync, self.name, parameters)
1215
1243
 
1216
- if PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE.value():
1217
- from prefect.new_flow_engine import run_flow, run_flow_sync
1244
+ from prefect.flow_engine import run_flow, run_flow_sync
1218
1245
 
1219
- run_kwargs = dict(
1220
- flow=self,
1221
- parameters=parameters,
1222
- wait_for=wait_for,
1223
- return_type=return_type,
1224
- )
1225
- if self.isasync:
1226
- # this returns an awaitable coroutine
1227
- return run_flow(**run_kwargs)
1228
- else:
1229
- return run_flow_sync(**run_kwargs)
1230
-
1231
- return enter_flow_run_engine_from_flow_call(
1232
- self,
1233
- parameters,
1246
+ run_kwargs = dict(
1247
+ flow=self,
1248
+ parameters=parameters,
1234
1249
  wait_for=wait_for,
1235
1250
  return_type=return_type,
1236
1251
  )
1237
-
1238
- @overload
1239
- def _run(self: "Flow[P, NoReturn]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
1240
- # `NoReturn` matches if a type can't be inferred for the function which stops a
1241
- # sync function from matching the `Coroutine` overload
1242
- ...
1243
-
1244
- @overload
1245
- def _run(
1246
- self: "Flow[P, Coroutine[Any, Any, T]]", *args: P.args, **kwargs: P.kwargs
1247
- ) -> Awaitable[T]:
1248
- ...
1249
-
1250
- @overload
1251
- def _run(self: "Flow[P, T]", *args: P.args, **kwargs: P.kwargs) -> State[T]:
1252
- ...
1253
-
1254
- def _run(
1255
- self,
1256
- *args: "P.args",
1257
- wait_for: Optional[Iterable[PrefectFuture]] = None,
1258
- **kwargs: "P.kwargs",
1259
- ):
1260
- """
1261
- Run the flow and return its final state.
1262
-
1263
- Examples:
1264
-
1265
- Run a flow and get the returned result
1266
-
1267
- >>> state = my_flow._run("marvin")
1268
- >>> state.result()
1269
- "goodbye marvin"
1270
- """
1271
- from prefect.engine import enter_flow_run_engine_from_flow_call
1272
-
1273
- # Convert the call args/kwargs to a parameter dict
1274
- parameters = get_call_parameters(self.fn, args, kwargs)
1275
-
1276
- return enter_flow_run_engine_from_flow_call(
1277
- self,
1278
- parameters,
1279
- wait_for=wait_for,
1280
- return_type="state",
1281
- )
1252
+ if self.isasync:
1253
+ # this returns an awaitable coroutine
1254
+ return run_flow(**run_kwargs)
1255
+ else:
1256
+ return run_flow_sync(**run_kwargs)
1282
1257
 
1283
1258
  @sync_compatible
1284
1259
  async def visualize(self, *args, **kwargs):
@@ -1291,6 +1266,16 @@ class Flow(Generic[P, R]):
1291
1266
  - GraphvizExecutableNotFoundError: If the `dot` executable isn't found.
1292
1267
  - FlowVisualizationError: If the flow can't be visualized for any other reason.
1293
1268
  """
1269
+ from prefect.utilities.visualization import (
1270
+ FlowVisualizationError,
1271
+ GraphvizExecutableNotFoundError,
1272
+ GraphvizImportError,
1273
+ TaskVizTracker,
1274
+ VisualizationUnsupportedError,
1275
+ build_task_dependencies,
1276
+ visualize_task_dependencies,
1277
+ )
1278
+
1294
1279
  if not PREFECT_UNIT_TEST_MODE:
1295
1280
  warnings.warn(
1296
1281
  "`flow.visualize()` will execute code inside of your flow that is not"
@@ -1343,7 +1328,7 @@ def flow(
1343
1328
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
1344
1329
  retries: Optional[int] = None,
1345
1330
  retry_delay_seconds: Optional[Union[int, float]] = None,
1346
- task_runner: BaseTaskRunner = ConcurrentTaskRunner,
1331
+ task_runner: Optional[TaskRunner] = None,
1347
1332
  description: str = None,
1348
1333
  timeout_seconds: Union[int, float] = None,
1349
1334
  validate_parameters: bool = True,
@@ -1375,7 +1360,7 @@ def flow(
1375
1360
  flow_run_name: Optional[Union[Callable[[], str], str]] = None,
1376
1361
  retries: int = None,
1377
1362
  retry_delay_seconds: Union[int, float] = None,
1378
- task_runner: BaseTaskRunner = ConcurrentTaskRunner,
1363
+ task_runner: Optional[TaskRunner] = None,
1379
1364
  description: str = None,
1380
1365
  timeout_seconds: Union[int, float] = None,
1381
1366
  validate_parameters: bool = True,
@@ -1553,6 +1538,23 @@ def flow(
1553
1538
  )
1554
1539
 
1555
1540
 
1541
+ def _raise_on_name_with_banned_characters(name: str) -> str:
1542
+ """
1543
+ Raise an InvalidNameError if the given name contains any invalid
1544
+ characters.
1545
+ """
1546
+ if name is None:
1547
+ return name
1548
+
1549
+ if not re.match(WITHOUT_BANNED_CHARACTERS, name):
1550
+ raise InvalidNameError(
1551
+ f"Name {name!r} contains an invalid character. "
1552
+ f"Must not contain any of: {BANNED_CHARACTERS}."
1553
+ )
1554
+
1555
+ return name
1556
+
1557
+
1556
1558
  # Add from_source so it is available on the flow function we all know and love
1557
1559
  flow.from_source = Flow.from_source
1558
1560
 
@@ -1799,6 +1801,87 @@ async def serve(
1799
1801
  await runner.start()
1800
1802
 
1801
1803
 
1804
+ @client_injector
1805
+ async def load_flow_from_flow_run(
1806
+ client: "PrefectClient",
1807
+ flow_run: "FlowRun",
1808
+ ignore_storage: bool = False,
1809
+ storage_base_path: Optional[str] = None,
1810
+ ) -> "Flow":
1811
+ """
1812
+ Load a flow from the location/script provided in a deployment's storage document.
1813
+
1814
+ If `ignore_storage=True` is provided, no pull from remote storage occurs. This flag
1815
+ is largely for testing, and assumes the flow is already available locally.
1816
+ """
1817
+ deployment = await client.read_deployment(flow_run.deployment_id)
1818
+
1819
+ if deployment.entrypoint is None:
1820
+ raise ValueError(
1821
+ f"Deployment {deployment.id} does not have an entrypoint and can not be run."
1822
+ )
1823
+
1824
+ run_logger = flow_run_logger(flow_run)
1825
+
1826
+ runner_storage_base_path = storage_base_path or os.environ.get(
1827
+ "PREFECT__STORAGE_BASE_PATH"
1828
+ )
1829
+
1830
+ # If there's no colon, assume it's a module path
1831
+ if ":" not in deployment.entrypoint:
1832
+ run_logger.debug(
1833
+ f"Importing flow code from module path {deployment.entrypoint}"
1834
+ )
1835
+ flow = await run_sync_in_worker_thread(
1836
+ load_flow_from_entrypoint, deployment.entrypoint
1837
+ )
1838
+ return flow
1839
+
1840
+ if not ignore_storage and not deployment.pull_steps:
1841
+ sys.path.insert(0, ".")
1842
+ if deployment.storage_document_id:
1843
+ storage_document = await client.read_block_document(
1844
+ deployment.storage_document_id
1845
+ )
1846
+ storage_block = Block._from_block_document(storage_document)
1847
+ else:
1848
+ basepath = deployment.path or Path(deployment.manifest_path).parent
1849
+ if runner_storage_base_path:
1850
+ basepath = str(basepath).replace(
1851
+ "$STORAGE_BASE_PATH", runner_storage_base_path
1852
+ )
1853
+ storage_block = LocalFileSystem(basepath=basepath)
1854
+
1855
+ from_path = (
1856
+ str(deployment.path).replace("$STORAGE_BASE_PATH", runner_storage_base_path)
1857
+ if runner_storage_base_path and deployment.path
1858
+ else deployment.path
1859
+ )
1860
+ run_logger.info(f"Downloading flow code from storage at {from_path!r}")
1861
+ await storage_block.get_directory(from_path=from_path, local_path=".")
1862
+
1863
+ if deployment.pull_steps:
1864
+ run_logger.debug(f"Running {len(deployment.pull_steps)} deployment pull steps")
1865
+ output = await run_steps(deployment.pull_steps)
1866
+ if output.get("directory"):
1867
+ run_logger.debug(f"Changing working directory to {output['directory']!r}")
1868
+ os.chdir(output["directory"])
1869
+
1870
+ import_path = relative_path_to_current_platform(deployment.entrypoint)
1871
+ # for backwards compat
1872
+ if deployment.manifest_path:
1873
+ with open(deployment.manifest_path, "r") as f:
1874
+ import_path = json.load(f)["import_path"]
1875
+ import_path = (
1876
+ Path(deployment.manifest_path).parent / import_path
1877
+ ).absolute()
1878
+ run_logger.debug(f"Importing flow code from '{import_path}'")
1879
+
1880
+ flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, str(import_path))
1881
+
1882
+ return flow
1883
+
1884
+
1802
1885
  def load_flow_argument_from_entrypoint(
1803
1886
  entrypoint: str, arg: str = "name"
1804
1887
  ) -> Optional[str]: