prefect-client 2.20.4__py3-none-any.whl → 3.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (288) hide show
  1. prefect/__init__.py +74 -110
  2. prefect/_internal/compatibility/deprecated.py +6 -115
  3. prefect/_internal/compatibility/experimental.py +4 -79
  4. prefect/_internal/compatibility/migration.py +166 -0
  5. prefect/_internal/concurrency/__init__.py +2 -2
  6. prefect/_internal/concurrency/api.py +1 -35
  7. prefect/_internal/concurrency/calls.py +0 -6
  8. prefect/_internal/concurrency/cancellation.py +0 -3
  9. prefect/_internal/concurrency/event_loop.py +0 -20
  10. prefect/_internal/concurrency/inspection.py +3 -3
  11. prefect/_internal/concurrency/primitives.py +1 -0
  12. prefect/_internal/concurrency/services.py +23 -0
  13. prefect/_internal/concurrency/threads.py +35 -0
  14. prefect/_internal/concurrency/waiters.py +0 -28
  15. prefect/_internal/integrations.py +7 -0
  16. prefect/_internal/pydantic/__init__.py +0 -45
  17. prefect/_internal/pydantic/annotations/pendulum.py +2 -2
  18. prefect/_internal/pydantic/v1_schema.py +21 -22
  19. prefect/_internal/pydantic/v2_schema.py +0 -2
  20. prefect/_internal/pydantic/v2_validated_func.py +18 -23
  21. prefect/_internal/pytz.py +1 -1
  22. prefect/_internal/retries.py +61 -0
  23. prefect/_internal/schemas/bases.py +45 -177
  24. prefect/_internal/schemas/fields.py +1 -43
  25. prefect/_internal/schemas/validators.py +47 -233
  26. prefect/agent.py +3 -695
  27. prefect/artifacts.py +173 -14
  28. prefect/automations.py +39 -4
  29. prefect/blocks/abstract.py +1 -1
  30. prefect/blocks/core.py +405 -153
  31. prefect/blocks/fields.py +2 -57
  32. prefect/blocks/notifications.py +43 -28
  33. prefect/blocks/redis.py +168 -0
  34. prefect/blocks/system.py +67 -20
  35. prefect/blocks/webhook.py +2 -9
  36. prefect/cache_policies.py +239 -0
  37. prefect/client/__init__.py +4 -0
  38. prefect/client/base.py +33 -27
  39. prefect/client/cloud.py +65 -20
  40. prefect/client/collections.py +1 -1
  41. prefect/client/orchestration.py +650 -442
  42. prefect/client/schemas/actions.py +115 -100
  43. prefect/client/schemas/filters.py +46 -52
  44. prefect/client/schemas/objects.py +228 -178
  45. prefect/client/schemas/responses.py +18 -36
  46. prefect/client/schemas/schedules.py +55 -36
  47. prefect/client/schemas/sorting.py +2 -0
  48. prefect/client/subscriptions.py +8 -7
  49. prefect/client/types/flexible_schedule_list.py +11 -0
  50. prefect/client/utilities.py +9 -6
  51. prefect/concurrency/asyncio.py +60 -11
  52. prefect/concurrency/context.py +24 -0
  53. prefect/concurrency/events.py +2 -2
  54. prefect/concurrency/services.py +46 -16
  55. prefect/concurrency/sync.py +51 -7
  56. prefect/concurrency/v1/asyncio.py +143 -0
  57. prefect/concurrency/v1/context.py +27 -0
  58. prefect/concurrency/v1/events.py +61 -0
  59. prefect/concurrency/v1/services.py +116 -0
  60. prefect/concurrency/v1/sync.py +92 -0
  61. prefect/context.py +246 -149
  62. prefect/deployments/__init__.py +33 -18
  63. prefect/deployments/base.py +10 -15
  64. prefect/deployments/deployments.py +2 -1048
  65. prefect/deployments/flow_runs.py +178 -0
  66. prefect/deployments/runner.py +72 -173
  67. prefect/deployments/schedules.py +31 -25
  68. prefect/deployments/steps/__init__.py +0 -1
  69. prefect/deployments/steps/core.py +7 -0
  70. prefect/deployments/steps/pull.py +15 -21
  71. prefect/deployments/steps/utility.py +2 -1
  72. prefect/docker/__init__.py +20 -0
  73. prefect/docker/docker_image.py +82 -0
  74. prefect/engine.py +15 -2475
  75. prefect/events/actions.py +17 -23
  76. prefect/events/cli/automations.py +20 -7
  77. prefect/events/clients.py +142 -80
  78. prefect/events/filters.py +14 -18
  79. prefect/events/related.py +74 -75
  80. prefect/events/schemas/__init__.py +0 -5
  81. prefect/events/schemas/automations.py +55 -46
  82. prefect/events/schemas/deployment_triggers.py +7 -197
  83. prefect/events/schemas/events.py +46 -65
  84. prefect/events/schemas/labelling.py +10 -14
  85. prefect/events/utilities.py +4 -5
  86. prefect/events/worker.py +23 -8
  87. prefect/exceptions.py +15 -0
  88. prefect/filesystems.py +30 -529
  89. prefect/flow_engine.py +827 -0
  90. prefect/flow_runs.py +379 -7
  91. prefect/flows.py +470 -360
  92. prefect/futures.py +382 -331
  93. prefect/infrastructure/__init__.py +5 -26
  94. prefect/infrastructure/base.py +3 -320
  95. prefect/infrastructure/provisioners/__init__.py +5 -3
  96. prefect/infrastructure/provisioners/cloud_run.py +13 -8
  97. prefect/infrastructure/provisioners/container_instance.py +14 -9
  98. prefect/infrastructure/provisioners/ecs.py +10 -8
  99. prefect/infrastructure/provisioners/modal.py +8 -5
  100. prefect/input/__init__.py +4 -0
  101. prefect/input/actions.py +2 -4
  102. prefect/input/run_input.py +9 -9
  103. prefect/logging/formatters.py +2 -4
  104. prefect/logging/handlers.py +9 -14
  105. prefect/logging/loggers.py +5 -5
  106. prefect/main.py +72 -0
  107. prefect/plugins.py +2 -64
  108. prefect/profiles.toml +16 -2
  109. prefect/records/__init__.py +1 -0
  110. prefect/records/base.py +223 -0
  111. prefect/records/filesystem.py +207 -0
  112. prefect/records/memory.py +178 -0
  113. prefect/records/result_store.py +64 -0
  114. prefect/results.py +577 -504
  115. prefect/runner/runner.py +117 -47
  116. prefect/runner/server.py +32 -34
  117. prefect/runner/storage.py +3 -12
  118. prefect/runner/submit.py +2 -10
  119. prefect/runner/utils.py +2 -2
  120. prefect/runtime/__init__.py +1 -0
  121. prefect/runtime/deployment.py +1 -0
  122. prefect/runtime/flow_run.py +40 -5
  123. prefect/runtime/task_run.py +1 -0
  124. prefect/serializers.py +28 -39
  125. prefect/server/api/collections_data/views/aggregate-worker-metadata.json +5 -14
  126. prefect/settings.py +209 -332
  127. prefect/states.py +160 -63
  128. prefect/task_engine.py +1478 -57
  129. prefect/task_runners.py +383 -287
  130. prefect/task_runs.py +240 -0
  131. prefect/task_worker.py +463 -0
  132. prefect/tasks.py +684 -374
  133. prefect/transactions.py +410 -0
  134. prefect/types/__init__.py +72 -86
  135. prefect/types/entrypoint.py +13 -0
  136. prefect/utilities/annotations.py +4 -3
  137. prefect/utilities/asyncutils.py +227 -148
  138. prefect/utilities/callables.py +137 -45
  139. prefect/utilities/collections.py +134 -86
  140. prefect/utilities/dispatch.py +27 -14
  141. prefect/utilities/dockerutils.py +11 -4
  142. prefect/utilities/engine.py +186 -32
  143. prefect/utilities/filesystem.py +4 -5
  144. prefect/utilities/importtools.py +26 -27
  145. prefect/utilities/pydantic.py +128 -38
  146. prefect/utilities/schema_tools/hydration.py +18 -1
  147. prefect/utilities/schema_tools/validation.py +30 -0
  148. prefect/utilities/services.py +35 -9
  149. prefect/utilities/templating.py +12 -2
  150. prefect/utilities/timeout.py +20 -5
  151. prefect/utilities/urls.py +195 -0
  152. prefect/utilities/visualization.py +1 -0
  153. prefect/variables.py +78 -59
  154. prefect/workers/__init__.py +0 -1
  155. prefect/workers/base.py +237 -244
  156. prefect/workers/block.py +5 -226
  157. prefect/workers/cloud.py +6 -0
  158. prefect/workers/process.py +265 -12
  159. prefect/workers/server.py +29 -11
  160. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/METADATA +28 -24
  161. prefect_client-3.0.0.dist-info/RECORD +201 -0
  162. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/WHEEL +1 -1
  163. prefect/_internal/pydantic/_base_model.py +0 -51
  164. prefect/_internal/pydantic/_compat.py +0 -82
  165. prefect/_internal/pydantic/_flags.py +0 -20
  166. prefect/_internal/pydantic/_types.py +0 -8
  167. prefect/_internal/pydantic/utilities/config_dict.py +0 -72
  168. prefect/_internal/pydantic/utilities/field_validator.py +0 -150
  169. prefect/_internal/pydantic/utilities/model_construct.py +0 -56
  170. prefect/_internal/pydantic/utilities/model_copy.py +0 -55
  171. prefect/_internal/pydantic/utilities/model_dump.py +0 -136
  172. prefect/_internal/pydantic/utilities/model_dump_json.py +0 -112
  173. prefect/_internal/pydantic/utilities/model_fields.py +0 -50
  174. prefect/_internal/pydantic/utilities/model_fields_set.py +0 -29
  175. prefect/_internal/pydantic/utilities/model_json_schema.py +0 -82
  176. prefect/_internal/pydantic/utilities/model_rebuild.py +0 -80
  177. prefect/_internal/pydantic/utilities/model_validate.py +0 -75
  178. prefect/_internal/pydantic/utilities/model_validate_json.py +0 -68
  179. prefect/_internal/pydantic/utilities/model_validator.py +0 -87
  180. prefect/_internal/pydantic/utilities/type_adapter.py +0 -71
  181. prefect/_vendor/fastapi/__init__.py +0 -25
  182. prefect/_vendor/fastapi/applications.py +0 -946
  183. prefect/_vendor/fastapi/background.py +0 -3
  184. prefect/_vendor/fastapi/concurrency.py +0 -44
  185. prefect/_vendor/fastapi/datastructures.py +0 -58
  186. prefect/_vendor/fastapi/dependencies/__init__.py +0 -0
  187. prefect/_vendor/fastapi/dependencies/models.py +0 -64
  188. prefect/_vendor/fastapi/dependencies/utils.py +0 -877
  189. prefect/_vendor/fastapi/encoders.py +0 -177
  190. prefect/_vendor/fastapi/exception_handlers.py +0 -40
  191. prefect/_vendor/fastapi/exceptions.py +0 -46
  192. prefect/_vendor/fastapi/logger.py +0 -3
  193. prefect/_vendor/fastapi/middleware/__init__.py +0 -1
  194. prefect/_vendor/fastapi/middleware/asyncexitstack.py +0 -25
  195. prefect/_vendor/fastapi/middleware/cors.py +0 -3
  196. prefect/_vendor/fastapi/middleware/gzip.py +0 -3
  197. prefect/_vendor/fastapi/middleware/httpsredirect.py +0 -3
  198. prefect/_vendor/fastapi/middleware/trustedhost.py +0 -3
  199. prefect/_vendor/fastapi/middleware/wsgi.py +0 -3
  200. prefect/_vendor/fastapi/openapi/__init__.py +0 -0
  201. prefect/_vendor/fastapi/openapi/constants.py +0 -2
  202. prefect/_vendor/fastapi/openapi/docs.py +0 -203
  203. prefect/_vendor/fastapi/openapi/models.py +0 -480
  204. prefect/_vendor/fastapi/openapi/utils.py +0 -485
  205. prefect/_vendor/fastapi/param_functions.py +0 -340
  206. prefect/_vendor/fastapi/params.py +0 -453
  207. prefect/_vendor/fastapi/py.typed +0 -0
  208. prefect/_vendor/fastapi/requests.py +0 -4
  209. prefect/_vendor/fastapi/responses.py +0 -40
  210. prefect/_vendor/fastapi/routing.py +0 -1331
  211. prefect/_vendor/fastapi/security/__init__.py +0 -15
  212. prefect/_vendor/fastapi/security/api_key.py +0 -98
  213. prefect/_vendor/fastapi/security/base.py +0 -6
  214. prefect/_vendor/fastapi/security/http.py +0 -172
  215. prefect/_vendor/fastapi/security/oauth2.py +0 -227
  216. prefect/_vendor/fastapi/security/open_id_connect_url.py +0 -34
  217. prefect/_vendor/fastapi/security/utils.py +0 -10
  218. prefect/_vendor/fastapi/staticfiles.py +0 -1
  219. prefect/_vendor/fastapi/templating.py +0 -3
  220. prefect/_vendor/fastapi/testclient.py +0 -1
  221. prefect/_vendor/fastapi/types.py +0 -3
  222. prefect/_vendor/fastapi/utils.py +0 -235
  223. prefect/_vendor/fastapi/websockets.py +0 -7
  224. prefect/_vendor/starlette/__init__.py +0 -1
  225. prefect/_vendor/starlette/_compat.py +0 -28
  226. prefect/_vendor/starlette/_exception_handler.py +0 -80
  227. prefect/_vendor/starlette/_utils.py +0 -88
  228. prefect/_vendor/starlette/applications.py +0 -261
  229. prefect/_vendor/starlette/authentication.py +0 -159
  230. prefect/_vendor/starlette/background.py +0 -43
  231. prefect/_vendor/starlette/concurrency.py +0 -59
  232. prefect/_vendor/starlette/config.py +0 -151
  233. prefect/_vendor/starlette/convertors.py +0 -87
  234. prefect/_vendor/starlette/datastructures.py +0 -707
  235. prefect/_vendor/starlette/endpoints.py +0 -130
  236. prefect/_vendor/starlette/exceptions.py +0 -60
  237. prefect/_vendor/starlette/formparsers.py +0 -276
  238. prefect/_vendor/starlette/middleware/__init__.py +0 -17
  239. prefect/_vendor/starlette/middleware/authentication.py +0 -52
  240. prefect/_vendor/starlette/middleware/base.py +0 -220
  241. prefect/_vendor/starlette/middleware/cors.py +0 -176
  242. prefect/_vendor/starlette/middleware/errors.py +0 -265
  243. prefect/_vendor/starlette/middleware/exceptions.py +0 -74
  244. prefect/_vendor/starlette/middleware/gzip.py +0 -113
  245. prefect/_vendor/starlette/middleware/httpsredirect.py +0 -19
  246. prefect/_vendor/starlette/middleware/sessions.py +0 -82
  247. prefect/_vendor/starlette/middleware/trustedhost.py +0 -64
  248. prefect/_vendor/starlette/middleware/wsgi.py +0 -147
  249. prefect/_vendor/starlette/py.typed +0 -0
  250. prefect/_vendor/starlette/requests.py +0 -328
  251. prefect/_vendor/starlette/responses.py +0 -347
  252. prefect/_vendor/starlette/routing.py +0 -933
  253. prefect/_vendor/starlette/schemas.py +0 -154
  254. prefect/_vendor/starlette/staticfiles.py +0 -248
  255. prefect/_vendor/starlette/status.py +0 -199
  256. prefect/_vendor/starlette/templating.py +0 -231
  257. prefect/_vendor/starlette/testclient.py +0 -804
  258. prefect/_vendor/starlette/types.py +0 -30
  259. prefect/_vendor/starlette/websockets.py +0 -193
  260. prefect/blocks/kubernetes.py +0 -119
  261. prefect/deprecated/__init__.py +0 -0
  262. prefect/deprecated/data_documents.py +0 -350
  263. prefect/deprecated/packaging/__init__.py +0 -12
  264. prefect/deprecated/packaging/base.py +0 -96
  265. prefect/deprecated/packaging/docker.py +0 -146
  266. prefect/deprecated/packaging/file.py +0 -92
  267. prefect/deprecated/packaging/orion.py +0 -80
  268. prefect/deprecated/packaging/serializers.py +0 -171
  269. prefect/events/instrument.py +0 -135
  270. prefect/infrastructure/container.py +0 -824
  271. prefect/infrastructure/kubernetes.py +0 -920
  272. prefect/infrastructure/process.py +0 -289
  273. prefect/manifests.py +0 -20
  274. prefect/new_flow_engine.py +0 -449
  275. prefect/new_task_engine.py +0 -423
  276. prefect/pydantic/__init__.py +0 -76
  277. prefect/pydantic/main.py +0 -39
  278. prefect/software/__init__.py +0 -2
  279. prefect/software/base.py +0 -50
  280. prefect/software/conda.py +0 -199
  281. prefect/software/pip.py +0 -122
  282. prefect/software/python.py +0 -52
  283. prefect/task_server.py +0 -322
  284. prefect_client-2.20.4.dist-info/RECORD +0 -294
  285. /prefect/{_internal/pydantic/utilities → client/types}/__init__.py +0 -0
  286. /prefect/{_vendor → concurrency/v1}/__init__.py +0 -0
  287. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/LICENSE +0 -0
  288. {prefect_client-2.20.4.dist-info → prefect_client-3.0.0.dist-info}/top_level.txt +0 -0
prefect/runner/runner.py CHANGED
@@ -44,7 +44,7 @@ import threading
44
44
  from copy import deepcopy
45
45
  from functools import partial
46
46
  from pathlib import Path
47
- from typing import Callable, Dict, Iterable, List, Optional, Set, Union
47
+ from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Set, Union
48
48
  from uuid import UUID, uuid4
49
49
 
50
50
  import anyio
@@ -64,23 +64,15 @@ from prefect.client.schemas.filters import (
64
64
  FlowRunFilterStateName,
65
65
  FlowRunFilterStateType,
66
66
  )
67
- from prefect.client.schemas.objects import (
68
- FlowRun,
69
- State,
70
- StateType,
71
- )
72
- from prefect.client.schemas.schedules import SCHEDULE_TYPES
73
- from prefect.deployments.deployments import load_flow_from_flow_run
74
- from prefect.deployments.runner import (
75
- EntrypointType,
76
- RunnerDeployment,
77
- )
78
- from prefect.deployments.schedules import FlexibleScheduleList
67
+ from prefect.client.schemas.objects import Flow as APIFlow
68
+ from prefect.client.schemas.objects import FlowRun, State, StateType
79
69
  from prefect.events import DeploymentTriggerTypes, TriggerTypes
70
+ from prefect.events.related import tags_as_related_resources
71
+ from prefect.events.schemas.events import RelatedResource
72
+ from prefect.events.utilities import emit_event
80
73
  from prefect.exceptions import Abort, ObjectNotFound
81
- from prefect.flows import Flow
74
+ from prefect.flows import Flow, load_flow_from_flow_run
82
75
  from prefect.logging.loggers import PrefectLogAdapter, flow_run_logger, get_logger
83
- from prefect.runner.server import start_webserver
84
76
  from prefect.runner.storage import RunnerStorage
85
77
  from prefect.settings import (
86
78
  PREFECT_API_URL,
@@ -90,6 +82,7 @@ from prefect.settings import (
90
82
  get_current_settings,
91
83
  )
92
84
  from prefect.states import Crashed, Pending, exception_to_failed_state
85
+ from prefect.types.entrypoint import EntrypointType
93
86
  from prefect.utilities.asyncutils import (
94
87
  asyncnullcontext,
95
88
  is_async_fn,
@@ -101,7 +94,16 @@ from prefect.utilities.processutils import (
101
94
  get_sys_executable,
102
95
  run_process,
103
96
  )
104
- from prefect.utilities.services import critical_service_loop
97
+ from prefect.utilities.services import (
98
+ critical_service_loop,
99
+ start_client_metrics_server,
100
+ )
101
+ from prefect.utilities.slugify import slugify
102
+
103
+ if TYPE_CHECKING:
104
+ from prefect.client.schemas.objects import Deployment
105
+ from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
106
+ from prefect.deployments.runner import RunnerDeployment
105
107
 
106
108
  __all__ = ["Runner"]
107
109
 
@@ -133,6 +135,7 @@ class Runner:
133
135
  Examples:
134
136
  Set up a Runner to manage the execute of scheduled flow runs for two flows:
135
137
  ```python
138
+ import asyncio
136
139
  from prefect import flow, Runner
137
140
 
138
141
  @flow
@@ -152,7 +155,7 @@ class Runner:
152
155
  # Run on a cron schedule
153
156
  runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
154
157
 
155
- runner.start()
158
+ asyncio.run(runner.start())
156
159
  ```
157
160
  """
158
161
  if name and ("/" in name or "%" in name):
@@ -169,30 +172,26 @@ class Runner:
169
172
  self.query_seconds = query_seconds or PREFECT_RUNNER_POLL_FREQUENCY.value()
170
173
  self._prefetch_seconds = prefetch_seconds
171
174
 
172
- self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
173
- self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
174
-
175
- self._limiter: Optional[anyio.CapacityLimiter] = anyio.CapacityLimiter(
176
- self.limit
177
- )
175
+ self._limiter: Optional[anyio.CapacityLimiter] = None
178
176
  self._client = get_client()
179
177
  self._submitting_flow_run_ids = set()
180
178
  self._cancelling_flow_run_ids = set()
181
179
  self._scheduled_task_scopes = set()
182
180
  self._deployment_ids: Set[UUID] = set()
183
- self._flow_run_process_map = dict()
181
+ self._flow_run_process_map: Dict[UUID, Dict] = dict()
184
182
 
185
183
  self._tmp_dir: Path = (
186
184
  Path(tempfile.gettempdir()) / "runner_storage" / str(uuid4())
187
185
  )
188
186
  self._storage_objs: List[RunnerStorage] = []
189
187
  self._deployment_storage_map: Dict[UUID, RunnerStorage] = {}
190
- self._loop = asyncio.get_event_loop()
188
+
189
+ self._loop: Optional[asyncio.AbstractEventLoop] = None
191
190
 
192
191
  @sync_compatible
193
192
  async def add_deployment(
194
193
  self,
195
- deployment: RunnerDeployment,
194
+ deployment: "RunnerDeployment",
196
195
  ) -> UUID:
197
196
  """
198
197
  Registers the deployment with the Prefect API and will monitor for work once
@@ -214,7 +213,7 @@ class Runner:
214
213
  async def add_flow(
215
214
  self,
216
215
  flow: Flow,
217
- name: str = None,
216
+ name: Optional[str] = None,
218
217
  interval: Optional[
219
218
  Union[
220
219
  Iterable[Union[int, float, datetime.timedelta]],
@@ -226,15 +225,13 @@ class Runner:
226
225
  cron: Optional[Union[Iterable[str], str]] = None,
227
226
  rrule: Optional[Union[Iterable[str], str]] = None,
228
227
  paused: Optional[bool] = None,
229
- schedules: Optional[FlexibleScheduleList] = None,
230
- schedule: Optional[SCHEDULE_TYPES] = None,
231
- is_schedule_active: Optional[bool] = None,
228
+ schedules: Optional["FlexibleScheduleList"] = None,
232
229
  parameters: Optional[dict] = None,
233
230
  triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
234
231
  description: Optional[str] = None,
235
232
  tags: Optional[List[str]] = None,
236
233
  version: Optional[str] = None,
237
- enforce_parameter_schema: bool = False,
234
+ enforce_parameter_schema: bool = True,
238
235
  entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
239
236
  ) -> UUID:
240
237
  """
@@ -251,11 +248,6 @@ class Runner:
251
248
  or a timedelta object. If a number is given, it will be interpreted as seconds.
252
249
  cron: A cron schedule of when to execute runs of this flow.
253
250
  rrule: An rrule schedule of when to execute runs of this flow.
254
- schedule: A schedule object of when to execute runs of this flow. Used for
255
- advanced scheduling options like timezone.
256
- is_schedule_active: Whether or not to set the schedule for this deployment as active. If
257
- not provided when creating a deployment, the schedule will be set as active. If not
258
- provided when updating a deployment, the schedule's activation will not be changed.
259
251
  triggers: A list of triggers that should kick of a run of this flow.
260
252
  parameters: A dictionary of default parameter values to pass to runs of this flow.
261
253
  description: A description for the created deployment. Defaults to the flow's
@@ -280,9 +272,7 @@ class Runner:
280
272
  cron=cron,
281
273
  rrule=rrule,
282
274
  schedules=schedules,
283
- schedule=schedule,
284
275
  paused=paused,
285
- is_schedule_active=is_schedule_active,
286
276
  triggers=triggers,
287
277
  parameters=parameters,
288
278
  description=description,
@@ -327,7 +317,6 @@ class Runner:
327
317
 
328
318
  sys.exit(0)
329
319
 
330
- @sync_compatible
331
320
  async def start(
332
321
  self, run_once: bool = False, webserver: Optional[bool] = None
333
322
  ) -> None:
@@ -345,6 +334,7 @@ class Runner:
345
334
  Initialize a Runner, add two flows, and serve them by starting the Runner:
346
335
 
347
336
  ```python
337
+ import asyncio
348
338
  from prefect import flow, Runner
349
339
 
350
340
  @flow
@@ -364,9 +354,11 @@ class Runner:
364
354
  # Run on a cron schedule
365
355
  runner.add_flow(goodbye_flow, schedule={"cron": "0 * * * *"})
366
356
 
367
- runner.start()
357
+ asyncio.run(runner.start())
368
358
  ```
369
359
  """
360
+ from prefect.runner.server import start_webserver
361
+
370
362
  _register_signal(signal.SIGTERM, self.handle_sigterm)
371
363
 
372
364
  webserver = webserver if webserver is not None else self.webserver
@@ -384,6 +376,8 @@ class Runner:
384
376
  )
385
377
  server_thread.start()
386
378
 
379
+ start_client_metrics_server()
380
+
387
381
  async with self as runner:
388
382
  async with self._loops_task_group as tg:
389
383
  for storage in self._storage_objs:
@@ -696,8 +690,9 @@ class Runner:
696
690
  """
697
691
  self._logger.info("Pausing all deployments...")
698
692
  for deployment_id in self._deployment_ids:
699
- self._logger.debug(f"Pausing deployment '{deployment_id}'")
700
693
  await self._client.set_deployment_paused_state(deployment_id, True)
694
+ self._logger.debug(f"Paused deployment '{deployment_id}'")
695
+
701
696
  self._logger.info("All deployments have been paused!")
702
697
 
703
698
  async def _get_and_submit_flow_runs(self):
@@ -819,8 +814,71 @@ class Runner:
819
814
  "message": state_msg or "Flow run was cancelled successfully."
820
815
  },
821
816
  )
817
+ try:
818
+ deployment = await self._client.read_deployment(flow_run.deployment_id)
819
+ except ObjectNotFound:
820
+ deployment = None
821
+ try:
822
+ flow = await self._client.read_flow(flow_run.flow_id)
823
+ except ObjectNotFound:
824
+ flow = None
825
+ self._emit_flow_run_cancelled_event(
826
+ flow_run=flow_run, flow=flow, deployment=deployment
827
+ )
822
828
  run_logger.info(f"Cancelled flow run '{flow_run.name}'!")
823
829
 
830
+ def _event_resource(self):
831
+ from prefect import __version__
832
+
833
+ return {
834
+ "prefect.resource.id": f"prefect.runner.{slugify(self.name)}",
835
+ "prefect.resource.name": self.name,
836
+ "prefect.version": __version__,
837
+ }
838
+
839
+ def _emit_flow_run_cancelled_event(
840
+ self,
841
+ flow_run: "FlowRun",
842
+ flow: "Optional[APIFlow]",
843
+ deployment: "Optional[Deployment]",
844
+ ):
845
+ related = []
846
+ tags = []
847
+ if deployment:
848
+ related.append(
849
+ {
850
+ "prefect.resource.id": f"prefect.deployment.{deployment.id}",
851
+ "prefect.resource.role": "deployment",
852
+ "prefect.resource.name": deployment.name,
853
+ }
854
+ )
855
+ tags.extend(deployment.tags)
856
+ if flow:
857
+ related.append(
858
+ {
859
+ "prefect.resource.id": f"prefect.flow.{flow.id}",
860
+ "prefect.resource.role": "flow",
861
+ "prefect.resource.name": flow.name,
862
+ }
863
+ )
864
+ related.append(
865
+ {
866
+ "prefect.resource.id": f"prefect.flow-run.{flow_run.id}",
867
+ "prefect.resource.role": "flow-run",
868
+ "prefect.resource.name": flow_run.name,
869
+ }
870
+ )
871
+ tags.extend(flow_run.tags)
872
+
873
+ related = [RelatedResource.model_validate(r) for r in related]
874
+ related += tags_as_related_resources(set(tags))
875
+
876
+ emit_event(
877
+ event="prefect.runner.cancelled-flow-run",
878
+ resource=self._event_resource(),
879
+ related=related,
880
+ )
881
+
824
882
  async def _get_scheduled_flow_runs(
825
883
  self,
826
884
  ) -> List["FlowRun"]:
@@ -957,7 +1015,7 @@ class Runner:
957
1015
  # If the run is not ready to submit, release the concurrency slot
958
1016
  self._release_limit_slot(flow_run.id)
959
1017
 
960
- self._submitting_flow_run_ids.remove(flow_run.id)
1018
+ self._submitting_flow_run_ids.discard(flow_run.id)
961
1019
 
962
1020
  async def _submit_run_and_capture_errors(
963
1021
  self,
@@ -1083,7 +1141,7 @@ class Runner:
1083
1141
  state_updates = state_updates or {}
1084
1142
  state_updates.setdefault("name", "Cancelled")
1085
1143
  state_updates.setdefault("type", StateType.CANCELLED)
1086
- state = flow_run.state.copy(update=state_updates)
1144
+ state = flow_run.state.model_copy(update=state_updates)
1087
1145
 
1088
1146
  await self._client.set_flow_run_state(flow_run.id, state, force=True)
1089
1147
 
@@ -1133,9 +1191,9 @@ class Runner:
1133
1191
  if state.is_cancelling():
1134
1192
  try:
1135
1193
  flow = await load_flow_from_flow_run(
1136
- flow_run, client=self._client, storage_base_path=str(self._tmp_dir)
1194
+ flow_run, storage_base_path=str(self._tmp_dir)
1137
1195
  )
1138
- hooks = flow.on_cancellation or []
1196
+ hooks = flow.on_cancellation_hooks or []
1139
1197
 
1140
1198
  await _run_hooks(hooks, flow_run, flow, state)
1141
1199
  except ObjectNotFound:
@@ -1154,9 +1212,9 @@ class Runner:
1154
1212
  """
1155
1213
  if state.is_crashed():
1156
1214
  flow = await load_flow_from_flow_run(
1157
- flow_run, client=self._client, storage_base_path=str(self._tmp_dir)
1215
+ flow_run, storage_base_path=str(self._tmp_dir)
1158
1216
  )
1159
- hooks = flow.on_crashed or []
1217
+ hooks = flow.on_crashed_hooks or []
1160
1218
 
1161
1219
  await _run_hooks(hooks, flow_run, flow, state)
1162
1220
 
@@ -1164,6 +1222,18 @@ class Runner:
1164
1222
  self._logger.debug("Starting runner...")
1165
1223
  self._client = get_client()
1166
1224
  self._tmp_dir.mkdir(parents=True)
1225
+
1226
+ self._limiter = anyio.CapacityLimiter(self.limit)
1227
+
1228
+ if not hasattr(self, "_loop") or not self._loop:
1229
+ self._loop = asyncio.get_event_loop()
1230
+
1231
+ if not hasattr(self, "_runs_task_group") or not self._runs_task_group:
1232
+ self._runs_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
1233
+
1234
+ if not hasattr(self, "_loops_task_group") or not self._loops_task_group:
1235
+ self._loops_task_group: anyio.abc.TaskGroup = anyio.create_task_group()
1236
+
1167
1237
  await self._client.__aenter__()
1168
1238
  await self._runs_task_group.__aenter__()
1169
1239
 
prefect/runner/server.py CHANGED
@@ -3,21 +3,19 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple
3
3
 
4
4
  import pendulum
5
5
  import uvicorn
6
- from prefect._vendor.fastapi import APIRouter, FastAPI, HTTPException, status
7
- from prefect._vendor.fastapi.responses import JSONResponse
6
+ from fastapi import APIRouter, FastAPI, HTTPException, status
7
+ from fastapi.responses import JSONResponse
8
8
  from typing_extensions import Literal
9
9
 
10
- from prefect._internal.pydantic import HAS_PYDANTIC_V2
11
10
  from prefect._internal.schemas.validators import validate_values_conform_to_schema
12
11
  from prefect.client.orchestration import get_client
13
12
  from prefect.exceptions import MissingFlowError, ScriptError
14
- from prefect.flows import Flow, load_flow_from_entrypoint, load_flows_from_script
13
+ from prefect.flows import Flow, load_flow_from_entrypoint
15
14
  from prefect.logging import get_logger
16
15
  from prefect.runner.utils import (
17
16
  inject_schemas_into_openapi,
18
17
  )
19
18
  from prefect.settings import (
20
- PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS,
21
19
  PREFECT_RUNNER_POLL_FREQUENCY,
22
20
  PREFECT_RUNNER_SERVER_HOST,
23
21
  PREFECT_RUNNER_SERVER_LOG_LEVEL,
@@ -25,15 +23,13 @@ from prefect.settings import (
25
23
  PREFECT_RUNNER_SERVER_PORT,
26
24
  )
27
25
  from prefect.utilities.asyncutils import sync_compatible
26
+ from prefect.utilities.importtools import load_script_as_module
28
27
 
29
28
  if TYPE_CHECKING:
30
29
  from prefect.client.schemas.responses import DeploymentResponse
31
30
  from prefect.runner import Runner
32
31
 
33
- if HAS_PYDANTIC_V2:
34
- from pydantic.v1 import BaseModel
35
- else:
36
- from pydantic import BaseModel
32
+ from pydantic import BaseModel
37
33
 
38
34
  logger = get_logger("webserver")
39
35
 
@@ -46,7 +42,7 @@ class RunnerGenericFlowRunRequest(BaseModel):
46
42
  parent_task_run_id: Optional[uuid.UUID] = None
47
43
 
48
44
 
49
- def perform_health_check(runner, delay_threshold: int = None) -> JSONResponse:
45
+ def perform_health_check(runner, delay_threshold: Optional[int] = None) -> JSONResponse:
50
46
  if delay_threshold is None:
51
47
  delay_threshold = (
52
48
  PREFECT_RUNNER_SERVER_MISSED_POLLS_TOLERANCE.value()
@@ -159,9 +155,12 @@ async def get_subflow_schemas(runner: "Runner") -> Dict[str, Dict]:
159
155
  continue
160
156
 
161
157
  script = deployment.entrypoint.split(":")[0]
162
- subflows = load_flows_from_script(script)
158
+ module = load_script_as_module(script)
159
+ subflows = [
160
+ obj for obj in module.__dict__.values() if isinstance(obj, Flow)
161
+ ]
163
162
  for flow in subflows:
164
- schemas[flow.name] = flow.parameters.dict()
163
+ schemas[flow.name] = flow.parameters.model_dump()
165
164
 
166
165
  return schemas
167
166
 
@@ -183,7 +182,7 @@ def _flow_schema_changed(flow: Flow, schemas: Dict[str, Dict]) -> bool:
183
182
  flow_name_with_dashes = flow.name.replace("_", "-")
184
183
 
185
184
  schema = schemas.get(flow.name, None) or schemas.get(flow_name_with_dashes, None)
186
- if schema is not None and flow.parameters.dict() != schema:
185
+ if schema is not None and flow.parameters.model_dump() != schema:
187
186
  return True
188
187
  return False
189
188
 
@@ -236,7 +235,7 @@ def _build_generic_endpoint_for_flows(
236
235
 
237
236
  return JSONResponse(
238
237
  status_code=status.HTTP_201_CREATED,
239
- content=flow_run.dict(json_compatible=True),
238
+ content=flow_run.model_dump(mode="json"),
240
239
  )
241
240
 
242
241
  return _create_flow_run_for_flow_from_fqn
@@ -261,29 +260,28 @@ async def build_server(runner: "Runner") -> FastAPI:
261
260
  router.add_api_route("/shutdown", shutdown(runner=runner), methods=["POST"])
262
261
  webserver.include_router(router)
263
262
 
264
- if PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS.value():
265
- deployments_router, deployment_schemas = await get_deployment_router(runner)
266
- webserver.include_router(deployments_router)
267
-
268
- subflow_schemas = await get_subflow_schemas(runner)
269
- webserver.add_api_route(
270
- "/flow/run",
271
- _build_generic_endpoint_for_flows(runner=runner, schemas=subflow_schemas),
272
- methods=["POST"],
273
- name="Run flow in background",
274
- description="Trigger any flow run as a background task on the runner.",
275
- summary="Run flow",
276
- )
277
-
278
- def customize_openapi():
279
- if webserver.openapi_schema:
280
- return webserver.openapi_schema
263
+ deployments_router, deployment_schemas = await get_deployment_router(runner)
264
+ webserver.include_router(deployments_router)
265
+
266
+ subflow_schemas = await get_subflow_schemas(runner)
267
+ webserver.add_api_route(
268
+ "/flow/run",
269
+ _build_generic_endpoint_for_flows(runner=runner, schemas=subflow_schemas),
270
+ methods=["POST"],
271
+ name="Run flow in background",
272
+ description="Trigger any flow run as a background task on the runner.",
273
+ summary="Run flow",
274
+ )
281
275
 
282
- openapi_schema = inject_schemas_into_openapi(webserver, deployment_schemas)
283
- webserver.openapi_schema = openapi_schema
276
+ def customize_openapi():
277
+ if webserver.openapi_schema:
284
278
  return webserver.openapi_schema
285
279
 
286
- webserver.openapi = customize_openapi
280
+ openapi_schema = inject_schemas_into_openapi(webserver, deployment_schemas)
281
+ webserver.openapi_schema = openapi_schema
282
+ return webserver.openapi_schema
283
+
284
+ webserver.openapi = customize_openapi
287
285
 
288
286
  return webserver
289
287
 
prefect/runner/storage.py CHANGED
@@ -8,20 +8,15 @@ from uuid import uuid4
8
8
 
9
9
  import fsspec
10
10
  from anyio import run_process
11
+ from pydantic import SecretStr
11
12
 
12
13
  from prefect._internal.concurrency.api import create_call, from_async
13
- from prefect._internal.pydantic import HAS_PYDANTIC_V2
14
14
  from prefect.blocks.core import Block, BlockNotSavedError
15
15
  from prefect.blocks.system import Secret
16
16
  from prefect.filesystems import ReadableDeploymentStorage, WritableDeploymentStorage
17
17
  from prefect.logging.loggers import get_logger
18
18
  from prefect.utilities.collections import visit_collection
19
19
 
20
- if HAS_PYDANTIC_V2:
21
- from pydantic.v1 import SecretStr
22
- else:
23
- from pydantic import SecretStr
24
-
25
20
 
26
21
  @runtime_checkable
27
22
  class RunnerStorage(Protocol):
@@ -159,7 +154,7 @@ class GitRepository:
159
154
  url_components = urlparse(self._url)
160
155
 
161
156
  credentials = (
162
- self._credentials.dict()
157
+ self._credentials.model_dump()
163
158
  if isinstance(self._credentials, Block)
164
159
  else deepcopy(self._credentials)
165
160
  )
@@ -396,7 +391,7 @@ class RemoteStorage:
396
391
  if hasattr(obj, "value"):
397
392
  return obj.value
398
393
  else:
399
- return obj.dict()
394
+ return obj.model_dump()
400
395
  return obj
401
396
 
402
397
  settings_with_block_values = visit_collection(
@@ -571,16 +566,12 @@ class BlockStorageAdapter:
571
566
  class LocalStorage:
572
567
  """
573
568
  Sets the working directory in the local filesystem.
574
-
575
569
  Parameters:
576
570
  Path: Local file path to set the working directory for the flow
577
-
578
571
  Examples:
579
572
  Sets the working directory for the local path to the flow:
580
-
581
573
  ```python
582
574
  from prefect.runner.storage import Localstorage
583
-
584
575
  storage = LocalStorage(
585
576
  path="/path/to/local/flow_directory",
586
577
  )
prefect/runner/submit.py CHANGED
@@ -14,7 +14,6 @@ from prefect.context import FlowRunContext
14
14
  from prefect.flows import Flow
15
15
  from prefect.logging import get_logger
16
16
  from prefect.settings import (
17
- PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS,
18
17
  PREFECT_RUNNER_PROCESS_LIMIT,
19
18
  PREFECT_RUNNER_SERVER_HOST,
20
19
  PREFECT_RUNNER_SERVER_PORT,
@@ -43,7 +42,7 @@ async def _submit_flow_to_runner(
43
42
  Returns:
44
43
  A `FlowRun` object representing the flow run that was submitted.
45
44
  """
46
- from prefect.engine import (
45
+ from prefect.utilities.engine import (
47
46
  _dynamic_key_for_task_run,
48
47
  collect_task_run_inputs,
49
48
  resolve_inputs,
@@ -90,7 +89,7 @@ async def _submit_flow_to_runner(
90
89
  )
91
90
  response.raise_for_status()
92
91
 
93
- return FlowRun.parse_obj(response.json())
92
+ return FlowRun.model_validate(response.json())
94
93
 
95
94
 
96
95
  @overload
@@ -131,13 +130,6 @@ async def submit_to_runner(
131
130
  "The `submit_to_runner` utility only supports submitting flows and tasks."
132
131
  )
133
132
 
134
- if not PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS.value():
135
- raise ValueError(
136
- "The `submit_to_runner` utility requires the `Runner` webserver to be"
137
- " running and built with extra endpoints enabled. To enable this, set the"
138
- " `PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS` setting to `True`."
139
- )
140
-
141
133
  parameters = parameters or {}
142
134
  if isinstance(parameters, List):
143
135
  return_single = False
prefect/runner/utils.py CHANGED
@@ -1,8 +1,8 @@
1
1
  from copy import deepcopy
2
2
  from typing import Any, Dict
3
3
 
4
- from prefect._vendor.fastapi import FastAPI
5
- from prefect._vendor.fastapi.openapi.utils import get_openapi
4
+ from fastapi import FastAPI
5
+ from fastapi.openapi.utils import get_openapi
6
6
 
7
7
  from prefect import __version__ as PREFECT_VERSION
8
8
 
@@ -8,6 +8,7 @@ Example usage:
8
8
  print(f"This script is running from deployment {deployment.id} with parameters {deployment.parameters}")
9
9
  ```
10
10
  """
11
+
11
12
  import prefect.runtime.deployment
12
13
  import prefect.runtime.flow_run
13
14
  import prefect.runtime.task_run
@@ -24,6 +24,7 @@ Available attributes:
24
24
  include default values set on the flow function, only the parameter values set on the deployment
25
25
  object or those directly provided via API for this run
26
26
  """
27
+
27
28
  import os
28
29
  from typing import Any, Dict, List, Optional
29
30
 
@@ -40,6 +40,7 @@ __all__ = [
40
40
  "parameters",
41
41
  "parent_flow_run_id",
42
42
  "parent_deployment_id",
43
+ "root_flow_run_id",
43
44
  "run_count",
44
45
  "api_url",
45
46
  "ui_url",
@@ -120,7 +121,7 @@ async def _get_flow_from_run(flow_run_id):
120
121
  return await client.read_flow(flow_run.flow_id)
121
122
 
122
123
 
123
- def get_id() -> str:
124
+ def get_id() -> Optional[str]:
124
125
  flow_run_ctx = FlowRunContext.get()
125
126
  task_run_ctx = TaskRunContext.get()
126
127
  if flow_run_ctx is not None:
@@ -254,11 +255,12 @@ def get_parent_flow_run_id() -> Optional[str]:
254
255
  parent_task_run = from_sync.call_soon_in_loop_thread(
255
256
  create_call(_get_task_run, parent_task_run_id)
256
257
  ).result()
257
- return parent_task_run.flow_run_id
258
+ return str(parent_task_run.flow_run_id) if parent_task_run.flow_run_id else None
259
+
258
260
  return None
259
261
 
260
262
 
261
- def get_parent_deployment_id() -> Dict[str, Any]:
263
+ def get_parent_deployment_id() -> Optional[str]:
262
264
  parent_flow_run_id = get_parent_flow_run_id()
263
265
  if parent_flow_run_id is None:
264
266
  return None
@@ -266,7 +268,39 @@ def get_parent_deployment_id() -> Dict[str, Any]:
266
268
  parent_flow_run = from_sync.call_soon_in_loop_thread(
267
269
  create_call(_get_flow_run, parent_flow_run_id)
268
270
  ).result()
269
- return parent_flow_run.deployment_id if parent_flow_run else None
271
+
272
+ if parent_flow_run:
273
+ return (
274
+ str(parent_flow_run.deployment_id)
275
+ if parent_flow_run.deployment_id
276
+ else None
277
+ )
278
+
279
+ return None
280
+
281
+
282
+ def get_root_flow_run_id() -> str:
283
+ run_id = get_id()
284
+ parent_flow_run_id = get_parent_flow_run_id()
285
+ if parent_flow_run_id is None:
286
+ return run_id
287
+
288
+ def _get_root_flow_run_id(flow_run_id):
289
+ flow_run = from_sync.call_soon_in_loop_thread(
290
+ create_call(_get_flow_run, flow_run_id)
291
+ ).result()
292
+
293
+ if flow_run.parent_task_run_id is None:
294
+ return str(flow_run_id)
295
+ else:
296
+ parent_task_run = from_sync.call_soon_in_loop_thread(
297
+ create_call(_get_task_run, flow_run.parent_task_run_id)
298
+ ).result()
299
+ return _get_root_flow_run_id(parent_task_run.flow_run_id)
300
+
301
+ root_flow_run_id = _get_root_flow_run_id(parent_flow_run_id)
302
+
303
+ return root_flow_run_id
270
304
 
271
305
 
272
306
  def get_flow_run_api_url() -> Optional[str]:
@@ -289,11 +323,12 @@ FIELDS = {
289
323
  "scheduled_start_time": get_scheduled_start_time,
290
324
  "name": get_name,
291
325
  "flow_name": get_flow_name,
292
- "flow_version": get_flow_version,
293
326
  "parameters": get_parameters,
294
327
  "parent_flow_run_id": get_parent_flow_run_id,
295
328
  "parent_deployment_id": get_parent_deployment_id,
329
+ "root_flow_run_id": get_root_flow_run_id,
296
330
  "run_count": get_run_count,
297
331
  "api_url": get_flow_run_api_url,
298
332
  "ui_url": get_flow_run_ui_url,
333
+ "flow_version": get_flow_version,
299
334
  }