prefect-client 2.18.0__py3-none-any.whl → 2.18.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. prefect/_internal/schemas/fields.py +31 -12
  2. prefect/automations.py +162 -0
  3. prefect/blocks/core.py +1 -1
  4. prefect/blocks/notifications.py +2 -2
  5. prefect/blocks/system.py +2 -3
  6. prefect/client/orchestration.py +309 -30
  7. prefect/client/schemas/objects.py +11 -8
  8. prefect/client/schemas/sorting.py +9 -0
  9. prefect/client/utilities.py +25 -3
  10. prefect/concurrency/asyncio.py +11 -5
  11. prefect/concurrency/events.py +3 -3
  12. prefect/concurrency/services.py +1 -1
  13. prefect/concurrency/sync.py +9 -5
  14. prefect/deployments/deployments.py +27 -18
  15. prefect/deployments/runner.py +34 -26
  16. prefect/engine.py +3 -1
  17. prefect/events/actions.py +2 -1
  18. prefect/events/cli/automations.py +207 -46
  19. prefect/events/clients.py +53 -20
  20. prefect/events/filters.py +31 -4
  21. prefect/events/instrument.py +40 -40
  22. prefect/events/related.py +2 -1
  23. prefect/events/schemas/automations.py +52 -7
  24. prefect/events/schemas/deployment_triggers.py +16 -228
  25. prefect/events/schemas/events.py +18 -11
  26. prefect/events/schemas/labelling.py +1 -1
  27. prefect/events/utilities.py +1 -1
  28. prefect/events/worker.py +10 -7
  29. prefect/flows.py +42 -24
  30. prefect/input/actions.py +9 -9
  31. prefect/input/run_input.py +51 -37
  32. prefect/new_flow_engine.py +444 -0
  33. prefect/new_task_engine.py +488 -0
  34. prefect/results.py +3 -2
  35. prefect/runner/runner.py +3 -2
  36. prefect/server/api/collections_data/views/aggregate-worker-metadata.json +45 -4
  37. prefect/settings.py +47 -0
  38. prefect/states.py +25 -19
  39. prefect/tasks.py +146 -19
  40. prefect/utilities/asyncutils.py +41 -0
  41. prefect/utilities/engine.py +6 -4
  42. prefect/utilities/schema_tools/validation.py +1 -1
  43. prefect/workers/process.py +2 -1
  44. {prefect_client-2.18.0.dist-info → prefect_client-2.18.2.dist-info}/METADATA +1 -1
  45. {prefect_client-2.18.0.dist-info → prefect_client-2.18.2.dist-info}/RECORD +48 -46
  46. prefect/concurrency/common.py +0 -0
  47. {prefect_client-2.18.0.dist-info → prefect_client-2.18.2.dist-info}/LICENSE +0 -0
  48. {prefect_client-2.18.0.dist-info → prefect_client-2.18.2.dist-info}/WHEEL +0 -0
  49. {prefect_client-2.18.0.dist-info → prefect_client-2.18.2.dist-info}/top_level.txt +0 -0
prefect/events/clients.py CHANGED
@@ -7,10 +7,11 @@ from typing import (
7
7
  ClassVar,
8
8
  Dict,
9
9
  List,
10
- Mapping,
10
+ MutableMapping,
11
11
  Optional,
12
12
  Tuple,
13
13
  Type,
14
+ cast,
14
15
  )
15
16
  from uuid import UUID
16
17
 
@@ -19,6 +20,7 @@ import orjson
19
20
  import pendulum
20
21
  from cachetools import TTLCache
21
22
  from typing_extensions import Self
23
+ from websockets import Subprotocol
22
24
  from websockets.client import WebSocketClientProtocol, connect
23
25
  from websockets.exceptions import (
24
26
  ConnectionClosed,
@@ -69,7 +71,7 @@ def get_events_client(
69
71
 
70
72
 
71
73
  def get_events_subscriber(
72
- filter: "EventFilter" = None,
74
+ filter: Optional["EventFilter"] = None,
73
75
  reconnection_attempts: int = 10,
74
76
  ) -> "PrefectEventSubscriber":
75
77
  api_url = PREFECT_API_URL.value()
@@ -105,7 +107,7 @@ class EventsClient(abc.ABC):
105
107
  async def _emit(self, event: Event) -> None: # pragma: no cover
106
108
  ...
107
109
 
108
- async def __aenter__(self) -> "EventsClient":
110
+ async def __aenter__(self) -> Self:
109
111
  self._in_context = True
110
112
  return self
111
113
 
@@ -153,7 +155,7 @@ class AssertingEventsClient(EventsClient):
153
155
  async def _emit(self, event: Event) -> None:
154
156
  self.events.append(event)
155
157
 
156
- async def __aenter__(self) -> "AssertingEventsClient":
158
+ async def __aenter__(self) -> Self:
157
159
  await super().__aenter__()
158
160
  self.events = []
159
161
  return self
@@ -227,7 +229,7 @@ class PrefectEventsClient(EventsClient):
227
229
 
228
230
  def __init__(
229
231
  self,
230
- api_url: str = None,
232
+ api_url: Optional[str] = None,
231
233
  reconnection_attempts: int = 10,
232
234
  checkpoint_every: int = 20,
233
235
  ):
@@ -343,8 +345,8 @@ class PrefectCloudEventsClient(PrefectEventsClient):
343
345
 
344
346
  def __init__(
345
347
  self,
346
- api_url: str = None,
347
- api_key: str = None,
348
+ api_url: Optional[str] = None,
349
+ api_key: Optional[str] = None,
348
350
  reconnection_attempts: int = 10,
349
351
  checkpoint_every: int = 20,
350
352
  ):
@@ -376,7 +378,7 @@ SEEN_EVENTS_TTL = 120
376
378
 
377
379
  class PrefectEventSubscriber:
378
380
  """
379
- Subscribes to a Prefect Cloud event stream, yielding events as they occur.
381
+ Subscribes to a Prefect event stream, yielding events as they occur.
380
382
 
381
383
  Example:
382
384
 
@@ -393,12 +395,14 @@ class PrefectEventSubscriber:
393
395
 
394
396
  _websocket: Optional[WebSocketClientProtocol]
395
397
  _filter: "EventFilter"
396
- _seen_events: Mapping[UUID, bool]
398
+ _seen_events: MutableMapping[UUID, bool]
399
+
400
+ _api_key: Optional[str]
397
401
 
398
402
  def __init__(
399
403
  self,
400
- api_url: str = None,
401
- filter: "EventFilter" = None,
404
+ api_url: Optional[str] = None,
405
+ filter: Optional["EventFilter"] = None,
402
406
  reconnection_attempts: int = 10,
403
407
  ):
404
408
  """
@@ -409,12 +413,12 @@ class PrefectEventSubscriber:
409
413
  the client should attempt to reconnect
410
414
  """
411
415
  if not api_url:
412
- api_url = PREFECT_API_URL.value()
416
+ api_url = cast(str, PREFECT_API_URL.value())
413
417
  self._api_key = None
414
418
 
415
419
  from prefect.events.filters import EventFilter
416
420
 
417
- self._filter = filter or EventFilter()
421
+ self._filter = filter or EventFilter() # type: ignore[call-arg]
418
422
  self._seen_events = TTLCache(maxsize=SEEN_EVENTS_SIZE, ttl=SEEN_EVENTS_TTL)
419
423
 
420
424
  socket_url = (
@@ -427,14 +431,14 @@ class PrefectEventSubscriber:
427
431
 
428
432
  self._connect = connect(
429
433
  socket_url,
430
- subprotocols=["prefect"],
434
+ subprotocols=[Subprotocol("prefect")],
431
435
  )
432
436
  self._websocket = None
433
437
  self._reconnection_attempts = reconnection_attempts
434
438
  if self._reconnection_attempts < 0:
435
439
  raise ValueError("reconnection_attempts must be a non-negative integer")
436
440
 
437
- async def __aenter__(self) -> "PrefectCloudEventSubscriber":
441
+ async def __aenter__(self) -> Self:
438
442
  # Don't handle any errors in the initial connection, because these are most
439
443
  # likely a permission or configuration issue that should propagate
440
444
  await self._reconnect()
@@ -498,7 +502,7 @@ class PrefectEventSubscriber:
498
502
  self._websocket = None
499
503
  await self._connect.__aexit__(exc_type, exc_val, exc_tb)
500
504
 
501
- def __aiter__(self) -> "PrefectCloudEventSubscriber":
505
+ def __aiter__(self) -> Self:
502
506
  return self
503
507
 
504
508
  async def __anext__(self) -> Event:
@@ -516,7 +520,7 @@ class PrefectEventSubscriber:
516
520
 
517
521
  while True:
518
522
  message = orjson.loads(await self._websocket.recv())
519
- event = Event.parse_obj(message["event"])
523
+ event: Event = Event.parse_obj(message["event"])
520
524
 
521
525
  if event.id in self._seen_events:
522
526
  continue
@@ -541,14 +545,15 @@ class PrefectEventSubscriber:
541
545
  # a standard load balancer timeout, but after that, just take a
542
546
  # beat to let things come back around.
543
547
  await asyncio.sleep(1)
548
+ raise StopAsyncIteration
544
549
 
545
550
 
546
551
  class PrefectCloudEventSubscriber(PrefectEventSubscriber):
547
552
  def __init__(
548
553
  self,
549
- api_url: str = None,
550
- api_key: str = None,
551
- filter: "EventFilter" = None,
554
+ api_url: Optional[str] = None,
555
+ api_key: Optional[str] = None,
556
+ filter: Optional["EventFilter"] = None,
552
557
  reconnection_attempts: int = 10,
553
558
  ):
554
559
  """
@@ -567,3 +572,31 @@ class PrefectCloudEventSubscriber(PrefectEventSubscriber):
567
572
  )
568
573
 
569
574
  self._api_key = api_key
575
+
576
+
577
+ class PrefectCloudAccountEventSubscriber(PrefectCloudEventSubscriber):
578
+ def __init__(
579
+ self,
580
+ api_url: Optional[str] = None,
581
+ api_key: Optional[str] = None,
582
+ filter: Optional["EventFilter"] = None,
583
+ reconnection_attempts: int = 10,
584
+ ):
585
+ """
586
+ Args:
587
+ api_url: The base URL for a Prefect Cloud workspace
588
+ api_key: The API of an actor with the manage_events scope
589
+ reconnection_attempts: When the client is disconnected, how many times
590
+ the client should attempt to reconnect
591
+ """
592
+ api_url, api_key = _get_api_url_and_key(api_url, api_key)
593
+
594
+ account_api_url, _, _ = api_url.partition("/workspaces/")
595
+
596
+ super().__init__(
597
+ api_url=account_api_url,
598
+ filter=filter,
599
+ reconnection_attempts=reconnection_attempts,
600
+ )
601
+
602
+ self._api_key = api_key
prefect/events/filters.py CHANGED
@@ -13,10 +13,37 @@ from .schemas.events import Event, Resource, ResourceSpecification
13
13
  if HAS_PYDANTIC_V2:
14
14
  from pydantic.v1 import Field, PrivateAttr
15
15
  else:
16
- from pydantic import Field, PrivateAttr
16
+ from pydantic import Field, PrivateAttr # type: ignore
17
17
 
18
18
 
19
- class EventDataFilter(PrefectBaseModel, extra="forbid"):
19
+ class AutomationFilterCreated(PrefectBaseModel):
20
+ """Filter by `Automation.created`."""
21
+
22
+ before_: Optional[DateTimeTZ] = Field(
23
+ default=None,
24
+ description="Only include automations created before this datetime",
25
+ )
26
+
27
+
28
+ class AutomationFilterName(PrefectBaseModel):
29
+ """Filter by `Automation.created`."""
30
+
31
+ any_: Optional[List[str]] = Field(
32
+ default=None,
33
+ description="Only include automations with names that match any of these strings",
34
+ )
35
+
36
+
37
+ class AutomationFilter(PrefectBaseModel):
38
+ name: Optional[AutomationFilterName] = Field(
39
+ default=None, description="Filter criteria for `Automation.name`"
40
+ )
41
+ created: Optional[AutomationFilterCreated] = Field(
42
+ default=None, description="Filter criteria for `Automation.created`"
43
+ )
44
+
45
+
46
+ class EventDataFilter(PrefectBaseModel, extra="forbid"): # type: ignore[call-arg]
20
47
  """A base class for filtering event data."""
21
48
 
22
49
  _top_level_filter: "EventFilter | None" = PrivateAttr(None)
@@ -203,7 +230,7 @@ class EventOrder(AutoEnum):
203
230
 
204
231
  class EventFilter(EventDataFilter):
205
232
  occurred: EventOccurredFilter = Field(
206
- default_factory=EventOccurredFilter,
233
+ default_factory=lambda: EventOccurredFilter(),
207
234
  description="Filter criteria for when the events occurred",
208
235
  )
209
236
  event: Optional[EventNameFilter] = Field(
@@ -220,7 +247,7 @@ class EventFilter(EventDataFilter):
220
247
  None, description="Filter criteria for the related resources of the event"
221
248
  )
222
249
  id: EventIDFilter = Field(
223
- default_factory=EventIDFilter,
250
+ default_factory=lambda: EventIDFilter(id=[]),
224
251
  description="Filter criteria for the events' ID",
225
252
  )
226
253
 
@@ -10,7 +10,9 @@ from typing import (
10
10
  Set,
11
11
  Tuple,
12
12
  Type,
13
+ TypeVar,
13
14
  Union,
15
+ cast,
14
16
  )
15
17
 
16
18
  from prefect.events import emit_event
@@ -41,45 +43,45 @@ def emit_instance_method_called_event(
41
43
  )
42
44
 
43
45
 
44
- def instrument_instance_method_call():
45
- def instrument(function):
46
- if is_instrumented(function):
47
- return function
46
+ F = TypeVar("F", bound=Callable)
48
47
 
49
- if inspect.iscoroutinefunction(function):
50
48
 
51
- @functools.wraps(function)
52
- async def inner(self, *args, **kwargs):
53
- success = True
54
- try:
55
- return await function(self, *args, **kwargs)
56
- except Exception as exc:
57
- success = False
58
- raise exc
59
- finally:
60
- emit_instance_method_called_event(
61
- instance=self, method_name=function.__name__, successful=success
62
- )
49
+ def instrument_instance_method_call(function: F) -> F:
50
+ if is_instrumented(function):
51
+ return function
63
52
 
64
- else:
53
+ if inspect.iscoroutinefunction(function):
65
54
 
66
- @functools.wraps(function)
67
- def inner(self, *args, **kwargs):
68
- success = True
69
- try:
70
- return function(self, *args, **kwargs)
71
- except Exception as exc:
72
- success = False
73
- raise exc
74
- finally:
75
- emit_instance_method_called_event(
76
- instance=self, method_name=function.__name__, successful=success
77
- )
55
+ @functools.wraps(function)
56
+ async def inner(self, *args, **kwargs):
57
+ success = True
58
+ try:
59
+ return await function(self, *args, **kwargs)
60
+ except Exception as exc:
61
+ success = False
62
+ raise exc
63
+ finally:
64
+ emit_instance_method_called_event(
65
+ instance=self, method_name=function.__name__, successful=success
66
+ )
78
67
 
79
- setattr(inner, "__events_instrumented__", True)
80
- return inner
68
+ else:
81
69
 
82
- return instrument
70
+ @functools.wraps(function)
71
+ def inner(self, *args, **kwargs):
72
+ success = True
73
+ try:
74
+ return function(self, *args, **kwargs)
75
+ except Exception as exc:
76
+ success = False
77
+ raise exc
78
+ finally:
79
+ emit_instance_method_called_event(
80
+ instance=self, method_name=function.__name__, successful=success
81
+ )
82
+
83
+ setattr(inner, "__events_instrumented__", True)
84
+ return cast(F, inner)
83
85
 
84
86
 
85
87
  def is_instrumented(function: Callable) -> bool:
@@ -119,17 +121,15 @@ def instrument_method_calls_on_class_instances(cls: Type) -> Type:
119
121
  """
120
122
 
121
123
  required_events_methods = ["_event_kind", "_event_method_called_resources"]
122
- for method in required_events_methods:
123
- if not hasattr(cls, method):
124
+ for method_name in required_events_methods:
125
+ if not hasattr(cls, method_name):
124
126
  raise RuntimeError(
125
- f"Unable to instrument class {cls}. Class must define {method!r}."
127
+ f"Unable to instrument class {cls}. Class must define {method_name!r}."
126
128
  )
127
129
 
128
- decorator = instrument_instance_method_call()
129
-
130
- for name, method in instrumentable_methods(
130
+ for method_name, method in instrumentable_methods(
131
131
  cls,
132
132
  exclude_methods=getattr(cls, "_events_excluded_methods", []),
133
133
  ):
134
- setattr(cls, name, decorator(method))
134
+ setattr(cls, method_name, instrument_instance_method_call(method))
135
135
  return cls
prefect/events/related.py CHANGED
@@ -57,6 +57,7 @@ async def related_resources_from_run_context(
57
57
  exclude: Optional[Set[str]] = None,
58
58
  ) -> List[RelatedResource]:
59
59
  from prefect.client.orchestration import get_client
60
+ from prefect.client.schemas.objects import FlowRun
60
61
  from prefect.context import FlowRunContext, TaskRunContext
61
62
 
62
63
  if exclude is None:
@@ -111,7 +112,7 @@ async def related_resources_from_run_context(
111
112
 
112
113
  flow_run = related_objects[0]["object"]
113
114
 
114
- if flow_run:
115
+ if isinstance(flow_run, FlowRun):
115
116
  related_objects += list(
116
117
  await asyncio.gather(
117
118
  _get_and_cache_related_object(
@@ -10,23 +10,24 @@ from typing import (
10
10
  Optional,
11
11
  Set,
12
12
  Union,
13
+ cast,
13
14
  )
14
15
  from uuid import UUID
15
16
 
16
17
  from typing_extensions import TypeAlias
17
18
 
18
19
  from prefect._internal.pydantic import HAS_PYDANTIC_V2
19
- from prefect._internal.schemas.validators import validate_trigger_within
20
20
 
21
21
  if HAS_PYDANTIC_V2:
22
- from pydantic.v1 import Field, root_validator, validator
22
+ from pydantic.v1 import Field, PrivateAttr, root_validator, validator
23
23
  from pydantic.v1.fields import ModelField
24
24
  else:
25
- from pydantic import Field, root_validator, validator
26
- from pydantic.fields import ModelField
25
+ from pydantic import Field, PrivateAttr, root_validator, validator # type: ignore
26
+ from pydantic.fields import ModelField # type: ignore
27
27
 
28
28
  from prefect._internal.schemas.bases import PrefectBaseModel
29
- from prefect.events.actions import ActionTypes
29
+ from prefect._internal.schemas.validators import validate_trigger_within
30
+ from prefect.events.actions import ActionTypes, RunDeployment
30
31
  from prefect.utilities.collections import AutoEnum
31
32
 
32
33
  from .events import ResourceSpecification
@@ -38,7 +39,7 @@ class Posture(AutoEnum):
38
39
  Metric = "Metric"
39
40
 
40
41
 
41
- class Trigger(PrefectBaseModel, abc.ABC, extra="ignore"):
42
+ class Trigger(PrefectBaseModel, abc.ABC, extra="ignore"): # type: ignore[call-arg]
42
43
  """
43
44
  Base class describing a set of criteria that must be satisfied in order to trigger
44
45
  an automation.
@@ -50,6 +51,50 @@ class Trigger(PrefectBaseModel, abc.ABC, extra="ignore"):
50
51
  def describe_for_cli(self, indent: int = 0) -> str:
51
52
  """Return a human-readable description of this trigger for the CLI"""
52
53
 
54
+ # The following allows the regular Trigger class to be used when serving or
55
+ # deploying flows, analogous to how the Deployment*Trigger classes work
56
+
57
+ _deployment_id: Optional[UUID] = PrivateAttr(default=None)
58
+
59
+ def set_deployment_id(self, deployment_id: UUID):
60
+ self._deployment_id = deployment_id
61
+
62
+ def owner_resource(self) -> Optional[str]:
63
+ return f"prefect.deployment.{self._deployment_id}"
64
+
65
+ def actions(self) -> List[ActionTypes]:
66
+ assert self._deployment_id
67
+ return [
68
+ RunDeployment(
69
+ source="selected",
70
+ deployment_id=self._deployment_id,
71
+ parameters=getattr(self, "parameters", None),
72
+ job_variables=getattr(self, "job_variables", None),
73
+ )
74
+ ]
75
+
76
+ def as_automation(self) -> "AutomationCore":
77
+ assert self._deployment_id
78
+
79
+ trigger: TriggerTypes = cast(TriggerTypes, self)
80
+
81
+ # This is one of the Deployment*Trigger classes, so translate it over to a
82
+ # plain Trigger
83
+ if hasattr(self, "trigger_type"):
84
+ trigger = self.trigger_type(**self.dict())
85
+
86
+ return AutomationCore(
87
+ name=(
88
+ getattr(self, "name", None)
89
+ or f"Automation for deployment {self._deployment_id}"
90
+ ),
91
+ description="",
92
+ enabled=getattr(self, "enabled", True),
93
+ trigger=trigger,
94
+ actions=self.actions(),
95
+ owner_resource=self.owner_resource(),
96
+ )
97
+
53
98
 
54
99
  class ResourceTrigger(Trigger, abc.ABC):
55
100
  """
@@ -346,7 +391,7 @@ CompoundTrigger.update_forward_refs()
346
391
  SequenceTrigger.update_forward_refs()
347
392
 
348
393
 
349
- class AutomationCore(PrefectBaseModel, extra="ignore"):
394
+ class AutomationCore(PrefectBaseModel, extra="ignore"): # type: ignore[call-arg]
350
395
  """Defines an action a user wants to take when a certain number of events
351
396
  do or don't happen to the matching resources"""
352
397