prefect-client 2.16.2__py3-none-any.whl → 2.16.3__py3-none-any.whl

prefect/events/schemas.py

@@ -1,8 +1,22 @@
+import abc
 from datetime import timedelta
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast
+from enum import Enum
+from typing import (
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Literal,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
 from uuid import UUID, uuid4
 
 import pendulum
+from typing_extensions import TypeAlias
 
 from prefect._internal.pydantic import HAS_PYDANTIC_V2
 
@@ -15,7 +29,6 @@ else:
 
 from prefect._internal.schemas.bases import PrefectBaseModel
 from prefect._internal.schemas.fields import DateTimeTZ
-from prefect._internal.schemas.transformations import FieldFrom, copy_model_fields
 from prefect.events.actions import ActionTypes, RunDeployment
 from prefect.utilities.collections import AutoEnum
 
@@ -27,6 +40,7 @@ MAXIMUM_RELATED_RESOURCES = 500
 class Posture(AutoEnum):
     Reactive = "Reactive"
     Proactive = "Proactive"
+    Metric = "Metric"
 
 
 class ResourceSpecification(PrefectBaseModel):
@@ -164,32 +178,56 @@ class Event(PrefectBaseModel):
         return value
 
 
-class Trigger(PrefectBaseModel):
-    """Defines the criteria for the events and conditions under which an Automation
-    will trigger an action"""
+class Trigger(PrefectBaseModel, abc.ABC):
+    """
+    Base class describing a set of criteria that must be satisfied in order to trigger
+    an automation.
+    """
+
+    class Config:
+        extra = Extra.ignore
+
+    type: str
+
+
+class ResourceTrigger(Trigger, abc.ABC):
+    """
+    Base class for triggers that may filter by the labels of resources.
+    """
+
+    type: str
 
     match: ResourceSpecification = Field(
         default_factory=lambda: ResourceSpecification(__root__={}),
-        description="Labels for resources which this Automation will match.",
+        description="Labels for resources which this trigger will match.",
     )
     match_related: ResourceSpecification = Field(
         default_factory=lambda: ResourceSpecification(__root__={}),
-        description="Labels for related resources which this Automation will match.",
+        description="Labels for related resources which this trigger will match.",
     )
 
+
+class EventTrigger(ResourceTrigger):
+    """
+    A trigger that fires based on the presence or absence of events within a given
+    period of time.
+    """
+
+    type: Literal["event"] = "event"
+
     after: Set[str] = Field(
         default_factory=set,
         description=(
-            "The event(s) which must first been seen to start this automation.  If "
-            "empty, then start this Automation immediately.  Events may include "
+            "The event(s) which must first been seen to fire this trigger.  If "
+            "empty, then fire this trigger immediately.  Events may include "
             "trailing wildcards, like `prefect.flow-run.*`"
         ),
     )
     expect: Set[str] = Field(
         default_factory=set,
         description=(
-            "The event(s) this automation is expecting to see.  If empty, this "
-            "automation will match any event.  Events may include trailing wildcards, "
+            "The event(s) this trigger is expecting to see.  If empty, this "
+            "trigger will match any event.  Events may include trailing wildcards, "
             "like `prefect.flow-run.*`"
         ),
     )
@@ -197,24 +235,29 @@ class Trigger(PrefectBaseModel):
     for_each: Set[str] = Field(
         default_factory=set,
         description=(
-            "Evaluate the Automation separately for each distinct value of these labels"
-            " on the resource"
+            "Evaluate the trigger separately for each distinct value of these labels "
+            "on the resource.  By default, labels refer to the primary resource of the "
+            "triggering event.  You may also refer to labels from related "
+            "resources by specifying `related:<role>:<label>`.  This will use the "
+            "value of that label for the first related resource in that role.  For "
+            'example, `"for_each": ["related:flow:prefect.resource.id"]` would '
+            "evaluate the trigger for each flow."
         ),
     )
-    posture: Posture = Field(
-        Posture.Reactive,
+    posture: Literal[Posture.Reactive, Posture.Proactive] = Field(  # type: ignore[valid-type]
+        ...,
         description=(
-            "The posture of this Automation, either Reactive or Proactive.  Reactive "
-            "automations respond to the _presence_ of the expected events, while "
-            "Proactive  automations respond to the _absence_ of those expected events."
+            "The posture of this trigger, either Reactive or Proactive.  Reactive "
+            "triggers respond to the _presence_ of the expected events, while "
+            "Proactive triggers respond to the _absence_ of those expected events."
         ),
     )
     threshold: int = Field(
         1,
         description=(
-            "The number of events required for this Automation to trigger (for "
-            "Reactive automations), or the number of events expected (for Proactive "
-            "automations)"
+            "The number of events required for this trigger to fire (for "
+            "Reactive triggers), or the number of events expected (for Proactive "
+            "triggers)"
         ),
     )
     within: timedelta = Field(
@@ -253,6 +296,86 @@ class Trigger(PrefectBaseModel):
         return values
 
 
+class MetricTriggerOperator(Enum):
+    LT = "<"
+    LTE = "<="
+    GT = ">"
+    GTE = ">="
+
+
+class PrefectMetric(Enum):
+    lateness = "lateness"
+    duration = "duration"
+    successes = "successes"
+
+
+class MetricTriggerQuery(PrefectBaseModel):
+    """Defines a subset of the Trigger subclass, which is specific
+    to Metric automations, that specify the query configurations
+    and breaching conditions for the Automation"""
+
+    name: PrefectMetric = Field(
+        ...,
+        description="The name of the metric to query.",
+    )
+    threshold: float = Field(
+        ...,
+        description=(
+            "The threshold value against which we'll compare " "the query result."
+        ),
+    )
+    operator: MetricTriggerOperator = Field(
+        ...,
+        description=(
+            "The comparative operator (LT / LTE / GT / GTE) used to compare "
+            "the query result against the threshold value."
+        ),
+    )
+    range: timedelta = Field(
+        timedelta(seconds=300),  # defaults to 5 minutes
+        minimum=300.0,
+        exclusiveMinimum=False,
+        description=(
+            "The lookback duration (seconds) for a metric query. This duration is "
+            "used to determine the time range over which the query will be executed. "
+            "The minimum value is 300 seconds (5 minutes)."
+        ),
+    )
+    firing_for: timedelta = Field(
+        timedelta(seconds=300),  # defaults to 5 minutes
+        minimum=300.0,
+        exclusiveMinimum=False,
+        description=(
+            "The duration (seconds) for which the metric query must breach "
+            "or resolve continuously before the state is updated and the "
+            "automation is triggered. "
+            "The minimum value is 300 seconds (5 minutes)."
+        ),
+    )
+
+
+class MetricTrigger(ResourceTrigger):
+    """
+    A trigger that fires based on the results of a metric query.
+    """
+
+    type: Literal["metric"] = "metric"
+
+    posture: Literal[Posture.Metric] = Field(  # type: ignore[valid-type]
+        Posture.Metric,
+        description="Periodically evaluate the configured metric query.",
+    )
+
+    metric: MetricTriggerQuery = Field(
+        ...,
+        description="The metric query to evaluate for this trigger. ",
+    )
+
+
+TriggerTypes: TypeAlias = Union[EventTrigger, MetricTrigger]
+"""The union of all concrete trigger types that a user may actually create"""
+
+
 class Automation(PrefectBaseModel):
     """Defines an action a user wants to take when a certain number of events
     do or don't happen to the matching resources"""
@@ -265,7 +388,7 @@ class Automation(PrefectBaseModel):
 
     enabled: bool = Field(True, description="Whether this automation will be evaluated")
 
-    trigger: Trigger = Field(
+    trigger: TriggerTypes = Field(
         ...,
         description=(
             "The criteria for which events this Automation covers and how it will "
@@ -286,30 +409,106 @@ class ExistingAutomation(Automation):
     id: UUID = Field(..., description="The ID of this automation")
 
 
-@copy_model_fields
-class ResourceTrigger(PrefectBaseModel):
+class AutomationCreateFromTrigger(PrefectBaseModel):
     name: Optional[str] = Field(
         None, description="The name to give to the automation created for this trigger."
     )
-    description: str = FieldFrom(Automation)
-    enabled: bool = FieldFrom(Automation)
+    description: str = Field("", description="A longer description of this automation")
+    enabled: bool = Field(True, description="Whether this automation will be evaluated")
+
+    # from ResourceTrigger
+
+    match: ResourceSpecification = Field(
+        default_factory=lambda: ResourceSpecification(__root__={}),
+        description="Labels for resources which this trigger will match.",
+    )
+    match_related: ResourceSpecification = Field(
+        default_factory=lambda: ResourceSpecification(__root__={}),
+        description="Labels for related resources which this trigger will match.",
+    )
+
+    # from both EventTrigger and MetricTrigger
+
+    posture: Posture = Field(
+        Posture.Reactive,
+        description=(
+            "The posture of this trigger, either Reactive, Proactive, or Metric. "
+            "Reactive triggers respond to the _presence_ of the expected events, while "
+            "Proactive triggers respond to the _absence_ of those expected events.  "
+            "Metric triggers periodically evaluate the configured metric query."
+        ),
+    )
 
-    match: ResourceSpecification = FieldFrom(Trigger)
-    match_related: ResourceSpecification = FieldFrom(Trigger)
-    after: Set[str] = FieldFrom(Trigger)
-    expect: Set[str] = FieldFrom(Trigger)
-    for_each: Set[str] = FieldFrom(Trigger)
-    posture: Posture = FieldFrom(Trigger)
-    threshold: int = FieldFrom(Trigger)
-    within: timedelta = FieldFrom(Trigger)
+    # from EventTrigger
+
+    after: Set[str] = Field(
+        default_factory=set,
+        description=(
+            "The event(s) which must first been seen to fire this trigger.  If "
+            "empty, then fire this trigger immediately.  Events may include "
+            "trailing wildcards, like `prefect.flow-run.*`"
+        ),
+    )
+    expect: Set[str] = Field(
+        default_factory=set,
+        description=(
+            "The event(s) this trigger is expecting to see.  If empty, this "
+            "trigger will match any event.  Events may include trailing wildcards, "
+            "like `prefect.flow-run.*`"
+        ),
+    )
+
+    for_each: Set[str] = Field(
+        default_factory=set,
+        description=(
+            "Evaluate the trigger separately for each distinct value of these labels "
+            "on the resource.  By default, labels refer to the primary resource of the "
+            "triggering event.  You may also refer to labels from related "
+            "resources by specifying `related:<role>:<label>`.  This will use the "
+            "value of that label for the first related resource in that role.  For "
+            'example, `"for_each": ["related:flow:prefect.resource.id"]` would '
+            "evaluate the trigger for each flow."
+        ),
+    )
+    threshold: int = Field(
+        1,
+        description=(
+            "The number of events required for this trigger to fire (for "
+            "Reactive triggers), or the number of events expected (for Proactive "
+            "triggers)"
+        ),
+    )
+    within: timedelta = Field(
+        timedelta(0),
+        minimum=0.0,
+        exclusiveMinimum=False,
+        description=(
+            "The time period over which the events must occur.  For Reactive triggers, "
+            "this may be as low as 0 seconds, but must be at least 10 seconds for "
+            "Proactive triggers"
+        ),
+    )
+
+    # from MetricTrigger
+
+    metric: Optional[MetricTriggerQuery] = Field(
+        None,
+        description="The metric query to evaluate for this trigger. ",
+    )
 
     def as_automation(self) -> Automation:
         assert self.name
-        return Automation(
-            name=self.name,
-            description=self.description,
-            enabled=self.enabled,
-            trigger=Trigger(
+
+        if self.posture == Posture.Metric:
+            trigger = MetricTrigger(
+                type="metric",
+                match=self.match,
+                match_related=self.match_related,
+                posture=self.posture,
+                metric=self.metric,
+            )
+        else:
+            trigger = EventTrigger(
                 match=self.match,
                 match_related=self.match_related,
                 after=self.after,
@@ -318,7 +517,13 @@ class ResourceTrigger(PrefectBaseModel):
                 posture=self.posture,
                 threshold=self.threshold,
                 within=self.within,
-            ),
+            )
+
+        return Automation(
+            name=self.name,
+            description=self.description,
+            enabled=self.enabled,
+            trigger=trigger,
             actions=self.actions(),
             owner_resource=self.owner_resource(),
         )
@@ -330,10 +535,15 @@ class ResourceTrigger(PrefectBaseModel):
         raise NotImplementedError
 
 
-@copy_model_fields
-class DeploymentTrigger(ResourceTrigger):
+class DeploymentTrigger(AutomationCreateFromTrigger):
     _deployment_id: Optional[UUID] = PrivateAttr(default=None)
-    parameters: Optional[Dict[str, Any]] = FieldFrom(RunDeployment)
+    parameters: Optional[Dict[str, Any]] = Field(
+        None,
+        description=(
+            "The parameters to pass to the deployment, or None to use the "
+            "deployment's default parameters"
+        ),
+    )
 
     def set_deployment_id(self, deployment_id: UUID):
         self._deployment_id = deployment_id

prefect/flows.py

@@ -183,6 +183,7 @@ class Flow(Generic[P, R]):
         on_completion: An optional list of callables to run when the flow enters a completed state.
         on_cancellation: An optional list of callables to run when the flow enters a cancelling state.
         on_crashed: An optional list of callables to run when the flow enters a crashed state.
+        on_running: An optional list of callables to run when the flow enters a running state.
     """
 
     # NOTE: These parameters (types, defaults, and docstrings) should be duplicated
@@ -212,6 +213,7 @@ class Flow(Generic[P, R]):
             List[Callable[[FlowSchema, FlowRun, State], None]]
         ] = None,
         on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
+        on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
     ):
         if name is not None and not isinstance(name, str):
             raise TypeError(
@@ -227,8 +229,20 @@ class Flow(Generic[P, R]):
             )
 
         # Validate if hook passed is list and contains callables
-        hook_categories = [on_completion, on_failure, on_cancellation, on_crashed]
-        hook_names = ["on_completion", "on_failure", "on_cancellation", "on_crashed"]
+        hook_categories = [
+            on_completion,
+            on_failure,
+            on_cancellation,
+            on_crashed,
+            on_running,
+        ]
+        hook_names = [
+            "on_completion",
+            "on_failure",
+            "on_cancellation",
+            "on_crashed",
+            "on_running",
+        ]
         for hooks, hook_name in zip(hook_categories, hook_names):
             if hooks is not None:
                 if not hooks:
@@ -350,6 +364,7 @@ class Flow(Generic[P, R]):
         self.on_failure = on_failure
         self.on_cancellation = on_cancellation
         self.on_crashed = on_crashed
+        self.on_running = on_running
 
         # Used for flows loaded from remote storage
         self._storage: Optional[RunnerStorage] = None
@@ -387,6 +402,7 @@ class Flow(Generic[P, R]):
             List[Callable[[FlowSchema, FlowRun, State], None]]
         ] = None,
         on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
+        on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
     ) -> Self:
         """
         Create a new flow from the current object, updating provided options.
@@ -415,6 +431,7 @@ class Flow(Generic[P, R]):
             on_completion: A new list of callables to run when the flow enters a completed state.
             on_cancellation: A new list of callables to run when the flow enters a cancelling state.
             on_crashed: A new list of callables to run when the flow enters a crashed state.
+            on_running: A new list of callables to run when the flow enters a running state.
 
         Returns:
             A new `Flow` instance.
@@ -484,6 +501,7 @@ class Flow(Generic[P, R]):
             on_failure=on_failure or self.on_failure,
             on_cancellation=on_cancellation or self.on_cancellation,
             on_crashed=on_crashed or self.on_crashed,
+            on_running=on_running or self.on_running,
         )
         new_flow._storage = self._storage
         new_flow._entrypoint = self._entrypoint
@@ -1333,6 +1351,7 @@ def flow(
         List[Callable[[FlowSchema, FlowRun, State], None]]
     ] = None,
     on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
+    on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
 ) -> Callable[[Callable[P, R]], Flow[P, R]]:
     ...
 
@@ -1360,6 +1379,7 @@ def flow(
         List[Callable[[FlowSchema, FlowRun, State], None]]
     ] = None,
     on_crashed: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
+    on_running: Optional[List[Callable[[FlowSchema, FlowRun, State], None]]] = None,
 ):
     """
     Decorator to designate a function as a Prefect workflow.
@@ -1423,6 +1443,8 @@ def flow(
         on_crashed: An optional list of functions to call when the flow run crashes. Each
             function should accept three arguments: the flow, the flow run, and the
             final state of the flow run.
+        on_running: An optional list of functions to call when the flow run is started. Each
+            function should accept three arguments: the flow, the flow run, and the current state
 
     Returns:
         A callable `Flow` object which, when called, will run the flow and return its
@@ -1485,6 +1507,7 @@ def flow(
                 on_failure=on_failure,
                 on_cancellation=on_cancellation,
                 on_crashed=on_crashed,
+                on_running=on_running,
             ),
         )
     else:
@@ -1510,6 +1533,7 @@ def flow(
                 on_failure=on_failure,
                 on_cancellation=on_cancellation,
                 on_crashed=on_crashed,
+                on_running=on_running,
             ),
         )
 

prefect/runner/runner.py

@@ -76,9 +76,9 @@ from prefect.client.schemas.schedules import SCHEDULE_TYPES
 from prefect.deployments.deployments import load_flow_from_flow_run
 from prefect.deployments.runner import (
     EntrypointType,
-    FlexibleScheduleList,
     RunnerDeployment,
 )
+from prefect.deployments.schedules import FlexibleScheduleList
 from prefect.engine import propose_state
 from prefect.events.schemas import DeploymentTrigger
 from prefect.exceptions import (
@@ -228,7 +228,7 @@ class Runner:
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,

prefect/server/api/collections_data/views/aggregate-worker-metadata.json

@@ -122,7 +122,8 @@
           "auto_deregister_task_definition": "{{ auto_deregister_task_definition }}",
           "vpc_id": "{{ vpc_id }}",
           "container_name": "{{ container_name }}",
-          "cluster": "{{ cluster }}"
+          "cluster": "{{ cluster }}",
+          "match_latest_revision_in_family": "{{ match_latest_revision_in_family }}"
         },
         "variables": {
           "description": "Variables for templating an ECS job.",
@@ -265,6 +266,12 @@
               "description": "If enabled, any task definitions that are created by this block will be deregistered. Existing task definitions linked by ARN will never be deregistered. Deregistering a task definition does not remove it from your AWS account, instead it will be marked as INACTIVE.",
               "default": false,
               "type": "boolean"
+            },
+            "match_latest_revision_in_family": {
+              "title": "Match Latest Revision In Family",
+              "description": "If enabled, the most recent active revision in the task definition family will be compared against the desired ECS task configuration. If they are equal, the existing task definition will be used instead of registering a new one. If no family is specified the default family \"prefect\" will be used.",
+              "default": false,
+              "type": "boolean"
             }
           },
           "definitions": {
@@ -1628,4 +1635,4 @@
       "type": "kubernetes"
     }
   }
-}
+}

prefect/settings.py

@@ -102,6 +102,8 @@ T = TypeVar("T")
 
 DEFAULT_PROFILES_PATH = Path(__file__).parent.joinpath("profiles.toml")
 
+REMOVED_EXPERIMENTAL_FLAGS = {"PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_SCHEDULING_UI"}
+
 
 class Setting(Generic[T]):
     """
@@ -1295,6 +1297,11 @@ PREFECT_EXPERIMENTAL_ENABLE_ARTIFACTS_ON_FLOW_RUN_GRAPH = Setting(bool, default=
 Whether or not to enable artifacts on the flow run graph.
 """
 
+PREFECT_EXPERIMENTAL_ENABLE_STATES_ON_FLOW_RUN_GRAPH = Setting(bool, default=False)
+"""
+Whether or not to enable flow run states on the flow run graph.
+"""
+
 PREFECT_EXPERIMENTAL_ENABLE_EVENTS_CLIENT = Setting(bool, default=True)
 """
 Whether or not to enable experimental Prefect work pools.
@@ -1335,6 +1342,11 @@ PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_CANCELLATION = Setting(bool, default=True)
 Whether or not to enable experimental enhanced flow run cancellation.
 """
 
+PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_DEPLOYMENT_PARAMETERS = Setting(bool, default=True)
+"""
+Whether or not to enable enhanced deployment parameters.
+"""
+
 PREFECT_EXPERIMENTAL_WARN_ENHANCED_CANCELLATION = Setting(bool, default=False)
 """
 Whether or not to warn when experimental enhanced flow run cancellation is used.
@@ -1513,11 +1525,6 @@ PREFECT_EXPERIMENTAL_ENABLE_WORK_QUEUE_STATUS = Setting(bool, default=True)
 Whether or not to enable experimental work queue status in-place of work queue health.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_SCHEDULING_UI = Setting(bool, default=True)
-"""
-Whether or not to enable the enhanced scheduling UI.
-"""
-
 
 # Defaults -----------------------------------------------------------------------------
 
@@ -2071,6 +2078,24 @@ class ProfilesCollection:
         )
 
 
+def _handle_removed_flags(profile_name: str, settings: dict) -> dict:
+    to_remove = [name for name in settings if name in REMOVED_EXPERIMENTAL_FLAGS]
+
+    for name in to_remove:
+        warnings.warn(
+            (
+                f"Experimental flag {name!r} has been removed, please "
+                f"update your {profile_name!r} profile."
+            ),
+            UserWarning,
+            stacklevel=3,
+        )
+
+        settings.pop(name)
+
+    return settings
+
+
 def _read_profiles_from(path: Path) -> ProfilesCollection:
     """
     Read profiles from a path into a new `ProfilesCollection`.
@@ -2087,10 +2112,10 @@ def _read_profiles_from(path: Path) -> ProfilesCollection:
     active_profile = contents.get("active")
     raw_profiles = contents.get("profiles", {})
 
-    profiles = [
-        Profile(name=name, settings=settings, source=path)
-        for name, settings in raw_profiles.items()
-    ]
+    profiles = []
+    for name, settings in raw_profiles.items():
+        settings = _handle_removed_flags(name, settings)
+        profiles.append(Profile(name=name, settings=settings, source=path))
 
     return ProfilesCollection(profiles, active=active_profile)