prefect-client 3.1.13__py3-none-any.whl → 3.1.14__py3-none-any.whl

prefect/_experimental/lineage.py

@@ -178,3 +178,66 @@ async def emit_result_write_event(
             downstream_resources=downstream_resources,
             direction_of_run_from_event="upstream",
         )
+
+
+async def emit_external_resource_lineage(
+    upstream_resources: Optional[UpstreamResources] = None,
+    downstream_resources: Optional[DownstreamResources] = None,
+) -> None:
+    """Emit lineage events connecting external resources to Prefect context resources.
+
+    This function emits events that place the current Prefect context resources
+    (like flow runs, task runs) as:
+    1. Downstream of any provided upstream external resources
+    2. Upstream of any provided downstream external resources
+
+    Args:
+        upstream_resources: Optional sequence of resources that are upstream of the
+            current Prefect context
+        downstream_resources: Optional sequence of resources that are downstream of
+            the current Prefect context
+    """
+    from prefect.client.orchestration import get_client
+
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    upstream_resources = list(upstream_resources) if upstream_resources else []
+    downstream_resources = list(downstream_resources) if downstream_resources else []
+
+    # Get the current Prefect context resources (flow runs, task runs, etc.)
+    async with get_client() as client:
+        context_resources = await related_resources_from_run_context(client)
+
+    # Add lineage group label to all resources
+    for res in upstream_resources + downstream_resources:
+        if "prefect.resource.lineage-group" not in res:
+            res["prefect.resource.lineage-group"] = "global"
+
+    # For each context resource, emit an event showing it as downstream of upstream resources
+    if upstream_resources:
+        for context_resource in context_resources:
+            emit_kwargs: Dict[str, Any] = {
+                "event": "prefect.resource.consumed",
+                "resource": context_resource,
+                "related": upstream_resources,
+            }
+            emit_event(**emit_kwargs)
+
+    # For each downstream resource, emit an event showing it as downstream of context resources
+    for downstream_resource in downstream_resources:
+        emit_kwargs: Dict[str, Any] = {
+            "event": "prefect.resource.produced",
+            "resource": downstream_resource,
+            "related": context_resources,
+        }
+        emit_event(**emit_kwargs)
+
+        # For each downstream resource, emit an event showing it as downstream of upstream resources
+        if upstream_resources:
+            direct_emit_kwargs = {
+                "event": "prefect.resource.direct-lineage",
+                "resource": downstream_resource,
+                "related": upstream_resources,
+            }
+            emit_event(**direct_emit_kwargs)

prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-01-17T08:46:53-0800",
+ "date": "2025-01-23T13:22:04-0800",
  "dirty": true,
  "error": null,
- "full-revisionid": "16e85ce3c281778f5ab6487a73377eed63bcac8b",
- "version": "3.1.13"
+ "full-revisionid": "5f1ebb57222bee1537b4d2c64b6dc9e3791ebafe",
+ "version": "3.1.14"
 }
 '''  # END VERSION_JSON
 

prefect/cache_policies.py

@@ -80,10 +80,12 @@ class CachePolicy:
         raise NotImplementedError
 
     def __sub__(self, other: str) -> "CachePolicy":
+        "No-op for all policies except Inputs and Compound"
+
+        # for interface compatibility
         if not isinstance(other, str):  # type: ignore[reportUnnecessaryIsInstance]
             raise TypeError("Can only subtract strings from key policies.")
-        new = Inputs(exclude=[other])
-        return CompoundCachePolicy(policies=[self, new])
+        return self
 
     def __add__(self, other: "CachePolicy") -> "CachePolicy":
         # adding _None is a no-op
@@ -157,6 +159,23 @@ class CompoundCachePolicy(CachePolicy):
 
     policies: list[CachePolicy] = field(default_factory=list)
 
+    def __post_init__(self) -> None:
+        # flatten any CompoundCachePolicies
+        self.policies = [
+            policy
+            for p in self.policies
+            for policy in (p.policies if isinstance(p, CompoundCachePolicy) else [p])
+        ]
+
+        # deduplicate any Inputs policies
+        inputs_policies = [p for p in self.policies if isinstance(p, Inputs)]
+        self.policies = [p for p in self.policies if not isinstance(p, Inputs)]
+        if inputs_policies:
+            all_excludes: set[str] = set()
+            for inputs_policy in inputs_policies:
+                all_excludes.update(inputs_policy.exclude)
+            self.policies.append(Inputs(exclude=sorted(all_excludes)))
+
     def compute_key(
         self,
         task_ctx: TaskRunContext,
@@ -178,6 +197,35 @@ class CompoundCachePolicy(CachePolicy):
             return None
         return hash_objects(*keys, raise_on_failure=True)
 
+    def __add__(self, other: "CachePolicy") -> "CachePolicy":
+        # Call the superclass add method to handle validation
+        super().__add__(other)
+
+        if isinstance(other, CompoundCachePolicy):
+            policies = [*self.policies, *other.policies]
+        else:
+            policies = [*self.policies, other]
+
+        return CompoundCachePolicy(
+            policies=policies,
+            key_storage=self.key_storage or other.key_storage,
+            isolation_level=self.isolation_level or other.isolation_level,
+            lock_manager=self.lock_manager or other.lock_manager,
+        )
+
+    def __sub__(self, other: str) -> "CachePolicy":
+        if not isinstance(other, str):  # type: ignore[reportUnnecessaryIsInstance]
+            raise TypeError("Can only subtract strings from key policies.")
+
+        inputs_policies = [p for p in self.policies if isinstance(p, Inputs)]
+
+        if inputs_policies:
+            new = Inputs(exclude=[other])
+            return CompoundCachePolicy(policies=[*self.policies, new])
+        else:
+            # no dependency on inputs already
+            return self
+
 
 @dataclass
 class _None(CachePolicy):

prefect/client/orchestration/_automations/client.py

@@ -45,6 +45,8 @@ class AutomationClient(BaseClient):
         return Automation.model_validate_list(response.json())
 
     def find_automation(self, id_or_name: "str | UUID") -> "Automation | None":
+        from uuid import UUID
+
         if isinstance(id_or_name, str):
             name = id_or_name
             try:
@@ -202,6 +204,8 @@ class AutomationAsyncClient(BaseAsyncClient):
         return Automation.model_validate_list(response.json())
 
     async def find_automation(self, id_or_name: "str | UUID") -> "Automation | None":
+        from uuid import UUID
+
         if isinstance(id_or_name, str):
             name = id_or_name
             try:

prefect/client/orchestration/_deployments/client.py

@@ -1031,8 +1031,8 @@ class DeploymentAsyncClient(BaseAsyncClient):
         deployment_ids: list["UUID"],
         scheduled_before: "datetime.datetime | None" = None,
         limit: int | None = None,
-    ) -> list["FlowRunResponse"]:
-        from prefect.client.schemas.responses import FlowRunResponse
+    ) -> list["FlowRun"]:
+        from prefect.client.schemas.objects import FlowRun
 
         body: dict[str, Any] = dict(deployment_ids=[str(id) for id in deployment_ids])
         if scheduled_before:
@@ -1046,7 +1046,7 @@ class DeploymentAsyncClient(BaseAsyncClient):
             json=body,
         )
 
-        return FlowRunResponse.model_validate_list(response.json())
+        return FlowRun.model_validate_list(response.json())
 
     async def create_flow_run_from_deployment(
         self,

prefect/context.py

@@ -337,10 +337,15 @@ class EngineContext(RunContext):
         flow: The flow instance associated with the run
         flow_run: The API metadata for the flow run
         task_runner: The task runner instance being used for the flow run
-        task_run_futures: A list of futures for task runs submitted within this flow run
-        task_run_states: A list of states for task runs created within this flow run
         task_run_results: A mapping of result ids to task run states for this flow run
-        flow_run_states: A list of states for flow runs created within this flow run
+        log_prints: Whether to log print statements from the flow run
+        parameters: The parameters passed to the flow run
+        detached: Flag indicating if context has been serialized and sent to remote infrastructure
+        result_store: The result store used to persist results
+        persist_result: Whether to persist the flow run result
+        task_run_dynamic_keys: Counter for task calls allowing unique keys
+        observed_flow_pauses: Counter for flow pauses
+        events: Events worker to emit events
     """
 
     flow: Optional["Flow[Any, Any]"] = None
@@ -373,7 +378,7 @@ class EngineContext(RunContext):
     __var__: ClassVar[ContextVar[Self]] = ContextVar("flow_run")
 
     def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
-        return self.model_dump(
+        serialized = self.model_dump(
             include={
                 "flow_run",
                 "flow",
@@ -381,13 +386,18 @@ class EngineContext(RunContext):
                 "log_prints",
                 "start_time",
                 "input_keyset",
-                "result_store",
                 "persist_result",
             },
             exclude_unset=True,
-            serialize_as_any=True,
             context={"include_secrets": include_secrets},
         )
+        if self.result_store:
+            serialized["result_store"] = self.result_store.model_dump(
+                serialize_as_any=True,
+                exclude_unset=True,
+                context={"include_secrets": include_secrets},
+            )
+        return serialized
 
 
 FlowRunContext = EngineContext  # for backwards compatibility

prefect/events/filters.py

@@ -1,7 +1,6 @@
-from typing import List, Optional, Tuple, cast
+from typing import Optional
 from uuid import UUID
 
-import pendulum
 from pydantic import Field
 
 from prefect._internal.schemas.bases import PrefectBaseModel
@@ -23,7 +22,7 @@ class AutomationFilterCreated(PrefectBaseModel):
 class AutomationFilterName(PrefectBaseModel):
     """Filter by `Automation.created`."""
 
-    any_: Optional[List[str]] = Field(
+    any_: Optional[list[str]] = Field(
         default=None,
         description="Only include automations with names that match any of these strings",
     )
@@ -41,8 +40,8 @@ class AutomationFilter(PrefectBaseModel):
 class EventDataFilter(PrefectBaseModel, extra="forbid"):  # type: ignore[call-arg]
     """A base class for filtering event data."""
 
-    def get_filters(self) -> List["EventDataFilter"]:
-        filters: List["EventDataFilter"] = [
+    def get_filters(self) -> list["EventDataFilter"]:
+        filters: list["EventDataFilter"] = [
             filter
             for filter in [getattr(self, name) for name in self.model_fields]
             if isinstance(filter, EventDataFilter)
@@ -60,14 +59,11 @@ class EventDataFilter(PrefectBaseModel, extra="forbid"):  # type: ignore[call-ar
 
 class EventOccurredFilter(EventDataFilter):
     since: DateTime = Field(
-        default_factory=lambda: cast(
-            DateTime,
-            pendulum.now("UTC").start_of("day").subtract(days=180),
-        ),
+        default_factory=lambda: DateTime.now("UTC").start_of("day").subtract(days=180),
         description="Only include events after this time (inclusive)",
     )
     until: DateTime = Field(
-        default_factory=lambda: cast(DateTime, pendulum.now("UTC")),
+        default_factory=lambda: DateTime.now("UTC"),
         description="Only include events prior to this time (inclusive)",
     )
 
@@ -76,18 +72,18 @@ class EventOccurredFilter(EventDataFilter):
 
 
 class EventNameFilter(EventDataFilter):
-    prefix: Optional[List[str]] = Field(
+    prefix: Optional[list[str]] = Field(
         default=None, description="Only include events matching one of these prefixes"
     )
-    exclude_prefix: Optional[List[str]] = Field(
+    exclude_prefix: Optional[list[str]] = Field(
         default=None, description="Exclude events matching one of these prefixes"
     )
 
-    name: Optional[List[str]] = Field(
+    name: Optional[list[str]] = Field(
         default=None,
         description="Only include events matching one of these names exactly",
     )
-    exclude_name: Optional[List[str]] = Field(
+    exclude_name: Optional[list[str]] = Field(
         default=None, description="Exclude events matching one of these names exactly"
     )
 
@@ -112,20 +108,20 @@ class EventNameFilter(EventDataFilter):
 
 
 class EventResourceFilter(EventDataFilter):
-    id: Optional[List[str]] = Field(
-        None, description="Only include events for resources with these IDs"
+    id: Optional[list[str]] = Field(
+        default=None, description="Only include events for resources with these IDs"
     )
-    id_prefix: Optional[List[str]] = Field(
-        None,
+    id_prefix: Optional[list[str]] = Field(
+        default=None,
         description=(
             "Only include events for resources with IDs starting with these prefixes."
         ),
     )
     labels: Optional[ResourceSpecification] = Field(
-        None, description="Only include events for resources with these labels"
+        default=None, description="Only include events for resources with these labels"
     )
     distinct: bool = Field(
-        False,
+        default=False,
         description="Only include events for distinct resources",
     )
 
@@ -148,35 +144,39 @@ class EventResourceFilter(EventDataFilter):
 
 
 class EventRelatedFilter(EventDataFilter):
-    id: Optional[List[str]] = Field(
-        None, description="Only include events for related resources with these IDs"
+    id: Optional[list[str]] = Field(
+        default=None,
+        description="Only include events for related resources with these IDs",
     )
-    role: Optional[List[str]] = Field(
-        None, description="Only include events for related resources in these roles"
+    role: Optional[list[str]] = Field(
+        default=None,
+        description="Only include events for related resources in these roles",
     )
-    resources_in_roles: Optional[List[Tuple[str, str]]] = Field(
-        None,
+    resources_in_roles: Optional[list[tuple[str, str]]] = Field(
+        default=None,
         description=(
             "Only include events with specific related resources in specific roles"
         ),
     )
     labels: Optional[ResourceSpecification] = Field(
-        None, description="Only include events for related resources with these labels"
+        default=None,
+        description="Only include events for related resources with these labels",
     )
 
 
 class EventAnyResourceFilter(EventDataFilter):
-    id: Optional[List[str]] = Field(
-        None, description="Only include events for resources with these IDs"
+    id: Optional[list[str]] = Field(
+        default=None, description="Only include events for resources with these IDs"
     )
-    id_prefix: Optional[List[str]] = Field(
-        None,
+    id_prefix: Optional[list[str]] = Field(
+        default=None,
         description=(
             "Only include events for resources with IDs starting with these prefixes"
         ),
     )
     labels: Optional[ResourceSpecification] = Field(
-        None, description="Only include events for related resources with these labels"
+        default=None,
+        description="Only include events for related resources with these labels",
     )
 
     def includes(self, event: Event) -> bool:
@@ -202,8 +202,8 @@ class EventAnyResourceFilter(EventDataFilter):
 
 
 class EventIDFilter(EventDataFilter):
-    id: Optional[List[UUID]] = Field(
-        None, description="Only include events with one of these IDs"
+    id: Optional[list[UUID]] = Field(
+        default=None, description="Only include events with one of these IDs"
     )
 
     def includes(self, event: Event) -> bool: