prefect 3.1.14.dev4__tar.gz → 3.1.15.dev2__tar.gz

{prefect-3.1.14.dev4/src/prefect.egg-info → prefect-3.1.15.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: prefect
-Version: 3.1.14.dev4
+Version: 3.1.15.dev2
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/_experimental/lineage.py

@@ -178,3 +178,67 @@ async def emit_result_write_event(
             downstream_resources=downstream_resources,
             direction_of_run_from_event="upstream",
         )
+
+
+async def emit_external_resource_lineage(
+    event_name: str = "prefect.lineage.event",
+    upstream_resources: Optional[UpstreamResources] = None,
+    downstream_resources: Optional[DownstreamResources] = None,
+) -> None:
+    """Emit lineage events connecting external resources to Prefect context resources.
+
+    This function emits events that place the current Prefect context resources
+    (like flow runs, task runs) as:
+    1. Downstream of any provided upstream external resources
+    2. Upstream of any provided downstream external resources
+
+    Args:
+        upstream_resources: Optional sequence of resources that are upstream of the
+            current Prefect context
+        downstream_resources: Optional sequence of resources that are downstream of
+            the current Prefect context
+    """
+    from prefect.client.orchestration import get_client
+
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    upstream_resources = list(upstream_resources) if upstream_resources else []
+    downstream_resources = list(downstream_resources) if downstream_resources else []
+
+    # Get the current Prefect context resources (flow runs, task runs, etc.)
+    async with get_client() as client:
+        context_resources = await related_resources_from_run_context(client)
+
+    # Add lineage group label to all resources
+    for res in upstream_resources + downstream_resources + context_resources:
+        if "prefect.resource.lineage-group" not in res:
+            res["prefect.resource.lineage-group"] = "global"
+
+    # For each context resource, emit an event showing it as downstream of upstream resources
+    if upstream_resources:
+        for context_resource in context_resources:
+            emit_kwargs: Dict[str, Any] = {
+                "event": "prefect.lineage.upstream-interaction",
+                "resource": context_resource,
+                "related": upstream_resources,
+            }
+            emit_event(**emit_kwargs)
+
+    # For each downstream resource, emit an event showing it as downstream of context resources
+    for downstream_resource in downstream_resources:
+        emit_kwargs: Dict[str, Any] = {
+            "event": "prefect.lineage.downstream-interaction",
+            "resource": downstream_resource,
+            "related": context_resources,
+        }
+        emit_event(**emit_kwargs)
+
+        # For each downstream resource, emit an event showing it as downstream of upstream resources
+        if upstream_resources:
+            direct_emit_kwargs = {
+                "event": event_name,
+                "resource": downstream_resource,
+                "related": upstream_resources,
+            }
+            emit_event(**direct_emit_kwargs)

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/_internal/concurrency/services.py

@@ -65,7 +65,7 @@ class _QueueServiceBase(abc.ABC, Generic[T]):
         # failure to process items. This is particularly relevant for services
         # which use an httpx client. See related issue at
         # https://github.com/python/cpython/issues/86813
-        threading._register_atexit(self._at_exit)  # pyright: ignore[reportUnknownVariableType, reportAttributeAccessIssue]
+        threading._register_atexit(self._at_exit)  # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
 
     def _at_exit(self) -> None:
         self.drain(at_exit=True)

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-01-22T15:30:28-0600",
+ "date": "2025-01-24T16:31:59-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "7548f1bca2328079cb3bf38f7b403d3568e0c02e",
- "version": "3.1.14.dev4"
+ "full-revisionid": "d5279c51b9a68a6baa251b213e1fcf7966426688",
+ "version": "3.1.15.dev2"
 }
 '''  # END VERSION_JSON
 

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/blocks/core.py

@@ -669,7 +669,7 @@ class Block(BaseModel, ABC):
         module_str = ".".join(qualified_name.split(".")[:-1])
         origin = cls.__pydantic_generic_metadata__.get("origin") or cls
         class_name = origin.__name__
-        block_variable_name = f'{cls.get_block_type_slug().replace("-", "_")}_block'
+        block_variable_name = f"{cls.get_block_type_slug().replace('-', '_')}_block"
 
         return dedent(
             f"""\

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/cache_policies.py

@@ -80,10 +80,12 @@ class CachePolicy:
         raise NotImplementedError
 
     def __sub__(self, other: str) -> "CachePolicy":
+        "No-op for all policies except Inputs and Compound"
+
+        # for interface compatibility
         if not isinstance(other, str):  # type: ignore[reportUnnecessaryIsInstance]
             raise TypeError("Can only subtract strings from key policies.")
-        new = Inputs(exclude=[other])
-        return CompoundCachePolicy(policies=[self, new])
+        return self
 
     def __add__(self, other: "CachePolicy") -> "CachePolicy":
         # adding _None is a no-op
@@ -214,8 +216,15 @@ class CompoundCachePolicy(CachePolicy):
     def __sub__(self, other: str) -> "CachePolicy":
         if not isinstance(other, str):  # type: ignore[reportUnnecessaryIsInstance]
             raise TypeError("Can only subtract strings from key policies.")
-        new = Inputs(exclude=[other])
-        return CompoundCachePolicy(policies=[*self.policies, new])
+
+        inputs_policies = [p for p in self.policies if isinstance(p, Inputs)]
+
+        if inputs_policies:
+            new = Inputs(exclude=[other])
+            return CompoundCachePolicy(policies=[*self.policies, new])
+        else:
+            # no dependency on inputs already
+            return self
 
 
 @dataclass

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/client/orchestration/_automations/client.py

@@ -45,6 +45,8 @@ class AutomationClient(BaseClient):
         return Automation.model_validate_list(response.json())
 
     def find_automation(self, id_or_name: "str | UUID") -> "Automation | None":
+        from uuid import UUID
+
         if isinstance(id_or_name, str):
             name = id_or_name
             try:
@@ -202,6 +204,8 @@ class AutomationAsyncClient(BaseAsyncClient):
         return Automation.model_validate_list(response.json())
 
     async def find_automation(self, id_or_name: "str | UUID") -> "Automation | None":
+        from uuid import UUID
+
         if isinstance(id_or_name, str):
             name = id_or_name
             try:

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/client/schemas/schedules.py

@@ -145,7 +145,7 @@ class CronSchedule(PrefectBaseModel):
 
     @field_validator("timezone")
     @classmethod
-    def valid_timezone(cls, v: Optional[str]) -> str:
+    def valid_timezone(cls, v: Optional[str]) -> Optional[str]:
         return default_timezone(v)
 
     @field_validator("cron")
@@ -276,7 +276,7 @@ class RRuleSchedule(PrefectBaseModel):
                 kwargs.update(
                     until=until.replace(tzinfo=timezone),
                 )
-            return rrule.replace(**kwargs)
+            return rrule.replace(**kwargs)  # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] missing type hints
 
         # update rrules
         localized_rrules: list[dateutil.rrule.rrule] = []
@@ -286,7 +286,7 @@ class RRuleSchedule(PrefectBaseModel):
             kwargs: dict[str, Any] = dict(dtstart=dtstart.replace(tzinfo=timezone))
             if until := _rrule_dt(rr, "_until"):
                 kwargs.update(until=until.replace(tzinfo=timezone))
-            localized_rrules.append(rr.replace(**kwargs))
+            localized_rrules.append(rr.replace(**kwargs))  # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] missing type hints
         setattr(rrule, "_rrule", localized_rrules)
 
         # update exrules
@@ -297,7 +297,7 @@ class RRuleSchedule(PrefectBaseModel):
             kwargs = dict(dtstart=dtstart.replace(tzinfo=timezone))
             if until := _rrule_dt(exr, "_until"):
                 kwargs.update(until=until.replace(tzinfo=timezone))
-            localized_exrules.append(exr.replace(**kwargs))
+            localized_exrules.append(exr.replace(**kwargs))  # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] missing type hints
         setattr(rrule, "_exrule", localized_exrules)
 
         # update rdates

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/context.py

@@ -337,10 +337,15 @@ class EngineContext(RunContext):
         flow: The flow instance associated with the run
         flow_run: The API metadata for the flow run
         task_runner: The task runner instance being used for the flow run
-        task_run_futures: A list of futures for task runs submitted within this flow run
-        task_run_states: A list of states for task runs created within this flow run
         task_run_results: A mapping of result ids to task run states for this flow run
-        flow_run_states: A list of states for flow runs created within this flow run
+        log_prints: Whether to log print statements from the flow run
+        parameters: The parameters passed to the flow run
+        detached: Flag indicating if context has been serialized and sent to remote infrastructure
+        result_store: The result store used to persist results
+        persist_result: Whether to persist the flow run result
+        task_run_dynamic_keys: Counter for task calls allowing unique keys
+        observed_flow_pauses: Counter for flow pauses
+        events: Events worker to emit events
     """
 
     flow: Optional["Flow[Any, Any]"] = None
@@ -373,7 +378,7 @@ class EngineContext(RunContext):
     __var__: ClassVar[ContextVar[Self]] = ContextVar("flow_run")
 
     def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
-        return self.model_dump(
+        serialized = self.model_dump(
             include={
                 "flow_run",
                 "flow",
@@ -381,13 +386,18 @@ class EngineContext(RunContext):
                 "log_prints",
                 "start_time",
                 "input_keyset",
-                "result_store",
                 "persist_result",
             },
             exclude_unset=True,
-            serialize_as_any=True,
             context={"include_secrets": include_secrets},
         )
+        if self.result_store:
+            serialized["result_store"] = self.result_store.model_dump(
+                serialize_as_any=True,
+                exclude_unset=True,
+                context={"include_secrets": include_secrets},
+            )
+        return serialized
 
 
 FlowRunContext = EngineContext  # for backwards compatibility

{prefect-3.1.14.dev4 → prefect-3.1.15.dev2}/src/prefect/events/clients.py

@@ -348,16 +348,26 @@ class PrefectEventsClient(EventsClient):
         await self._connect.__aexit__(exc_type, exc_val, exc_tb)
         return await super().__aexit__(exc_type, exc_val, exc_tb)
 
+    def _log_debug(self, message: str, *args, **kwargs) -> None:
+        message = f"EventsClient(id={id(self)}): " + message
+        logger.debug(message, *args, **kwargs)
+
     async def _reconnect(self) -> None:
+        logger.debug("Reconnecting websocket connection.")
+
         if self._websocket:
             self._websocket = None
             await self._connect.__aexit__(None, None, None)
+            logger.debug("Cleared existing websocket connection.")
 
         try:
+            logger.debug("Opening websocket connection.")
             self._websocket = await self._connect.__aenter__()
             # make sure we have actually connected
+            logger.debug("Pinging to ensure websocket connected.")
             pong = await self._websocket.ping()
             await pong
+            logger.debug("Pong received. Websocket connected.")
         except Exception as e:
             # The client is frequently run in a background thread
             # so we log an additional warning to ensure
@@ -375,11 +385,13 @@ class PrefectEventsClient(EventsClient):
             raise
 
         events_to_resend = self._unconfirmed_events
+        logger.debug("Resending %s unconfirmed events.", len(events_to_resend))
         # Clear the unconfirmed events here, because they are going back through emit
         # and will be added again through the normal checkpointing process
         self._unconfirmed_events = []
         for event in events_to_resend:
             await self.emit(event)
+        logger.debug("Finished resending unconfirmed events.")
 
     async def _checkpoint(self, event: Event) -> None:
         assert self._websocket
@@ -387,11 +399,20 @@ class PrefectEventsClient(EventsClient):
         self._unconfirmed_events.append(event)
 
         unconfirmed_count = len(self._unconfirmed_events)
+
+        logger.debug(
+            "Added event id=%s to unconfirmed events list. "
+            "There are now %s unconfirmed events.",
+            event.id,
+            unconfirmed_count,
+        )
         if unconfirmed_count < self._checkpoint_every:
             return
 
+        logger.debug("Pinging to checkpoint unconfirmed events.")
         pong = await self._websocket.ping()
         await pong
+        self._log_debug("Pong received. Events checkpointed.")
 
         # once the pong returns, we know for sure that we've sent all the messages
         # we had enqueued prior to that.  There could be more that came in after, so
@@ -401,7 +422,9 @@ class PrefectEventsClient(EventsClient):
         EVENT_WEBSOCKET_CHECKPOINTS.labels(self.client_name).inc()
 
     async def _emit(self, event: Event) -> None:
+        self._log_debug("Emitting event id=%s.", event.id)
         for i in range(self._reconnection_attempts + 1):
+            self._log_debug("Emit reconnection attempt %s.", i)
             try:
                 # If we're here and the websocket is None, then we've had a failure in a
                 # previous reconnection attempt.
@@ -410,14 +433,18 @@ class PrefectEventsClient(EventsClient):
                 # from a ConnectionClosed, so reconnect now, resending any unconfirmed
                 # events before we send this one.
                 if not self._websocket or i > 0:
+                    self._log_debug("Attempting websocket reconnection.")
                     await self._reconnect()
                     assert self._websocket
 
+                self._log_debug("Sending event id=%s.", event.id)
                 await self._websocket.send(event.model_dump_json())
+                self._log_debug("Checkpointing event id=%s.", event.id)
                 await self._checkpoint(event)
 
                 return
             except ConnectionClosed:
+                self._log_debug("Got ConnectionClosed error.")
                 if i == self._reconnection_attempts:
                     # this was our final chance, raise the most recent error
                     raise
@@ -426,6 +453,9 @@ class PrefectEventsClient(EventsClient):
                     # let the first two attempts happen quickly in case this is just
                     # a standard load balancer timeout, but after that, just take a
                     # beat to let things come back around.
+                    logger.debug(
+                        "Sleeping for 1 second before next reconnection attempt."
+                    )
                     await asyncio.sleep(1)