prefect-client 3.1.13__py3-none-any.whl → 3.1.15__py3-none-any.whl

prefect/__main__.py

@@ -0,0 +1,4 @@
+from prefect.cli import app
+
+if __name__ == "__main__":
+    app()

prefect/_experimental/lineage.py

@@ -1,15 +1,16 @@
 from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Sequence, Union
 
+from typing_extensions import TypeAlias
+
 from prefect.events.related import related_resources_from_run_context
-from prefect.events.schemas.events import RelatedResource, Resource
+from prefect.events.schemas.events import RelatedResource
 from prefect.events.utilities import emit_event
 from prefect.settings import get_current_settings
 
 if TYPE_CHECKING:
     from prefect.results import ResultStore
 
-UpstreamResources = Sequence[Union[RelatedResource, dict[str, str]]]
-DownstreamResources = Sequence[Union[Resource, dict[str, str]]]
+LineageResources: TypeAlias = Sequence[Union[RelatedResource, dict[str, str]]]
 
 # Map block types to their URI schemes
 STORAGE_URI_SCHEMES = {
@@ -47,8 +48,8 @@ def get_result_resource_uri(
 
 async def emit_lineage_event(
     event_name: str,
-    upstream_resources: Optional[UpstreamResources] = None,
-    downstream_resources: Optional[DownstreamResources] = None,
+    upstream_resources: Optional[LineageResources] = None,
+    downstream_resources: Optional[LineageResources] = None,
     direction_of_run_from_event: Literal["upstream", "downstream"] = "downstream",
 ) -> None:
     """Emit lineage events showing relationships between resources.
@@ -73,7 +74,14 @@ async def emit_lineage_event(
     downstream_resources = list(downstream_resources) if downstream_resources else []
 
     async with get_client() as client:
-        related_resources = await related_resources_from_run_context(client)
+        context_resources = await related_resources_from_run_context(client)
+
+    tag_resources = [
+        res for res in context_resources if res.get("prefect.resource.role") == "tag"
+    ]
+    context_resources = [
+        res for res in context_resources if res.get("prefect.resource.role") != "tag"
+    ]
 
     # NOTE: We handle adding run-related resources to the event here instead of in
     # the EventsWorker because not all run-related resources are upstream from
@@ -82,9 +90,9 @@ async def emit_lineage_event(
     # lineage-related events, tracks upstream resources only. For downstream
     # resources, we need to emit an event for each downstream resource.
     if direction_of_run_from_event == "downstream":
-        downstream_resources.extend(related_resources)
+        downstream_resources.extend(context_resources)
     else:
-        upstream_resources.extend(related_resources)
+        upstream_resources.extend(context_resources)
 
     # We want to consider all resources upstream and downstream of the event as
     # lineage-related, including flows, flow runs, etc., so we add the label to
@@ -101,7 +109,7 @@ async def emit_lineage_event(
         emit_kwargs: Dict[str, Any] = {
             "event": event_name,
             "resource": resource,
-            "related": upstream_resources,
+            "related": upstream_resources + tag_resources,
         }
 
         emit_event(**emit_kwargs)
@@ -110,7 +118,7 @@ async def emit_lineage_event(
 async def emit_result_read_event(
     store: "ResultStore",
     result_key: str,
-    downstream_resources: Optional[DownstreamResources] = None,
+    downstream_resources: Optional[LineageResources] = None,
     cached: bool = False,
 ) -> None:
     """
@@ -150,7 +158,7 @@ async def emit_result_read_event(
 async def emit_result_write_event(
     store: "ResultStore",
     result_key: str,
-    upstream_resources: Optional[UpstreamResources] = None,
+    upstream_resources: Optional[LineageResources] = None,
 ) -> None:
     """
     Emit a lineage event showing a task or flow result was written.
@@ -178,3 +186,76 @@ async def emit_result_write_event(
             downstream_resources=downstream_resources,
             direction_of_run_from_event="upstream",
         )
+
+
+async def emit_external_resource_lineage(
+    event_name: str = "prefect.lineage.event",
+    upstream_resources: Optional[LineageResources] = None,
+    downstream_resources: Optional[LineageResources] = None,
+    context_resources: Optional[LineageResources] = None,
+) -> None:
+    """Emit lineage events connecting external resources to Prefect context resources.
+
+    This function emits events that place the current Prefect context resources
+    (like flow runs, task runs) as:
+    1. Downstream of any provided upstream external resources
+    2. Upstream of any provided downstream external resources
+
+    Args:
+        upstream_resources: Optional sequence of resources that are upstream of the
+            current Prefect context
+        downstream_resources: Optional sequence of resources that are downstream of
+            the current Prefect context
+    """
+    from prefect.client.orchestration import get_client
+
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    upstream_resources = list(upstream_resources) if upstream_resources else []
+    downstream_resources = list(downstream_resources) if downstream_resources else []
+
+    # Get the current Prefect context resources (flow runs, task runs, etc.)
+    if not context_resources:
+        async with get_client() as client:
+            context_resources = await related_resources_from_run_context(client)
+
+    tag_resources = [
+        res for res in context_resources if res.get("prefect.resource.role") == "tag"
+    ]
+    context_resources = [
+        res for res in context_resources if res.get("prefect.resource.role") != "tag"
+    ]
+
+    # Add lineage group label to all resources
+    for res in upstream_resources + downstream_resources + context_resources:
+        if "prefect.resource.lineage-group" not in res:
+            res["prefect.resource.lineage-group"] = "global"
+
+    # For each context resource, emit an event showing it as downstream of upstream resources
+    if upstream_resources:
+        for context_resource in context_resources:
+            emit_kwargs: Dict[str, Any] = {
+                "event": "prefect.lineage.upstream-interaction",
+                "resource": context_resource,
+                "related": upstream_resources + tag_resources,
+            }
+            emit_event(**emit_kwargs)
+
+    # For each downstream resource, emit an event showing it as downstream of context resources
+    for downstream_resource in downstream_resources:
+        emit_kwargs: Dict[str, Any] = {
+            "event": "prefect.lineage.downstream-interaction",
+            "resource": downstream_resource,
+            "related": context_resources + tag_resources,
+        }
+        emit_event(**emit_kwargs)
+
+        # For each downstream resource, emit an event showing it as downstream of upstream resources
+        if upstream_resources:
+            direct_emit_kwargs = {
+                "event": event_name,
+                "resource": downstream_resource,
+                "related": upstream_resources + tag_resources,
+            }
+            emit_event(**direct_emit_kwargs)

prefect/_internal/concurrency/services.py

@@ -65,7 +65,7 @@ class _QueueServiceBase(abc.ABC, Generic[T]):
         # failure to process items. This is particularly relevant for services
         # which use an httpx client. See related issue at
         # https://github.com/python/cpython/issues/86813
-        threading._register_atexit(self._at_exit)  # pyright: ignore[reportUnknownVariableType, reportAttributeAccessIssue]
+        threading._register_atexit(self._at_exit)  # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
 
     def _at_exit(self) -> None:
         self.drain(at_exit=True)

prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2025-01-17T08:46:53-0800",
+ "date": "2025-01-30T11:31:29-0800",
  "dirty": true,
  "error": null,
- "full-revisionid": "16e85ce3c281778f5ab6487a73377eed63bcac8b",
- "version": "3.1.13"
+ "full-revisionid": "3ac3d54885a6157989efd79cbfc0d681b4bb7e0c",
+ "version": "3.1.15"
 }
 '''  # END VERSION_JSON