prefect-client 3.1.5__tar.gz → 3.1.7__tar.gz

{prefect-client-3.1.5 → prefect-client-3.1.7}/MANIFEST.in

@@ -25,9 +25,9 @@ include src/prefect/server/api/collections_data/views/*.json
 
 # Migrations
 include src/prefect/server/database/alembic.ini
-include src/prefect/server/database/migrations/*
-include src/prefect/server/database/migrations/versions/*
-include src/prefect/server/database/migrations/versions/*/*
+include src/prefect/server/database/_migrations/*
+include src/prefect/server/database/_migrations/versions/*
+include src/prefect/server/database/_migrations/versions/*/*
 
 # SQL templates
 graft src/prefect/server/database/sql

{prefect-client-3.1.5/src/prefect_client.egg-info → prefect-client-3.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.1.5
+Version: 3.1.7
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.
@@ -126,7 +126,9 @@ Start with our [friendly tutorial](https://docs.prefect.io/tutorials) or explore
 
 ## Join the community
 
-Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system. The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design. All community forums, including code contributions, issue discussions, and slack messages are subject to our [Code of Conduct](https://discourse.prefect.io/faq).
+Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system. 
+The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design. 
+All community forums, including code contributions, issue discussions, and Slack messages are subject to our [Code of Conduct](https://github.com/PrefectHQ/prefect/blob/main/CODE_OF_CONDUCT.md).
 
 ## Contribute
 

{prefect-client-3.1.5 → prefect-client-3.1.7}/README.md

@@ -98,7 +98,9 @@ Start with our [friendly tutorial](https://docs.prefect.io/tutorials) or explore
 
 ## Join the community
 
-Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system. The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design. All community forums, including code contributions, issue discussions, and slack messages are subject to our [Code of Conduct](https://discourse.prefect.io/faq).
+Prefect is made possible by the fastest growing community of thousands of friendly data engineers. Join us in building a new kind of workflow system. 
+The [Prefect Slack community](https://prefect.io/slack) is a fantastic place to learn more about Prefect, ask questions, or get help with workflow design. 
+All community forums, including code contributions, issue discussions, and Slack messages are subject to our [Code of Conduct](https://github.com/PrefectHQ/prefect/blob/main/CODE_OF_CONDUCT.md).
 
 ## Contribute
 

{prefect-client-3.1.5 → prefect-client-3.1.7}/requirements-client.txt

@@ -26,6 +26,7 @@ pydantic_extra_types >= 2.8.2, < 3.0.0
 pydantic_settings > 2.2.1
 python_dateutil >= 2.8.2, < 3.0.0
 python-slugify >= 5.0, < 9.0
+python-socks[asyncio] >= 2.5.3, < 3.0
 pyyaml >= 5.4.1, < 7.0.0
 rfc3339-validator >= 0.1.4, < 0.2.0
 rich >= 11.0, < 14.0

{prefect-client-3.1.5 → prefect-client-3.1.7}/setup.cfg

@@ -70,6 +70,7 @@ branch = True
 [coverage:report]
 exclude_lines = 
 	@(abc\.)?abstractmethod
+	if TYPE_CHECKING:
 omit = 
 	src/prefect/server/database/migrations/versions/*
 ignore_errors = True

{prefect-client-3.1.5 → prefect-client-3.1.7}/src/prefect/__init__.py

@@ -38,6 +38,7 @@ if TYPE_CHECKING:
         Transaction,
         unmapped,
         serve,
+        aserve,
         deploy,
         pause_flow_run,
         resume_flow_run,
@@ -66,6 +67,7 @@ _public_api: dict[str, tuple[str, str]] = {
     "Transaction": (__spec__.parent, ".main"),
     "unmapped": (__spec__.parent, ".main"),
     "serve": (__spec__.parent, ".main"),
+    "aserve": (__spec__.parent, ".main"),
     "deploy": (__spec__.parent, ".main"),
     "pause_flow_run": (__spec__.parent, ".main"),
     "resume_flow_run": (__spec__.parent, ".main"),
@@ -86,6 +88,7 @@ __all__ = [
     "Transaction",
     "unmapped",
     "serve",
+    "aserve",
     "deploy",
     "pause_flow_run",
     "resume_flow_run",

prefect-client-3.1.7/src/prefect/_experimental/lineage.py

@@ -0,0 +1,181 @@
+from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Sequence, Union
+
+from prefect.events.related import related_resources_from_run_context
+from prefect.events.schemas.events import RelatedResource, Resource
+from prefect.events.utilities import emit_event
+from prefect.settings import get_current_settings
+
+if TYPE_CHECKING:
+    from prefect.results import ResultStore
+
+UpstreamResources = Sequence[Union[RelatedResource, dict[str, str]]]
+DownstreamResources = Sequence[Union[Resource, dict[str, str]]]
+
+# Map block types to their URI schemes
+STORAGE_URI_SCHEMES = {
+    "local-file-system": "file://{path}",
+    "s3-bucket": "s3://{storage.bucket_name}/{path}",
+    "gcs-bucket": "gs://{storage.bucket}/{path}",
+    "azure-blob-storage": "azure-blob://{storage.container_name}/{path}",
+}
+
+
+def get_result_resource_uri(
+    store: "ResultStore",
+    key: str,
+) -> Optional[str]:
+    """
+    Generate a URI for a result based on its storage backend.
+
+    Args:
+        store: A `ResultStore` instance.
+        key: The key of the result to generate a URI for.
+    """
+    storage = store.result_storage
+    if storage is None:
+        return
+
+    path = store._resolved_key_path(key)
+
+    block_type = storage.get_block_type_slug()
+    if block_type and block_type in STORAGE_URI_SCHEMES:
+        return STORAGE_URI_SCHEMES[block_type].format(storage=storage, path=path)
+
+    # Generic fallback
+    return f"prefect://{block_type}/{path}"
+
+
+async def emit_lineage_event(
+    event_name: str,
+    upstream_resources: Optional[UpstreamResources] = None,
+    downstream_resources: Optional[DownstreamResources] = None,
+    direction_of_run_from_event: Literal["upstream", "downstream"] = "downstream",
+) -> None:
+    """Emit lineage events showing relationships between resources.
+
+    Args:
+        event_name: The name of the event to emit
+        upstream_resources: Optional list of RelatedResources that were upstream of
+            the event
+        downstream_resources: Optional list of Resources that were downstream
+            of the event
+        direction_of_run_from_event: The direction of the current run from
+            the event. E.g., if we're in a flow run and
+            `direction_of_run_from_event` is "downstream", then the flow run is
+            considered downstream of the resource's event.
+    """
+    from prefect.client.orchestration import get_client  # Avoid a circular import
+
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    upstream_resources = list(upstream_resources) if upstream_resources else []
+    downstream_resources = list(downstream_resources) if downstream_resources else []
+
+    async with get_client() as client:
+        related_resources = await related_resources_from_run_context(client)
+
+    # NOTE: We handle adding run-related resources to the event here instead of in
+    # the EventsWorker because not all run-related resources are upstream from
+    # every lineage event (they might be downstream). The EventsWorker only adds
+    # related resources to the "related" field in the event, which, for
+    # lineage-related events, tracks upstream resources only. For downstream
+    # resources, we need to emit an event for each downstream resource.
+    if direction_of_run_from_event == "downstream":
+        downstream_resources.extend(related_resources)
+    else:
+        upstream_resources.extend(related_resources)
+
+    # Emit an event for each downstream resource. This is necessary because
+    # our event schema allows one primary resource and many related resources,
+    # and for the purposes of lineage, related resources can only represent
+    # upstream resources.
+    for resource in downstream_resources:
+        # Downstream lineage resources need to have the
+        # prefect.resource.lineage-group label. All upstram resources from a
+        # downstream resource with this label will be considered lineage-related
+        # resources.
+        if "prefect.resource.lineage-group" not in resource:
+            resource["prefect.resource.lineage-group"] = "global"
+
+        emit_kwargs: Dict[str, Any] = {
+            "event": event_name,
+            "resource": resource,
+            "related": upstream_resources,
+        }
+
+        emit_event(**emit_kwargs)
+
+
+async def emit_result_read_event(
+    store: "ResultStore",
+    result_key: str,
+    downstream_resources: Optional[DownstreamResources] = None,
+    cached: bool = False,
+) -> None:
+    """
+    Emit a lineage event showing a task or flow result was read.
+
+    Args:
+        store: A `ResultStore` instance.
+        result_key: The key of the result to generate a URI for.
+        downstream_resources: List of resources that were
+            downstream of the event's resource.
+    """
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    result_resource_uri = get_result_resource_uri(store, result_key)
+    if result_resource_uri:
+        upstream_resources = [
+            RelatedResource(
+                root={
+                    "prefect.resource.id": result_resource_uri,
+                    "prefect.resource.role": "result",
+                }
+            )
+        ]
+        event_name = "prefect.result.read"
+        if cached:
+            event_name += ".cached"
+
+        await emit_lineage_event(
+            event_name=event_name,
+            upstream_resources=upstream_resources,
+            downstream_resources=downstream_resources,
+            direction_of_run_from_event="downstream",
+        )
+
+
+async def emit_result_write_event(
+    store: "ResultStore",
+    result_key: str,
+    upstream_resources: Optional[UpstreamResources] = None,
+) -> None:
+    """
+    Emit a lineage event showing a task or flow result was written.
+
+    Args:
+        store: A `ResultStore` instance.
+        result_key: The key of the result to generate a URI for.
+        upstream_resources: Optional list of resources that were
+            upstream of the event's resource.
+    """
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    result_resource_uri = get_result_resource_uri(store, result_key)
+    if result_resource_uri:
+        downstream_resources = [
+            {
+                "prefect.resource.id": result_resource_uri,
+                "prefect.resource.role": "result",
+                "prefect.resource.lineage-group": "global",
+            }
+        ]
+        await emit_lineage_event(
+            event_name="prefect.result.write",
+            upstream_resources=upstream_resources,
+            downstream_resources=downstream_resources,
+            direction_of_run_from_event="upstream",
+        )

{prefect-client-3.1.5 → prefect-client-3.1.7}/src/prefect/_internal/compatibility/async_dispatch.py

@@ -1,11 +1,12 @@
 import asyncio
 import inspect
 from functools import wraps
-from typing import Any, Callable, Coroutine, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Callable, Coroutine, Optional, TypeVar, Union
 
 from typing_extensions import ParamSpec
 
-from prefect.tasks import Task
+if TYPE_CHECKING:
+    from prefect.tasks import Task
 
 R = TypeVar("R")
 P = ParamSpec("P")
@@ -13,20 +14,46 @@ P = ParamSpec("P")
 
 def is_in_async_context() -> bool:
     """
-    Returns True if called from within an async context (coroutine or running event loop)
+    Returns True if called from within an async context.
+
+    An async context is one of:
+        - a coroutine
+        - a running event loop
+        - a task or flow that is async
     """
+    from prefect.context import get_run_context
+    from prefect.exceptions import MissingContextError
+
     try:
-        asyncio.get_running_loop()
-        return True
-    except RuntimeError:
-        return False
+        run_ctx = get_run_context()
+        parent_obj = getattr(run_ctx, "task", None)
+        if not parent_obj:
+            parent_obj = getattr(run_ctx, "flow", None)
+        return getattr(parent_obj, "isasync", True)
+    except MissingContextError:
+        # not in an execution context, make best effort to
+        # decide whether to syncify
+        try:
+            asyncio.get_running_loop()
+            return True
+        except RuntimeError:
+            return False
 
 
-def _is_acceptable_callable(obj: Union[Callable, Task]) -> bool:
+def _is_acceptable_callable(
+    obj: Union[Callable[P, R], "Task[P, R]", classmethod],
+) -> bool:
     if inspect.iscoroutinefunction(obj):
         return True
-    if isinstance(obj, Task) and inspect.iscoroutinefunction(obj.fn):
+
+    # Check if a task or flow. Need to avoid importing `Task` or `Flow` here
+    # due to circular imports.
+    if (fn := getattr(obj, "fn", None)) and inspect.iscoroutinefunction(fn):
         return True
+
+    if isinstance(obj, classmethod) and inspect.iscoroutinefunction(obj.__func__):
+        return True
+
     return False
 
 
@@ -56,6 +83,8 @@ def async_dispatch(
 
             if should_run_sync:
                 return sync_fn(*args, **kwargs)
+            if isinstance(async_impl, classmethod):
+                return async_impl.__func__(*args, **kwargs)
             return async_impl(*args, **kwargs)
 
         return wrapper  # type: ignore

{prefect-client-3.1.5 → prefect-client-3.1.7}/src/prefect/_internal/compatibility/migration.py

@@ -86,7 +86,7 @@ REMOVED_IN_V3 = {
 # See src/prefect/filesystems.py for an example
 
 
-def import_string_class_method(new_location: str) -> Callable:
+def import_string_class_method(new_location: str) -> Callable[..., Any]:
     """
     Handle moved class methods.
 

{prefect-client-3.1.5 → prefect-client-3.1.7}/src/prefect/_internal/concurrency/api.py

@@ -6,51 +6,48 @@ import abc
 import asyncio
 import concurrent.futures
 import contextlib
-from typing import (
-    Awaitable,
-    Callable,
-    ContextManager,
-    Iterable,
-    Optional,
-    TypeVar,
-    Union,
-)
+from collections.abc import Awaitable, Iterable
+from contextlib import AbstractContextManager
+from typing import Any, Callable, Optional, Union, cast
 
-from typing_extensions import ParamSpec
+from typing_extensions import ParamSpec, TypeAlias, TypeVar
 
 from prefect._internal.concurrency.threads import (
     WorkerThread,
     get_global_loop,
     in_global_loop,
 )
-from prefect._internal.concurrency.waiters import (
-    AsyncWaiter,
-    Call,
-    SyncWaiter,
-)
+from prefect._internal.concurrency.waiters import AsyncWaiter, Call, SyncWaiter
 
 P = ParamSpec("P")
-T = TypeVar("T")
-Future = Union[concurrent.futures.Future, asyncio.Future]
+T = TypeVar("T", infer_variance=True)
+Future = Union[concurrent.futures.Future[T], asyncio.Future[T]]
+
+_SyncOrAsyncCallable: TypeAlias = Callable[P, Union[T, Awaitable[T]]]
 
 
-def create_call(__fn: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> Call[T]:
-    return Call.new(__fn, *args, **kwargs)
+def create_call(
+    __fn: _SyncOrAsyncCallable[P, T], *args: P.args, **kwargs: P.kwargs
+) -> Call[T]:
+    return Call[T].new(__fn, *args, **kwargs)
 
 
-def _cast_to_call(call_like: Union[Callable[[], T], Call[T]]) -> Call[T]:
+def cast_to_call(
+    call_like: Union["_SyncOrAsyncCallable[[], T]", Call[T]],
+) -> Call[T]:
     if isinstance(call_like, Call):
-        return call_like
+        return cast(Call[T], call_like)
     else:
         return create_call(call_like)
 
 
 class _base(abc.ABC):
-    @abc.abstractstaticmethod
+    @staticmethod
+    @abc.abstractmethod
     def wait_for_call_in_loop_thread(
-        __call: Union[Callable[[], T], Call[T]],
+        __call: Union["_SyncOrAsyncCallable[[], Any]", Call[T]],
         timeout: Optional[float] = None,
-        done_callbacks: Optional[Iterable[Call]] = None,
+        done_callbacks: Optional[Iterable[Call[Any]]] = None,
     ) -> T:
         """
         Schedule a function in the global worker thread and wait for completion.
@@ -59,11 +56,12 @@ class _base(abc.ABC):
         """
         raise NotImplementedError()
 
-    @abc.abstractstaticmethod
+    @staticmethod
+    @abc.abstractmethod
     def wait_for_call_in_new_thread(
-        __call: Union[Callable[[], T], Call[T]],
+        __call: Union["_SyncOrAsyncCallable[[], T]", Call[T]],
         timeout: Optional[float] = None,
-        done_callbacks: Optional[Iterable[Call]] = None,
+        done_callbacks: Optional[Iterable[Call[Any]]] = None,
     ) -> T:
         """
         Schedule a function in a new worker thread.
@@ -74,14 +72,15 @@ class _base(abc.ABC):
 
     @staticmethod
     def call_soon_in_new_thread(
-        __call: Union[Callable[[], T], Call[T]], timeout: Optional[float] = None
+        __call: Union["_SyncOrAsyncCallable[[], T]", Call[T]],
+        timeout: Optional[float] = None,
     ) -> Call[T]:
         """
         Schedule a call for execution in a new worker thread.
 
         Returns the submitted call.
         """
-        call = _cast_to_call(__call)
+        call = cast_to_call(__call)
         runner = WorkerThread(run_once=True)
         call.set_timeout(timeout)
         runner.submit(call)
@@ -89,7 +88,7 @@ class _base(abc.ABC):
 
     @staticmethod
     def call_soon_in_loop_thread(
-        __call: Union[Callable[[], Awaitable[T]], Call[Awaitable[T]]],
+        __call: Union["_SyncOrAsyncCallable[[], T]", Call[T]],
         timeout: Optional[float] = None,
     ) -> Call[T]:
         """
@@ -97,7 +96,7 @@ class _base(abc.ABC):
 
         Returns the submitted call.
         """
-        call = _cast_to_call(__call)
+        call = cast_to_call(__call)
         runner = get_global_loop()
         call.set_timeout(timeout)
         runner.submit(call)
@@ -116,7 +115,7 @@ class _base(abc.ABC):
 
     @staticmethod
     def call_in_loop_thread(
-        __call: Union[Callable[[], Awaitable[T]], Call[Awaitable[T]]],
+        __call: Union[Callable[[], Awaitable[T]], Call[T]],
         timeout: Optional[float] = None,
     ) -> T:
         """
@@ -130,12 +129,12 @@ class _base(abc.ABC):
 class from_async(_base):
     @staticmethod
     async def wait_for_call_in_loop_thread(
-        __call: Union[Callable[[], Awaitable[T]], Call[Awaitable[T]]],
+        __call: Union[Callable[[], Awaitable[T]], Call[T]],
         timeout: Optional[float] = None,
-        done_callbacks: Optional[Iterable[Call]] = None,
-        contexts: Optional[Iterable[ContextManager]] = None,
-    ) -> Awaitable[T]:
-        call = _cast_to_call(__call)
+        done_callbacks: Optional[Iterable[Call[Any]]] = None,
+        contexts: Optional[Iterable[AbstractContextManager[Any]]] = None,
+    ) -> T:
+        call = cast_to_call(__call)
         waiter = AsyncWaiter(call)
         for callback in done_callbacks or []:
             waiter.add_done_callback(callback)
@@ -150,9 +149,9 @@ class from_async(_base):
     async def wait_for_call_in_new_thread(
         __call: Union[Callable[[], T], Call[T]],
         timeout: Optional[float] = None,
-        done_callbacks: Optional[Iterable[Call]] = None,
+        done_callbacks: Optional[Iterable[Call[Any]]] = None,
     ) -> T:
-        call = _cast_to_call(__call)
+        call = cast_to_call(__call)
         waiter = AsyncWaiter(call=call)
         for callback in done_callbacks or []:
             waiter.add_done_callback(callback)
@@ -169,7 +168,7 @@ class from_async(_base):
 
     @staticmethod
     def call_in_loop_thread(
-        __call: Union[Callable[[], Awaitable[T]], Call[Awaitable[T]]],
+        __call: Union[Callable[[], Awaitable[T]], Call[T]],
         timeout: Optional[float] = None,
     ) -> Awaitable[T]:
         call = _base.call_soon_in_loop_thread(__call, timeout=timeout)
@@ -181,13 +180,13 @@ class from_sync(_base):
     def wait_for_call_in_loop_thread(
         __call: Union[
             Callable[[], Awaitable[T]],
-            Union[Callable[[], Awaitable[T]], Call[Awaitable[T]]],
+            Call[T],
         ],
         timeout: Optional[float] = None,
-        done_callbacks: Optional[Iterable[Call]] = None,
-        contexts: Optional[Iterable[ContextManager]] = None,
-    ) -> Awaitable[T]:
-        call = _cast_to_call(__call)
+        done_callbacks: Optional[Iterable[Call[T]]] = None,
+        contexts: Optional[Iterable[AbstractContextManager[Any]]] = None,
+    ) -> T:
+        call = cast_to_call(__call)
         waiter = SyncWaiter(call)
         _base.call_soon_in_loop_thread(call, timeout=timeout)
         for callback in done_callbacks or []:
@@ -202,9 +201,9 @@ class from_sync(_base):
     def wait_for_call_in_new_thread(
         __call: Union[Callable[[], T], Call[T]],
         timeout: Optional[float] = None,
-        done_callbacks: Optional[Iterable[Call]] = None,
-    ) -> Call[T]:
-        call = _cast_to_call(__call)
+        done_callbacks: Optional[Iterable[Call[T]]] = None,
+    ) -> T:
+        call = cast_to_call(__call)
         waiter = SyncWaiter(call=call)
         for callback in done_callbacks or []:
             waiter.add_done_callback(callback)
@@ -214,20 +213,21 @@ class from_sync(_base):
 
     @staticmethod
     def call_in_new_thread(
-        __call: Union[Callable[[], T], Call[T]], timeout: Optional[float] = None
+        __call: Union["_SyncOrAsyncCallable[[], T]", Call[T]],
+        timeout: Optional[float] = None,
     ) -> T:
         call = _base.call_soon_in_new_thread(__call, timeout=timeout)
         return call.result()
 
     @staticmethod
     def call_in_loop_thread(
-        __call: Union[Callable[[], Awaitable[T]], Call[Awaitable[T]]],
+        __call: Union["_SyncOrAsyncCallable[[], T]", Call[T]],
         timeout: Optional[float] = None,
-    ) -> T:
+    ) -> Union[Awaitable[T], T]:
         if in_global_loop():
             # Avoid deadlock where the call is submitted to the loop then the loop is
             # blocked waiting for the call
-            call = _cast_to_call(__call)
+            call = cast_to_call(__call)
             return call()
 
         call = _base.call_soon_in_loop_thread(__call, timeout=timeout)