prefect-client 3.4.5.dev4__py3-none-any.whl → 3.4.6.dev1__py3-none-any.whl

prefect/_build_info.py

@@ -1,5 +1,5 @@
 # Generated by versioningit
-__version__ = "3.4.5.dev4"
-__build_date__ = "2025-06-05 08:09:20.805818+00:00"
-__git_commit__ = "92cd0a62e663cace1fdbb654f3533c368622d8c6"
+__version__ = "3.4.6.dev1"
+__build_date__ = "2025-06-10 08:09:30.415238+00:00"
+__git_commit__ = "cd3c98b5dbb76efc127c1ee052df55873c1723dc"
 __dirty__ = False

prefect/assets/__init__.py

@@ -0,0 +1,4 @@
+from prefect.assets.core import Asset, AssetProperties, add_asset_metadata
+from prefect.assets.materialize import materialize
+
+__all__ = ["Asset", "AssetProperties", "materialize", "add_asset_metadata"]

prefect/assets/core.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar, Optional
+
+from pydantic import ConfigDict, Field
+
+from prefect._internal.schemas.bases import PrefectBaseModel
+from prefect.types import URILike
+
+
+class AssetProperties(PrefectBaseModel):
+    """
+    Metadata properties to configure on an Asset
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(frozen=True)
+
+    name: Optional[str] = Field(
+        default=None, description="Human readable name of the Asset."
+    )
+    url: Optional[str] = Field(
+        default=None, description="Visitable url to view the Asset."
+    )
+    description: Optional[str] = Field(
+        default=None, description="Description of the Asset."
+    )
+    owners: Optional[list[str]] = Field(
+        default=None, description="Owners of the Asset."
+    )
+
+
+class Asset(PrefectBaseModel):
+    """
+    Assets are objects that represent materialized data,
+    providing a way to track lineage and dependencies.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(frozen=True)
+
+    key: URILike
+    properties: Optional[AssetProperties] = Field(
+        default=None,
+        description="Properties of the asset. "
+        "Setting this will overwrite properties of a known asset.",
+    )
+
+    def __repr__(self) -> str:
+        return f"Asset(key={self.key!r})"
+
+    def __hash__(self) -> int:
+        return hash(self.key)
+
+    def add_metadata(self, metadata: dict[str, Any]) -> None:
+        from prefect.context import AssetContext
+
+        asset_ctx = AssetContext.get()
+        if not asset_ctx:
+            raise RuntimeError(
+                "Unable add Asset metadata when not inside of an AssetContext"
+            )
+
+        asset_ctx.add_asset_metadata(self.key, metadata)
+
+
+def add_asset_metadata(asset: str | Asset, metadata: dict[str, Any]) -> None:
+    from prefect.context import AssetContext
+
+    asset_ctx = AssetContext.get()
+    if not asset_ctx:
+        raise RuntimeError(
+            "Unable to call `add_asset_metadata` when not inside of an AssetContext"
+        )
+
+    asset_key = asset if isinstance(asset, str) else asset.key
+    asset_ctx.add_asset_metadata(asset_key, metadata)

prefect/assets/materialize.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable, TypeVar, Union
+
+from typing_extensions import ParamSpec, Unpack
+
+from .core import Asset
+
+T = TypeVar("T")
+P = ParamSpec("P")
+R = TypeVar("R")
+
+if TYPE_CHECKING:
+    from prefect.tasks import MaterializingTask, TaskOptions
+
+
+def materialize(
+    *assets: Union[str, Asset],
+    by: str | None = None,
+    **task_kwargs: Unpack[TaskOptions],
+) -> Callable[[Callable[P, R]], MaterializingTask[P, R]]:
+    """
+    Decorator for materializing assets.
+
+    Args:
+        *assets: Assets to materialize
+        by: An optional tool that is ultimately responsible for materializing the asset e.g. "dbt" or "spark"
+        **task_kwargs: Additional task configuration
+    """
+    if not assets:
+        raise TypeError(
+            "materialize requires at least one asset argument, e.g. `@materialize(asset)`"
+        )
+
+    from prefect.tasks import MaterializingTask
+
+    def decorator(fn: Callable[P, R]) -> MaterializingTask[P, R]:
+        return MaterializingTask(
+            fn=fn, assets=assets, materialized_by=by, **task_kwargs
+        )
+
+    return decorator

prefect/context.py

@@ -7,13 +7,23 @@ For more user-accessible information about the current run, see [`prefect.runtim
 """
 
 import asyncio
+import json
 import os
 import sys
 import warnings
 from collections.abc import AsyncGenerator, Generator, Mapping
 from contextlib import ExitStack, asynccontextmanager, contextmanager
 from contextvars import ContextVar, Token
-from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ClassVar,
+    Optional,
+    TypeVar,
+    Union,
+)
+from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 from typing_extensions import Self
@@ -21,6 +31,7 @@ from typing_extensions import Self
 import prefect.settings
 import prefect.types._datetime
 from prefect._internal.compatibility.migration import getattr_migration
+from prefect.assets import Asset
 from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
 from prefect.client.schemas import FlowRun, TaskRun
 from prefect.events.worker import EventsWorker
@@ -48,9 +59,15 @@ if TYPE_CHECKING:
     from prefect.tasks import Task
 
 
-def serialize_context() -> dict[str, Any]:
+def serialize_context(
+    asset_ctx_kwargs: Union[dict[str, Any], None] = None,
+) -> dict[str, Any]:
     """
     Serialize the current context for use in a remote execution environment.
+
+    Optionally provide asset_ctx_kwargs to create new AssetContext, that will be used
+    in the remote execution environment. This is useful for TaskRunners, who rely on creating the
+    task run in the remote environment.
     """
     flow_run_context = EngineContext.get()
     task_run_context = TaskRunContext.get()
@@ -62,6 +79,11 @@ def serialize_context() -> dict[str, Any]:
         "task_run_context": task_run_context.serialize() if task_run_context else {},
         "tags_context": tags_context.serialize() if tags_context else {},
         "settings_context": settings_context.serialize() if settings_context else {},
+        "asset_context": AssetContext.from_task_and_inputs(
+            **asset_ctx_kwargs
+        ).serialize()
+        if asset_ctx_kwargs
+        else {},
     }
 
 
@@ -112,6 +134,9 @@ def hydrated_context(
             # Set up tags context
             if tags_context := serialized_context.get("tags_context"):
                 stack.enter_context(tags(*tags_context["current_tags"]))
+            # Set up asset context
+            if asset_context := serialized_context.get("asset_context"):
+                stack.enter_context(AssetContext(**asset_context))
         yield
 
 
@@ -373,6 +398,10 @@ class EngineContext(RunContext):
     # Holds the ID of the object returned by the task run and task run state
     task_run_results: dict[int, State] = Field(default_factory=dict)
 
+    # Tracking information needed to track asset linage between
+    # tasks and materialization
+    task_run_assets: dict[UUID, set[Asset]] = Field(default_factory=dict)
+
     # Events worker to emit events
     events: Optional[EventsWorker] = None
 
@@ -443,6 +472,214 @@ class TaskRunContext(RunContext):
         )
 
 
+class AssetContext(ContextModel):
+    """
+    The asset context for a materializing task run. Contains all asset-related information needed
+    for asset event emission and downstream asset dependency propagation.
+
+    Attributes:
+        direct_asset_dependencies: Assets that this task directly depends on (from task.asset_deps)
+        downstream_assets: Assets that this task will create/materialize (from MaterializingTask.assets)
+        upstream_assets: Assets from upstream task dependencies
+        materialized_by: Tool that materialized the assets (from MaterializingTask.materialized_by)
+        task_run_id: ID of the associated task run
+        materialization_metadata: Metadata for materialized assets
+    """
+
+    direct_asset_dependencies: set[Asset] = Field(default_factory=set)
+    downstream_assets: set[Asset] = Field(default_factory=set)
+    upstream_assets: set[Asset] = Field(default_factory=set)
+    materialized_by: Optional[str] = None
+    task_run_id: Optional[UUID] = None
+    materialization_metadata: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+    __var__: ClassVar[ContextVar[Self]] = ContextVar("asset_context")
+
+    @classmethod
+    def from_task_and_inputs(
+        cls,
+        task: "Task[Any, Any]",
+        task_run_id: UUID,
+        task_inputs: Optional[dict[str, set[Any]]] = None,
+    ) -> "AssetContext":
+        """
+        Create an AssetContext from a task and its resolved inputs.
+
+        Args:
+            task: The task instance
+            task_run_id: The task run ID
+            task_inputs: The resolved task inputs (TaskRunResult objects)
+
+        Returns:
+            Configured AssetContext
+        """
+        from prefect.client.schemas import TaskRunResult
+        from prefect.tasks import MaterializingTask
+
+        upstream_assets: set[Asset] = set()
+
+        # Get upstream assets from engine context instead of TaskRunResult.assets
+        flow_ctx = FlowRunContext.get()
+        if task_inputs and flow_ctx:
+            for inputs in task_inputs.values():
+                for task_input in inputs:
+                    if isinstance(task_input, TaskRunResult):
+                        # Look up assets in the engine context
+                        task_assets = flow_ctx.task_run_assets.get(task_input.id)
+                        if task_assets:
+                            upstream_assets.update(task_assets)
+
+        ctx = cls(
+            direct_asset_dependencies=set(task.asset_deps)
+            if task.asset_deps
+            else set(),
+            downstream_assets=set(task.assets)
+            if isinstance(task, MaterializingTask) and task.assets
+            else set(),
+            upstream_assets=upstream_assets,
+            materialized_by=task.materialized_by
+            if isinstance(task, MaterializingTask)
+            else None,
+            task_run_id=task_run_id,
+        )
+        ctx.update_tracked_assets()
+
+        return ctx
+
+    def add_asset_metadata(self, asset_key: str, metadata: dict[str, Any]) -> None:
+        """
+        Add metadata for a materialized asset.
+
+        Args:
+            asset_key: The asset key
+            metadata: Metadata dictionary to add
+
+        Raises:
+            ValueError: If asset_key is not in downstream_assets
+        """
+        downstream_keys = {asset.key for asset in self.downstream_assets}
+        if asset_key not in downstream_keys:
+            raise ValueError(
+                "Can only add metadata to assets that are arguments to @materialize"
+            )
+
+        existing = self.materialization_metadata.get(asset_key, {})
+        self.materialization_metadata[asset_key] = existing | metadata
+
+    @staticmethod
+    def asset_as_resource(asset: Asset) -> dict[str, str]:
+        """Convert Asset to event resource format."""
+        resource = {"prefect.resource.id": asset.key}
+
+        if asset.properties:
+            properties_dict = asset.properties.model_dump(exclude_unset=True)
+
+            if "name" in properties_dict:
+                resource["prefect.resource.name"] = properties_dict["name"]
+
+            if "description" in properties_dict:
+                resource["prefect.asset.description"] = properties_dict["description"]
+
+            if "url" in properties_dict:
+                resource["prefect.asset.url"] = properties_dict["url"]
+
+            if "owners" in properties_dict:
+                resource["prefect.asset.owners"] = json.dumps(properties_dict["owners"])
+
+        return resource
+
+    @staticmethod
+    def asset_as_related(asset: Asset) -> dict[str, str]:
+        """Convert Asset to event related format."""
+        return {
+            "prefect.resource.id": asset.key,
+            "prefect.resource.role": "asset",
+        }
+
+    @staticmethod
+    def related_materialized_by(by: str) -> dict[str, str]:
+        """Create a related resource for the tool that performed the materialization"""
+        return {
+            "prefect.resource.id": by,
+            "prefect.resource.role": "asset-materialized-by",
+        }
+
+    def emit_events(self, state: State) -> None:
+        """
+        Emit asset events
+        """
+
+        from prefect.events import emit_event
+
+        if state.name == "Cached":
+            return
+        elif state.is_failed():
+            event_status = "failed"
+        elif state.is_completed():
+            event_status = "succeeded"
+        else:
+            return
+
+        # If we have no downstream assets, this not a materialization
+        if not self.downstream_assets:
+            return
+
+        # Emit reference events for all upstream assets (direct + inherited)
+        all_upstream_assets = self.upstream_assets | self.direct_asset_dependencies
+        for asset in all_upstream_assets:
+            emit_event(
+                event="prefect.asset.referenced",
+                resource=self.asset_as_resource(asset),
+                related=[],
+            )
+
+        # Emit materialization events for downstream assets
+        upstream_related = [self.asset_as_related(a) for a in all_upstream_assets]
+
+        if self.materialized_by:
+            upstream_related.append(self.related_materialized_by(self.materialized_by))
+
+        for asset in self.downstream_assets:
+            emit_event(
+                event=f"prefect.asset.materialization.{event_status}",
+                resource=self.asset_as_resource(asset),
+                related=upstream_related,
+                payload=self.materialization_metadata.get(asset.key),
+            )
+
+    def update_tracked_assets(self) -> None:
+        """
+        Update the flow run context with assets that should be propagated downstream.
+        """
+        if not (flow_run_context := FlowRunContext.get()):
+            return
+
+        if not self.task_run_id:
+            return
+
+        if self.downstream_assets:
+            # MaterializingTask: propagate the downstream assets (what we create)
+            assets_for_downstream = set(self.downstream_assets)
+        else:
+            # Regular task: propagate upstream assets + direct dependencies
+            assets_for_downstream = set(
+                self.upstream_assets | self.direct_asset_dependencies
+            )
+
+        flow_run_context.task_run_assets[self.task_run_id] = assets_for_downstream
+
+    def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
+        """Serialize the AssetContext for distributed execution."""
+        return self.model_dump(
+            # use json serialization so fields that are
+            # sets of pydantic models are serialized
+            mode="json",
+            exclude_unset=True,
+            serialize_as_any=True,
+            context={"include_secrets": include_secrets},
+        )
+
+
 class TagsContext(ContextModel):
     """
     The context for `prefect.tags` management.

prefect/deployments/runner.py

@@ -1222,14 +1222,14 @@ async def deploy(
                 " or specify a remote storage location for the flow with `.from_source`."
                 " If you are attempting to deploy a flow to a local process work pool,"
                 " consider using `flow.serve` instead. See the documentation for more"
-                " information: https://docs.prefect.io/latest/deploy/run-flows-in-local-processes"
+                " information: https://docs.prefect.io/latest/how-to-guides/deployments/run-flows-in-local-processes"
             )
         elif work_pool.type == "process" and not ignore_warnings:
             console.print(
                 "Looks like you're deploying to a process work pool. If you're creating a"
                 " deployment for local development, calling `.serve` on your flow is a great"
                 " way to get started. See the documentation for more information:"
-                " https://docs.prefect.io/latest/deploy/run-flows-in-local-processes "
+                " https://docs.prefect.io/latest/how-to-guides/deployments/run-flows-in-local-processes "
                 " Set `ignore_warnings=True` to suppress this message.",
                 style="yellow",
             )

prefect/events/clients.py

@@ -628,7 +628,7 @@ class PrefectEventSubscriber:
         try:
             await self._reconnect()
         finally:
-            EVENT_WEBSOCKET_CONNECTIONS.labels(self.client_name, "out", "initial")
+            EVENT_WEBSOCKET_CONNECTIONS.labels(self.client_name, "out", "initial").inc()
         return self
 
     async def _reconnect(self) -> None:
@@ -709,7 +709,7 @@ class PrefectEventSubscriber:
                     finally:
                         EVENT_WEBSOCKET_CONNECTIONS.labels(
                             self.client_name, "out", "reconnect"
-                        )
+                        ).inc()
                     assert self._websocket
 
                 while True:

prefect/runner/server.py

@@ -257,7 +257,7 @@ def _build_generic_endpoint_for_flows(
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 async def build_server(runner: "Runner") -> FastAPI:
     """
@@ -306,7 +306,7 @@ async def build_server(runner: "Runner") -> FastAPI:
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 def start_webserver(runner: "Runner", log_level: str | None = None) -> None:
     """

prefect/runner/submit.py

@@ -124,7 +124,7 @@ def submit_to_runner(
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 @sync_compatible
 async def submit_to_runner(
@@ -196,7 +196,7 @@ async def submit_to_runner(
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 @sync_compatible
 async def wait_for_submitted_runs(

prefect/server/api/events.py

@@ -51,7 +51,7 @@ async def create_events(
     """
     Record a batch of Events.
 
-    For more information, see https://docs.prefect.io/v3/automate/events/events.
+    For more information, see https://docs.prefect.io/v3/concepts/events.
     """
     if ephemeral_request:
         await EventsPipeline().process_events(events)

prefect/server/api/task_workers.py

@@ -23,7 +23,7 @@ async def read_task_workers(
     """
     Read active task workers. Optionally filter by task keys.
 
-    For more information, see https://docs.prefect.io/v3/develop/deferred-tasks.
+    For more information, see https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks.
     """
 
     if task_worker_filter and task_worker_filter.task_keys:

prefect/settings/profiles.py

@@ -1,17 +1,14 @@
 from __future__ import annotations
 
-import inspect
 import warnings
 from pathlib import Path
 from typing import (
     Annotated,
     Any,
     ClassVar,
-    Dict,
     Iterable,
     Iterator,
     Optional,
-    Union,
 )
 
 import toml
@@ -20,16 +17,15 @@ from pydantic import (
     BeforeValidator,
     ConfigDict,
     Field,
-    TypeAdapter,
     ValidationError,
 )
-from pydantic_settings import BaseSettings
 
 from prefect.exceptions import ProfileSettingsValidationError
 from prefect.settings.constants import DEFAULT_PROFILES_PATH
 from prefect.settings.context import get_current_settings
 from prefect.settings.legacy import Setting, _get_settings_fields
 from prefect.settings.models.root import Settings
+from prefect.utilities.collections import set_in_dict
 
 
 def _cast_settings(
@@ -69,7 +65,7 @@ class Profile(BaseModel):
     )
     source: Optional[Path] = None
 
-    def to_environment_variables(self) -> Dict[str, str]:
+    def to_environment_variables(self) -> dict[str, str]:
         """Convert the profile settings to a dictionary of environment variables."""
         return {
             setting.name: str(value)
@@ -78,23 +74,40 @@ class Profile(BaseModel):
         }
 
     def validate_settings(self) -> None:
-        errors: list[tuple[Setting, ValidationError]] = []
+        """
+        Validate all settings in this profile by creating a partial Settings object
+        with the nested structure properly constructed using accessor paths.
+        """
+        if not self.settings:
+            return
+
+        nested_settings: dict[str, Any] = {}
+
         for setting, value in self.settings.items():
-            try:
-                model_fields = Settings.model_fields
-                annotation = None
-                for section in setting.accessor.split("."):
-                    annotation = model_fields[section].annotation
-                    if inspect.isclass(annotation) and issubclass(
-                        annotation, BaseSettings
-                    ):
-                        model_fields = annotation.model_fields
-
-                TypeAdapter(annotation).validate_python(value)
-            except ValidationError as e:
-                errors.append((setting, e))
-        if errors:
-            raise ProfileSettingsValidationError(errors)
+            set_in_dict(nested_settings, setting.accessor, value)
+
+        try:
+            Settings.model_validate(nested_settings)
+        except ValidationError as e:
+            errors: list[tuple[Setting, ValidationError]] = []
+
+            for error in e.errors():
+                error_path = ".".join(str(loc) for loc in error["loc"])
+
+                for setting in self.settings.keys():
+                    if setting.accessor == error_path:
+                        errors.append(
+                            (
+                                setting,
+                                ValidationError.from_exception_data(
+                                    "ValidationError", [error]
+                                ),
+                            )
+                        )
+                        break
+
+            if errors:
+                raise ProfileSettingsValidationError(errors)
 
 
 class ProfilesCollection:
@@ -106,9 +119,7 @@ class ProfilesCollection:
     The collection may store the name of the active profile.
     """
 
-    def __init__(
-        self, profiles: Iterable[Profile], active: Optional[str] = None
-    ) -> None:
+    def __init__(self, profiles: Iterable[Profile], active: str | None = None) -> None:
         self.profiles_by_name: dict[str, Profile] = {
             profile.name: profile for profile in profiles
         }
@@ -122,7 +133,7 @@ class ProfilesCollection:
         return set(self.profiles_by_name.keys())
 
     @property
-    def active_profile(self) -> Optional[Profile]:
+    def active_profile(self) -> Profile | None:
         """
         Retrieve the active profile in this collection.
         """
@@ -130,7 +141,7 @@ class ProfilesCollection:
             return None
         return self[self.active_name]
 
-    def set_active(self, name: Optional[str], check: bool = True) -> None:
+    def set_active(self, name: str | None, check: bool = True) -> None:
         """
         Set the active profile name in the collection.
 
@@ -145,7 +156,7 @@ class ProfilesCollection:
         self,
         name: str,
         settings: dict[Setting, Any],
-        source: Optional[Path] = None,
+        source: Path | None = None,
     ) -> Profile:
         """
         Add a profile to the collection or update the existing on if the name is already
@@ -201,7 +212,7 @@ class ProfilesCollection:
         """
         self.profiles_by_name.pop(name)
 
-    def without_profile_source(self, path: Optional[Path]) -> "ProfilesCollection":
+    def without_profile_source(self, path: Path | None) -> "ProfilesCollection":
         """
         Remove profiles that were loaded from a given path.
 
@@ -367,7 +378,7 @@ def load_profile(name: str) -> Profile:
 
 
 def update_current_profile(
-    settings: Dict[Union[str, Setting], Any],
+    settings: dict[str | Setting, Any],
 ) -> Profile:
     """
     Update the persisted data for the profile currently in-use.