prefect 3.4.5.dev4__tar.gz → 3.4.5.dev5__tar.gz

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.4.5.dev4
+Version: 3.4.5.dev5
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -154,11 +154,11 @@ Description-Content-Type: text/markdown
         Quickstart
     </a>
     ·
-    <a href="https://docs.prefect.io/v3/develop/index?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
+    <a href="https://docs.prefect.io/v3/how-to-guides/workflows/write-and-run?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
         Build workflows
     </a>
     ·
-    <a href="https://docs.prefect.io/v3/deploy/index?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
+    <a href="https://docs.prefect.io/v3/concepts/deployments?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
         Deploy workflows
     </a>
     ·

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/README.md

@@ -32,11 +32,11 @@
         Quickstart
     </a>
     ·
-    <a href="https://docs.prefect.io/v3/develop/index?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
+    <a href="https://docs.prefect.io/v3/how-to-guides/workflows/write-and-run?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
         Build workflows
     </a>
     ·
-    <a href="https://docs.prefect.io/v3/deploy/index?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
+    <a href="https://docs.prefect.io/v3/concepts/deployments?utm_source=oss&utm_medium=oss&utm_campaign=oss_gh_repo&utm_term=none&utm_content=none">
         Deploy workflows
     </a>
     ·

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/pyproject.toml

@@ -192,6 +192,7 @@ markdown-docs = [
     "prefect[gitlab]",
     "prefect[kubernetes]",
     "prefect[ray]",
+    "prefect[redis]",
     "prefect[shell]",
     "prefect[slack]",
     "prefect[snowflake]",

prefect-3.4.5.dev5/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.4.5.dev5"
+__build_date__ = "2025-06-06 08:09:50.121651+00:00"
+__git_commit__ = "6fd843adf3aa002e8907b0ba1b96be0af8259b2d"
+__dirty__ = False

prefect-3.4.5.dev5/src/prefect/assets/__init__.py

@@ -0,0 +1,4 @@
+from prefect.assets.core import Asset, AssetProperties, add_asset_metadata
+from prefect.assets.materialize import materialize
+
+__all__ = ["Asset", "AssetProperties", "materialize", "add_asset_metadata"]

prefect-3.4.5.dev5/src/prefect/assets/core.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+from typing import Any, ClassVar, Optional
+
+from pydantic import ConfigDict, Field
+
+from prefect._internal.schemas.bases import PrefectBaseModel
+from prefect.types import URILike
+
+
+class AssetProperties(PrefectBaseModel):
+    """
+    Metadata properties to configure on an Asset
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(frozen=True)
+
+    name: Optional[str] = Field(
+        default=None, description="Human readable name of the Asset."
+    )
+    url: Optional[str] = Field(
+        default=None, description="Visitable url to view the Asset."
+    )
+    description: Optional[str] = Field(
+        default=None, description="Description of the Asset."
+    )
+    owners: Optional[list[str]] = Field(
+        default=None, description="Owners of the Asset."
+    )
+
+
+class Asset(PrefectBaseModel):
+    """
+    Assets are objects that represent materialized data,
+    providing a way to track lineage and dependencies.
+    """
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(frozen=True)
+
+    key: URILike
+    properties: Optional[AssetProperties] = Field(
+        default=None,
+        description="Properties of the asset. "
+        "Setting this will overwrite properties of a known asset.",
+    )
+
+    def __repr__(self) -> str:
+        return f"Asset(key={self.key!r})"
+
+    def add_metadata(self, metadata: dict[str, Any]) -> None:
+        from prefect.context import AssetContext
+
+        asset_ctx = AssetContext.get()
+        if not asset_ctx:
+            raise RuntimeError(
+                "Unable add Asset metadata when not inside of an AssetContext"
+            )
+
+        asset_ctx.add_asset_metadata(self.key, metadata)
+
+
+def add_asset_metadata(asset_key: str, metadata: dict[str, Any]) -> None:
+    from prefect.context import AssetContext
+
+    asset_ctx = AssetContext.get()
+    if not asset_ctx:
+        raise RuntimeError(
+            "Unable to call `add_asset_metadata` when not inside of an AssetContext"
+        )
+
+    asset_ctx.add_asset_metadata(asset_key, metadata)

prefect-3.4.5.dev5/src/prefect/assets/materialize.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Callable, TypeVar, Union
+
+from typing_extensions import ParamSpec, Unpack
+
+from .core import Asset
+
+T = TypeVar("T")
+P = ParamSpec("P")
+R = TypeVar("R")
+
+if TYPE_CHECKING:
+    from prefect.tasks import MaterializingTask, TaskOptions
+
+
+def materialize(
+    *assets: Union[str, Asset],
+    by: str | None = None,
+    **task_kwargs: Unpack[TaskOptions],
+) -> Callable[[Callable[P, R]], MaterializingTask[P, R]]:
+    """
+    Decorator for materializing assets.
+
+    Args:
+        *assets: Assets to materialize
+        by: An optional tool that is ultimately responsible for materializing the asset e.g. "dbt" or "spark"
+        **task_kwargs: Additional task configuration
+    """
+    if not assets:
+        raise TypeError(
+            "materialize requires at least one asset argument, e.g. `@materialize(asset)`"
+        )
+
+    from prefect.tasks import MaterializingTask
+
+    def decorator(fn: Callable[P, R]) -> MaterializingTask[P, R]:
+        return MaterializingTask(
+            fn=fn, assets=assets, materialized_by=by, **task_kwargs
+        )
+
+    return decorator

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/src/prefect/context.py

@@ -7,13 +7,23 @@ For more user-accessible information about the current run, see [`prefect.runtim
 """
 
 import asyncio
+import json
 import os
 import sys
 import warnings
 from collections.abc import AsyncGenerator, Generator, Mapping
 from contextlib import ExitStack, asynccontextmanager, contextmanager
 from contextvars import ContextVar, Token
-from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    ClassVar,
+    Optional,
+    TypeVar,
+    Union,
+)
+from uuid import UUID
 
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 from typing_extensions import Self
@@ -21,6 +31,7 @@ from typing_extensions import Self
 import prefect.settings
 import prefect.types._datetime
 from prefect._internal.compatibility.migration import getattr_migration
+from prefect.assets import Asset
 from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_client
 from prefect.client.schemas import FlowRun, TaskRun
 from prefect.events.worker import EventsWorker
@@ -48,9 +59,15 @@ if TYPE_CHECKING:
     from prefect.tasks import Task
 
 
-def serialize_context() -> dict[str, Any]:
+def serialize_context(
+    asset_ctx_kwargs: Union[dict[str, Any], None] = None,
+) -> dict[str, Any]:
     """
     Serialize the current context for use in a remote execution environment.
+
+    Optionally provide asset_ctx_kwargs to create new AssetContext, that will be used
+    in the remote execution environment. This is useful for TaskRunners, who rely on creating the
+    task run in the remote environment.
     """
     flow_run_context = EngineContext.get()
     task_run_context = TaskRunContext.get()
@@ -62,6 +79,11 @@ def serialize_context() -> dict[str, Any]:
         "task_run_context": task_run_context.serialize() if task_run_context else {},
         "tags_context": tags_context.serialize() if tags_context else {},
         "settings_context": settings_context.serialize() if settings_context else {},
+        "asset_context": AssetContext.from_task_and_inputs(
+            **asset_ctx_kwargs
+        ).serialize()
+        if asset_ctx_kwargs
+        else {},
     }
 
 
@@ -112,6 +134,9 @@ def hydrated_context(
             # Set up tags context
             if tags_context := serialized_context.get("tags_context"):
                 stack.enter_context(tags(*tags_context["current_tags"]))
+            # Set up asset context
+            if asset_context := serialized_context.get("asset_context"):
+                stack.enter_context(AssetContext(**asset_context))
         yield
 
 
@@ -373,6 +398,10 @@ class EngineContext(RunContext):
     # Holds the ID of the object returned by the task run and task run state
     task_run_results: dict[int, State] = Field(default_factory=dict)
 
+    # Tracking information needed to track asset linage between
+    # tasks and materialization
+    task_run_assets: dict[UUID, list[Asset]] = Field(default_factory=dict)
+
     # Events worker to emit events
     events: Optional[EventsWorker] = None
 
@@ -443,6 +472,201 @@ class TaskRunContext(RunContext):
         )
 
 
+class AssetContext(ContextModel):
+    """
+    The asset context for a materializing task run. Contains all asset-related information needed
+    for asset event emission and downstream asset dependency propagation.
+
+    Attributes:
+        direct_asset_dependencies: Assets that this task directly depends on (from task.asset_deps)
+        downstream_assets: Assets that this task will create/materialize (from MaterializingTask.assets)
+        upstream_assets: Assets from upstream task dependencies
+        materialized_by: Tool that materialized the assets (from MaterializingTask.materialized_by)
+        task_run_id: ID of the associated task run
+        materialization_metadata: Metadata for materialized assets
+    """
+
+    direct_asset_dependencies: list[Asset] = Field(default_factory=list)
+    downstream_assets: list[Asset] = Field(default_factory=list)
+    upstream_assets: list[Asset] = Field(default_factory=list)
+    materialized_by: Optional[str] = None
+    task_run_id: Optional[UUID] = None
+    materialization_metadata: dict[str, dict[str, Any]] = Field(default_factory=dict)
+
+    __var__: ClassVar[ContextVar[Self]] = ContextVar("asset_context")
+
+    @classmethod
+    def from_task_and_inputs(
+        cls,
+        task: "Task[Any, Any]",
+        task_run_id: UUID,
+        task_inputs: Optional[dict[str, set[Any]]] = None,
+    ) -> "AssetContext":
+        """
+        Create an AssetContext from a task and its resolved inputs.
+
+        Args:
+            task: The task instance
+            task_run_id: The task run ID
+            task_inputs: The resolved task inputs (TaskRunResult objects)
+
+        Returns:
+            Configured AssetContext
+        """
+        from prefect.client.schemas import TaskRunResult
+        from prefect.tasks import MaterializingTask
+
+        upstream_assets: list[Asset] = []
+
+        # Get upstream assets from engine context instead of TaskRunResult.assets
+        flow_ctx = FlowRunContext.get()
+        if task_inputs and flow_ctx:
+            for inputs in task_inputs.values():
+                for task_input in inputs:
+                    if isinstance(task_input, TaskRunResult):
+                        # Look up assets in the engine context
+                        task_assets = flow_ctx.task_run_assets.get(task_input.id)
+                        if task_assets:
+                            upstream_assets.extend(task_assets)
+
+        ctx = cls(
+            direct_asset_dependencies=task.asset_deps[:] if task.asset_deps else [],
+            downstream_assets=task.assets[:]
+            if isinstance(task, MaterializingTask) and task.assets
+            else [],
+            upstream_assets=upstream_assets,
+            materialized_by=task.materialized_by
+            if isinstance(task, MaterializingTask)
+            else None,
+            task_run_id=task_run_id,
+        )
+        ctx.update_tracked_assets()
+
+        return ctx
+
+    def add_asset_metadata(self, asset_key: str, metadata: dict[str, Any]) -> None:
+        """
+        Add metadata for a materialized asset.
+
+        Args:
+            asset_key: The asset key
+            metadata: Metadata dictionary to add
+        """
+
+        existing = self.materialization_metadata.get(asset_key, {})
+        self.materialization_metadata[asset_key] = existing | metadata
+
+    @staticmethod
+    def asset_as_resource(asset: Asset) -> dict[str, str]:
+        """Convert Asset to event resource format."""
+        resource = {"prefect.resource.id": asset.key}
+
+        if asset.properties:
+            properties_dict = asset.properties.model_dump(exclude_unset=True)
+
+            if "name" in properties_dict:
+                resource["prefect.resource.name"] = properties_dict["name"]
+
+            if "description" in properties_dict:
+                resource["prefect.asset.description"] = properties_dict["description"]
+
+            if "url" in properties_dict:
+                resource["prefect.asset.url"] = properties_dict["url"]
+
+            if "owners" in properties_dict:
+                resource["prefect.asset.owners"] = json.dumps(properties_dict["owners"])
+
+        return resource
+
+    @staticmethod
+    def asset_as_related(asset: Asset) -> dict[str, str]:
+        """Convert Asset to event related format."""
+        return {
+            "prefect.resource.id": asset.key,
+            "prefect.resource.role": "asset",
+        }
+
+    @staticmethod
+    def related_materialized_by(by: str) -> dict[str, str]:
+        """Create a related resource for the tool that performed the materialization"""
+        return {
+            "prefect.resource.id": by,
+            "prefect.resource.role": "asset-materialized-by",
+        }
+
+    def emit_events(self, state: State) -> None:
+        """
+        Emit asset reference and materialization events based on task completion.
+        """
+
+        from prefect.events import emit_event
+
+        if state.name == "Cached":
+            return
+        if state.is_failed():
+            event_status = "failed"
+        elif state.is_completed():
+            event_status = "succeeded"
+        else:
+            return
+
+        asset_deps_related: list[Asset] = []
+
+        # Emit reference events for direct asset dependencies
+        for asset in self.direct_asset_dependencies:
+            emit_event(
+                event=f"prefect.asset.reference.{event_status}",
+                resource=self.asset_as_resource(asset),
+                related=[],
+            )
+            asset_deps_related.append(self.asset_as_related(asset))
+
+        # Emit materialization events for downstream assets
+        if self.downstream_assets:
+            upstream_related = [self.asset_as_related(a) for a in self.upstream_assets]
+            all_related = upstream_related + asset_deps_related
+
+            if self.materialized_by:
+                all_related.append(self.related_materialized_by(self.materialized_by))
+
+            for asset in self.downstream_assets:
+                emit_event(
+                    event=f"prefect.asset.materialization.{event_status}",
+                    resource=self.asset_as_resource(asset),
+                    related=all_related,
+                    payload=self.materialization_metadata.get(asset.key),
+                )
+
+    def update_tracked_assets(self) -> None:
+        """
+        Update the flow run context with assets that should be propagated downstream.
+        """
+        if not (flow_run_context := FlowRunContext.get()):
+            return
+
+        if not self.task_run_id:
+            return
+
+        if self.downstream_assets:
+            # MaterializingTask: propagate the downstream assets (what we create)
+            assets_for_downstream = self.downstream_assets[:]
+        else:
+            # Regular task: propagate upstream assets + direct dependencies
+            assets_for_downstream = (
+                list(self.upstream_assets) + self.direct_asset_dependencies
+            )
+
+        flow_run_context.task_run_assets[self.task_run_id] = assets_for_downstream
+
+    def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
+        """Serialize the AssetContext for distributed execution."""
+        return self.model_dump(
+            exclude_unset=True,
+            serialize_as_any=True,
+            context={"include_secrets": include_secrets},
+        )
+
+
 class TagsContext(ContextModel):
     """
     The context for `prefect.tags` management.

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/src/prefect/deployments/runner.py

@@ -1222,14 +1222,14 @@ async def deploy(
                 " or specify a remote storage location for the flow with `.from_source`."
                 " If you are attempting to deploy a flow to a local process work pool,"
                 " consider using `flow.serve` instead. See the documentation for more"
-                " information: https://docs.prefect.io/latest/deploy/run-flows-in-local-processes"
+                " information: https://docs.prefect.io/latest/how-to-guides/deployments/run-flows-in-local-processes"
             )
         elif work_pool.type == "process" and not ignore_warnings:
             console.print(
                 "Looks like you're deploying to a process work pool. If you're creating a"
                 " deployment for local development, calling `.serve` on your flow is a great"
                 " way to get started. See the documentation for more information:"
-                " https://docs.prefect.io/latest/deploy/run-flows-in-local-processes "
+                " https://docs.prefect.io/latest/how-to-guides/deployments/run-flows-in-local-processes "
                 " Set `ignore_warnings=True` to suppress this message.",
                 style="yellow",
             )

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/src/prefect/runner/server.py

@@ -257,7 +257,7 @@ def _build_generic_endpoint_for_flows(
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 async def build_server(runner: "Runner") -> FastAPI:
     """
@@ -306,7 +306,7 @@ async def build_server(runner: "Runner") -> FastAPI:
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 def start_webserver(runner: "Runner", log_level: str | None = None) -> None:
     """

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/src/prefect/runner/submit.py

@@ -124,7 +124,7 @@ def submit_to_runner(
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 @sync_compatible
 async def submit_to_runner(
@@ -196,7 +196,7 @@ async def submit_to_runner(
 @deprecated_callable(
     start_date=datetime(2025, 4, 1),
     end_date=datetime(2025, 10, 1),
-    help="Use background tasks (https://docs.prefect.io/v3/develop/deferred-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
+    help="Use background tasks (https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks) or `run_deployment` and `.serve` instead of submitting runs to the Runner webserver.",
 )
 @sync_compatible
 async def wait_for_submitted_runs(

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/src/prefect/server/api/events.py

@@ -51,7 +51,7 @@ async def create_events(
     """
     Record a batch of Events.
 
-    For more information, see https://docs.prefect.io/v3/automate/events/events.
+    For more information, see https://docs.prefect.io/v3/concepts/events.
     """
     if ephemeral_request:
         await EventsPipeline().process_events(events)

{prefect-3.4.5.dev4 → prefect-3.4.5.dev5}/src/prefect/server/api/task_workers.py

@@ -23,7 +23,7 @@ async def read_task_workers(
     """
     Read active task workers. Optionally filter by task keys.
 
-    For more information, see https://docs.prefect.io/v3/develop/deferred-tasks.
+    For more information, see https://docs.prefect.io/v3/concepts/flows-and-tasks#background-tasks.
     """
 
     if task_worker_filter and task_worker_filter.task_keys: