prefect-client 3.4.5.dev5__tar.gz → 3.4.6.dev2__tar.gz

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect-client
-Version: 3.4.5.dev5
+Version: 3.4.6.dev2
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -44,7 +44,7 @@ Requires-Dist: packaging<25.1,>=21.3
 Requires-Dist: pathspec>=0.8.0
 Requires-Dist: pendulum<4,>=3.0.0; python_version < '3.13'
 Requires-Dist: prometheus-client>=0.20.0
-Requires-Dist: pydantic!=2.10.0,<3.0.0,>=2.9
+Requires-Dist: pydantic!=2.10.0,!=2.11.0,!=2.11.1,!=2.11.2,!=2.11.3,!=2.11.4,<3.0.0,>=2.9
 Requires-Dist: pydantic-core<3.0.0,>=2.12.0
 Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
 Requires-Dist: pydantic-settings!=2.9.0,<3.0.0,>2.2.1

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/pyproject.toml

@@ -48,7 +48,7 @@ dependencies = [
     "pathspec>=0.8.0",
     "pendulum>=3.0.0,<4; python_version<'3.13'",
     "prometheus-client>=0.20.0",
-    "pydantic>=2.9,<3.0.0,!=2.10.0",
+    "pydantic>=2.9,<3.0.0,!=2.10.0,!=2.11.0,!=2.11.1,!=2.11.2,!=2.11.3,!=2.11.4",
     "pydantic_core>=2.12.0,<3.0.0",
     "pydantic_extra_types>=2.8.2,<3.0.0",
     "pydantic_settings>2.2.1,<3.0.0,!=2.9.0",

prefect_client-3.4.6.dev2/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.4.6.dev2"
+__build_date__ = "2025-06-11 08:09:26.576114+00:00"
+__git_commit__ = "4e82c8d6b8d529a8d8804a58781a68023465c7d4"
+__dirty__ = False

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/src/prefect/assets/core.py

@@ -47,6 +47,9 @@ class Asset(PrefectBaseModel):
     def __repr__(self) -> str:
         return f"Asset(key={self.key!r})"
 
+    def __hash__(self) -> int:
+        return hash(self.key)
+
     def add_metadata(self, metadata: dict[str, Any]) -> None:
         from prefect.context import AssetContext
 
@@ -59,7 +62,7 @@ class Asset(PrefectBaseModel):
         asset_ctx.add_asset_metadata(self.key, metadata)
 
 
-def add_asset_metadata(asset_key: str, metadata: dict[str, Any]) -> None:
+def add_asset_metadata(asset: str | Asset, metadata: dict[str, Any]) -> None:
     from prefect.context import AssetContext
 
     asset_ctx = AssetContext.get()
@@ -68,4 +71,5 @@ def add_asset_metadata(asset_key: str, metadata: dict[str, Any]) -> None:
             "Unable to call `add_asset_metadata` when not inside of an AssetContext"
         )
 
+    asset_key = asset if isinstance(asset, str) else asset.key
     asset_ctx.add_asset_metadata(asset_key, metadata)

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/src/prefect/context.py

@@ -400,7 +400,7 @@ class EngineContext(RunContext):
 
     # Tracking information needed to track asset linage between
     # tasks and materialization
-    task_run_assets: dict[UUID, list[Asset]] = Field(default_factory=dict)
+    task_run_assets: dict[UUID, set[Asset]] = Field(default_factory=dict)
 
     # Events worker to emit events
     events: Optional[EventsWorker] = None
@@ -486,12 +486,13 @@ class AssetContext(ContextModel):
         materialization_metadata: Metadata for materialized assets
     """
 
-    direct_asset_dependencies: list[Asset] = Field(default_factory=list)
-    downstream_assets: list[Asset] = Field(default_factory=list)
-    upstream_assets: list[Asset] = Field(default_factory=list)
+    direct_asset_dependencies: set[Asset] = Field(default_factory=set)
+    downstream_assets: set[Asset] = Field(default_factory=set)
+    upstream_assets: set[Asset] = Field(default_factory=set)
     materialized_by: Optional[str] = None
     task_run_id: Optional[UUID] = None
     materialization_metadata: dict[str, dict[str, Any]] = Field(default_factory=dict)
+    copy_to_child_ctx: bool = False
 
     __var__: ClassVar[ContextVar[Self]] = ContextVar("asset_context")
 
@@ -501,6 +502,7 @@ class AssetContext(ContextModel):
         task: "Task[Any, Any]",
         task_run_id: UUID,
         task_inputs: Optional[dict[str, set[Any]]] = None,
+        copy_to_child_ctx: bool = False,
     ) -> "AssetContext":
         """
         Create an AssetContext from a task and its resolved inputs.
@@ -509,6 +511,7 @@ class AssetContext(ContextModel):
             task: The task instance
             task_run_id: The task run ID
             task_inputs: The resolved task inputs (TaskRunResult objects)
+            copy_to_child_ctx: Whether this context should be copied on a child AssetContext
 
         Returns:
             Configured AssetContext
@@ -516,29 +519,35 @@ class AssetContext(ContextModel):
         from prefect.client.schemas import TaskRunResult
         from prefect.tasks import MaterializingTask
 
-        upstream_assets: list[Asset] = []
+        upstream_assets: set[Asset] = set()
 
-        # Get upstream assets from engine context instead of TaskRunResult.assets
         flow_ctx = FlowRunContext.get()
         if task_inputs and flow_ctx:
-            for inputs in task_inputs.values():
+            for name, inputs in task_inputs.items():
+                # Parent task runs are not dependencies
+                # that we want to track
+                if name == "__parents__":
+                    continue
+
                 for task_input in inputs:
                     if isinstance(task_input, TaskRunResult):
-                        # Look up assets in the engine context
                         task_assets = flow_ctx.task_run_assets.get(task_input.id)
                         if task_assets:
-                            upstream_assets.extend(task_assets)
+                            upstream_assets.update(task_assets)
 
         ctx = cls(
-            direct_asset_dependencies=task.asset_deps[:] if task.asset_deps else [],
-            downstream_assets=task.assets[:]
+            direct_asset_dependencies=set(task.asset_deps)
+            if task.asset_deps
+            else set(),
+            downstream_assets=set(task.assets)
             if isinstance(task, MaterializingTask) and task.assets
-            else [],
+            else set(),
             upstream_assets=upstream_assets,
             materialized_by=task.materialized_by
             if isinstance(task, MaterializingTask)
             else None,
             task_run_id=task_run_id,
+            copy_to_child_ctx=copy_to_child_ctx,
         )
         ctx.update_tracked_assets()
 
@@ -551,7 +560,15 @@ class AssetContext(ContextModel):
         Args:
             asset_key: The asset key
             metadata: Metadata dictionary to add
+
+        Raises:
+            ValueError: If asset_key is not in downstream_assets
         """
+        downstream_keys = {asset.key for asset in self.downstream_assets}
+        if asset_key not in downstream_keys:
+            raise ValueError(
+                "Can only add metadata to assets that are arguments to @materialize"
+            )
 
         existing = self.materialization_metadata.get(asset_key, {})
         self.materialization_metadata[asset_key] = existing | metadata
@@ -596,46 +613,46 @@ class AssetContext(ContextModel):
 
     def emit_events(self, state: State) -> None:
         """
-        Emit asset reference and materialization events based on task completion.
+        Emit asset events
         """
 
         from prefect.events import emit_event
 
         if state.name == "Cached":
             return
-        if state.is_failed():
+        elif state.is_failed():
             event_status = "failed"
         elif state.is_completed():
             event_status = "succeeded"
         else:
             return
 
-        asset_deps_related: list[Asset] = []
+        # If we have no downstream assets, this not a materialization
+        if not self.downstream_assets:
+            return
 
-        # Emit reference events for direct asset dependencies
-        for asset in self.direct_asset_dependencies:
+        # Emit reference events for all upstream assets (direct + inherited)
+        all_upstream_assets = self.upstream_assets | self.direct_asset_dependencies
+        for asset in all_upstream_assets:
             emit_event(
-                event=f"prefect.asset.reference.{event_status}",
+                event="prefect.asset.referenced",
                 resource=self.asset_as_resource(asset),
                 related=[],
             )
-            asset_deps_related.append(self.asset_as_related(asset))
 
         # Emit materialization events for downstream assets
-        if self.downstream_assets:
-            upstream_related = [self.asset_as_related(a) for a in self.upstream_assets]
-            all_related = upstream_related + asset_deps_related
-
-            if self.materialized_by:
-                all_related.append(self.related_materialized_by(self.materialized_by))
-
-            for asset in self.downstream_assets:
-                emit_event(
-                    event=f"prefect.asset.materialization.{event_status}",
-                    resource=self.asset_as_resource(asset),
-                    related=all_related,
-                    payload=self.materialization_metadata.get(asset.key),
-                )
+        upstream_related = [self.asset_as_related(a) for a in all_upstream_assets]
+
+        if self.materialized_by:
+            upstream_related.append(self.related_materialized_by(self.materialized_by))
+
+        for asset in self.downstream_assets:
+            emit_event(
+                event=f"prefect.asset.materialization.{event_status}",
+                resource=self.asset_as_resource(asset),
+                related=upstream_related,
+                payload=self.materialization_metadata.get(asset.key),
+            )
 
     def update_tracked_assets(self) -> None:
         """
@@ -649,11 +666,11 @@ class AssetContext(ContextModel):
 
         if self.downstream_assets:
             # MaterializingTask: propagate the downstream assets (what we create)
-            assets_for_downstream = self.downstream_assets[:]
+            assets_for_downstream = set(self.downstream_assets)
         else:
             # Regular task: propagate upstream assets + direct dependencies
-            assets_for_downstream = (
-                list(self.upstream_assets) + self.direct_asset_dependencies
+            assets_for_downstream = set(
+                self.upstream_assets | self.direct_asset_dependencies
             )
 
         flow_run_context.task_run_assets[self.task_run_id] = assets_for_downstream
@@ -661,6 +678,9 @@ class AssetContext(ContextModel):
     def serialize(self: Self, include_secrets: bool = True) -> dict[str, Any]:
         """Serialize the AssetContext for distributed execution."""
         return self.model_dump(
+            # use json serialization so fields that are
+            # sets of pydantic models are serialized
+            mode="json",
             exclude_unset=True,
             serialize_as_any=True,
             context={"include_secrets": include_secrets},

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/src/prefect/events/clients.py

@@ -628,7 +628,7 @@ class PrefectEventSubscriber:
         try:
             await self._reconnect()
         finally:
-            EVENT_WEBSOCKET_CONNECTIONS.labels(self.client_name, "out", "initial")
+            EVENT_WEBSOCKET_CONNECTIONS.labels(self.client_name, "out", "initial").inc()
         return self
 
     async def _reconnect(self) -> None:
@@ -709,7 +709,7 @@ class PrefectEventSubscriber:
                     finally:
                         EVENT_WEBSOCKET_CONNECTIONS.labels(
                             self.client_name, "out", "reconnect"
-                        )
+                        ).inc()
                     assert self._websocket
 
                 while True:

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/src/prefect/settings/profiles.py

@@ -1,17 +1,14 @@
 from __future__ import annotations
 
-import inspect
 import warnings
 from pathlib import Path
 from typing import (
     Annotated,
     Any,
     ClassVar,
-    Dict,
     Iterable,
     Iterator,
     Optional,
-    Union,
 )
 
 import toml
@@ -20,16 +17,15 @@ from pydantic import (
     BeforeValidator,
     ConfigDict,
     Field,
-    TypeAdapter,
     ValidationError,
 )
-from pydantic_settings import BaseSettings
 
 from prefect.exceptions import ProfileSettingsValidationError
 from prefect.settings.constants import DEFAULT_PROFILES_PATH
 from prefect.settings.context import get_current_settings
 from prefect.settings.legacy import Setting, _get_settings_fields
 from prefect.settings.models.root import Settings
+from prefect.utilities.collections import set_in_dict
 
 
 def _cast_settings(
@@ -69,7 +65,7 @@ class Profile(BaseModel):
     )
     source: Optional[Path] = None
 
-    def to_environment_variables(self) -> Dict[str, str]:
+    def to_environment_variables(self) -> dict[str, str]:
         """Convert the profile settings to a dictionary of environment variables."""
         return {
             setting.name: str(value)
@@ -78,23 +74,40 @@ class Profile(BaseModel):
         }
 
     def validate_settings(self) -> None:
-        errors: list[tuple[Setting, ValidationError]] = []
+        """
+        Validate all settings in this profile by creating a partial Settings object
+        with the nested structure properly constructed using accessor paths.
+        """
+        if not self.settings:
+            return
+
+        nested_settings: dict[str, Any] = {}
+
         for setting, value in self.settings.items():
-            try:
-                model_fields = Settings.model_fields
-                annotation = None
-                for section in setting.accessor.split("."):
-                    annotation = model_fields[section].annotation
-                    if inspect.isclass(annotation) and issubclass(
-                        annotation, BaseSettings
-                    ):
-                        model_fields = annotation.model_fields
-
-                TypeAdapter(annotation).validate_python(value)
-            except ValidationError as e:
-                errors.append((setting, e))
-        if errors:
-            raise ProfileSettingsValidationError(errors)
+            set_in_dict(nested_settings, setting.accessor, value)
+
+        try:
+            Settings.model_validate(nested_settings)
+        except ValidationError as e:
+            errors: list[tuple[Setting, ValidationError]] = []
+
+            for error in e.errors():
+                error_path = ".".join(str(loc) for loc in error["loc"])
+
+                for setting in self.settings.keys():
+                    if setting.accessor == error_path:
+                        errors.append(
+                            (
+                                setting,
+                                ValidationError.from_exception_data(
+                                    "ValidationError", [error]
+                                ),
+                            )
+                        )
+                        break
+
+            if errors:
+                raise ProfileSettingsValidationError(errors)
 
 
 class ProfilesCollection:
@@ -106,9 +119,7 @@ class ProfilesCollection:
     The collection may store the name of the active profile.
     """
 
-    def __init__(
-        self, profiles: Iterable[Profile], active: Optional[str] = None
-    ) -> None:
+    def __init__(self, profiles: Iterable[Profile], active: str | None = None) -> None:
         self.profiles_by_name: dict[str, Profile] = {
             profile.name: profile for profile in profiles
         }
@@ -122,7 +133,7 @@ class ProfilesCollection:
         return set(self.profiles_by_name.keys())
 
     @property
-    def active_profile(self) -> Optional[Profile]:
+    def active_profile(self) -> Profile | None:
         """
         Retrieve the active profile in this collection.
         """
@@ -130,7 +141,7 @@ class ProfilesCollection:
             return None
         return self[self.active_name]
 
-    def set_active(self, name: Optional[str], check: bool = True) -> None:
+    def set_active(self, name: str | None, check: bool = True) -> None:
         """
         Set the active profile name in the collection.
 
@@ -145,7 +156,7 @@ class ProfilesCollection:
         self,
         name: str,
         settings: dict[Setting, Any],
-        source: Optional[Path] = None,
+        source: Path | None = None,
     ) -> Profile:
         """
         Add a profile to the collection or update the existing on if the name is already
@@ -201,7 +212,7 @@ class ProfilesCollection:
         """
         self.profiles_by_name.pop(name)
 
-    def without_profile_source(self, path: Optional[Path]) -> "ProfilesCollection":
+    def without_profile_source(self, path: Path | None) -> "ProfilesCollection":
         """
         Remove profiles that were loaded from a given path.
 
@@ -367,7 +378,7 @@ def load_profile(name: str) -> Profile:
 
 
 def update_current_profile(
-    settings: Dict[Union[str, Setting], Any],
+    settings: dict[str | Setting, Any],
 ) -> Profile:
     """
     Update the persisted data for the profile currently in-use.

{prefect_client-3.4.5.dev5 → prefect_client-3.4.6.dev2}/src/prefect/task_engine.py

@@ -454,8 +454,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 result = state.data
 
             link_state_to_result(new_state, result)
-            if asset_context := AssetContext.get():
-                asset_context.emit_events(new_state)
 
         # emit a state change event
         self._last_event = emit_task_run_state_change_event(
@@ -641,15 +639,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
             else:
                 persist_result = should_persist_result()
 
-            asset_context = AssetContext.get()
-            if not asset_context:
-                asset_context = AssetContext.from_task_and_inputs(
-                    task=self.task,
-                    task_run_id=self.task_run.id,
-                    task_inputs=self.task_run.task_inputs,
-                )
-                stack.enter_context(asset_context)
-
             stack.enter_context(
                 TaskRunContext(
                     task=self.task,
@@ -672,6 +661,24 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
 
             yield
 
+    @contextmanager
+    def asset_context(self):
+        parent_asset_ctx = AssetContext.get()
+
+        if parent_asset_ctx and parent_asset_ctx.copy_to_child_ctx:
+            asset_ctx = parent_asset_ctx.model_copy()
+            asset_ctx.copy_to_child_ctx = False
+        else:
+            asset_ctx = AssetContext.from_task_and_inputs(
+                self.task, self.task_run.id, self.task_run.task_inputs
+            )
+
+        with asset_ctx as ctx:
+            try:
+                yield
+            finally:
+                ctx.emit_events(self.state)
+
     @contextmanager
     def initialize_run(
         self,
@@ -1032,8 +1039,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 result = new_state.data
 
             link_state_to_result(new_state, result)
-            if asset_context := AssetContext.get():
-                asset_context.emit_events(new_state)
 
         # emit a state change event
         self._last_event = emit_task_run_state_change_event(
@@ -1219,15 +1224,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
             else:
                 persist_result = should_persist_result()
 
-            asset_context = AssetContext.get()
-            if not asset_context:
-                asset_context = AssetContext.from_task_and_inputs(
-                    task=self.task,
-                    task_run_id=self.task_run.id,
-                    task_inputs=self.task_run.task_inputs,
-                )
-                stack.enter_context(asset_context)
-
             stack.enter_context(
                 TaskRunContext(
                     task=self.task,
@@ -1249,6 +1245,24 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
 
             yield
 
+    @asynccontextmanager
+    async def asset_context(self):
+        parent_asset_ctx = AssetContext.get()
+
+        if parent_asset_ctx and parent_asset_ctx.copy_to_child_ctx:
+            asset_ctx = parent_asset_ctx.model_copy()
+            asset_ctx.copy_to_child_ctx = False
+        else:
+            asset_ctx = AssetContext.from_task_and_inputs(
+                self.task, self.task_run.id, self.task_run.task_inputs
+            )
+
+        with asset_ctx as ctx:
+            try:
+                yield
+            finally:
+                ctx.emit_events(self.state)
+
     @asynccontextmanager
     async def initialize_run(
         self,
@@ -1465,7 +1479,11 @@ def run_task_sync(
     with engine.start(task_run_id=task_run_id, dependencies=dependencies):
         while engine.is_running():
             run_coro_as_sync(engine.wait_until_ready())
-            with engine.run_context(), engine.transaction_context() as txn:
+            with (
+                engine.asset_context(),
+                engine.run_context(),
+                engine.transaction_context() as txn,
+            ):
                 engine.call_task_fn(txn)
 
     return engine.state if return_type == "state" else engine.result()
@@ -1492,7 +1510,11 @@ async def run_task_async(
     async with engine.start(task_run_id=task_run_id, dependencies=dependencies):
         while engine.is_running():
             await engine.wait_until_ready()
-            async with engine.run_context(), engine.transaction_context() as txn:
+            async with (
+                engine.asset_context(),
+                engine.run_context(),
+                engine.transaction_context() as txn,
+            ):
                 await engine.call_task_fn(txn)
 
     return engine.state if return_type == "state" else await engine.result()
@@ -1522,7 +1544,11 @@ def run_generator_task_sync(
     with engine.start(task_run_id=task_run_id, dependencies=dependencies):
         while engine.is_running():
             run_coro_as_sync(engine.wait_until_ready())
-            with engine.run_context(), engine.transaction_context() as txn:
+            with (
+                engine.asset_context(),
+                engine.run_context(),
+                engine.transaction_context() as txn,
+            ):
                 # TODO: generators should default to commit_mode=OFF
                 # because they are dynamic by definition
                 # for now we just prevent this branch explicitly
@@ -1576,7 +1602,11 @@ async def run_generator_task_async(
     async with engine.start(task_run_id=task_run_id, dependencies=dependencies):
         while engine.is_running():
             await engine.wait_until_ready()
-            async with engine.run_context(), engine.transaction_context() as txn:
+            async with (
+                engine.asset_context(),
+                engine.run_context(),
+                engine.transaction_context() as txn,
+            ):
                 # TODO: generators should default to commit_mode=OFF
                 # because they are dynamic by definition
                 # for now we just prevent this branch explicitly