prefect-client 3.1.1__py3-none-any.whl → 3.1.3__py3-none-any.whl

prefect/_internal/compatibility/async_dispatch.py

@@ -1,73 +1,63 @@
 import asyncio
 import inspect
 from functools import wraps
-from typing import Any, Callable, Coroutine, Protocol, TypeVar, Union
+from typing import Any, Callable, Coroutine, Optional, TypeVar, Union
 
 from typing_extensions import ParamSpec
 
+from prefect.tasks import Task
+
 R = TypeVar("R")
 P = ParamSpec("P")
 
 
-class AsyncDispatchable(Protocol[P, R]):
-    """Protocol for functions decorated with async_dispatch."""
-
-    def __call__(
-        self, *args: P.args, **kwargs: P.kwargs
-    ) -> Union[R, Coroutine[Any, Any, R]]:
-        ...
-
-    aio: Callable[P, Coroutine[Any, Any, R]]
-    sync: Callable[P, R]
-
-
 def is_in_async_context() -> bool:
-    """Check if we're in an async context."""
+    """
+    Returns True if called from within an async context (coroutine or running event loop)
+    """
     try:
-        # First check if we're in a coroutine
-        if asyncio.current_task() is not None:
-            return True
-
-        # Check if we have a loop and it's running
-        loop = asyncio.get_event_loop()
-        return loop.is_running()
+        asyncio.get_running_loop()
+        return True
     except RuntimeError:
         return False
 
 
+def _is_acceptable_callable(obj: Union[Callable, Task]) -> bool:
+    if inspect.iscoroutinefunction(obj):
+        return True
+    if isinstance(obj, Task) and inspect.iscoroutinefunction(obj.fn):
+        return True
+    return False
+
+
 def async_dispatch(
     async_impl: Callable[P, Coroutine[Any, Any, R]],
-) -> Callable[[Callable[P, R]], AsyncDispatchable[P, R]]:
+) -> Callable[[Callable[P, R]], Callable[P, Union[R, Coroutine[Any, Any, R]]]]:
     """
-    Decorator that adds async compatibility to a sync function.
-
-    The decorated function will:
-    - Return a coroutine when in an async context (detected via running event loop)
-    - Run synchronously when in a sync context
-    - Provide .aio for explicit async access
-    - Provide .sync for explicit sync access
+    Decorator that dispatches to either sync or async implementation based on context.
 
     Args:
-        async_impl: The async implementation to dispatch to when async execution
-                   is needed
+        async_impl: The async implementation to dispatch to when in async context
     """
-    if not inspect.iscoroutinefunction(async_impl):
-        raise TypeError(
-            "async_impl must be an async function to dispatch in async contexts"
-        )
 
-    def decorator(sync_fn: Callable[P, R]) -> AsyncDispatchable[P, R]:
+    def decorator(
+        sync_fn: Callable[P, R],
+    ) -> Callable[P, Union[R, Coroutine[Any, Any, R]]]:
+        if not _is_acceptable_callable(async_impl):
+            raise TypeError("async_impl must be an async function")
+
         @wraps(sync_fn)
         def wrapper(
-            *args: P.args, **kwargs: P.kwargs
+            *args: P.args,
+            _sync: Optional[bool] = None,  # type: ignore
+            **kwargs: P.kwargs,
         ) -> Union[R, Coroutine[Any, Any, R]]:
-            if is_in_async_context():
-                return async_impl(*args, **kwargs)
-            return sync_fn(*args, **kwargs)
+            should_run_sync = _sync if _sync is not None else not is_in_async_context()
+
+            if should_run_sync:
+                return sync_fn(*args, **kwargs)
+            return async_impl(*args, **kwargs)
 
-        # Attach both async and sync implementations directly
-        wrapper.aio = async_impl
-        wrapper.sync = sync_fn
         return wrapper  # type: ignore
 
     return decorator

prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-11-08T12:38:16-0800",
+ "date": "2024-11-19T15:25:34-0600",
  "dirty": true,
  "error": null,
- "full-revisionid": "6b50a2b9f9d4ebf59703c55e1156c6f79151f1c3",
- "version": "3.1.1"
+ "full-revisionid": "39b6028cea9f2b0437546ba13cc08bb3bf6d94a4",
+ "version": "3.1.3"
 }
 '''  # END VERSION_JSON
 

prefect/blocks/notifications.py

@@ -494,6 +494,7 @@ class OpsgenieWebhook(AbstractAppriseNotificationBlock):
                 entity=self.entity,
                 batch=self.batch,
                 tags=self.tags,
+                action="new",
             ).url()
         )
         self._start_apprise_client(url)

prefect/cache_policies.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional,
 from typing_extensions import Self
 
 from prefect.context import TaskRunContext
+from prefect.exceptions import HashError
 from prefect.utilities.hashing import hash_objects
 
 if TYPE_CHECKING:
@@ -223,7 +224,6 @@ class TaskSource(CachePolicy):
                 lines = task_ctx.task.fn.__code__.co_code
             else:
                 raise
-
         return hash_objects(lines, raise_on_failure=True)
 
 
@@ -293,7 +293,18 @@ class Inputs(CachePolicy):
             if key not in exclude:
                 hashed_inputs[key] = val
 
-        return hash_objects(hashed_inputs, raise_on_failure=True)
+        try:
+            return hash_objects(hashed_inputs, raise_on_failure=True)
+        except HashError as exc:
+            msg = (
+                f"{exc}\n\n"
+                "This often occurs when task inputs contain objects that cannot be cached "
+                "like locks, file handles, or other system resources.\n\n"
+                "To resolve this, you can:\n"
+                "  1. Exclude these arguments by defining a custom `cache_key_fn`\n"
+                "  2. Disable caching by passing `cache_policy=NONE`\n"
+            )
+            raise ValueError(msg) from exc
 
     def __sub__(self, other: str) -> "CachePolicy":
         if not isinstance(other, str):

prefect/client/cloud.py

@@ -1,5 +1,6 @@
 import re
 from typing import Any, Dict, List, Optional, cast
+from uuid import UUID
 
 import anyio
 import httpx
@@ -12,6 +13,7 @@ from prefect.client.base import PrefectHttpxAsyncClient
 from prefect.client.schemas.objects import (
     IPAllowlist,
     IPAllowlistMyAccessResponse,
+    KeyValueLabels,
     Workspace,
 )
 from prefect.exceptions import ObjectNotFound, PrefectException
@@ -151,6 +153,25 @@ class CloudClient:
         response = await self.get(f"{self.account_base_url}/ip_allowlist/my_access")
         return IPAllowlistMyAccessResponse.model_validate(response)
 
+    async def update_flow_run_labels(
+        self, flow_run_id: UUID, labels: KeyValueLabels
+    ) -> httpx.Response:
+        """
+        Update the labels for a flow run.
+
+        Args:
+            flow_run_id: The identifier for the flow run to update.
+            labels: A dictionary of labels to update for the flow run.
+
+        Returns:
+            an `httpx.Response` object from the PATCH request
+        """
+
+        return await self._client.patch(
+            f"{self.workspace_base_url}/flow_runs/{flow_run_id}/labels",
+            json=labels,
+        )
+
     async def __aenter__(self):
         await self._client.__aenter__()
         return self

prefect/client/orchestration.py

@@ -2613,7 +2613,7 @@ class PrefectClient:
             "heartbeat_interval_seconds": heartbeat_interval_seconds,
         }
         if worker_metadata:
-            params["worker_metadata"] = worker_metadata.model_dump(mode="json")
+            params["metadata"] = worker_metadata.model_dump(mode="json")
         if get_worker_id:
             params["return_id"] = get_worker_id
 

prefect/client/schemas/objects.py

@@ -23,6 +23,9 @@ from pydantic import (
     HttpUrl,
     IPvAnyNetwork,
     SerializationInfo,
+    StrictBool,
+    StrictFloat,
+    StrictInt,
     Tag,
     field_validator,
     model_serializer,
@@ -68,6 +71,8 @@ if TYPE_CHECKING:
 
 R = TypeVar("R", default=Any)
 
+KeyValueLabels = dict[str, Union[StrictBool, StrictInt, StrictFloat, str]]
+
 
 DEFAULT_BLOCK_SCHEMA_VERSION = "non-versioned"
 DEFAULT_AGENT_WORK_POOL_NAME = "default-agent-pool"
@@ -492,6 +497,9 @@ class FlowRunPolicy(PrefectBaseModel):
     resuming: Optional[bool] = Field(
         default=False, description="Indicates if this run is resuming from a pause."
     )
+    retry_type: Optional[Literal["in_process", "reschedule"]] = Field(
+        default=None, description="The type of retry this run is undergoing."
+    )
 
     @model_validator(mode="before")
     @classmethod
@@ -555,6 +563,11 @@ class FlowRun(ObjectBaseModel):
         description="A list of tags on the flow run",
         examples=[["tag-1", "tag-2"]],
     )
+    labels: KeyValueLabels = Field(
+        default_factory=dict,
+        description="Prefect Cloud: A dictionary of key-value labels. Values can be strings, numbers, or booleans.",
+        examples=[{"key": "value1", "key2": 42}],
+    )
     parent_task_run_id: Optional[UUID] = Field(
         default=None,
         description=(

prefect/events/filters.py

@@ -83,17 +83,18 @@ class EventOccurredFilter(EventDataFilter):
 
 class EventNameFilter(EventDataFilter):
     prefix: Optional[List[str]] = Field(
-        None, description="Only include events matching one of these prefixes"
+        default=None, description="Only include events matching one of these prefixes"
     )
     exclude_prefix: Optional[List[str]] = Field(
-        None, description="Exclude events matching one of these prefixes"
+        default=None, description="Exclude events matching one of these prefixes"
     )
 
     name: Optional[List[str]] = Field(
-        None, description="Only include events matching one of these names exactly"
+        default=None,
+        description="Only include events matching one of these names exactly",
     )
     exclude_name: Optional[List[str]] = Field(
-        None, description="Exclude events matching one of these names exactly"
+        default=None, description="Exclude events matching one of these names exactly"
     )
 
     def includes(self, event: Event) -> bool:
@@ -230,17 +231,20 @@ class EventFilter(EventDataFilter):
         description="Filter criteria for when the events occurred",
     )
     event: Optional[EventNameFilter] = Field(
-        None,
+        default=None,
         description="Filter criteria for the event name",
     )
     any_resource: Optional[EventAnyResourceFilter] = Field(
-        None, description="Filter criteria for any resource involved in the event"
+        default=None,
+        description="Filter criteria for any resource involved in the event",
     )
     resource: Optional[EventResourceFilter] = Field(
-        None, description="Filter criteria for the resource of the event"
+        default=None,
+        description="Filter criteria for the resource of the event",
     )
     related: Optional[EventRelatedFilter] = Field(
-        None, description="Filter criteria for the related resources of the event"
+        default=None,
+        description="Filter criteria for the related resources of the event",
     )
     id: EventIDFilter = Field(
         default_factory=lambda: EventIDFilter(id=[]),
@@ -248,6 +252,6 @@ class EventFilter(EventDataFilter):
     )
 
     order: EventOrder = Field(
-        EventOrder.DESC,
+        default=EventOrder.DESC,
         description="The order to return filtered events",
     )

prefect/exceptions.py

@@ -443,3 +443,7 @@ class ProfileSettingsValidationError(PrefectException):
 
     def __init__(self, errors: List[Tuple[Any, ValidationError]]) -> None:
         self.errors = errors
+
+
+class HashError(PrefectException):
+    """Raised when hashing objects fails"""

prefect/flow_engine.py

@@ -22,8 +22,11 @@ from typing import (
 )
 from uuid import UUID
 
+from opentelemetry import trace
+from opentelemetry.trace import Tracer, get_tracer
 from typing_extensions import ParamSpec
 
+import prefect
 from prefect import Task
 from prefect.client.orchestration import SyncPrefectClient, get_client
 from prefect.client.schemas import FlowRun, TaskRun
@@ -124,6 +127,10 @@ class FlowRunEngine(Generic[P, R]):
     _client: Optional[SyncPrefectClient] = None
     short_circuit: bool = False
     _flow_run_name_set: bool = False
+    _tracer: Tracer = field(
+        default_factory=lambda: get_tracer("prefect", prefect.__version__)
+    )
+    _span: Optional[trace.Span] = None
 
     def __post_init__(self):
         if self.flow is None and self.flow_run_id is None:
@@ -233,6 +240,17 @@ class FlowRunEngine(Generic[P, R]):
         self.flow_run.state = state  # type: ignore
         self.flow_run.state_name = state.name  # type: ignore
         self.flow_run.state_type = state.type  # type: ignore
+
+        if self._span:
+            self._span.add_event(
+                state.name,
+                {
+                    "prefect.state.message": state.message or "",
+                    "prefect.state.type": state.type,
+                    "prefect.state.name": state.name or state.type,
+                    "prefect.state.id": str(state.id),
+                },
+            )
         return state
 
     def result(self, raise_on_failure: bool = True) -> "Union[R, State, None]":
@@ -281,6 +299,9 @@ class FlowRunEngine(Generic[P, R]):
         )
         self.set_state(terminal_state)
         self._return_value = resolved_result
+
+        self._end_span_on_success()
+
         return result
 
     def handle_exception(
@@ -311,6 +332,9 @@ class FlowRunEngine(Generic[P, R]):
             )
             state = self.set_state(Running())
         self._raised = exc
+
+        self._end_span_on_error(exc, state.message)
+
         return state
 
     def handle_timeout(self, exc: TimeoutError) -> None:
@@ -329,6 +353,8 @@ class FlowRunEngine(Generic[P, R]):
         self.set_state(state)
         self._raised = exc
 
+        self._end_span_on_error(exc, message)
+
     def handle_crash(self, exc: BaseException) -> None:
         state = run_coro_as_sync(exception_to_crashed_state(exc))
         self.logger.error(f"Crash detected! {state.message}")
@@ -336,6 +362,23 @@ class FlowRunEngine(Generic[P, R]):
         self.set_state(state, force=True)
         self._raised = exc
 
+        self._end_span_on_error(exc, state.message)
+
+    def _end_span_on_success(self):
+        if not self._span:
+            return
+        self._span.set_status(trace.Status(trace.StatusCode.OK))
+        self._span.end(time.time_ns())
+        self._span = None
+
+    def _end_span_on_error(self, exc: BaseException, description: Optional[str]):
+        if not self._span:
+            return
+        self._span.record_exception(exc)
+        self._span.set_status(trace.Status(trace.StatusCode.ERROR, description))
+        self._span.end(time.time_ns())
+        self._span = None
+
     def load_subflow_run(
         self,
         parent_task_run: TaskRun,
@@ -578,6 +621,18 @@ class FlowRunEngine(Generic[P, R]):
                     flow_version=self.flow.version,
                     empirical_policy=self.flow_run.empirical_policy,
                 )
+
+            self._span = self._tracer.start_span(
+                name=self.flow_run.name,
+                attributes={
+                    **self.flow_run.labels,
+                    "prefect.run.type": "flow",
+                    "prefect.run.id": str(self.flow_run.id),
+                    "prefect.tags": self.flow_run.tags,
+                    "prefect.flow.name": self.flow.name,
+                },
+            )
+
             try:
                 yield self
 
@@ -632,7 +687,7 @@ class FlowRunEngine(Generic[P, R]):
 
     @contextmanager
     def start(self) -> Generator[None, None, None]:
-        with self.initialize_run():
+        with self.initialize_run(), trace.use_span(self._span):
             self.begin_run()
 
             if self.state.is_running():

prefect/logging/handlers.py

@@ -32,7 +32,6 @@ from prefect.settings import (
     PREFECT_LOGGING_TO_API_BATCH_SIZE,
     PREFECT_LOGGING_TO_API_MAX_LOG_SIZE,
     PREFECT_LOGGING_TO_API_WHEN_MISSING_FLOW,
-    get_current_settings,
 )
 
 
@@ -241,10 +240,12 @@ class APILogHandler(logging.Handler):
 
 class WorkerAPILogHandler(APILogHandler):
     def emit(self, record: logging.LogRecord):
-        if get_current_settings().experiments.worker_logging_to_api_enabled:
-            super().emit(record)
-        else:
+        # Open-source API servers do not currently support worker logs, and
+        # worker logs only have an associated worker ID when connected to Cloud,
+        # so we won't send worker logs to the API unless they have a worker ID.
+        if not getattr(record, "worker_id", None):
             return
+        super().emit(record)
 
     def prepare(self, record: logging.LogRecord) -> Dict[str, Any]:
         """

prefect/settings/base.py

@@ -17,6 +17,7 @@ from pydantic_settings import (
 
 from prefect.settings.sources import (
     EnvFilterSettingsSource,
+    FilteredDotEnvSettingsSource,
     PrefectTomlConfigSettingsSource,
     ProfileSettingsTomlLoader,
     PyprojectTomlConfigSettingsSource,
@@ -43,13 +44,14 @@ class PrefectBaseSettings(BaseSettings):
         See https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources
         """
         env_filter = set()
-        for field in settings_cls.model_fields.values():
+        for field_name, field in settings_cls.model_fields.items():
             if field.validation_alias is not None and isinstance(
                 field.validation_alias, AliasChoices
             ):
                 for alias in field.validation_alias.choices:
                     if isinstance(alias, AliasPath) and len(alias.path) > 0:
                         env_filter.add(alias.path[0])
+            env_filter.add(field_name)
         return (
             init_settings,
             EnvFilterSettingsSource(
@@ -62,7 +64,18 @@ class PrefectBaseSettings(BaseSettings):
                 env_parse_enums=cls.model_config.get("env_parse_enums"),
                 env_filter=list(env_filter),
             ),
-            dotenv_settings,
+            FilteredDotEnvSettingsSource(
+                settings_cls,
+                env_file=cls.model_config.get("env_file"),
+                env_file_encoding=cls.model_config.get("env_file_encoding"),
+                case_sensitive=cls.model_config.get("case_sensitive"),
+                env_prefix=cls.model_config.get("env_prefix"),
+                env_nested_delimiter=cls.model_config.get("env_nested_delimiter"),
+                env_ignore_empty=cls.model_config.get("env_ignore_empty"),
+                env_parse_none_str=cls.model_config.get("env_parse_none_str"),
+                env_parse_enums=cls.model_config.get("env_parse_enums"),
+                env_blacklist=list(env_filter),
+            ),
             file_secret_settings,
             PrefectTomlConfigSettingsSource(settings_cls),
             PyprojectTomlConfigSettingsSource(settings_cls),
@@ -92,7 +105,7 @@ class PrefectBaseSettings(BaseSettings):
             elif (value := env.get(key)) is not None:
                 env_variables[
                     f"{self.model_config.get('env_prefix')}{key.upper()}"
-                ] = str(value)
+                ] = _to_environment_variable_value(value)
         return env_variables
 
     @model_serializer(
@@ -191,3 +204,9 @@ def _build_settings_config(
         pyproject_toml_table_header=("tool", "prefect", *path),
         json_schema_extra=_add_environment_variables,
     )
+
+
+def _to_environment_variable_value(value: Any) -> str:
+    if isinstance(value, (list, set, tuple)):
+        return ",".join(str(v) for v in value)
+    return str(value)

prefect/settings/models/experiments.py

@@ -18,11 +18,6 @@ class ExperimentsSettings(PrefectBaseSettings):
         ),
     )
 
-    worker_logging_to_api_enabled: bool = Field(
-        default=False,
-        description="Enables the logging of worker logs to Prefect Cloud.",
-    )
-
     telemetry_enabled: bool = Field(
         default=False,
         description="Enables sending telemetry to Prefect Cloud.",

prefect/settings/sources.py

@@ -10,10 +10,15 @@ from pydantic import AliasChoices
 from pydantic.fields import FieldInfo
 from pydantic_settings import (
     BaseSettings,
+    DotEnvSettingsSource,
     EnvSettingsSource,
     PydanticBaseSettingsSource,
 )
-from pydantic_settings.sources import ConfigFileSourceMixin
+from pydantic_settings.sources import (
+    ENV_FILE_SENTINEL,
+    ConfigFileSourceMixin,
+    DotenvType,
+)
 
 from prefect.settings.constants import DEFAULT_PREFECT_HOME, DEFAULT_PROFILES_PATH
 from prefect.utilities.collections import get_from_dict
@@ -61,6 +66,44 @@ class EnvFilterSettingsSource(EnvSettingsSource):
                 }
 
 
+class FilteredDotEnvSettingsSource(DotEnvSettingsSource):
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        env_file: Optional[DotenvType] = ENV_FILE_SENTINEL,
+        env_file_encoding: Optional[str] = None,
+        case_sensitive: Optional[bool] = None,
+        env_prefix: Optional[str] = None,
+        env_nested_delimiter: Optional[str] = None,
+        env_ignore_empty: Optional[bool] = None,
+        env_parse_none_str: Optional[str] = None,
+        env_parse_enums: Optional[bool] = None,
+        env_blacklist: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__(
+            settings_cls,
+            env_file,
+            env_file_encoding,
+            case_sensitive,
+            env_prefix,
+            env_nested_delimiter,
+            env_ignore_empty,
+            env_parse_none_str,
+            env_parse_enums,
+        )
+        self.env_blacklist = env_blacklist
+        if self.env_blacklist:
+            if isinstance(self.env_vars, dict):
+                for key in self.env_blacklist:
+                    self.env_vars.pop(key, None)
+            else:
+                self.env_vars = {
+                    key: value
+                    for key, value in self.env_vars.items()  # type: ignore
+                    if key.lower() not in env_blacklist
+                }
+
+
 class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
     """
     Custom pydantic settings source to load profile settings from a toml file.
@@ -111,21 +154,34 @@ class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
     ) -> Tuple[Any, str, bool]:
         """Concrete implementation to get the field value from the profile settings"""
         if field.validation_alias:
+            # Use validation alias as the key to ensure profile value does not
+            # higher priority sources. Lower priority sources that use the
+            # field name can override higher priority sources that use the
+            # validation alias as seen in https://github.com/PrefectHQ/prefect/issues/15981
             if isinstance(field.validation_alias, str):
                 value = self.profile_settings.get(field.validation_alias.upper())
                 if value is not None:
-                    return value, field_name, self.field_is_complex(field)
+                    return value, field.validation_alias, self.field_is_complex(field)
             elif isinstance(field.validation_alias, AliasChoices):
+                value = None
+                lowest_priority_alias = next(
+                    choice
+                    for choice in reversed(field.validation_alias.choices)
+                    if isinstance(choice, str)
+                )
                 for alias in field.validation_alias.choices:
                     if not isinstance(alias, str):
                         continue
                     value = self.profile_settings.get(alias.upper())
                     if value is not None:
-                        return value, field_name, self.field_is_complex(field)
-
-        value = self.profile_settings.get(
-            f"{self.config.get('env_prefix','')}{field_name.upper()}"
-        )
+                        return (
+                            value,
+                            lowest_priority_alias,
+                            self.field_is_complex(field),
+                        )
+
+        name = f"{self.config.get('env_prefix','')}{field_name.upper()}"
+        value = self.profile_settings.get(name)
         return value, field_name, self.field_is_complex(field)
 
     def __call__(self) -> Dict[str, Any]:
@@ -164,7 +220,22 @@ class TomlConfigSettingsSourceBase(PydanticBaseSettingsSource, ConfigFileSourceM
             # if the value is a dict, it is likely a nested settings object and a nested
             # source will handle it
             value = None
-        return value, field_name, self.field_is_complex(field)
+        name = field_name
+        # Use validation alias as the key to ensure profile value does not
+        # higher priority sources. Lower priority sources that use the
+        # field name can override higher priority sources that use the
+        # validation alias as seen in https://github.com/PrefectHQ/prefect/issues/15981
+        if value is not None:
+            if field.validation_alias and isinstance(field.validation_alias, str):
+                name = field.validation_alias
+            elif field.validation_alias and isinstance(
+                field.validation_alias, AliasChoices
+            ):
+                for alias in reversed(field.validation_alias.choices):
+                    if isinstance(alias, str):
+                        name = alias
+                        break
+        return value, name, self.field_is_complex(field)
 
     def __call__(self) -> Dict[str, Any]:
         """Called by pydantic to get the settings from our custom source"""