prefect-client 3.0.10__py3-none-any.whl → 3.0.11__py3-none-any.whl

prefect/settings/sources.py

@@ -0,0 +1,162 @@
+import os
+import sys
+import warnings
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Type
+
+import toml
+from pydantic import AliasChoices
+from pydantic.fields import FieldInfo
+from pydantic_settings import (
+    BaseSettings,
+    EnvSettingsSource,
+    PydanticBaseSettingsSource,
+)
+
+from prefect.settings.constants import DEFAULT_PREFECT_HOME, DEFAULT_PROFILES_PATH
+
+
+class EnvFilterSettingsSource(EnvSettingsSource):
+    """
+    Custom pydantic settings source to filter out specific environment variables.
+
+    All validation aliases are loaded from environment variables by default. We use
+    `AliasPath` to maintain the ability set fields via model initialization, but those
+    shouldn't be loaded from environment variables. This loader allows use to say which
+    environment variables should be ignored.
+    """
+
+    def __init__(
+        self,
+        settings_cls: type[BaseSettings],
+        case_sensitive: Optional[bool] = None,
+        env_prefix: Optional[str] = None,
+        env_nested_delimiter: Optional[str] = None,
+        env_ignore_empty: Optional[bool] = None,
+        env_parse_none_str: Optional[str] = None,
+        env_parse_enums: Optional[bool] = None,
+        env_filter: Optional[List[str]] = None,
+    ) -> None:
+        super().__init__(
+            settings_cls,
+            case_sensitive,
+            env_prefix,
+            env_nested_delimiter,
+            env_ignore_empty,
+            env_parse_none_str,
+            env_parse_enums,
+        )
+        if env_filter:
+            if isinstance(self.env_vars, dict):
+                for key in env_filter:
+                    self.env_vars.pop(key, None)
+            else:
+                self.env_vars = {
+                    key: value
+                    for key, value in self.env_vars.items()
+                    if key.lower() not in env_filter
+                }
+
+
+class ProfileSettingsTomlLoader(PydanticBaseSettingsSource):
+    """
+    Custom pydantic settings source to load profile settings from a toml file.
+
+    See https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources
+    """
+
+    def __init__(self, settings_cls: Type[BaseSettings]):
+        super().__init__(settings_cls)
+        self.settings_cls = settings_cls
+        self.profiles_path = _get_profiles_path()
+        self.profile_settings = self._load_profile_settings()
+
+    def _load_profile_settings(self) -> Dict[str, Any]:
+        """Helper method to load the profile settings from the profiles.toml file"""
+
+        if not self.profiles_path.exists():
+            return {}
+
+        try:
+            all_profile_data = toml.load(self.profiles_path)
+        except toml.TomlDecodeError:
+            warnings.warn(
+                f"Failed to load profiles from {self.profiles_path}. Please ensure the file is valid TOML."
+            )
+            return {}
+
+        if (
+            sys.argv[0].endswith("/prefect")
+            and len(sys.argv) >= 3
+            and sys.argv[1] == "--profile"
+        ):
+            active_profile = sys.argv[2]
+
+        else:
+            active_profile = os.environ.get("PREFECT_PROFILE") or all_profile_data.get(
+                "active"
+            )
+
+        profiles_data = all_profile_data.get("profiles", {})
+
+        if not active_profile or active_profile not in profiles_data:
+            return {}
+        return profiles_data[active_profile]
+
+    def get_field_value(
+        self, field: FieldInfo, field_name: str
+    ) -> Tuple[Any, str, bool]:
+        """Concrete implementation to get the field value from the profile settings"""
+        if field.validation_alias:
+            if isinstance(field.validation_alias, str):
+                value = self.profile_settings.get(field.validation_alias.upper())
+                if value is not None:
+                    return value, field_name, self.field_is_complex(field)
+            elif isinstance(field.validation_alias, AliasChoices):
+                for alias in field.validation_alias.choices:
+                    if not isinstance(alias, str):
+                        continue
+                    value = self.profile_settings.get(alias.upper())
+                    if value is not None:
+                        return value, field_name, self.field_is_complex(field)
+
+        value = self.profile_settings.get(
+            f"{self.config.get('env_prefix','')}{field_name.upper()}"
+        )
+        return value, field_name, self.field_is_complex(field)
+
+    def __call__(self) -> Dict[str, Any]:
+        """Called by pydantic to get the settings from our custom source"""
+        if _is_test_mode():
+            return {}
+        profile_settings: Dict[str, Any] = {}
+        for field_name, field in self.settings_cls.model_fields.items():
+            value, key, is_complex = self.get_field_value(field, field_name)
+            if value is not None:
+                prepared_value = self.prepare_field_value(
+                    field_name, field, value, is_complex
+                )
+                profile_settings[key] = prepared_value
+        return profile_settings
+
+
+def _is_test_mode() -> bool:
+    """Check if the current process is in test mode."""
+    return bool(
+        os.getenv("PREFECT_TEST_MODE")
+        or os.getenv("PREFECT_UNIT_TEST_MODE")
+        or os.getenv("PREFECT_TESTING_UNIT_TEST_MODE")
+        or os.getenv("PREFECT_TESTING_TEST_MODE")
+    )
+
+
+def _get_profiles_path() -> Path:
+    """Helper to get the profiles path"""
+
+    if _is_test_mode():
+        return DEFAULT_PROFILES_PATH
+    if env_path := os.getenv("PREFECT_PROFILES_PATH"):
+        return Path(env_path)
+    if not (DEFAULT_PREFECT_HOME / "profiles.toml").exists():
+        return DEFAULT_PROFILES_PATH
+    return DEFAULT_PREFECT_HOME / "profiles.toml"

prefect/task_engine.py

@@ -1,3 +1,4 @@
+import asyncio
 import inspect
 import logging
 import threading
@@ -184,6 +185,22 @@ class BaseTaskRunEngine(Generic[P, R]):
 
         self.parameters = resolved_parameters
 
+    def _set_custom_task_run_name(self):
+        from prefect.utilities.engine import _resolve_custom_task_run_name
+
+        # update the task run name if necessary
+        if not self._task_name_set and self.task.task_run_name:
+            task_run_name = _resolve_custom_task_run_name(
+                task=self.task, parameters=self.parameters or {}
+            )
+
+            self.logger.extra["task_run_name"] = task_run_name
+            self.logger.debug(
+                f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
+            )
+            self.task_run.name = task_run_name
+            self._task_name_set = True
+
     def _wait_for_dependencies(self):
         if not self.wait_for:
             return
@@ -291,7 +308,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 data=exc,
                 message=f"Task run encountered unexpected exception: {repr(exc)}",
             )
-            if inspect.iscoroutinefunction(retry_condition):
+            if asyncio.iscoroutinefunction(retry_condition):
                 should_retry = run_coro_as_sync(
                     retry_condition(self.task, self.task_run, state)
                 )
@@ -335,7 +352,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                     f" {state.name!r}"
                 )
                 result = hook(task, task_run, state)
-                if inspect.isawaitable(result):
+                if asyncio.iscoroutine(result):
                     run_coro_as_sync(result)
             except Exception:
                 self.logger.error(
@@ -348,6 +365,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
     def begin_run(self):
         try:
             self._resolve_parameters()
+            self._set_custom_task_run_name()
             self._wait_for_dependencies()
         except UpstreamTaskError as upstream_exc:
             state = self.set_state(
@@ -419,7 +437,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 # Avoid fetching the result unless it is cached, otherwise we defeat
                 # the purpose of disabling `cache_result_in_memory`
                 result = state.result(raise_on_failure=False, fetch=True)
-                if inspect.isawaitable(result):
+                if asyncio.iscoroutine(result):
                     result = run_coro_as_sync(result)
             elif isinstance(state.data, ResultRecord):
                 result = state.data.result
@@ -443,7 +461,7 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
             # if the return value is a BaseResult, we need to fetch it
             if isinstance(self._return_value, BaseResult):
                 _result = self._return_value.get()
-                if inspect.isawaitable(_result):
+                if asyncio.iscoroutine(_result):
                     _result = run_coro_as_sync(_result)
                 return _result
             elif isinstance(self._return_value, ResultRecord):
@@ -577,7 +595,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
     @contextmanager
     def setup_run_context(self, client: Optional[SyncPrefectClient] = None):
         from prefect.utilities.engine import (
-            _resolve_custom_task_run_name,
             should_log_prints,
         )
 
@@ -609,18 +626,6 @@ class SyncTaskRunEngine(BaseTaskRunEngine[P, R]):
 
             self.logger = task_run_logger(task_run=self.task_run, task=self.task)  # type: ignore
 
-            # update the task run name if necessary
-            if not self._task_name_set and self.task.task_run_name:
-                task_run_name = _resolve_custom_task_run_name(
-                    task=self.task, parameters=self.parameters
-                )
-
-                self.logger.extra["task_run_name"] = task_run_name
-                self.logger.debug(
-                    f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
-                )
-                self.task_run.name = task_run_name
-                self._task_name_set = True
             yield
 
     @contextmanager
@@ -813,7 +818,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
                 data=exc,
                 message=f"Task run encountered unexpected exception: {repr(exc)}",
             )
-            if inspect.iscoroutinefunction(retry_condition):
+            if asyncio.iscoroutinefunction(retry_condition):
                 should_retry = await retry_condition(self.task, self.task_run, state)
             elif inspect.isfunction(retry_condition):
                 should_retry = retry_condition(self.task, self.task_run, state)
@@ -869,6 +874,7 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
     async def begin_run(self):
         try:
             self._resolve_parameters()
+            self._set_custom_task_run_name()
             self._wait_for_dependencies()
         except UpstreamTaskError as upstream_exc:
             state = await self.set_state(
@@ -1091,7 +1097,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
     @asynccontextmanager
     async def setup_run_context(self, client: Optional[PrefectClient] = None):
         from prefect.utilities.engine import (
-            _resolve_custom_task_run_name,
             should_log_prints,
         )
 
@@ -1122,16 +1127,6 @@ class AsyncTaskRunEngine(BaseTaskRunEngine[P, R]):
 
             self.logger = task_run_logger(task_run=self.task_run, task=self.task)  # type: ignore
 
-            if not self._task_name_set and self.task.task_run_name:
-                task_run_name = _resolve_custom_task_run_name(
-                    task=self.task, parameters=self.parameters
-                )
-                self.logger.extra["task_run_name"] = task_run_name
-                self.logger.debug(
-                    f"Renamed task run {self.task_run.name!r} to {task_run_name!r}"
-                )
-                self.task_run.name = task_run_name
-                self._task_name_set = True
             yield
 
     @asynccontextmanager

prefect/task_runners.py

@@ -29,6 +29,7 @@ from prefect.futures import (
     PrefectFutureList,
 )
 from prefect.logging.loggers import get_logger, get_run_logger
+from prefect.settings import PREFECT_TASK_RUNNER_THREAD_POOL_MAX_WORKERS
 from prefect.utilities.annotations import allow_failure, quote, unmapped
 from prefect.utilities.callables import (
     collapse_variadic_parameters,
@@ -220,7 +221,11 @@ class ThreadPoolTaskRunner(TaskRunner[PrefectConcurrentFuture]):
     def __init__(self, max_workers: Optional[int] = None):
         super().__init__()
         self._executor: Optional[ThreadPoolExecutor] = None
-        self._max_workers = sys.maxsize if max_workers is None else max_workers
+        self._max_workers = (
+            (PREFECT_TASK_RUNNER_THREAD_POOL_MAX_WORKERS.value() or sys.maxsize)
+            if max_workers is None
+            else max_workers
+        )
         self._cancel_events: Dict[uuid.UUID, threading.Event] = {}
 
     def duplicate(self) -> "ThreadPoolTaskRunner":

prefect/tasks.py

@@ -4,6 +4,7 @@ Module containing the base workflow task class and decorator - for most use case
 # This file requires type-checking with pyright because mypy does not yet support PEP612
 # See https://github.com/python/mypy/issues/8645
 
+import asyncio
 import datetime
 import inspect
 from copy import copy
@@ -309,7 +310,9 @@ class Task(Generic[P, R]):
             Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
         ] = None,
         cache_expiration: Optional[datetime.timedelta] = None,
-        task_run_name: Optional[Union[Callable[[], str], str]] = None,
+        task_run_name: Optional[
+            Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
+        ] = None,
         retries: Optional[int] = None,
         retry_delay_seconds: Optional[
             Union[
@@ -370,7 +373,7 @@ class Task(Generic[P, R]):
 
         # the task is considered async if its function is async or an async
         # generator
-        self.isasync = inspect.iscoroutinefunction(
+        self.isasync = asyncio.iscoroutinefunction(
             self.fn
         ) or inspect.isasyncgenfunction(self.fn)
 
@@ -530,7 +533,9 @@ class Task(Generic[P, R]):
         cache_key_fn: Optional[
             Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
         ] = None,
-        task_run_name: Optional[Union[Callable[[], str], str, Type[NotSet]]] = NotSet,
+        task_run_name: Optional[
+            Union[Callable[[], str], Callable[[Dict[str, Any]], str], str, Type[NotSet]]
+        ] = NotSet,
         cache_expiration: Optional[datetime.timedelta] = None,
         retries: Union[int, Type[NotSet]] = NotSet,
         retry_delay_seconds: Union[
@@ -1059,6 +1064,8 @@ class Task(Generic[P, R]):
         task runner. This call only blocks execution while the task is being submitted,
         once it is submitted, the flow function will continue executing.
 
+        This method is always synchronous, even if the underlying user function is asynchronous.
+
         Args:
             *args: Arguments to run the task with
             return_state: Return the result of the flow run wrapped in a
@@ -1111,7 +1118,7 @@ class Task(Generic[P, R]):
             >>>
             >>> @flow
             >>> async def my_flow():
-            >>>     await my_async_task.submit()
+            >>>     my_async_task.submit()
 
             Run a sync task in an async flow
 
@@ -1169,51 +1176,73 @@ class Task(Generic[P, R]):
 
     @overload
     def map(
-        self: "Task[P, NoReturn]",
-        *args: P.args,
-        **kwargs: P.kwargs,
-    ) -> PrefectFutureList[PrefectFuture[NoReturn]]:
+        self: "Task[P, R]",
+        *args: Any,
+        return_state: Literal[True],
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
+        deferred: bool = ...,
+        **kwargs: Any,
+    ) -> List[State[R]]:
         ...
 
     @overload
     def map(
-        self: "Task[P, Coroutine[Any, Any, T]]",
-        *args: P.args,
-        **kwargs: P.kwargs,
-    ) -> PrefectFutureList[PrefectFuture[T]]:
+        self: "Task[P, R]",
+        *args: Any,
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
+        deferred: bool = ...,
+        **kwargs: Any,
+    ) -> PrefectFutureList[R]:
         ...
 
     @overload
     def map(
-        self: "Task[P, T]",
-        *args: P.args,
-        **kwargs: P.kwargs,
-    ) -> PrefectFutureList[PrefectFuture[T]]:
+        self: "Task[P, R]",
+        *args: Any,
+        return_state: Literal[True],
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
+        deferred: bool = ...,
+        **kwargs: Any,
+    ) -> List[State[R]]:
         ...
 
     @overload
     def map(
-        self: "Task[P, Coroutine[Any, Any, T]]",
-        *args: P.args,
-        return_state: Literal[True],
-        **kwargs: P.kwargs,
-    ) -> PrefectFutureList[State[T]]:
+        self: "Task[P, R]",
+        *args: Any,
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
+        deferred: bool = ...,
+        **kwargs: Any,
+    ) -> PrefectFutureList[R]:
         ...
 
     @overload
     def map(
-        self: "Task[P, T]",
-        *args: P.args,
+        self: "Task[P, Coroutine[Any, Any, R]]",
+        *args: Any,
         return_state: Literal[True],
-        **kwargs: P.kwargs,
-    ) -> PrefectFutureList[State[T]]:
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
+        deferred: bool = ...,
+        **kwargs: Any,
+    ) -> List[State[R]]:
+        ...
+
+    @overload
+    def map(
+        self: "Task[P, Coroutine[Any, Any, R]]",
+        *args: Any,
+        return_state: Literal[False],
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = ...,
+        deferred: bool = ...,
+        **kwargs: Any,
+    ) -> PrefectFutureList[R]:
         ...
 
     def map(
         self,
         *args: Any,
         return_state: bool = False,
-        wait_for: Optional[Iterable[PrefectFuture]] = None,
+        wait_for: Optional[Iterable[Union[PrefectFuture[T], T]]] = None,
         deferred: bool = False,
         **kwargs: Any,
     ):
@@ -1234,6 +1263,8 @@ class Task(Generic[P, R]):
         also blocks while the tasks are being submitted, once they are
         submitted, the flow function will continue executing.
 
+        This method is always synchronous, even if the underlying user function is asynchronous.
+
         Args:
             *args: Iterable and static arguments to run the tasks with
             return_state: Return a list of Prefect States that wrap the results
@@ -1556,7 +1587,9 @@ def task(
         Callable[["TaskRunContext", Dict[str, Any]], Optional[str]]
     ] = None,
     cache_expiration: Optional[datetime.timedelta] = None,
-    task_run_name: Optional[Union[Callable[[], str], str]] = None,
+    task_run_name: Optional[
+        Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
+    ] = None,
     retries: int = 0,
     retry_delay_seconds: Union[
         float,
@@ -1593,7 +1626,9 @@ def task(
         Callable[["TaskRunContext", Dict[str, Any]], Optional[str]], None
     ] = None,
     cache_expiration: Optional[datetime.timedelta] = None,
-    task_run_name: Optional[Union[Callable[[], str], str]] = None,
+    task_run_name: Optional[
+        Union[Callable[[], str], Callable[[Dict[str, Any]], str], str]
+    ] = None,
     retries: Optional[int] = None,
     retry_delay_seconds: Union[
         float, int, List[float], Callable[[int], List[float]], None

prefect/utilities/asyncutils.py

@@ -83,7 +83,7 @@ def is_async_fn(
     while hasattr(func, "__wrapped__"):
         func = func.__wrapped__
 
-    return inspect.iscoroutinefunction(func)
+    return asyncio.iscoroutinefunction(func)
 
 
 def is_async_gen_fn(func):

prefect/utilities/engine.py

@@ -503,7 +503,7 @@ def propose_state_sync(
             # Avoid fetching the result unless it is cached, otherwise we defeat
             # the purpose of disabling `cache_result_in_memory`
             result = state.result(raise_on_failure=False, fetch=True)
-            if inspect.isawaitable(result):
+            if asyncio.iscoroutine(result):
                 result = run_coro_as_sync(result)
         elif isinstance(state.data, ResultRecord):
             result = state.data.result
@@ -685,7 +685,15 @@ def _resolve_custom_flow_run_name(flow: Flow, parameters: Dict[str, Any]) -> str
 
 def _resolve_custom_task_run_name(task: Task, parameters: Dict[str, Any]) -> str:
     if callable(task.task_run_name):
-        task_run_name = task.task_run_name()
+        sig = inspect.signature(task.task_run_name)
+
+        # If the callable accepts a 'parameters' kwarg, pass the entire parameters dict
+        if "parameters" in sig.parameters:
+            task_run_name = task.task_run_name(parameters=parameters)
+        else:
+            # If it doesn't expect parameters, call it without arguments
+            task_run_name = task.task_run_name()
+
         if not isinstance(task_run_name, str):
             raise TypeError(
                 f"Callable {task.task_run_name} for 'task_run_name' returned type"
@@ -870,7 +878,7 @@ def resolve_to_final_result(expr, context):
         )
 
     _result = state.result(raise_on_failure=False, fetch=True)
-    if inspect.isawaitable(_result):
+    if asyncio.iscoroutine(_result):
         _result = run_coro_as_sync(_result)
     return _result
 

prefect/utilities/services.py

@@ -10,7 +10,7 @@ import anyio
 import httpx
 
 from prefect.logging.loggers import get_logger
-from prefect.settings import PREFECT_CLIENT_ENABLE_METRICS, PREFECT_CLIENT_METRICS_PORT
+from prefect.settings import PREFECT_CLIENT_METRICS_ENABLED, PREFECT_CLIENT_METRICS_PORT
 from prefect.utilities.collections import distinct
 from prefect.utilities.math import clamped_poisson_interval
 
@@ -160,8 +160,8 @@ _metrics_server: Optional[Tuple[WSGIServer, threading.Thread]] = None
 
 def start_client_metrics_server():
     """Start the process-wide Prometheus metrics server for client metrics (if enabled
-    with `PREFECT_CLIENT_ENABLE_METRICS`) on the port `PREFECT_CLIENT_METRICS_PORT`."""
-    if not PREFECT_CLIENT_ENABLE_METRICS:
+    with `PREFECT_CLIENT_METRICS_ENABLED`) on the port `PREFECT_CLIENT_METRICS_PORT`."""
+    if not PREFECT_CLIENT_METRICS_ENABLED:
         return
 
     global _metrics_server

prefect/workers/base.py

@@ -1,5 +1,5 @@
 import abc
-import inspect
+import asyncio
 import threading
 from contextlib import AsyncExitStack
 from functools import partial
@@ -137,6 +137,12 @@ class BaseJobConfiguration(BaseModel):
         variables = cls._get_base_config_defaults(
             variables_schema.get("properties", {})
         )
+
+        # copy variable defaults for `env` to job config before they're replaced by
+        # deployment overrides
+        if variables.get("env"):
+            job_config["env"] = variables.get("env")
+
         variables.update(values)
 
         # deep merge `env`
@@ -1078,7 +1084,7 @@ class BaseWorker(abc.ABC):
                 task_status.started()
 
             result = fn(*args, **kwargs)
-            if inspect.iscoroutine(result):
+            if asyncio.iscoroutine(result):
                 await result
 
         await self._runs_task_group.start(wrapper)

{prefect_client-3.0.10.dist-info → prefect_client-3.0.11.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.0.10
+Version: 3.0.11
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.
@@ -47,7 +47,7 @@ Requires-Dist: prometheus-client>=0.20.0
 Requires-Dist: pydantic<3.0.0,>=2.7
 Requires-Dist: pydantic-core<3.0.0,>=2.12.0
 Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
-Requires-Dist: pydantic-settings
+Requires-Dist: pydantic-settings>2.2.1
 Requires-Dist: python-dateutil<3.0.0,>=2.8.2
 Requires-Dist: python-slugify<9.0,>=5.0
 Requires-Dist: pyyaml<7.0.0,>=5.4.1