prefect-client 3.0.11__tar.gz → 3.1.1__tar.gz

{prefect-client-3.0.11/src/prefect_client.egg-info → prefect-client-3.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.0.11
+Version: 3.1.1
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.

{prefect-client-3.0.11 → prefect-client-3.1.1}/requirements-client.txt

@@ -3,7 +3,7 @@ asgi-lifespan >= 1.0, < 3.0
 cachetools >= 5.3, < 6.0
 cloudpickle >= 2.0, < 4.0
 coolname >= 1.0.4, < 3.0.0
-croniter >= 1.0.12, < 4.0.0
+croniter >= 1.0.12, < 5.0.0
 exceptiongroup >= 1.0.0
 fastapi >= 0.111.0, < 1.0.0
 fsspec >= 2022.5.0

{prefect-client-3.0.11 → prefect-client-3.1.1}/requirements-dev.txt

@@ -12,7 +12,6 @@ pluggy >= 1.4.0
 pytest >= 8.3
 pytest-asyncio >= 0.24
 pytest-benchmark
-pytest-codspeed
 pytest-cov
 pytest-env
 pytest-flakefinder
@@ -37,3 +36,12 @@ mkdocs
 mkdocs-material
 mkdocstrings[python]
 mkdocs-gen-files
+
+# OpenTelemetry
+# Other than the `test-utils` package these versions should match the versions
+# in `requirements-otel.txt`
+opentelemetry-distro >= 0.48b0, < 1.0.0
+opentelemetry-exporter-otlp >= 1.27.0, < 2.0.0
+opentelemetry-instrumentation >= 0.48b0, < 1.0.0
+opentelemetry-instrumentation-logging >= 0.48b0, < 1.0.0
+opentelemetry-test-utils >= 0.48b0, < 1.0.0

{prefect-client-3.0.11 → prefect-client-3.1.1}/requirements.txt

@@ -12,7 +12,8 @@ graphviz >= 0.20.1
 jinja2 >= 3.0.0, < 4.0.0
 jinja2-humanize-extension >= 0.4.0
 humanize >= 4.9.0, < 5.0.0
+opentelemetry-api >= 1.27.0, < 2.0.0
 pytz >= 2021.1, < 2025
 readchar >= 4.0.0, < 5.0.0
 sqlalchemy[asyncio] >= 2.0, < 3.0.0
-typer >= 0.12.0, != 0.12.2, < 0.13.0
+typer >= 0.12.0, != 0.12.2, < 0.14.0

prefect-client-3.1.1/src/prefect/_internal/compatibility/async_dispatch.py

@@ -0,0 +1,73 @@
+import asyncio
+import inspect
+from functools import wraps
+from typing import Any, Callable, Coroutine, Protocol, TypeVar, Union
+
+from typing_extensions import ParamSpec
+
+R = TypeVar("R")
+P = ParamSpec("P")
+
+
+class AsyncDispatchable(Protocol[P, R]):
+    """Protocol for functions decorated with async_dispatch."""
+
+    def __call__(
+        self, *args: P.args, **kwargs: P.kwargs
+    ) -> Union[R, Coroutine[Any, Any, R]]:
+        ...
+
+    aio: Callable[P, Coroutine[Any, Any, R]]
+    sync: Callable[P, R]
+
+
+def is_in_async_context() -> bool:
+    """Check if we're in an async context."""
+    try:
+        # First check if we're in a coroutine
+        if asyncio.current_task() is not None:
+            return True
+
+        # Check if we have a loop and it's running
+        loop = asyncio.get_event_loop()
+        return loop.is_running()
+    except RuntimeError:
+        return False
+
+
+def async_dispatch(
+    async_impl: Callable[P, Coroutine[Any, Any, R]],
+) -> Callable[[Callable[P, R]], AsyncDispatchable[P, R]]:
+    """
+    Decorator that adds async compatibility to a sync function.
+
+    The decorated function will:
+    - Return a coroutine when in an async context (detected via running event loop)
+    - Run synchronously when in a sync context
+    - Provide .aio for explicit async access
+    - Provide .sync for explicit sync access
+
+    Args:
+        async_impl: The async implementation to dispatch to when async execution
+                   is needed
+    """
+    if not inspect.iscoroutinefunction(async_impl):
+        raise TypeError(
+            "async_impl must be an async function to dispatch in async contexts"
+        )
+
+    def decorator(sync_fn: Callable[P, R]) -> AsyncDispatchable[P, R]:
+        @wraps(sync_fn)
+        def wrapper(
+            *args: P.args, **kwargs: P.kwargs
+        ) -> Union[R, Coroutine[Any, Any, R]]:
+            if is_in_async_context():
+                return async_impl(*args, **kwargs)
+            return sync_fn(*args, **kwargs)
+
+        # Attach both async and sync implementations directly
+        wrapper.aio = async_impl
+        wrapper.sync = sync_fn
+        return wrapper  # type: ignore
+
+    return decorator

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-10-24T17:36:06-0500",
+ "date": "2024-11-08T12:38:16-0800",
  "dirty": true,
  "error": null,
- "full-revisionid": "a17ccfcf2be76360f13806d7aca1e2cf9fa3741e",
- "version": "3.0.11"
+ "full-revisionid": "6b50a2b9f9d4ebf59703c55e1156c6f79151f1c3",
+ "version": "3.1.1"
 }
 '''  # END VERSION_JSON
 

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/cache_policies.py

@@ -175,7 +175,7 @@ class CompoundCachePolicy(CachePolicy):
                 keys.append(policy_key)
         if not keys:
             return None
-        return hash_objects(*keys)
+        return hash_objects(*keys, raise_on_failure=True)
 
 
 @dataclass
@@ -224,7 +224,7 @@ class TaskSource(CachePolicy):
             else:
                 raise
 
-        return hash_objects(lines)
+        return hash_objects(lines, raise_on_failure=True)
 
 
 @dataclass
@@ -242,7 +242,7 @@ class FlowParameters(CachePolicy):
     ) -> Optional[str]:
         if not flow_parameters:
             return None
-        return hash_objects(flow_parameters)
+        return hash_objects(flow_parameters, raise_on_failure=True)
 
 
 @dataclass
@@ -293,7 +293,7 @@ class Inputs(CachePolicy):
             if key not in exclude:
                 hashed_inputs[key] = val
 
-        return hash_objects(hashed_inputs)
+        return hash_objects(hashed_inputs, raise_on_failure=True)
 
     def __sub__(self, other: str) -> "CachePolicy":
         if not isinstance(other, str):

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/client/orchestration.py

@@ -99,6 +99,7 @@ from prefect.client.schemas.objects import (
     TaskRunResult,
     Variable,
     Worker,
+    WorkerMetadata,
     WorkPool,
     WorkQueue,
     WorkQueueStatusDetail,
@@ -134,6 +135,7 @@ from prefect.settings import (
     PREFECT_CLOUD_API_URL,
     PREFECT_SERVER_ALLOW_EPHEMERAL_MODE,
     PREFECT_TESTING_UNIT_TEST_MODE,
+    get_current_settings,
 )
 
 if TYPE_CHECKING:
@@ -2594,22 +2596,44 @@ class PrefectClient:
         work_pool_name: str,
         worker_name: str,
         heartbeat_interval_seconds: Optional[float] = None,
-    ):
+        get_worker_id: bool = False,
+        worker_metadata: Optional[WorkerMetadata] = None,
+    ) -> Optional[UUID]:
         """
         Sends a worker heartbeat for a given work pool.
 
         Args:
             work_pool_name: The name of the work pool to heartbeat against.
             worker_name: The name of the worker sending the heartbeat.
+            return_id: Whether to return the worker ID. Note: will return `None` if the connected server does not support returning worker IDs, even if `return_id` is `True`.
+            worker_metadata: Metadata about the worker to send to the server.
         """
-        await self._client.post(
+        params = {
+            "name": worker_name,
+            "heartbeat_interval_seconds": heartbeat_interval_seconds,
+        }
+        if worker_metadata:
+            params["worker_metadata"] = worker_metadata.model_dump(mode="json")
+        if get_worker_id:
+            params["return_id"] = get_worker_id
+
+        resp = await self._client.post(
             f"/work_pools/{work_pool_name}/workers/heartbeat",
-            json={
-                "name": worker_name,
-                "heartbeat_interval_seconds": heartbeat_interval_seconds,
-            },
+            json=params,
         )
 
+        if (
+            (
+                self.server_type == ServerType.CLOUD
+                or get_current_settings().testing.test_mode
+            )
+            and get_worker_id
+            and resp.status_code == 200
+        ):
+            return UUID(resp.text)
+        else:
+            return None
+
     async def read_workers_for_work_pool(
         self,
         work_pool_name: str,

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/client/schemas/actions.py

@@ -85,18 +85,10 @@ class DeploymentScheduleCreate(ActionBaseModel):
     active: bool = Field(
         default=True, description="Whether or not the schedule is active."
     )
-    max_active_runs: Optional[PositiveInteger] = Field(
-        default=None,
-        description="The maximum number of active runs for the schedule.",
-    )
     max_scheduled_runs: Optional[PositiveInteger] = Field(
         default=None,
         description="The maximum number of scheduled runs for the schedule.",
     )
-    catchup: bool = Field(
-        default=False,
-        description="Whether or not a worker should catch up on Late runs for the schedule.",
-    )
 
     @field_validator("max_scheduled_runs")
     @classmethod
@@ -114,21 +106,11 @@ class DeploymentScheduleUpdate(ActionBaseModel):
         default=True, description="Whether or not the schedule is active."
     )
 
-    max_active_runs: Optional[PositiveInteger] = Field(
-        default=None,
-        description="The maximum number of active runs for the schedule.",
-    )
-
     max_scheduled_runs: Optional[PositiveInteger] = Field(
         default=None,
         description="The maximum number of scheduled runs for the schedule.",
     )
 
-    catchup: Optional[bool] = Field(
-        default=None,
-        description="Whether or not a worker should catch up on Late runs for the schedule.",
-    )
-
     @field_validator("max_scheduled_runs")
     @classmethod
     def validate_max_scheduled_runs(cls, v):
@@ -580,6 +562,17 @@ class LogCreate(ActionBaseModel):
     timestamp: DateTime = Field(default=..., description="The log timestamp.")
     flow_run_id: Optional[UUID] = Field(None)
     task_run_id: Optional[UUID] = Field(None)
+    worker_id: Optional[UUID] = Field(None)
+
+    def model_dump(self, *args, **kwargs):
+        """
+        The worker_id field is only included in logs sent to Prefect Cloud.
+        If it's unset, we should not include it in the log payload.
+        """
+        data = super().model_dump(*args, **kwargs)
+        if self.worker_id is None:
+            data.pop("worker_id")
+        return data
 
 
 class WorkPoolCreate(ActionBaseModel):

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/client/schemas/objects.py

@@ -32,6 +32,7 @@ from pydantic.functional_validators import ModelWrapValidatorHandler
 from pydantic_extra_types.pendulum_dt import DateTime
 from typing_extensions import Literal, Self, TypeVar
 
+from prefect._internal.compatibility import deprecated
 from prefect._internal.compatibility.migration import getattr_migration
 from prefect._internal.schemas.bases import ObjectBaseModel, PrefectBaseModel
 from prefect._internal.schemas.fields import CreatedBy, UpdatedBy
@@ -186,7 +187,7 @@ class StateDetails(PrefectBaseModel):
 
 
 def data_discriminator(x: Any) -> str:
-    if isinstance(x, dict) and "type" in x:
+    if isinstance(x, dict) and "type" in x and x["type"] != "unpersisted":
         return "BaseResult"
     elif isinstance(x, dict) and "storage_key" in x:
         return "ResultRecordMetadata"
@@ -220,10 +221,17 @@ class State(ObjectBaseModel, Generic[R]):
     def result(self: "State[R]", raise_on_failure: bool = False) -> Union[R, Exception]:
         ...
 
+    @deprecated.deprecated_parameter(
+        "fetch",
+        when=lambda fetch: fetch is not True,
+        start_date="Oct 2024",
+        end_date="Jan 2025",
+        help="Please ensure you are awaiting the call to `result()` when calling in an async context.",
+    )
     def result(
         self,
         raise_on_failure: bool = True,
-        fetch: Optional[bool] = None,
+        fetch: bool = True,
         retry_result_failure: bool = True,
     ) -> Union[R, Exception]:
         """
@@ -248,22 +256,6 @@ class State(ObjectBaseModel, Generic[R]):
             The result of the run
 
         Examples:
-            >>> from prefect import flow, task
-            >>> @task
-            >>> def my_task(x):
-            >>>     return x
-
-            Get the result from a task future in a flow
-
-            >>> @flow
-            >>> def my_flow():
-            >>>     future = my_task("hello")
-            >>>     state = future.wait()
-            >>>     result = state.result()
-            >>>     print(result)
-            >>> my_flow()
-            hello
-
             Get the result from a flow state
 
             >>> @flow
@@ -307,7 +299,7 @@ class State(ObjectBaseModel, Generic[R]):
             >>>     raise ValueError("oh no!")
             >>> my_flow.deploy("my_deployment/my_flow")
             >>> flow_run = run_deployment("my_deployment/my_flow")
-            >>> await flow_run.state.result(raise_on_failure=True, fetch=True) # Raises `ValueError("oh no!")`
+            >>> await flow_run.state.result(raise_on_failure=True) # Raises `ValueError("oh no!")`
         """
         from prefect.states import get_state_result
 
@@ -365,6 +357,12 @@ class State(ObjectBaseModel, Generic[R]):
                 self.state_details.scheduled_time = DateTime.now("utc")
         return self
 
+    @model_validator(mode="after")
+    def set_unpersisted_results_to_none(self) -> Self:
+        if isinstance(self.data, dict) and self.data.get("type") == "unpersisted":
+            self.data = None
+        return self
+
     def is_scheduled(self) -> bool:
         return self.type == StateType.SCHEDULED
 
@@ -1061,18 +1059,10 @@ class DeploymentSchedule(ObjectBaseModel):
     active: bool = Field(
         default=True, description="Whether or not the schedule is active."
     )
-    max_active_runs: Optional[PositiveInteger] = Field(
-        default=None,
-        description="The maximum number of active runs for the schedule.",
-    )
     max_scheduled_runs: Optional[PositiveInteger] = Field(
         default=None,
         description="The maximum number of scheduled runs for the schedule.",
     )
-    catchup: bool = Field(
-        default=False,
-        description="Whether or not a worker should catch up on Late runs for the schedule.",
-    )
 
 
 class Deployment(ObjectBaseModel):
@@ -1699,3 +1689,24 @@ class CsrfToken(ObjectBaseModel):
 
 
 __getattr__ = getattr_migration(__name__)
+
+
+class Integration(PrefectBaseModel):
+    """A representation of an installed Prefect integration."""
+
+    name: str = Field(description="The name of the Prefect integration.")
+    version: str = Field(description="The version of the Prefect integration.")
+
+
+class WorkerMetadata(PrefectBaseModel):
+    """
+    Worker metadata.
+
+    We depend on the structure of `integrations`, but otherwise, worker classes
+    should support flexible metadata.
+    """
+
+    integrations: List[Integration] = Field(
+        default=..., description="Prefect integrations installed in the worker."
+    )
+    model_config = ConfigDict(extra="allow")

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/context.py

@@ -38,7 +38,11 @@ from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_c
 from prefect.client.schemas import FlowRun, TaskRun
 from prefect.events.worker import EventsWorker
 from prefect.exceptions import MissingContextError
-from prefect.results import ResultStore, get_default_persist_setting
+from prefect.results import (
+    ResultStore,
+    get_default_persist_setting,
+    get_default_persist_setting_for_tasks,
+)
 from prefect.settings import Profile, Settings
 from prefect.settings.legacy import _get_settings_fields
 from prefect.states import State
@@ -397,7 +401,7 @@ class TaskRunContext(RunContext):
 
     # Result handling
     result_store: ResultStore
-    persist_result: bool = Field(default_factory=get_default_persist_setting)
+    persist_result: bool = Field(default_factory=get_default_persist_setting_for_tasks)
 
     __var__ = ContextVar("task_run")
 

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/deployments/base.py

@@ -272,8 +272,6 @@ def _format_deployment_for_saving_to_prefect_file(
                 schedule_config = deployment_schedule.schedule.model_dump()
 
             schedule_config["active"] = deployment_schedule.active
-            schedule_config["max_active_runs"] = deployment_schedule.max_active_runs
-            schedule_config["catchup"] = deployment_schedule.catchup
             schedules.append(schedule_config)
 
         deployment["schedules"] = schedules

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/deployments/schedules.py

@@ -14,15 +14,11 @@ FlexibleScheduleList = Sequence[
 def create_deployment_schedule_create(
     schedule: "SCHEDULE_TYPES",
     active: Optional[bool] = True,
-    max_active_runs: Optional[int] = None,
-    catchup: bool = False,
 ) -> DeploymentScheduleCreate:
     """Create a DeploymentScheduleCreate object from common schedule parameters."""
     return DeploymentScheduleCreate(
         schedule=schedule,
         active=active if active is not None else True,
-        max_active_runs=max_active_runs,
-        catchup=catchup,
     )
 
 

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/logging/handlers.py

@@ -32,6 +32,7 @@ from prefect.settings import (
     PREFECT_LOGGING_TO_API_BATCH_SIZE,
     PREFECT_LOGGING_TO_API_MAX_LOG_SIZE,
     PREFECT_LOGGING_TO_API_WHEN_MISSING_FLOW,
+    get_current_settings,
 )
 
 
@@ -180,6 +181,7 @@ class APILogHandler(logging.Handler):
         """
         flow_run_id = getattr(record, "flow_run_id", None)
         task_run_id = getattr(record, "task_run_id", None)
+        worker_id = getattr(record, "worker_id", None)
 
         if not flow_run_id:
             try:
@@ -215,6 +217,7 @@ class APILogHandler(logging.Handler):
         log = LogCreate(
             flow_run_id=flow_run_id if is_uuid_like else None,
             task_run_id=task_run_id,
+            worker_id=worker_id,
             name=record.name,
             level=record.levelno,
             timestamp=pendulum.from_timestamp(
@@ -236,6 +239,44 @@ class APILogHandler(logging.Handler):
         return len(json.dumps(log).encode())
 
 
+class WorkerAPILogHandler(APILogHandler):
+    def emit(self, record: logging.LogRecord):
+        if get_current_settings().experiments.worker_logging_to_api_enabled:
+            super().emit(record)
+        else:
+            return
+
+    def prepare(self, record: logging.LogRecord) -> Dict[str, Any]:
+        """
+        Convert a `logging.LogRecord` to the API `LogCreate` schema and serialize.
+
+        This will add in the worker id to the log.
+
+        Logs exceeding the maximum size will be dropped.
+        """
+
+        worker_id = getattr(record, "worker_id", None)
+
+        log = LogCreate(
+            worker_id=worker_id,
+            name=record.name,
+            level=record.levelno,
+            timestamp=pendulum.from_timestamp(
+                getattr(record, "created", None) or time.time()
+            ),
+            message=self.format(record),
+        ).model_dump(mode="json")
+
+        log_size = log["__payload_size__"] = self._get_payload_size(log)
+        if log_size > PREFECT_LOGGING_TO_API_MAX_LOG_SIZE.value():
+            raise ValueError(
+                f"Log of size {log_size} is greater than the max size of "
+                f"{PREFECT_LOGGING_TO_API_MAX_LOG_SIZE.value()}"
+            )
+
+        return log
+
+
 class PrefectConsoleHandler(logging.StreamHandler):
     def __init__(
         self,

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/logging/loggers.py

@@ -12,6 +12,7 @@ from typing_extensions import Self
 import prefect
 from prefect.exceptions import MissingContextError
 from prefect.logging.filters import ObfuscateApiKeyFilter
+from prefect.telemetry.logging import add_telemetry_log_handler
 
 if TYPE_CHECKING:
     from prefect.client.schemas import FlowRun as ClientFlowRun
@@ -19,6 +20,7 @@ if TYPE_CHECKING:
     from prefect.context import RunContext
     from prefect.flows import Flow
     from prefect.tasks import Task
+    from prefect.workers.base import BaseWorker
 
 
 class PrefectLogAdapter(logging.LoggerAdapter):
@@ -75,6 +77,8 @@ def get_logger(name: Optional[str] = None) -> logging.Logger:
     obfuscate_api_key_filter = ObfuscateApiKeyFilter()
     logger.addFilter(obfuscate_api_key_filter)
 
+    add_telemetry_log_handler(logger=logger)
+
     return logger
 
 
@@ -136,6 +140,12 @@ def get_run_logger(
     else:
         raise MissingContextError("There is no active flow or task run context.")
 
+    if isinstance(logger, logging.LoggerAdapter):
+        assert isinstance(logger.logger, logging.Logger)
+        add_telemetry_log_handler(logger.logger)
+    else:
+        add_telemetry_log_handler(logger)
+
     return logger
 
 
@@ -205,6 +215,29 @@ def task_run_logger(
     )
 
 
+def get_worker_logger(worker: "BaseWorker", name: Optional[str] = None):
+    """
+    Create a worker logger with the worker's metadata attached.
+
+    If the worker has a backend_id, it will be attached to the log records.
+    If the worker does not have a backend_id a basic logger will be returned.
+    If the worker does not have a backend_id attribute, a basic logger will be returned.
+    """
+
+    worker_log_name = name or f"workers.{worker.__class__.type}.{worker.name.lower()}"
+
+    worker_id = getattr(worker, "backend_id", None)
+    if worker_id:
+        return PrefectLogAdapter(
+            get_logger(worker_log_name),
+            extra={
+                "worker_id": str(worker.backend_id),
+            },
+        )
+    else:
+        return get_logger(worker_log_name)
+
+
 @contextmanager
 def disable_logger(name: str):
     """

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/logging/logging.yml

@@ -69,6 +69,10 @@ handlers:
     class: logging.StreamHandler
     formatter: debug
 
+  worker_api:
+    level: 0
+    class: prefect.logging.handlers.WorkerAPILogHandler
+
 loggers:
   prefect:
     level: "${PREFECT_LOGGING_LEVEL}"
@@ -86,6 +90,10 @@ loggers:
     level: NOTSET
     handlers: [api]
 
+  prefect.workers:
+    level: NOTSET
+    handlers: [worker_api]
+
   prefect.server:
     level: "${PREFECT_SERVER_LOGGING_LEVEL}"
 
@@ -102,9 +110,13 @@ loggers:
 
   uvicorn:
     level: "${PREFECT_SERVER_LOGGING_LEVEL}"
+    handlers: [console]
+    propagate: false
 
   fastapi:
     level: "${PREFECT_SERVER_LOGGING_LEVEL}"
+    handlers: [console]
+    propagate: false
 
 # The root logger: any logger without propagation disabled sends to here as well
 root:

{prefect-client-3.0.11 → prefect-client-3.1.1}/src/prefect/main.py

@@ -57,6 +57,11 @@ prefect.logging.get_logger("profiles").debug(
     f"Using profile {prefect.context.get_settings_context().profile.name!r}"
 )
 
+# Configure telemetry
+import prefect.telemetry.bootstrap
+
+prefect.telemetry.bootstrap.setup_telemetry()
+
 
 from prefect._internal.compatibility.deprecated import (
     inject_renamed_module_alias_finder,