prefect-client 3.1.0__tar.gz → 3.1.2__tar.gz

{prefect-client-3.1.0/src/prefect_client.egg-info → prefect-client-3.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.1.0
+Version: 3.1.2
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.

{prefect-client-3.1.0 → prefect-client-3.1.2}/requirements-dev.txt

@@ -36,3 +36,12 @@ mkdocs
 mkdocs-material
 mkdocstrings[python]
 mkdocs-gen-files
+
+# OpenTelemetry
+# Other than the `test-utils` package these versions should match the versions
+# in `requirements-otel.txt`
+opentelemetry-distro >= 0.48b0, < 1.0.0
+opentelemetry-exporter-otlp >= 1.27.0, < 2.0.0
+opentelemetry-instrumentation >= 0.48b0, < 1.0.0
+opentelemetry-instrumentation-logging >= 0.48b0, < 1.0.0
+opentelemetry-test-utils >= 0.48b0, < 1.0.0

{prefect-client-3.1.0 → prefect-client-3.1.2}/requirements.txt

@@ -16,4 +16,4 @@ opentelemetry-api >= 1.27.0, < 2.0.0
 pytz >= 2021.1, < 2025
 readchar >= 4.0.0, < 5.0.0
 sqlalchemy[asyncio] >= 2.0, < 3.0.0
-typer >= 0.12.0, != 0.12.2, < 0.13.0
+typer >= 0.12.0, != 0.12.2, < 0.14.0

prefect-client-3.1.2/src/prefect/_internal/compatibility/async_dispatch.py

@@ -0,0 +1,73 @@
+import asyncio
+import inspect
+from functools import wraps
+from typing import Any, Callable, Coroutine, Protocol, TypeVar, Union
+
+from typing_extensions import ParamSpec
+
+R = TypeVar("R")
+P = ParamSpec("P")
+
+
+class AsyncDispatchable(Protocol[P, R]):
+    """Protocol for functions decorated with async_dispatch."""
+
+    def __call__(
+        self, *args: P.args, **kwargs: P.kwargs
+    ) -> Union[R, Coroutine[Any, Any, R]]:
+        ...
+
+    aio: Callable[P, Coroutine[Any, Any, R]]
+    sync: Callable[P, R]
+
+
+def is_in_async_context() -> bool:
+    """Check if we're in an async context."""
+    try:
+        # First check if we're in a coroutine
+        if asyncio.current_task() is not None:
+            return True
+
+        # Check if we have a loop and it's running
+        loop = asyncio.get_event_loop()
+        return loop.is_running()
+    except RuntimeError:
+        return False
+
+
+def async_dispatch(
+    async_impl: Callable[P, Coroutine[Any, Any, R]],
+) -> Callable[[Callable[P, R]], AsyncDispatchable[P, R]]:
+    """
+    Decorator that adds async compatibility to a sync function.
+
+    The decorated function will:
+    - Return a coroutine when in an async context (detected via running event loop)
+    - Run synchronously when in a sync context
+    - Provide .aio for explicit async access
+    - Provide .sync for explicit sync access
+
+    Args:
+        async_impl: The async implementation to dispatch to when async execution
+                   is needed
+    """
+    if not inspect.iscoroutinefunction(async_impl):
+        raise TypeError(
+            "async_impl must be an async function to dispatch in async contexts"
+        )
+
+    def decorator(sync_fn: Callable[P, R]) -> AsyncDispatchable[P, R]:
+        @wraps(sync_fn)
+        def wrapper(
+            *args: P.args, **kwargs: P.kwargs
+        ) -> Union[R, Coroutine[Any, Any, R]]:
+            if is_in_async_context():
+                return async_impl(*args, **kwargs)
+            return sync_fn(*args, **kwargs)
+
+        # Attach both async and sync implementations directly
+        wrapper.aio = async_impl
+        wrapper.sync = sync_fn
+        return wrapper  # type: ignore
+
+    return decorator

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-10-31T12:43:46-0700",
+ "date": "2024-11-12T13:38:45-0800",
  "dirty": true,
  "error": null,
- "full-revisionid": "a83ba39b095e5945140ab6313e39dbc56056afe1",
- "version": "3.1.0"
+ "full-revisionid": "02b99f0a756f0395e4292952dcb2d963fc839750",
+ "version": "3.1.2"
 }
 '''  # END VERSION_JSON
 

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/client/cloud.py

@@ -1,5 +1,6 @@
 import re
 from typing import Any, Dict, List, Optional, cast
+from uuid import UUID
 
 import anyio
 import httpx
@@ -12,6 +13,7 @@ from prefect.client.base import PrefectHttpxAsyncClient
 from prefect.client.schemas.objects import (
     IPAllowlist,
     IPAllowlistMyAccessResponse,
+    KeyValueLabels,
     Workspace,
 )
 from prefect.exceptions import ObjectNotFound, PrefectException
@@ -151,6 +153,25 @@ class CloudClient:
         response = await self.get(f"{self.account_base_url}/ip_allowlist/my_access")
         return IPAllowlistMyAccessResponse.model_validate(response)
 
+    async def update_flow_run_labels(
+        self, flow_run_id: UUID, labels: KeyValueLabels
+    ) -> httpx.Response:
+        """
+        Update the labels for a flow run.
+
+        Args:
+            flow_run_id: The identifier for the flow run to update.
+            labels: A dictionary of labels to update for the flow run.
+
+        Returns:
+            an `httpx.Response` object from the PATCH request
+        """
+
+        return await self._client.patch(
+            f"{self.workspace_base_url}/flow_runs/{flow_run_id}/labels",
+            json=labels,
+        )
+
     async def __aenter__(self):
         await self._client.__aenter__()
         return self

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/client/orchestration.py

@@ -99,6 +99,7 @@ from prefect.client.schemas.objects import (
     TaskRunResult,
     Variable,
     Worker,
+    WorkerMetadata,
     WorkPool,
     WorkQueue,
     WorkQueueStatusDetail,
@@ -2596,6 +2597,7 @@ class PrefectClient:
         worker_name: str,
         heartbeat_interval_seconds: Optional[float] = None,
         get_worker_id: bool = False,
+        worker_metadata: Optional[WorkerMetadata] = None,
     ) -> Optional[UUID]:
         """
         Sends a worker heartbeat for a given work pool.
@@ -2604,20 +2606,20 @@ class PrefectClient:
             work_pool_name: The name of the work pool to heartbeat against.
             worker_name: The name of the worker sending the heartbeat.
             return_id: Whether to return the worker ID. Note: will return `None` if the connected server does not support returning worker IDs, even if `return_id` is `True`.
+            worker_metadata: Metadata about the worker to send to the server.
         """
-
+        params = {
+            "name": worker_name,
+            "heartbeat_interval_seconds": heartbeat_interval_seconds,
+        }
+        if worker_metadata:
+            params["metadata"] = worker_metadata.model_dump(mode="json")
         if get_worker_id:
-            return_dict = {"return_id": get_worker_id}
-        else:
-            return_dict = {}
+            params["return_id"] = get_worker_id
 
         resp = await self._client.post(
             f"/work_pools/{work_pool_name}/workers/heartbeat",
-            json={
-                "name": worker_name,
-                "heartbeat_interval_seconds": heartbeat_interval_seconds,
-            }
-            | return_dict,
+            json=params,
         )
 
         if (

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/client/schemas/objects.py

@@ -23,6 +23,9 @@ from pydantic import (
     HttpUrl,
     IPvAnyNetwork,
     SerializationInfo,
+    StrictBool,
+    StrictFloat,
+    StrictInt,
     Tag,
     field_validator,
     model_serializer,
@@ -68,6 +71,8 @@ if TYPE_CHECKING:
 
 R = TypeVar("R", default=Any)
 
+KeyValueLabels = dict[str, Union[StrictBool, StrictInt, StrictFloat, str]]
+
 
 DEFAULT_BLOCK_SCHEMA_VERSION = "non-versioned"
 DEFAULT_AGENT_WORK_POOL_NAME = "default-agent-pool"
@@ -1689,3 +1694,24 @@ class CsrfToken(ObjectBaseModel):
 
 
 __getattr__ = getattr_migration(__name__)
+
+
+class Integration(PrefectBaseModel):
+    """A representation of an installed Prefect integration."""
+
+    name: str = Field(description="The name of the Prefect integration.")
+    version: str = Field(description="The version of the Prefect integration.")
+
+
+class WorkerMetadata(PrefectBaseModel):
+    """
+    Worker metadata.
+
+    We depend on the structure of `integrations`, but otherwise, worker classes
+    should support flexible metadata.
+    """
+
+    integrations: List[Integration] = Field(
+        default=..., description="Prefect integrations installed in the worker."
+    )
+    model_config = ConfigDict(extra="allow")

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/logging/handlers.py

@@ -32,6 +32,7 @@ from prefect.settings import (
     PREFECT_LOGGING_TO_API_BATCH_SIZE,
     PREFECT_LOGGING_TO_API_MAX_LOG_SIZE,
     PREFECT_LOGGING_TO_API_WHEN_MISSING_FLOW,
+    get_current_settings,
 )
 
 
@@ -180,6 +181,7 @@ class APILogHandler(logging.Handler):
         """
         flow_run_id = getattr(record, "flow_run_id", None)
         task_run_id = getattr(record, "task_run_id", None)
+        worker_id = getattr(record, "worker_id", None)
 
         if not flow_run_id:
             try:
@@ -215,6 +217,7 @@ class APILogHandler(logging.Handler):
         log = LogCreate(
             flow_run_id=flow_run_id if is_uuid_like else None,
             task_run_id=task_run_id,
+            worker_id=worker_id,
             name=record.name,
             level=record.levelno,
             timestamp=pendulum.from_timestamp(
@@ -236,6 +239,44 @@ class APILogHandler(logging.Handler):
         return len(json.dumps(log).encode())
 
 
+class WorkerAPILogHandler(APILogHandler):
+    def emit(self, record: logging.LogRecord):
+        if get_current_settings().experiments.worker_logging_to_api_enabled:
+            super().emit(record)
+        else:
+            return
+
+    def prepare(self, record: logging.LogRecord) -> Dict[str, Any]:
+        """
+        Convert a `logging.LogRecord` to the API `LogCreate` schema and serialize.
+
+        This will add in the worker id to the log.
+
+        Logs exceeding the maximum size will be dropped.
+        """
+
+        worker_id = getattr(record, "worker_id", None)
+
+        log = LogCreate(
+            worker_id=worker_id,
+            name=record.name,
+            level=record.levelno,
+            timestamp=pendulum.from_timestamp(
+                getattr(record, "created", None) or time.time()
+            ),
+            message=self.format(record),
+        ).model_dump(mode="json")
+
+        log_size = log["__payload_size__"] = self._get_payload_size(log)
+        if log_size > PREFECT_LOGGING_TO_API_MAX_LOG_SIZE.value():
+            raise ValueError(
+                f"Log of size {log_size} is greater than the max size of "
+                f"{PREFECT_LOGGING_TO_API_MAX_LOG_SIZE.value()}"
+            )
+
+        return log
+
+
 class PrefectConsoleHandler(logging.StreamHandler):
     def __init__(
         self,

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/logging/loggers.py

@@ -12,6 +12,7 @@ from typing_extensions import Self
 import prefect
 from prefect.exceptions import MissingContextError
 from prefect.logging.filters import ObfuscateApiKeyFilter
+from prefect.telemetry.logging import add_telemetry_log_handler
 
 if TYPE_CHECKING:
     from prefect.client.schemas import FlowRun as ClientFlowRun
@@ -19,6 +20,7 @@ if TYPE_CHECKING:
     from prefect.context import RunContext
     from prefect.flows import Flow
     from prefect.tasks import Task
+    from prefect.workers.base import BaseWorker
 
 
 class PrefectLogAdapter(logging.LoggerAdapter):
@@ -75,6 +77,8 @@ def get_logger(name: Optional[str] = None) -> logging.Logger:
     obfuscate_api_key_filter = ObfuscateApiKeyFilter()
     logger.addFilter(obfuscate_api_key_filter)
 
+    add_telemetry_log_handler(logger=logger)
+
     return logger
 
 
@@ -136,6 +140,12 @@ def get_run_logger(
     else:
         raise MissingContextError("There is no active flow or task run context.")
 
+    if isinstance(logger, logging.LoggerAdapter):
+        assert isinstance(logger.logger, logging.Logger)
+        add_telemetry_log_handler(logger.logger)
+    else:
+        add_telemetry_log_handler(logger)
+
     return logger
 
 
@@ -205,6 +215,29 @@ def task_run_logger(
     )
 
 
+def get_worker_logger(worker: "BaseWorker", name: Optional[str] = None):
+    """
+    Create a worker logger with the worker's metadata attached.
+
+    If the worker has a backend_id, it will be attached to the log records.
+    If the worker does not have a backend_id a basic logger will be returned.
+    If the worker does not have a backend_id attribute, a basic logger will be returned.
+    """
+
+    worker_log_name = name or f"workers.{worker.__class__.type}.{worker.name.lower()}"
+
+    worker_id = getattr(worker, "backend_id", None)
+    if worker_id:
+        return PrefectLogAdapter(
+            get_logger(worker_log_name),
+            extra={
+                "worker_id": str(worker.backend_id),
+            },
+        )
+    else:
+        return get_logger(worker_log_name)
+
+
 @contextmanager
 def disable_logger(name: str):
     """

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/logging/logging.yml

@@ -69,6 +69,10 @@ handlers:
     class: logging.StreamHandler
     formatter: debug
 
+  worker_api:
+    level: 0
+    class: prefect.logging.handlers.WorkerAPILogHandler
+
 loggers:
   prefect:
     level: "${PREFECT_LOGGING_LEVEL}"
@@ -86,6 +90,10 @@ loggers:
     level: NOTSET
     handlers: [api]
 
+  prefect.workers:
+    level: NOTSET
+    handlers: [worker_api]
+
   prefect.server:
     level: "${PREFECT_SERVER_LOGGING_LEVEL}"
 

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/main.py

@@ -57,6 +57,11 @@ prefect.logging.get_logger("profiles").debug(
     f"Using profile {prefect.context.get_settings_context().profile.name!r}"
 )
 
+# Configure telemetry
+import prefect.telemetry.bootstrap
+
+prefect.telemetry.bootstrap.setup_telemetry()
+
 
 from prefect._internal.compatibility.deprecated import (
     inject_renamed_module_alias_finder,

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/settings/base.py

@@ -17,6 +17,7 @@ from pydantic_settings import (
 
 from prefect.settings.sources import (
     EnvFilterSettingsSource,
+    FilteredDotEnvSettingsSource,
     PrefectTomlConfigSettingsSource,
     ProfileSettingsTomlLoader,
     PyprojectTomlConfigSettingsSource,
@@ -43,13 +44,14 @@ class PrefectBaseSettings(BaseSettings):
         See https://docs.pydantic.dev/latest/concepts/pydantic_settings/#customise-settings-sources
         """
         env_filter = set()
-        for field in settings_cls.model_fields.values():
+        for field_name, field in settings_cls.model_fields.items():
             if field.validation_alias is not None and isinstance(
                 field.validation_alias, AliasChoices
             ):
                 for alias in field.validation_alias.choices:
                     if isinstance(alias, AliasPath) and len(alias.path) > 0:
                         env_filter.add(alias.path[0])
+            env_filter.add(field_name)
         return (
             init_settings,
             EnvFilterSettingsSource(
@@ -62,7 +64,18 @@ class PrefectBaseSettings(BaseSettings):
                 env_parse_enums=cls.model_config.get("env_parse_enums"),
                 env_filter=list(env_filter),
             ),
-            dotenv_settings,
+            FilteredDotEnvSettingsSource(
+                settings_cls,
+                env_file=cls.model_config.get("env_file"),
+                env_file_encoding=cls.model_config.get("env_file_encoding"),
+                case_sensitive=cls.model_config.get("case_sensitive"),
+                env_prefix=cls.model_config.get("env_prefix"),
+                env_nested_delimiter=cls.model_config.get("env_nested_delimiter"),
+                env_ignore_empty=cls.model_config.get("env_ignore_empty"),
+                env_parse_none_str=cls.model_config.get("env_parse_none_str"),
+                env_parse_enums=cls.model_config.get("env_parse_enums"),
+                env_blacklist=list(env_filter),
+            ),
             file_secret_settings,
             PrefectTomlConfigSettingsSource(settings_cls),
             PyprojectTomlConfigSettingsSource(settings_cls),
@@ -92,7 +105,7 @@ class PrefectBaseSettings(BaseSettings):
             elif (value := env.get(key)) is not None:
                 env_variables[
                     f"{self.model_config.get('env_prefix')}{key.upper()}"
-                ] = str(value)
+                ] = _to_environment_variable_value(value)
         return env_variables
 
     @model_serializer(
@@ -191,3 +204,9 @@ def _build_settings_config(
         pyproject_toml_table_header=("tool", "prefect", *path),
         json_schema_extra=_add_environment_variables,
     )
+
+
+def _to_environment_variable_value(value: Any) -> str:
+    if isinstance(value, (list, set, tuple)):
+        return ",".join(str(v) for v in value)
+    return str(value)

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/settings/models/api.py

@@ -25,7 +25,7 @@ class APISettings(PrefectBaseSettings):
     )
     tls_insecure_skip_verify: bool = Field(
         default=False,
-        description="If `True`, disables SSL checking to allow insecure requests. This is recommended only during development, e.g. when using self-signed certificates.",
+        description="If `True`, disables SSL checking to allow insecure requests. Setting to False is recommended only during development. For example, when using self-signed certificates.",
     )
     ssl_cert_file: Optional[str] = Field(
         default=os.environ.get("SSL_CERT_FILE"),

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/settings/models/experiments.py

@@ -22,3 +22,8 @@ class ExperimentsSettings(PrefectBaseSettings):
         default=False,
         description="Enables the logging of worker logs to Prefect Cloud.",
     )
+
+    telemetry_enabled: bool = Field(
+        default=False,
+        description="Enables sending telemetry to Prefect Cloud.",
+    )

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/settings/models/logging.py

@@ -1,11 +1,18 @@
+from functools import partial
 from pathlib import Path
 from typing import Annotated, Literal, Optional, Union
 
-from pydantic import AfterValidator, AliasChoices, AliasPath, Field, model_validator
+from pydantic import (
+    AliasChoices,
+    AliasPath,
+    BeforeValidator,
+    Field,
+    model_validator,
+)
 from typing_extensions import Self
 
 from prefect.settings.base import PrefectBaseSettings, _build_settings_config
-from prefect.types import LogLevel
+from prefect.types import LogLevel, validate_set_T_from_delim_string
 
 
 def max_log_size_smaller_than_batch_size(values):
@@ -97,7 +104,7 @@ class LoggingSettings(PrefectBaseSettings):
 
     extra_loggers: Annotated[
         Union[str, list[str], None],
-        AfterValidator(lambda v: [n.strip() for n in v.split(",")] if v else []),
+        BeforeValidator(partial(validate_set_T_from_delim_string, type_=str)),
     ] = Field(
         default=None,
         description="Additional loggers to attach to Prefect logging at runtime.",

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/settings/models/root.py

@@ -72,7 +72,7 @@ class Settings(PrefectBaseSettings):
 
     client: ClientSettings = Field(
         default_factory=ClientSettings,
-        description="Settings for for controlling API client behavior",
+        description="Settings for controlling API client behavior",
     )
 
     cloud: CloudSettings = Field(

{prefect-client-3.1.0 → prefect-client-3.1.2}/src/prefect/settings/models/server/events.py

@@ -128,7 +128,7 @@ class ServerEventsSettings(PrefectBaseSettings):
 
     messaging_cache: str = Field(
         default="prefect.server.utilities.messaging.memory",
-        description="Which cache implementation to use for the events system.  Should point to a module that exports a Cache class.",
+        description="Which cache implementation to use for the events system. Should point to a module that exports a Cache class.",
         validation_alias=AliasChoices(
             AliasPath("messaging_cache"),
             "prefect_server_events_messaging_cache",