prefect-client 2.16.7__py3-none-any.whl → 2.16.9__py3-none-any.whl

prefect/_internal/schemas/bases.py

@@ -1,6 +1,7 @@
 """
 Utilities for creating and working with Prefect REST API schemas.
 """
+
 import datetime
 import json
 import os
@@ -23,7 +24,6 @@ else:
     from pydantic import BaseModel, Field, SecretField
     from pydantic.json import custom_pydantic_encoder
 
-from prefect._internal.compatibility.experimental import experiment_enabled
 from prefect._internal.schemas.fields import DateTimeTZ
 from prefect._internal.schemas.serializers import orjson_dumps_extra_compatible
 
@@ -131,22 +131,6 @@ class PrefectBaseModel(BaseModel):
             dict
         """
 
-        experimental_fields = [
-            field
-            for _, field in self.__fields__.items()
-            if field.field_info.extra.get("experimental")
-        ]
-        experimental_fields_to_exclude = [
-            field.name
-            for field in experimental_fields
-            if not experiment_enabled(field.field_info.extra["experimental-group"])
-        ]
-
-        if experimental_fields_to_exclude:
-            kwargs["exclude"] = (kwargs.get("exclude") or set()).union(
-                experimental_fields_to_exclude
-            )
-
         if json_compatible and shallow:
             raise ValueError(
                 "`json_compatible` can only be applied to the entire object."

prefect/_internal/schemas/validators.py

@@ -1,18 +1,33 @@
+"""
+This module contains a collection of functions that are used to validate the
+values of fields in Pydantic models. These functions are used as validators in
+Pydantic models to ensure that the values of fields conform to the expected
+format.
+This will be subject to consolidation and refactoring over the next few months.
+"""
+
 import datetime
 import json
 import logging
 import re
+import sys
 import urllib.parse
+import warnings
+from copy import copy
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union
 
 import jsonschema
 import pendulum
+import yaml
 
 from prefect._internal.pydantic import HAS_PYDANTIC_V2
+from prefect._internal.pydantic._flags import USE_PYDANTIC_V2
 from prefect._internal.schemas.fields import DateTimeTZ
 from prefect.exceptions import InvalidNameError, InvalidRepositoryURLError
 from prefect.utilities.annotations import NotSet
+from prefect.utilities.dockerutils import get_prefect_image_name
+from prefect.utilities.filesystem import relative_path_to_current_platform
 from prefect.utilities.importtools import from_qualified_name
 from prefect.utilities.names import generate_slug
 from prefect.utilities.pydantic import JsonPatch
@@ -27,9 +42,11 @@ if TYPE_CHECKING:
     from prefect.utilities.callables import ParameterSchema
 
     if HAS_PYDANTIC_V2:
-        from pydantic.v1.fields import ModelField
-    else:
-        from pydantic.fields import ModelField
+        if USE_PYDANTIC_V2:
+            # TODO: we need to account for rewriting the validator to not use ModelField
+            pass
+        if not USE_PYDANTIC_V2:
+            from pydantic.v1.fields import ModelField
 
 
 def raise_on_name_with_banned_characters(name: str) -> str:
@@ -191,6 +208,30 @@ def handle_openapi_schema(value: Optional["ParameterSchema"]) -> "ParameterSchem
     return value
 
 
+def validate_parameters_conform_to_schema(value: dict, values: dict) -> dict:
+    """Validate that the parameters conform to the parameter schema."""
+    if values.get("enforce_parameter_schema"):
+        validate_values_conform_to_schema(
+            value, values.get("parameter_openapi_schema"), ignore_required=True
+        )
+    return value
+
+
+def validate_parameter_openapi_schema(value: dict, values: dict) -> dict:
+    """Validate that the parameter_openapi_schema is a valid json schema."""
+    if values.get("enforce_parameter_schema"):
+        validate_schema(value)
+    return value
+
+
+def return_none_schedule(v: Optional[Union[str, dict]]) -> Optional[Union[str, dict]]:
+    from prefect.client.schemas.schedules import NoSchedule
+
+    if isinstance(v, NoSchedule):
+        return None
+    return v
+
+
 ### SCHEDULE SCHEMA VALIDATORS ###
 
 
@@ -246,6 +287,91 @@ def reconcile_schedules(cls, values: dict) -> dict:
     return values
 
 
+# TODO: consolidate with above if possible
+def reconcile_schedules_runner(values: dict) -> dict:
+    """
+    Similar to above, we reconcile the `schedule` and `schedules` fields in a deployment.
+    """
+    from prefect.deployments.schedules import (
+        create_minimal_deployment_schedule,
+        normalize_to_minimal_deployment_schedules,
+    )
+
+    schedule = values.get("schedule")
+    schedules = values.get("schedules")
+
+    if schedules is None and schedule is not None:
+        values["schedules"] = [create_minimal_deployment_schedule(schedule)]
+    elif schedules is not None and len(schedules) > 0:
+        values["schedules"] = normalize_to_minimal_deployment_schedules(schedules)
+
+    return values
+
+
+def set_deployment_schedules(values: dict) -> dict:
+    from prefect.server.schemas.actions import DeploymentScheduleCreate
+
+    if not values.get("schedules") and values.get("schedule"):
+        values["schedules"] = [
+            DeploymentScheduleCreate(
+                schedule=values["schedule"],
+                active=values["is_schedule_active"],
+            )
+        ]
+
+    return values
+
+
+def remove_old_deployment_fields(values: dict) -> dict:
+    # 2.7.7 removed worker_pool_queue_id in lieu of worker_pool_name and
+    # worker_pool_queue_name. Those fields were later renamed to work_pool_name
+    # and work_queue_name. This validator removes old fields provided
+    # by older clients to avoid 422 errors.
+    values_copy = copy(values)
+    worker_pool_queue_id = values_copy.pop("worker_pool_queue_id", None)
+    worker_pool_name = values_copy.pop("worker_pool_name", None)
+    worker_pool_queue_name = values_copy.pop("worker_pool_queue_name", None)
+    work_pool_queue_name = values_copy.pop("work_pool_queue_name", None)
+    if worker_pool_queue_id:
+        warnings.warn(
+            (
+                "`worker_pool_queue_id` is no longer supported for creating or updating "
+                "deployments. Please use `work_pool_name` and "
+                "`work_queue_name` instead."
+            ),
+            UserWarning,
+        )
+    if worker_pool_name or worker_pool_queue_name or work_pool_queue_name:
+        warnings.warn(
+            (
+                "`worker_pool_name`, `worker_pool_queue_name`, and "
+                "`work_pool_name` are"
+                "no longer supported for creating or updating "
+                "deployments. Please use `work_pool_name` and "
+                "`work_queue_name` instead."
+            ),
+            UserWarning,
+        )
+    return values_copy
+
+
+def reconcile_paused_deployment(values):
+    paused = values.get("paused")
+    is_schedule_active = values.get("is_schedule_active")
+
+    if paused is not None:
+        values["paused"] = paused
+        values["is_schedule_active"] = not paused
+    elif is_schedule_active is not None:
+        values["paused"] = not is_schedule_active
+        values["is_schedule_active"] = is_schedule_active
+    else:
+        values["paused"] = False
+        values["is_schedule_active"] = True
+
+    return values
+
+
 def interval_schedule_must_be_positive(v: datetime.timedelta) -> datetime.timedelta:
     if v.total_seconds() <= 0:
         raise ValueError("The interval must be positive")
@@ -473,6 +599,9 @@ def set_default_image(values: dict) -> dict:
     return values
 
 
+### STATE SCHEMA VALIDATORS ###
+
+
 def get_or_create_state_name(v: str, values: dict) -> str:
     """If a name is not provided, use the type"""
 
@@ -483,6 +612,23 @@ def get_or_create_state_name(v: str, values: dict) -> str:
     return v
 
 
+def set_default_scheduled_time(cls, values: dict) -> dict:
+    """
+    TODO: This should throw an error instead of setting a default but is out of
+            scope for https://github.com/PrefectHQ/orion/pull/174/ and can be rolled
+            into work refactoring state initialization
+    """
+    from prefect.server.schemas.states import StateType
+
+    if values.get("type") == StateType.SCHEDULED:
+        state_details = values.setdefault(
+            "state_details", cls.__fields__["state_details"].get_default()
+        )
+        if not state_details.scheduled_time:
+            state_details.scheduled_time = pendulum.now("utc")
+    return values
+
+
 def get_or_create_run_name(name):
     return name or generate_slug(2)
 
@@ -558,6 +704,48 @@ def validate_picklelib(value: str) -> str:
     return value
 
 
+def validate_picklelib_version(values: dict) -> dict:
+    """
+    Infers a default value for `picklelib_version` if null or ensures it matches
+    the version retrieved from the `pickelib`.
+    """
+    picklelib = values.get("picklelib")
+    picklelib_version = values.get("picklelib_version")
+
+    if not picklelib:
+        raise ValueError("Unable to check version of unrecognized picklelib module")
+
+    pickler = from_qualified_name(picklelib)
+    pickler_version = getattr(pickler, "__version__", None)
+
+    if not picklelib_version:
+        values["picklelib_version"] = pickler_version
+    elif picklelib_version != pickler_version:
+        warnings.warn(
+            (
+                f"Mismatched {picklelib!r} versions. Found {pickler_version} in the"
+                f" environment but {picklelib_version} was requested. This may"
+                " cause the serializer to fail."
+            ),
+            RuntimeWarning,
+            stacklevel=3,
+        )
+
+    return values
+
+
+def validate_picklelib_and_modules(values: dict) -> dict:
+    """
+    Prevents modules from being specified if picklelib is not cloudpickle
+    """
+    if values.get("picklelib") != "cloudpickle" and values.get("pickle_modules"):
+        raise ValueError(
+            "`pickle_modules` cannot be used without 'cloudpickle'. Got"
+            f" {values.get('picklelib')!r}."
+        )
+    return values
+
+
 def validate_dump_kwargs(value: dict) -> dict:
     # `default` is set by `object_encoder`. A user provided callable would make this
     # class unserializable anyway.
@@ -607,3 +795,236 @@ def validate_compressionlib(value: str) -> str:
         )
 
     return value
+
+
+# TODO: if we use this elsewhere we can change the error message to be more generic
+def list_length_50_or_less(v: Optional[List[float]]) -> Optional[List[float]]:
+    if isinstance(v, list) and (len(v) > 50):
+        raise ValueError("Can not configure more than 50 retry delays per task.")
+    return v
+
+
+# TODO: if we use this elsewhere we can change the error message to be more generic
+def validate_not_negative(v: Optional[float]) -> Optional[float]:
+    if v is not None and v < 0:
+        raise ValueError("`retry_jitter_factor` must be >= 0.")
+    return v
+
+
+def validate_message_template_variables(v: Optional[str]) -> Optional[str]:
+    from prefect.client.schemas.objects import FLOW_RUN_NOTIFICATION_TEMPLATE_KWARGS
+
+    if v is not None:
+        try:
+            v.format(**{k: "test" for k in FLOW_RUN_NOTIFICATION_TEMPLATE_KWARGS})
+        except KeyError as exc:
+            raise ValueError(f"Invalid template variable provided: '{exc.args[0]}'")
+    return v
+
+
+def validate_default_queue_id_not_none(v: Optional[str]) -> Optional[str]:
+    if v is None:
+        raise ValueError(
+            "`default_queue_id` is a required field. If you are "
+            "creating a new WorkPool and don't have a queue "
+            "ID yet, use the `actions.WorkPoolCreate` model instead."
+        )
+    return v
+
+
+def validate_max_metadata_length(
+    v: Optional[Dict[str, Any]],
+) -> Optional[Dict[str, Any]]:
+    max_metadata_length = 500
+    if not isinstance(v, dict):
+        return v
+    for key in v.keys():
+        if len(str(v[key])) > max_metadata_length:
+            v[key] = str(v[key])[:max_metadata_length] + "..."
+    return v
+
+
+### DOCKER SCHEMA VALIDATORS ###
+
+
+def validate_registry_url(value: Optional[str]) -> Optional[str]:
+    if isinstance(value, str):
+        if "://" not in value:
+            return "https://" + value
+    return value
+
+
+def convert_labels_to_docker_format(labels: Dict[str, str]) -> Dict[str, str]:
+    labels = labels or {}
+    new_labels = {}
+    for name, value in labels.items():
+        if "/" in name:
+            namespace, key = name.split("/", maxsplit=1)
+            new_namespace = ".".join(reversed(namespace.split(".")))
+            new_labels[f"{new_namespace}.{key}"] = value
+        else:
+            new_labels[name] = value
+    return new_labels
+
+
+def check_volume_format(volumes: List[str]) -> List[str]:
+    for volume in volumes:
+        if ":" not in volume:
+            raise ValueError(
+                "Invalid volume specification. "
+                f"Expected format 'path:container_path', but got {volume!r}"
+            )
+
+    return volumes
+
+
+def assign_default_base_image(values: Mapping[str, Any]) -> Mapping[str, Any]:
+    from prefect.software.conda import CondaEnvironment
+
+    if not values.get("base_image") and not values.get("dockerfile"):
+        values["base_image"] = get_prefect_image_name(
+            flavor=(
+                "conda"
+                if isinstance(values.get("python_environment"), CondaEnvironment)
+                else None
+            )
+        )
+    return values
+
+
+def base_image_xor_dockerfile(values: Mapping[str, Any]):
+    if values.get("base_image") and values.get("dockerfile"):
+        raise ValueError(
+            "Either `base_image` or `dockerfile` should be provided, but not both"
+        )
+    return values
+
+
+def set_default_python_environment(values: Mapping[str, Any]) -> Mapping[str, Any]:
+    from prefect.software.python import PythonEnvironment
+
+    if values.get("base_image") and not values.get("python_environment"):
+        values["python_environment"] = PythonEnvironment.from_environment()
+    return values
+
+
+### SETTINGS SCHEMA VALIDATORS ###
+
+
+def validate_settings(value: dict) -> dict:
+    from prefect.settings import SETTING_VARIABLES, Setting
+
+    if value is None:
+        return value
+
+    # Cast string setting names to variables
+    validated = {}
+    for setting, val in value.items():
+        if isinstance(setting, str) and setting in SETTING_VARIABLES:
+            validated[SETTING_VARIABLES[setting]] = val
+        elif isinstance(setting, Setting):
+            validated[setting] = val
+        else:
+            raise ValueError(f"Unknown setting {setting!r}.")
+
+    return validated
+
+
+def validate_yaml(value: Union[str, dict]) -> dict:
+    if isinstance(value, str):
+        return yaml.safe_load(value)
+    return value
+
+
+### TASK RUN SCHEMA VALIDATORS ###
+
+
+def validate_cache_key_length(cache_key: Optional[str]) -> Optional[str]:
+    from prefect.settings import (
+        PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH,
+    )
+
+    if cache_key and len(cache_key) > PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH.value():
+        raise ValueError(
+            "Cache key exceeded maximum allowed length of"
+            f" {PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH.value()} characters."
+        )
+    return cache_key
+
+
+def set_run_policy_deprecated_fields(values: dict) -> dict:
+    """
+    If deprecated fields are provided, populate the corresponding new fields
+    to preserve orchestration behavior.
+    """
+    if not values.get("retries", None) and values.get("max_retries", 0) != 0:
+        values["retries"] = values["max_retries"]
+
+    if (
+        not values.get("retry_delay", None)
+        and values.get("retry_delay_seconds", 0) != 0
+    ):
+        values["retry_delay"] = values["retry_delay_seconds"]
+
+    return values
+
+
+### PYTHON ENVIRONMENT SCHEMA VALIDATORS ###
+
+
+def infer_python_version(value: Optional[str]) -> Optional[str]:
+    if value is None:
+        return f"{sys.version_info.major}.{sys.version_info.minor}"
+    return value
+
+
+def return_v_or_none(v: Optional[str]) -> Optional[str]:
+    """Make sure that empty strings are treated as None"""
+    if not v:
+        return None
+    return v
+
+
+### INFRASTRUCTURE BLOCK SCHEMA VALIDATORS ###
+
+
+def validate_block_is_infrastructure(v: "Block") -> "Block":
+    from prefect.infrastructure.base import Infrastructure
+
+    print("v: ", v)
+    if not isinstance(v, Infrastructure):
+        raise TypeError("Provided block is not a valid infrastructure block.")
+
+    return v
+
+
+### BLOCK SCHEMA VALIDATORS ###
+
+
+def validate_parent_and_ref_diff(values: dict) -> dict:
+    parent_id = values.get("parent_block_document_id")
+    ref_id = values.get("reference_block_document_id")
+    if parent_id and ref_id and parent_id == ref_id:
+        raise ValueError(
+            "`parent_block_document_id` and `reference_block_document_id` cannot be"
+            " the same"
+        )
+    return values
+
+
+def validate_name_present_on_nonanonymous_blocks(values: dict) -> dict:
+    # anonymous blocks may have no name prior to actually being
+    # stored in the database
+    if not values.get("is_anonymous") and not values.get("name"):
+        raise ValueError("Names must be provided for block documents.")
+    return values
+
+
+### PROCESS JOB CONFIGURATION VALIDATORS ###
+
+
+def validate_command(v: str) -> Path:
+    """Make sure that the working directory is formatted for the current platform."""
+    if v:
+        return relative_path_to_current_platform(v)
+    return v

prefect/blocks/kubernetes.py

@@ -3,7 +3,9 @@ from typing import TYPE_CHECKING, Dict, Type
 
 import yaml
 
+from prefect._internal.compatibility.deprecated import deprecated_class
 from prefect._internal.pydantic import HAS_PYDANTIC_V2
+from prefect._internal.schemas.validators import validate_yaml
 
 if HAS_PYDANTIC_V2:
     from pydantic.v1 import Field, validator
@@ -23,6 +25,10 @@ else:
     kubernetes = lazy_import("kubernetes")
 
 
+@deprecated_class(
+    start_date="Mar 2024",
+    help="Use the KubernetesClusterConfig block from prefect-kubernetes instead.",
+)
 class KubernetesClusterConfig(Block):
     """
     Stores configuration for interaction with Kubernetes clusters.
@@ -55,9 +61,7 @@ class KubernetesClusterConfig(Block):
 
     @validator("config", pre=True)
     def parse_yaml_config(cls, value):
-        if isinstance(value, str):
-            return yaml.safe_load(value)
-        return value
+        return validate_yaml(value)
 
     @classmethod
     def from_file(cls: Type[Self], path: Path = None, context_name: str = None) -> Self:

prefect/client/cloud.py

@@ -63,7 +63,7 @@ class CloudClient:
         self,
         host: str,
         api_key: str,
-        httpx_settings: dict = None,
+        httpx_settings: Optional[Dict[str, Any]] = None,
     ) -> None:
         httpx_settings = httpx_settings or dict()
         httpx_settings.setdefault("headers", dict())

prefect/client/orchestration.py

@@ -216,7 +216,7 @@ class PrefectClient:
         *,
         api_key: str = None,
         api_version: str = None,
-        httpx_settings: dict = None,
+        httpx_settings: Optional[Dict[str, Any]] = None,
     ) -> None:
         httpx_settings = httpx_settings.copy() if httpx_settings else {}
         httpx_settings.setdefault("headers", {})
@@ -523,8 +523,8 @@ class PrefectClient:
         self,
         deployment_id: UUID,
         *,
-        parameters: Dict[str, Any] = None,
-        context: dict = None,
+        parameters: Optional[Dict[str, Any]] = None,
+        context: Optional[Dict[str, Any]] = None,
         state: prefect.states.State = None,
         name: str = None,
         tags: Iterable[str] = None,
@@ -608,8 +608,8 @@ class PrefectClient:
         self,
         flow: "FlowObject",
         name: str = None,
-        parameters: Dict[str, Any] = None,
-        context: dict = None,
+        parameters: Optional[Dict[str, Any]] = None,
+        context: Optional[Dict[str, Any]] = None,
         tags: Iterable[str] = None,
         parent_task_run_id: UUID = None,
         state: "prefect.states.State" = None,
@@ -1578,7 +1578,7 @@ class PrefectClient:
         version: str = None,
         schedule: SCHEDULE_TYPES = None,
         schedules: List[DeploymentScheduleCreate] = None,
-        parameters: Dict[str, Any] = None,
+        parameters: Optional[Dict[str, Any]] = None,
         description: str = None,
         work_queue_name: str = None,
         work_pool_name: str = None,
@@ -1588,8 +1588,8 @@ class PrefectClient:
         path: str = None,
         entrypoint: str = None,
         infrastructure_document_id: UUID = None,
-        infra_overrides: Dict[str, Any] = None,
-        parameter_openapi_schema: dict = None,
+        infra_overrides: Optional[Dict[str, Any]] = None,
+        parameter_openapi_schema: Optional[Dict[str, Any]] = None,
         is_schedule_active: Optional[bool] = None,
         paused: Optional[bool] = None,
         pull_steps: Optional[List[dict]] = None,