prefect-client 2.16.8__py3-none-any.whl → 2.17.0__py3-none-any.whl

prefect/_internal/schemas/validators.py

@@ -1,18 +1,33 @@
+"""
+This module contains a collection of functions that are used to validate the
+values of fields in Pydantic models. These functions are used as validators in
+Pydantic models to ensure that the values of fields conform to the expected
+format.
+This will be subject to consolidation and refactoring over the next few months.
+"""
+
 import datetime
 import json
 import logging
 import re
+import sys
 import urllib.parse
+import warnings
+from copy import copy
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Tuple, Union
 
 import jsonschema
 import pendulum
+import yaml
 
 from prefect._internal.pydantic import HAS_PYDANTIC_V2
+from prefect._internal.pydantic._flags import USE_PYDANTIC_V2
 from prefect._internal.schemas.fields import DateTimeTZ
 from prefect.exceptions import InvalidNameError, InvalidRepositoryURLError
 from prefect.utilities.annotations import NotSet
+from prefect.utilities.dockerutils import get_prefect_image_name
+from prefect.utilities.filesystem import relative_path_to_current_platform
 from prefect.utilities.importtools import from_qualified_name
 from prefect.utilities.names import generate_slug
 from prefect.utilities.pydantic import JsonPatch
@@ -27,9 +42,11 @@ if TYPE_CHECKING:
     from prefect.utilities.callables import ParameterSchema
 
     if HAS_PYDANTIC_V2:
-        from pydantic.v1.fields import ModelField
-    else:
-        from pydantic.fields import ModelField
+        if USE_PYDANTIC_V2:
+            # TODO: we need to account for rewriting the validator to not use ModelField
+            pass
+        if not USE_PYDANTIC_V2:
+            from pydantic.v1.fields import ModelField
 
 
 def raise_on_name_with_banned_characters(name: str) -> str:
@@ -191,6 +208,30 @@ def handle_openapi_schema(value: Optional["ParameterSchema"]) -> "ParameterSchem
     return value
 
 
+def validate_parameters_conform_to_schema(value: dict, values: dict) -> dict:
+    """Validate that the parameters conform to the parameter schema."""
+    if values.get("enforce_parameter_schema"):
+        validate_values_conform_to_schema(
+            value, values.get("parameter_openapi_schema"), ignore_required=True
+        )
+    return value
+
+
+def validate_parameter_openapi_schema(value: dict, values: dict) -> dict:
+    """Validate that the parameter_openapi_schema is a valid json schema."""
+    if values.get("enforce_parameter_schema"):
+        validate_schema(value)
+    return value
+
+
+def return_none_schedule(v: Optional[Union[str, dict]]) -> Optional[Union[str, dict]]:
+    from prefect.client.schemas.schedules import NoSchedule
+
+    if isinstance(v, NoSchedule):
+        return None
+    return v
+
+
 ### SCHEDULE SCHEMA VALIDATORS ###
 
 
@@ -246,6 +287,91 @@ def reconcile_schedules(cls, values: dict) -> dict:
     return values
 
 
+# TODO: consolidate with above if possible
+def reconcile_schedules_runner(values: dict) -> dict:
+    """
+    Similar to above, we reconcile the `schedule` and `schedules` fields in a deployment.
+    """
+    from prefect.deployments.schedules import (
+        create_minimal_deployment_schedule,
+        normalize_to_minimal_deployment_schedules,
+    )
+
+    schedule = values.get("schedule")
+    schedules = values.get("schedules")
+
+    if schedules is None and schedule is not None:
+        values["schedules"] = [create_minimal_deployment_schedule(schedule)]
+    elif schedules is not None and len(schedules) > 0:
+        values["schedules"] = normalize_to_minimal_deployment_schedules(schedules)
+
+    return values
+
+
+def set_deployment_schedules(values: dict) -> dict:
+    from prefect.server.schemas.actions import DeploymentScheduleCreate
+
+    if not values.get("schedules") and values.get("schedule"):
+        values["schedules"] = [
+            DeploymentScheduleCreate(
+                schedule=values["schedule"],
+                active=values["is_schedule_active"],
+            )
+        ]
+
+    return values
+
+
+def remove_old_deployment_fields(values: dict) -> dict:
+    # 2.7.7 removed worker_pool_queue_id in lieu of worker_pool_name and
+    # worker_pool_queue_name. Those fields were later renamed to work_pool_name
+    # and work_queue_name. This validator removes old fields provided
+    # by older clients to avoid 422 errors.
+    values_copy = copy(values)
+    worker_pool_queue_id = values_copy.pop("worker_pool_queue_id", None)
+    worker_pool_name = values_copy.pop("worker_pool_name", None)
+    worker_pool_queue_name = values_copy.pop("worker_pool_queue_name", None)
+    work_pool_queue_name = values_copy.pop("work_pool_queue_name", None)
+    if worker_pool_queue_id:
+        warnings.warn(
+            (
+                "`worker_pool_queue_id` is no longer supported for creating or updating "
+                "deployments. Please use `work_pool_name` and "
+                "`work_queue_name` instead."
+            ),
+            UserWarning,
+        )
+    if worker_pool_name or worker_pool_queue_name or work_pool_queue_name:
+        warnings.warn(
+            (
+                "`worker_pool_name`, `worker_pool_queue_name`, and "
+                "`work_pool_name` are"
+                "no longer supported for creating or updating "
+                "deployments. Please use `work_pool_name` and "
+                "`work_queue_name` instead."
+            ),
+            UserWarning,
+        )
+    return values_copy
+
+
+def reconcile_paused_deployment(values):
+    paused = values.get("paused")
+    is_schedule_active = values.get("is_schedule_active")
+
+    if paused is not None:
+        values["paused"] = paused
+        values["is_schedule_active"] = not paused
+    elif is_schedule_active is not None:
+        values["paused"] = not is_schedule_active
+        values["is_schedule_active"] = is_schedule_active
+    else:
+        values["paused"] = False
+        values["is_schedule_active"] = True
+
+    return values
+
+
 def interval_schedule_must_be_positive(v: datetime.timedelta) -> datetime.timedelta:
     if v.total_seconds() <= 0:
         raise ValueError("The interval must be positive")
@@ -473,6 +599,9 @@ def set_default_image(values: dict) -> dict:
     return values
 
 
+### STATE SCHEMA VALIDATORS ###
+
+
 def get_or_create_state_name(v: str, values: dict) -> str:
     """If a name is not provided, use the type"""
 
@@ -483,6 +612,23 @@ def get_or_create_state_name(v: str, values: dict) -> str:
     return v
 
 
+def set_default_scheduled_time(cls, values: dict) -> dict:
+    """
+    TODO: This should throw an error instead of setting a default but is out of
+            scope for https://github.com/PrefectHQ/orion/pull/174/ and can be rolled
+            into work refactoring state initialization
+    """
+    from prefect.server.schemas.states import StateType
+
+    if values.get("type") == StateType.SCHEDULED:
+        state_details = values.setdefault(
+            "state_details", cls.__fields__["state_details"].get_default()
+        )
+        if not state_details.scheduled_time:
+            state_details.scheduled_time = pendulum.now("utc")
+    return values
+
+
 def get_or_create_run_name(name):
     return name or generate_slug(2)
 
@@ -558,6 +704,48 @@ def validate_picklelib(value: str) -> str:
     return value
 
 
+def validate_picklelib_version(values: dict) -> dict:
+    """
+    Infers a default value for `picklelib_version` if null or ensures it matches
+    the version retrieved from the `pickelib`.
+    """
+    picklelib = values.get("picklelib")
+    picklelib_version = values.get("picklelib_version")
+
+    if not picklelib:
+        raise ValueError("Unable to check version of unrecognized picklelib module")
+
+    pickler = from_qualified_name(picklelib)
+    pickler_version = getattr(pickler, "__version__", None)
+
+    if not picklelib_version:
+        values["picklelib_version"] = pickler_version
+    elif picklelib_version != pickler_version:
+        warnings.warn(
+            (
+                f"Mismatched {picklelib!r} versions. Found {pickler_version} in the"
+                f" environment but {picklelib_version} was requested. This may"
+                " cause the serializer to fail."
+            ),
+            RuntimeWarning,
+            stacklevel=3,
+        )
+
+    return values
+
+
+def validate_picklelib_and_modules(values: dict) -> dict:
+    """
+    Prevents modules from being specified if picklelib is not cloudpickle
+    """
+    if values.get("picklelib") != "cloudpickle" and values.get("pickle_modules"):
+        raise ValueError(
+            "`pickle_modules` cannot be used without 'cloudpickle'. Got"
+            f" {values.get('picklelib')!r}."
+        )
+    return values
+
+
 def validate_dump_kwargs(value: dict) -> dict:
     # `default` is set by `object_encoder`. A user provided callable would make this
     # class unserializable anyway.
@@ -607,3 +795,236 @@ def validate_compressionlib(value: str) -> str:
         )
 
     return value
+
+
+# TODO: if we use this elsewhere we can change the error message to be more generic
+def list_length_50_or_less(v: Optional[List[float]]) -> Optional[List[float]]:
+    if isinstance(v, list) and (len(v) > 50):
+        raise ValueError("Can not configure more than 50 retry delays per task.")
+    return v
+
+
+# TODO: if we use this elsewhere we can change the error message to be more generic
+def validate_not_negative(v: Optional[float]) -> Optional[float]:
+    if v is not None and v < 0:
+        raise ValueError("`retry_jitter_factor` must be >= 0.")
+    return v
+
+
+def validate_message_template_variables(v: Optional[str]) -> Optional[str]:
+    from prefect.client.schemas.objects import FLOW_RUN_NOTIFICATION_TEMPLATE_KWARGS
+
+    if v is not None:
+        try:
+            v.format(**{k: "test" for k in FLOW_RUN_NOTIFICATION_TEMPLATE_KWARGS})
+        except KeyError as exc:
+            raise ValueError(f"Invalid template variable provided: '{exc.args[0]}'")
+    return v
+
+
+def validate_default_queue_id_not_none(v: Optional[str]) -> Optional[str]:
+    if v is None:
+        raise ValueError(
+            "`default_queue_id` is a required field. If you are "
+            "creating a new WorkPool and don't have a queue "
+            "ID yet, use the `actions.WorkPoolCreate` model instead."
+        )
+    return v
+
+
+def validate_max_metadata_length(
+    v: Optional[Dict[str, Any]],
+) -> Optional[Dict[str, Any]]:
+    max_metadata_length = 500
+    if not isinstance(v, dict):
+        return v
+    for key in v.keys():
+        if len(str(v[key])) > max_metadata_length:
+            v[key] = str(v[key])[:max_metadata_length] + "..."
+    return v
+
+
+### DOCKER SCHEMA VALIDATORS ###
+
+
+def validate_registry_url(value: Optional[str]) -> Optional[str]:
+    if isinstance(value, str):
+        if "://" not in value:
+            return "https://" + value
+    return value
+
+
+def convert_labels_to_docker_format(labels: Dict[str, str]) -> Dict[str, str]:
+    labels = labels or {}
+    new_labels = {}
+    for name, value in labels.items():
+        if "/" in name:
+            namespace, key = name.split("/", maxsplit=1)
+            new_namespace = ".".join(reversed(namespace.split(".")))
+            new_labels[f"{new_namespace}.{key}"] = value
+        else:
+            new_labels[name] = value
+    return new_labels
+
+
+def check_volume_format(volumes: List[str]) -> List[str]:
+    for volume in volumes:
+        if ":" not in volume:
+            raise ValueError(
+                "Invalid volume specification. "
+                f"Expected format 'path:container_path', but got {volume!r}"
+            )
+
+    return volumes
+
+
+def assign_default_base_image(values: Mapping[str, Any]) -> Mapping[str, Any]:
+    from prefect.software.conda import CondaEnvironment
+
+    if not values.get("base_image") and not values.get("dockerfile"):
+        values["base_image"] = get_prefect_image_name(
+            flavor=(
+                "conda"
+                if isinstance(values.get("python_environment"), CondaEnvironment)
+                else None
+            )
+        )
+    return values
+
+
+def base_image_xor_dockerfile(values: Mapping[str, Any]):
+    if values.get("base_image") and values.get("dockerfile"):
+        raise ValueError(
+            "Either `base_image` or `dockerfile` should be provided, but not both"
+        )
+    return values
+
+
+def set_default_python_environment(values: Mapping[str, Any]) -> Mapping[str, Any]:
+    from prefect.software.python import PythonEnvironment
+
+    if values.get("base_image") and not values.get("python_environment"):
+        values["python_environment"] = PythonEnvironment.from_environment()
+    return values
+
+
+### SETTINGS SCHEMA VALIDATORS ###
+
+
+def validate_settings(value: dict) -> dict:
+    from prefect.settings import SETTING_VARIABLES, Setting
+
+    if value is None:
+        return value
+
+    # Cast string setting names to variables
+    validated = {}
+    for setting, val in value.items():
+        if isinstance(setting, str) and setting in SETTING_VARIABLES:
+            validated[SETTING_VARIABLES[setting]] = val
+        elif isinstance(setting, Setting):
+            validated[setting] = val
+        else:
+            raise ValueError(f"Unknown setting {setting!r}.")
+
+    return validated
+
+
+def validate_yaml(value: Union[str, dict]) -> dict:
+    if isinstance(value, str):
+        return yaml.safe_load(value)
+    return value
+
+
+### TASK RUN SCHEMA VALIDATORS ###
+
+
+def validate_cache_key_length(cache_key: Optional[str]) -> Optional[str]:
+    from prefect.settings import (
+        PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH,
+    )
+
+    if cache_key and len(cache_key) > PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH.value():
+        raise ValueError(
+            "Cache key exceeded maximum allowed length of"
+            f" {PREFECT_API_TASK_CACHE_KEY_MAX_LENGTH.value()} characters."
+        )
+    return cache_key
+
+
+def set_run_policy_deprecated_fields(values: dict) -> dict:
+    """
+    If deprecated fields are provided, populate the corresponding new fields
+    to preserve orchestration behavior.
+    """
+    if not values.get("retries", None) and values.get("max_retries", 0) != 0:
+        values["retries"] = values["max_retries"]
+
+    if (
+        not values.get("retry_delay", None)
+        and values.get("retry_delay_seconds", 0) != 0
+    ):
+        values["retry_delay"] = values["retry_delay_seconds"]
+
+    return values
+
+
+### PYTHON ENVIRONMENT SCHEMA VALIDATORS ###
+
+
+def infer_python_version(value: Optional[str]) -> Optional[str]:
+    if value is None:
+        return f"{sys.version_info.major}.{sys.version_info.minor}"
+    return value
+
+
+def return_v_or_none(v: Optional[str]) -> Optional[str]:
+    """Make sure that empty strings are treated as None"""
+    if not v:
+        return None
+    return v
+
+
+### INFRASTRUCTURE BLOCK SCHEMA VALIDATORS ###
+
+
+def validate_block_is_infrastructure(v: "Block") -> "Block":
+    from prefect.infrastructure.base import Infrastructure
+
+    print("v: ", v)
+    if not isinstance(v, Infrastructure):
+        raise TypeError("Provided block is not a valid infrastructure block.")
+
+    return v
+
+
+### BLOCK SCHEMA VALIDATORS ###
+
+
+def validate_parent_and_ref_diff(values: dict) -> dict:
+    parent_id = values.get("parent_block_document_id")
+    ref_id = values.get("reference_block_document_id")
+    if parent_id and ref_id and parent_id == ref_id:
+        raise ValueError(
+            "`parent_block_document_id` and `reference_block_document_id` cannot be"
+            " the same"
+        )
+    return values
+
+
+def validate_name_present_on_nonanonymous_blocks(values: dict) -> dict:
+    # anonymous blocks may have no name prior to actually being
+    # stored in the database
+    if not values.get("is_anonymous") and not values.get("name"):
+        raise ValueError("Names must be provided for block documents.")
+    return values
+
+
+### PROCESS JOB CONFIGURATION VALIDATORS ###
+
+
+def validate_command(v: str) -> Path:
+    """Make sure that the working directory is formatted for the current platform."""
+    if v:
+        return relative_path_to_current_platform(v)
+    return v

prefect/agent.py

@@ -445,7 +445,7 @@ class PrefectAgent:
         # attributes of the infrastructure block
         doc_dict = infra_document.dict()
         infra_dict = doc_dict.get("data", {})
-        for override, value in (deployment.infra_overrides or {}).items():
+        for override, value in (deployment.job_variables or {}).items():
             nested_fields = override.split(".")
             data = infra_dict
             for field in nested_fields[:-1]:

prefect/blocks/kubernetes.py

@@ -3,7 +3,9 @@ from typing import TYPE_CHECKING, Dict, Type
 
 import yaml
 
+from prefect._internal.compatibility.deprecated import deprecated_class
 from prefect._internal.pydantic import HAS_PYDANTIC_V2
+from prefect._internal.schemas.validators import validate_yaml
 
 if HAS_PYDANTIC_V2:
     from pydantic.v1 import Field, validator
@@ -23,6 +25,10 @@ else:
     kubernetes = lazy_import("kubernetes")
 
 
+@deprecated_class(
+    start_date="Mar 2024",
+    help="Use the KubernetesClusterConfig block from prefect-kubernetes instead.",
+)
 class KubernetesClusterConfig(Block):
     """
     Stores configuration for interaction with Kubernetes clusters.
@@ -55,9 +61,7 @@ class KubernetesClusterConfig(Block):
 
     @validator("config", pre=True)
     def parse_yaml_config(cls, value):
-        if isinstance(value, str):
-            return yaml.safe_load(value)
-        return value
+        return validate_yaml(value)
 
     @classmethod
     def from_file(cls: Type[Self], path: Path = None, context_name: str = None) -> Self:

prefect/blocks/notifications.py

@@ -77,7 +77,7 @@ class AppriseNotificationBlock(AbstractAppriseNotificationBlock, ABC):
         default=...,
         title="Webhook URL",
         description="Incoming webhook URL used to send notifications.",
-        example="https://hooks.example.com/XXX",
+        examples=["https://hooks.example.com/XXX"],
     )
 
 
@@ -105,7 +105,7 @@ class SlackWebhook(AppriseNotificationBlock):
         default=...,
         title="Webhook URL",
         description="Slack incoming webhook URL used to send notifications.",
-        example="https://hooks.slack.com/XXX",
+        examples=["https://hooks.slack.com/XXX"],
     )
 
 
@@ -131,9 +131,9 @@ class MicrosoftTeamsWebhook(AppriseNotificationBlock):
         ...,
         title="Webhook URL",
         description="The Teams incoming webhook URL used to send notifications.",
-        example=(
+        examples=[
             "https://your-org.webhook.office.com/webhookb2/XXX/IncomingWebhook/YYY/ZZZ"
-        ),
+        ],
     )
 
 
@@ -222,7 +222,7 @@ class PagerDutyWebHook(AbstractAppriseNotificationBlock):
     custom_details: Optional[Dict[str, str]] = Field(
         default=None,
         description="Additional details to include as part of the payload.",
-        example='{"disk_space_left": "145GB"}',
+        examples=['{"disk_space_left": "145GB"}'],
     )
 
     def block_initialization(self) -> None:
@@ -283,14 +283,14 @@ class TwilioSMS(AbstractAppriseNotificationBlock):
     from_phone_number: str = Field(
         default=...,
         description="The valid Twilio phone number to send the message from.",
-        example="18001234567",
+        examples=["18001234567"],
     )
 
     to_phone_numbers: List[str] = Field(
         default=...,
         description="A list of valid Twilio phone number(s) to send the message to.",
         # not wrapped in brackets because of the way UI displays examples; in code should be ["18004242424"]
-        example="18004242424",
+        examples=["18004242424"],
     )
 
     def block_initialization(self) -> None:
@@ -366,7 +366,7 @@ class OpsgenieWebhook(AbstractAppriseNotificationBlock):
             "A comma-separated list of tags you can associate with your Opsgenie"
             " message."
         ),
-        example='["tag1", "tag2"]',
+        examples=['["tag1", "tag2"]'],
     )
 
     priority: Optional[str] = Field(
@@ -388,7 +388,7 @@ class OpsgenieWebhook(AbstractAppriseNotificationBlock):
     details: Optional[Dict[str, str]] = Field(
         default=None,
         description="Additional details composed of key/values pairs.",
-        example='{"key1": "value1", "key2": "value2"}',
+        examples=['{"key1": "value1", "key2": "value2"}'],
     )
 
     def block_initialization(self) -> None:
@@ -445,7 +445,7 @@ class MattermostWebhook(AbstractAppriseNotificationBlock):
     hostname: str = Field(
         default=...,
         description="The hostname of your Mattermost server.",
-        example="Mattermost.example.com",
+        examples=["Mattermost.example.com"],
     )
 
     token: SecretStr = Field(
@@ -617,7 +617,7 @@ class CustomWebhookNotificationBlock(NotificationBlock):
     url: str = Field(
         title="Webhook URL",
         description="The webhook URL.",
-        example="https://hooks.slack.com/XXX",
+        examples=["https://hooks.slack.com/XXX"],
     )
 
     method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"] = Field(
@@ -631,10 +631,10 @@ class CustomWebhookNotificationBlock(NotificationBlock):
         default=None,
         title="JSON Data",
         description="Send json data as payload.",
-        example=(
+        examples=[
             '{"text": "{{subject}}\\n{{body}}", "title": "{{name}}", "token":'
             ' "{{tokenFromSecrets}}"}'
-        ),
+        ],
     )
     form_data: Optional[Dict[str, str]] = Field(
         default=None,
@@ -642,10 +642,10 @@ class CustomWebhookNotificationBlock(NotificationBlock):
         description=(
             "Send form data as payload. Should not be used together with _JSON Data_."
         ),
-        example=(
+        examples=[
             '{"text": "{{subject}}\\n{{body}}", "title": "{{name}}", "token":'
             ' "{{tokenFromSecrets}}"}'
-        ),
+        ],
     )
 
     headers: Optional[Dict[str, str]] = Field(None, description="Custom headers.")
@@ -659,7 +659,7 @@ class CustomWebhookNotificationBlock(NotificationBlock):
         default_factory=lambda: SecretDict(dict()),
         title="Custom Secret Values",
         description="A dictionary of secret values to be substituted in other configs.",
-        example='{"tokenFromSecrets":"SomeSecretToken"}',
+        examples=['{"tokenFromSecrets":"SomeSecretToken"}'],
     )
 
     def _build_request_args(self, body: str, subject: Optional[str]):
@@ -753,14 +753,14 @@ class SendgridEmail(AbstractAppriseNotificationBlock):
     sender_email: str = Field(
         title="Sender email id",
         description="The sender email id.",
-        example="test-support@gmail.com",
+        examples=["test-support@gmail.com"],
     )
 
     to_emails: List[str] = Field(
         default=...,
         title="Recipient emails",
         description="Email ids of all recipients.",
-        example='"recipient1@gmail.com"',
+        examples=['"recipient1@gmail.com"'],
     )
 
     def block_initialization(self) -> None:

prefect/blocks/webhook.py

@@ -36,7 +36,7 @@ class Webhook(Block):
         default=...,
         title="Webhook URL",
         description="The webhook URL.",
-        example="https://hooks.slack.com/XXX",
+        examples=["https://hooks.slack.com/XXX"],
     )
 
     headers: SecretDict = Field(