prefect-client 2.17.1__py3-none-any.whl → 2.18.0__py3-none-any.whl

prefect/serializers.py

@@ -13,9 +13,10 @@ bytes to an object respectively.
 
 import abc
 import base64
-from typing import Any, Dict, Generic, Optional, TypeVar
+from typing import Any, Dict, Generic, Optional, Type, TypeVar
+
+from typing_extensions import Literal, Self
 
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
 from prefect._internal.schemas.validators import (
     cast_type_names_to_serializers,
     validate_compressionlib,
@@ -24,20 +25,29 @@ from prefect._internal.schemas.validators import (
     validate_picklelib,
     validate_picklelib_version,
 )
+from prefect.pydantic import HAS_PYDANTIC_V2
+from prefect.utilities.dispatch import get_dispatch_key, lookup_type, register_base_type
+from prefect.utilities.importtools import from_qualified_name, to_qualified_name
+from prefect.utilities.pydantic import custom_pydantic_encoder
 
 if HAS_PYDANTIC_V2:
-    import pydantic.v1 as pydantic
-    from pydantic.v1 import BaseModel
-    from pydantic.v1.json import pydantic_encoder
+    from pydantic.v1 import (
+        BaseModel,
+        Field,
+        ValidationError,
+        parse_obj_as,
+        root_validator,
+        validator,
+    )
 else:
-    import pydantic
-    from pydantic import BaseModel
-    from pydantic.json import pydantic_encoder
-
-from typing_extensions import Literal
-
-from prefect.utilities.importtools import from_qualified_name, to_qualified_name
-from prefect.utilities.pydantic import add_type_dispatch
+    from pydantic import (
+        BaseModel,
+        Field,
+        ValidationError,
+        parse_obj_as,
+        root_validator,
+        validator,
+    )
 
 D = TypeVar("D")
 
@@ -53,7 +63,7 @@ def prefect_json_object_encoder(obj: Any) -> Any:
     else:
         return {
             "__class__": to_qualified_name(obj.__class__),
-            "data": pydantic_encoder(obj),
+            "data": custom_pydantic_encoder({}, obj),
         }
 
 
@@ -63,21 +73,35 @@ def prefect_json_object_decoder(result: dict):
     with `prefect_json_object_encoder`
     """
     if "__class__" in result:
-        return pydantic.parse_obj_as(
-            from_qualified_name(result["__class__"]), result["data"]
-        )
+        return parse_obj_as(from_qualified_name(result["__class__"]), result["data"])
     elif "__exc_type__" in result:
         return from_qualified_name(result["__exc_type__"])(result["message"])
     else:
         return result
 
 
-@add_type_dispatch
+@register_base_type
 class Serializer(BaseModel, Generic[D], abc.ABC):
     """
     A serializer that can encode objects of type 'D' into bytes.
     """
 
+    def __init__(self, **data: Any) -> None:
+        type_string = get_dispatch_key(self) if type(self) != Serializer else "__base__"
+        data.setdefault("type", type_string)
+        super().__init__(**data)
+
+    def __new__(cls: Type[Self], **kwargs) -> Self:
+        if "type" in kwargs:
+            try:
+                subcls = lookup_type(cls, dispatch_key=kwargs["type"])
+            except KeyError as exc:
+                raise ValidationError(errors=[exc], model=cls)
+
+            return super().__new__(subcls)
+        else:
+            return super().__new__(cls)
+
     type: str
 
     @abc.abstractmethod
@@ -91,6 +115,10 @@ class Serializer(BaseModel, Generic[D], abc.ABC):
     class Config:
         extra = "forbid"
 
+    @classmethod
+    def __dispatch_key__(cls):
+        return cls.__fields__.get("type").get_default()
+
 
 class PickleSerializer(Serializer):
     """
@@ -107,11 +135,11 @@ class PickleSerializer(Serializer):
     picklelib: str = "cloudpickle"
     picklelib_version: str = None
 
-    @pydantic.validator("picklelib")
+    @validator("picklelib")
     def check_picklelib(cls, value):
         return validate_picklelib(value)
 
-    @pydantic.root_validator
+    @root_validator
     def check_picklelib_version(cls, values):
         return validate_picklelib_version(values)
 
@@ -135,16 +163,17 @@ class JSONSerializer(Serializer):
     """
 
     type: Literal["json"] = "json"
+
     jsonlib: str = "json"
-    object_encoder: Optional[str] = pydantic.Field(
+    object_encoder: Optional[str] = Field(
         default="prefect.serializers.prefect_json_object_encoder",
         description=(
             "An optional callable to use when serializing objects that are not "
             "supported by the JSON encoder. By default, this is set to a callable that "
-            "adds support for all types supported by Pydantic."
+            "adds support for all types supported by "
         ),
     )
-    object_decoder: Optional[str] = pydantic.Field(
+    object_decoder: Optional[str] = Field(
         default="prefect.serializers.prefect_json_object_decoder",
         description=(
             "An optional callable to use when deserializing objects. This callable "
@@ -153,14 +182,14 @@ class JSONSerializer(Serializer):
             "by our default `object_encoder`."
         ),
     )
-    dumps_kwargs: Dict[str, Any] = pydantic.Field(default_factory=dict)
-    loads_kwargs: Dict[str, Any] = pydantic.Field(default_factory=dict)
+    dumps_kwargs: Dict[str, Any] = Field(default_factory=dict)
+    loads_kwargs: Dict[str, Any] = Field(default_factory=dict)
 
-    @pydantic.validator("dumps_kwargs")
+    @validator("dumps_kwargs")
     def dumps_kwargs_cannot_contain_default(cls, value):
         return validate_dump_kwargs(value)
 
-    @pydantic.validator("loads_kwargs")
+    @validator("loads_kwargs")
     def loads_kwargs_cannot_contain_object_hook(cls, value):
         return validate_load_kwargs(value)
 
@@ -200,11 +229,11 @@ class CompressedSerializer(Serializer):
     serializer: Serializer
     compressionlib: str = "lzma"
 
-    @pydantic.validator("serializer", pre=True)
+    @validator("serializer", pre=True)
     def validate_serializer(cls, value):
         return cast_type_names_to_serializers(value)
 
-    @pydantic.validator("compressionlib")
+    @validator("compressionlib")
     def check_compressionlib(cls, value):
         return validate_compressionlib(value)
 
@@ -225,7 +254,8 @@ class CompressedPickleSerializer(CompressedSerializer):
     """
 
     type: Literal["compressed/pickle"] = "compressed/pickle"
-    serializer: Serializer = pydantic.Field(default_factory=PickleSerializer)
+
+    serializer: Serializer = Field(default_factory=PickleSerializer)
 
 
 class CompressedJSONSerializer(CompressedSerializer):
@@ -234,4 +264,5 @@ class CompressedJSONSerializer(CompressedSerializer):
     """
 
     type: Literal["compressed/json"] = "compressed/json"
-    serializer: Serializer = pydantic.Field(default_factory=JSONSerializer)
+
+    serializer: Serializer = Field(default_factory=JSONSerializer)

prefect/settings.py

@@ -109,6 +109,8 @@ REMOVED_EXPERIMENTAL_FLAGS = {
     "PREFECT_EXPERIMENTAL_ENABLE_ENHANCED_DEPLOYMENT_PARAMETERS",
     "PREFECT_EXPERIMENTAL_ENABLE_EVENTS_CLIENT",
     "PREFECT_EXPERIMENTAL_WARN_EVENTS_CLIENT",
+    "PREFECT_EXPERIMENTAL_ENABLE_FLOW_RUN_INFRA_OVERRIDES",
+    "PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES",
 }
 
 
@@ -1542,16 +1544,6 @@ a task server should move a task from PENDING to RUNNING very quickly, so runs s
 PENDING for a while is a sign that the task server may have crashed.
 """
 
-PREFECT_EXPERIMENTAL_ENABLE_FLOW_RUN_INFRA_OVERRIDES = Setting(bool, default=False)
-"""
-Whether or not to enable infrastructure overrides made on flow runs.
-"""
-
-PREFECT_EXPERIMENTAL_WARN_FLOW_RUN_INFRA_OVERRIDES = Setting(bool, default=True)
-"""
-Whether or not to warn infrastructure when experimental flow runs overrides are used.
-"""
-
 PREFECT_EXPERIMENTAL_ENABLE_EXTRA_RUNNER_ENDPOINTS = Setting(bool, default=False)
 """
 Whether or not to enable experimental worker webserver endpoints.
@@ -1701,6 +1693,10 @@ PREFECT_API_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL = Setting(float, default=5,
 The maximum number of seconds between flushes of the event persister.
 """
 
+PREFECT_API_EVENTS_STREAM_OUT_ENABLED = Setting(bool, default=True)
+"""
+Whether or not to allow streaming events out of via websockets.
+"""
 
 # Deprecated settings ------------------------------------------------------------------
 

prefect/types/__init__.py

@@ -0,0 +1,90 @@
+from dataclasses import dataclass
+from typing import Any, Callable, ClassVar, Generator
+
+from pydantic_core import core_schema, CoreSchema, SchemaValidator
+from typing_extensions import Self
+from datetime import timedelta
+
+
+@dataclass
+class NonNegativeInteger(int):
+    schema: ClassVar[CoreSchema] = core_schema.int_schema(ge=0)
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+        yield cls.validate
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[..., Any]
+    ) -> CoreSchema:
+        return cls.schema
+
+    @classmethod
+    def validate(cls, v: Any) -> Self:
+        return SchemaValidator(schema=cls.schema).validate_python(v)
+
+
+@dataclass
+class PositiveInteger(int):
+    schema: ClassVar[CoreSchema] = core_schema.int_schema(gt=0)
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+        yield cls.validate
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[..., Any]
+    ) -> CoreSchema:
+        return cls.schema
+
+    @classmethod
+    def validate(cls, v: Any) -> Self:
+        return SchemaValidator(schema=cls.schema).validate_python(v)
+
+
+@dataclass
+class NonNegativeDuration(timedelta):
+    schema: ClassVar = core_schema.timedelta_schema(ge=timedelta(seconds=0))
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+        yield cls.validate
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[..., Any]
+    ) -> CoreSchema:
+        return cls.schema
+
+    @classmethod
+    def validate(cls, v: Any) -> Self:
+        return SchemaValidator(schema=cls.schema).validate_python(v)
+
+
+@dataclass
+class PositiveDuration(timedelta):
+    schema: ClassVar = core_schema.timedelta_schema(gt=timedelta(seconds=0))
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+        yield cls.validate
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[..., Any]
+    ) -> CoreSchema:
+        return cls.schema
+
+    @classmethod
+    def validate(cls, v: Any) -> Self:
+        return SchemaValidator(schema=cls.schema).validate_python(v)
+
+
+__all__ = [
+    "NonNegativeInteger",
+    "PositiveInteger",
+    "NonNegativeDuration",
+    "PositiveDuration",
+]

prefect/utilities/pydantic.py

@@ -1,24 +1,25 @@
 from functools import partial
-from typing import Any, Callable, Generic, Type, TypeVar, cast, overload
-
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-
-if HAS_PYDANTIC_V2:
-    import pydantic.v1 as pydantic
-else:
-    import pydantic
+from typing import Any, Callable, Dict, Generic, Optional, Type, TypeVar, cast, overload
 
 from jsonpatch import JsonPatch as JsonPatchBase
+from pydantic_core import to_jsonable_python
 from typing_extensions import Self
 
+from prefect._internal.pydantic.utilities.model_dump import model_dump
+from prefect.pydantic import HAS_PYDANTIC_V2
 from prefect.utilities.dispatch import get_dispatch_key, lookup_type, register_base_type
 from prefect.utilities.importtools import from_qualified_name, to_qualified_name
 
+if HAS_PYDANTIC_V2:
+    import pydantic.v1 as pydantic_v1
+else:
+    import pydantic as pydantic_v1
+
 D = TypeVar("D", bound=Any)
-M = TypeVar("M", bound=pydantic.BaseModel)
+M = TypeVar("M", bound=pydantic_v1.BaseModel)
 
 
-def _reduce_model(model: pydantic.BaseModel):
+def _reduce_model(model: pydantic_v1.BaseModel):
     """
     Helper for serializing a cythonized model with cloudpickle.
 
@@ -82,7 +83,7 @@ def add_cloudpickle_reduction(__model_cls: Type[M] = None, **kwargs: Any):
         )
 
 
-def get_class_fields_only(model: Type[pydantic.BaseModel]) -> set:
+def get_class_fields_only(model: Type[pydantic_v1.BaseModel]) -> set:
     """
     Gets all the field names defined on the model class but not any parent classes.
     Any fields that are on the parent but redefined on the subclass are included.
@@ -91,7 +92,7 @@ def get_class_fields_only(model: Type[pydantic.BaseModel]) -> set:
     parent_class_fields = set()
 
     for base in model.__class__.__bases__:
-        if issubclass(base, pydantic.BaseModel):
+        if issubclass(base, pydantic_v1.BaseModel):
             parent_class_fields.update(base.__annotations__.keys())
 
     return (subclass_class_fields - parent_class_fields) | (
@@ -134,7 +135,7 @@ def add_type_dispatch(model_cls: Type[M]) -> Type[M]:
 
     elif defines_dispatch_key and not defines_type_field:
         # Add a type field to store the value of the dispatch key
-        model_cls.__fields__["type"] = pydantic.fields.ModelField(
+        model_cls.__fields__["type"] = pydantic_v1.fields.ModelField(
             name="type",
             type_=str,
             required=True,
@@ -180,7 +181,7 @@ def add_type_dispatch(model_cls: Type[M]) -> Type[M]:
             try:
                 subcls = lookup_type(cls, dispatch_key=kwargs["type"])
             except KeyError as exc:
-                raise pydantic.ValidationError(errors=[exc], model=cls)
+                raise pydantic_v1.ValidationError(errors=[exc], model=cls)
             return cls_new(subcls)
         else:
             return cls_new(cls)
@@ -206,7 +207,7 @@ class PartialModel(Generic[M]):
     a field already has a value.
 
     Example:
-        >>> class MyModel(pydantic.BaseModel):
+        >>> class MyModel(pydantic_v1.BaseModel):
         >>>     x: int
         >>>     y: str
         >>>     z: float
@@ -267,3 +268,21 @@ class JsonPatch(JsonPatchBase):
                 },
             }
         )
+
+
+def custom_pydantic_encoder(
+    type_encoders: Optional[Dict[Any, Callable[[Type[Any]], Any]]], obj: Any
+) -> Any:
+    # Check the class type and its superclasses for a matching encoder
+    for base in obj.__class__.__mro__[:-1]:
+        try:
+            encoder = type_encoders[base]
+        except KeyError:
+            continue
+
+        return encoder(obj)
+    else:  # We have exited the for loop without finding a suitable encoder
+        if isinstance(obj, pydantic_v1.BaseModel):
+            return model_dump(obj, mode="json")
+        else:
+            return to_jsonable_python(obj)

prefect/utilities/schema_tools/hydration.py

@@ -1,41 +1,53 @@
 import json
 from typing import Any, Callable, Dict, Optional
 
-from prefect._internal.pydantic import HAS_PYDANTIC_V2
-
-if HAS_PYDANTIC_V2:
-    from pydantic.v1 import BaseModel, Field
-else:
-    from pydantic import BaseModel, Field
-
+import jinja2
+from pydantic import BaseModel, Field
 from sqlalchemy.ext.asyncio import AsyncSession
 from typing_extensions import TypeAlias
 
-from prefect.server.models.variables import read_variables
+from prefect.server.utilities.user_templates import (
+    TemplateSecurityError,
+    render_user_template_sync,
+    validate_user_template,
+)
 
 
 class HydrationContext(BaseModel):
     workspace_variables: Dict[str, str] = Field(default_factory=dict)
+    render_workspace_variables: bool = Field(default=False)
     raise_on_error: bool = Field(default=False)
+    render_jinja: bool = Field(default=False)
+    jinja_context: Dict[str, Any] = Field(default_factory=dict)
 
     @classmethod
     async def build(
         cls,
         session: AsyncSession,
         raise_on_error: bool = False,
+        render_jinja: bool = False,
+        render_workspace_variables: bool = False,
     ) -> "HydrationContext":
-        variables = await read_variables(
-            session=session,
-        )
+        from prefect.server.models.variables import read_variables
+
+        if render_workspace_variables:
+            variables = await read_variables(
+                session=session,
+            )
+        else:
+            variables = []
+
         return cls(
             workspace_variables={
                 variable.name: variable.value for variable in variables
             },
             raise_on_error=raise_on_error,
+            render_jinja=render_jinja,
+            render_workspace_variables=render_workspace_variables,
         )
 
 
-Handler: TypeAlias = Callable[[Dict, HydrationContext], Any]
+Handler: TypeAlias = Callable[[dict, HydrationContext], Any]
 PrefectKind: TypeAlias = Optional[str]
 
 _handlers: Dict[PrefectKind, Handler] = {}
@@ -93,6 +105,12 @@ class ValueNotFound(KeyNotFound):
         return "value"
 
 
+class TemplateNotFound(KeyNotFound):
+    @property
+    def key(self):
+        return "template"
+
+
 class VariableNameNotFound(KeyNotFound):
     @property
     def key(self):
@@ -108,6 +126,15 @@ class InvalidJSON(HydrationError):
         return message
 
 
+class InvalidJinja(HydrationError):
+    @property
+    def message(self):
+        message = "Invalid jinja"
+        if self.detail:
+            message += f": {self.detail}"
+        return message
+
+
 class WorkspaceVariableNotFound(HydrationError):
     @property
     def variable_name(self) -> str:
@@ -116,7 +143,25 @@ class WorkspaceVariableNotFound(HydrationError):
 
     @property
     def message(self):
-        return f"Variable '{self.detail}' not found."
+        return f"Variable '{self.detail}' not found in workspace."
+
+
+class WorkspaceVariable(Placeholder):
+    def __init__(self, variable_name: str):
+        self.variable_name = variable_name
+
+    def __eq__(self, other):
+        return (
+            isinstance(other, type(self)) and self.variable_name == other.variable_name
+        )
+
+
+class ValidJinja(Placeholder):
+    def __init__(self, template: str):
+        self.template = template
+
+    def __eq__(self, other):
+        return isinstance(other, type(self)) and self.template == other.template
 
 
 def handler(kind: PrefectKind) -> Callable:
@@ -127,7 +172,7 @@ def handler(kind: PrefectKind) -> Callable:
     return decorator
 
 
-def call_handler(kind: PrefectKind, obj: Dict, ctx: HydrationContext) -> Any:
+def call_handler(kind: PrefectKind, obj: dict, ctx: HydrationContext) -> Any:
     if kind not in _handlers:
         return (obj or {}).get("value", None)
 
@@ -138,7 +183,7 @@ def call_handler(kind: PrefectKind, obj: Dict, ctx: HydrationContext) -> Any:
 
 
 @handler("none")
-def null_handler(obj: Dict, ctx: HydrationContext):
+def null_handler(obj: dict, ctx: HydrationContext):
     if "value" in obj:
         # null handler is a pass through, so we want to continue to hydrate
         return _hydrate(obj["value"], ctx)
@@ -147,7 +192,7 @@ def null_handler(obj: Dict, ctx: HydrationContext):
 
 
 @handler("json")
-def json_handler(obj: Dict, ctx: HydrationContext):
+def json_handler(obj: dict, ctx: HydrationContext):
     if "value" in obj:
         if isinstance(obj["value"], dict):
             dehydrated_json = _hydrate(obj["value"], ctx)
@@ -167,14 +212,38 @@ def json_handler(obj: Dict, ctx: HydrationContext):
         return RemoveValue()
 
 
+@handler("jinja")
+def jinja_handler(obj: dict, ctx: HydrationContext):
+    if "template" in obj:
+        if isinstance(obj["template"], dict):
+            dehydrated_jinja = _hydrate(obj["template"], ctx)
+        else:
+            dehydrated_jinja = obj["template"]
+
+        try:
+            validate_user_template(dehydrated_jinja)
+        except (jinja2.exceptions.TemplateSyntaxError, TemplateSecurityError) as exc:
+            return InvalidJinja(detail=str(exc))
+
+        if ctx.render_jinja:
+            return render_user_template_sync(dehydrated_jinja, ctx.jinja_context)
+        else:
+            return ValidJinja(template=dehydrated_jinja)
+    else:
+        return TemplateNotFound()
+
+
 @handler("workspace_variable")
-def workspace_variable_handler(obj: Dict, ctx: HydrationContext):
+def workspace_variable_handler(obj: dict, ctx: HydrationContext):
     if "variable_name" in obj:
         if isinstance(obj["variable_name"], dict):
             dehydrated_variable = _hydrate(obj["variable_name"], ctx)
         else:
             dehydrated_variable = obj["variable_name"]
 
+        if not ctx.render_workspace_variables:
+            return WorkspaceVariable(variable_name=obj["variable_name"])
+
         if dehydrated_variable in ctx.workspace_variables:
             return ctx.workspace_variables[dehydrated_variable]
         else:
@@ -191,7 +260,7 @@ def workspace_variable_handler(obj: Dict, ctx: HydrationContext):
         return RemoveValue()
 
 
-def hydrate(obj: Dict, ctx: Optional[HydrationContext] = None):
+def hydrate(obj: dict, ctx: Optional[HydrationContext] = None):
     res = _hydrate(obj, ctx)
 
     if _remove_value(res):
@@ -200,7 +269,7 @@ def hydrate(obj: Dict, ctx: Optional[HydrationContext] = None):
     return res
 
 
-def _hydrate(obj, ctx: Optional[HydrationContext] = None):
+def _hydrate(obj, ctx: Optional[HydrationContext] = None) -> Any:
     if ctx is None:
         ctx = HydrationContext()
 

prefect/variables.py

@@ -30,11 +30,11 @@ class Variable(VariableRequest):
         """
         Sets a new variable. If one exists with the same name, user must pass `overwrite=True`
         ```
-            from prefect import variables
+            from prefect.variables import Variable
 
             @flow
             def my_flow():
-                var = variables.Variable.set(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
+                var = Variable.set(name="my_var",value="test_value", tags=["hi", "there"], overwrite=True)
         ```
         or
         ```
@@ -69,11 +69,11 @@ class Variable(VariableRequest):
         """
         Get a variable by name. If doesn't exist return the default.
         ```
-            from prefect import variables
+            from prefect.variables import Variable
 
             @flow
             def my_flow():
-                var = variables.Variable.get("my_var")
+                var = Variable.get("my_var")
         ```
         or
         ```

{prefect_client-2.17.1.dist-info → prefect_client-2.18.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 2.17.1
+Version: 2.18.0
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.