prefect-client 3.0.0rc18__py3-none-any.whl → 3.0.0rc20__py3-none-any.whl

prefect/results.py

@@ -17,7 +17,15 @@ from typing import (
 )
 from uuid import UUID
 
-from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, ValidationError
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    PrivateAttr,
+    ValidationError,
+    model_serializer,
+    model_validator,
+)
 from pydantic_core import PydanticUndefinedType
 from pydantic_extra_types.pendulum_dt import DateTime
 from typing_extensions import ParamSpec, Self
@@ -25,13 +33,13 @@ from typing_extensions import ParamSpec, Self
 import prefect
 from prefect.blocks.core import Block
 from prefect.client.utilities import inject_client
-from prefect.exceptions import MissingResult
+from prefect.exceptions import SerializationError
 from prefect.filesystems import (
     LocalFileSystem,
     WritableFileSystem,
 )
 from prefect.logging import get_logger
-from prefect.serializers import Serializer
+from prefect.serializers import PickleSerializer, Serializer
 from prefect.settings import (
     PREFECT_DEFAULT_RESULT_STORAGE_BLOCK,
     PREFECT_LOCAL_STORAGE_PATH,
@@ -332,22 +340,15 @@ class ResultFactory(BaseModel):
         obj: R,
         key: Optional[str] = None,
         expiration: Optional[DateTime] = None,
-        defer_persistence: bool = False,
     ) -> Union[R, "BaseResult[R]"]:
         """
         Create a result type for the given object.
 
-        If persistence is disabled, the object is wrapped in an `UnpersistedResult` and
-        returned.
-
         If persistence is enabled the object is serialized, persisted to storage, and a reference is returned.
         """
         # Null objects are "cached" in memory at no cost
         should_cache_object = self.cache_result_in_memory or obj is None
 
-        if not self.persist_result:
-            return await UnpersistedResult.create(obj, cache_object=should_cache_object)
-
         if key:
 
             def key_fn():
@@ -365,23 +366,198 @@ class ResultFactory(BaseModel):
             serializer=self.serializer,
             cache_object=should_cache_object,
             expiration=expiration,
-            defer_persistence=defer_persistence,
+            serialize_to_none=not self.persist_result,
         )
 
+    # TODO: These two methods need to find a new home
+
     @sync_compatible
     async def store_parameters(self, identifier: UUID, parameters: Dict[str, Any]):
-        data = self.serializer.dumps(parameters)
-        blob = PersistedResultBlob(serializer=self.serializer, data=data)
+        record = ResultRecord(
+            result=parameters,
+            metadata=ResultRecordMetadata(
+                serializer=self.serializer, storage_key=str(identifier)
+            ),
+        )
         await self.storage_block.write_path(
-            f"parameters/{identifier}", content=blob.to_bytes()
+            f"parameters/{identifier}", content=record.serialize()
         )
 
     @sync_compatible
     async def read_parameters(self, identifier: UUID) -> Dict[str, Any]:
-        blob = PersistedResultBlob.model_validate_json(
+        record = ResultRecord.deserialize(
             await self.storage_block.read_path(f"parameters/{identifier}")
         )
-        return self.serializer.loads(blob.data)
+        return record.result
+
+
+class ResultRecordMetadata(BaseModel):
+    """
+    Metadata for a result record.
+    """
+
+    storage_key: Optional[str] = Field(
+        default=None
+    )  # optional for backwards compatibility
+    expiration: Optional[DateTime] = Field(default=None)
+    serializer: Serializer = Field(default_factory=PickleSerializer)
+    prefect_version: str = Field(default=prefect.__version__)
+
+    def dump_bytes(self) -> bytes:
+        """
+        Serialize the metadata to bytes.
+
+        Returns:
+            bytes: the serialized metadata
+        """
+        return self.model_dump_json(serialize_as_any=True).encode()
+
+    @classmethod
+    def load_bytes(cls, data: bytes) -> "ResultRecordMetadata":
+        """
+        Deserialize metadata from bytes.
+
+        Args:
+            data: the serialized metadata
+
+        Returns:
+            ResultRecordMetadata: the deserialized metadata
+        """
+        return cls.model_validate_json(data)
+
+
+class ResultRecord(BaseModel, Generic[R]):
+    """
+    A record of a result.
+    """
+
+    metadata: ResultRecordMetadata
+    result: R
+
+    @property
+    def expiration(self) -> Optional[DateTime]:
+        return self.metadata.expiration
+
+    @property
+    def serializer(self) -> Serializer:
+        return self.metadata.serializer
+
+    def serialize_result(self) -> bytes:
+        try:
+            data = self.serializer.dumps(self.result)
+        except Exception as exc:
+            extra_info = (
+                'You can try a different serializer (e.g. result_serializer="json") '
+                "or disabling persistence (persist_result=False) for this flow or task."
+            )
+            # check if this is a known issue with cloudpickle and pydantic
+            # and add extra information to help the user recover
+
+            if (
+                isinstance(exc, TypeError)
+                and isinstance(self.result, BaseModel)
+                and str(exc).startswith("cannot pickle")
+            ):
+                try:
+                    from IPython import get_ipython
+
+                    if get_ipython() is not None:
+                        extra_info = inspect.cleandoc(
+                            """
+                            This is a known issue in Pydantic that prevents
+                            locally-defined (non-imported) models from being
+                            serialized by cloudpickle in IPython/Jupyter
+                            environments. Please see
+                            https://github.com/pydantic/pydantic/issues/8232 for
+                            more information. To fix the issue, either: (1) move
+                            your Pydantic class definition to an importable
+                            location, (2) use the JSON serializer for your flow
+                            or task (`result_serializer="json"`), or (3)
+                            disable result persistence for your flow or task
+                            (`persist_result=False`).
+                            """
+                        ).replace("\n", " ")
+                except ImportError:
+                    pass
+            raise SerializationError(
+                f"Failed to serialize object of type {type(self.result).__name__!r} with "
+                f"serializer {self.serializer.type!r}. {extra_info}"
+            ) from exc
+
+        return data
+
+    @model_validator(mode="before")
+    @classmethod
+    def coerce_old_format(cls, value: Any):
+        if isinstance(value, dict):
+            if "data" in value:
+                value["result"] = value.pop("data")
+            if "metadata" not in value:
+                value["metadata"] = {}
+            if "expiration" in value:
+                value["metadata"]["expiration"] = value.pop("expiration")
+            if "serializer" in value:
+                value["metadata"]["serializer"] = value.pop("serializer")
+            if "prefect_version" in value:
+                value["metadata"]["prefect_version"] = value.pop("prefect_version")
+        return value
+
+    def serialize_metadata(self) -> bytes:
+        return self.metadata.dump_bytes()
+
+    def serialize(
+        self,
+    ) -> bytes:
+        """
+        Serialize the record to bytes.
+
+        Returns:
+            bytes: the serialized record
+
+        """
+        return (
+            self.model_copy(update={"result": self.serialize_result()})
+            .model_dump_json(serialize_as_any=True)
+            .encode()
+        )
+
+    @classmethod
+    def deserialize(cls, data: bytes) -> "ResultRecord[R]":
+        """
+        Deserialize a record from bytes.
+
+        Args:
+            data: the serialized record
+
+        Returns:
+            ResultRecord: the deserialized record
+        """
+        instance = cls.model_validate_json(data)
+        if isinstance(instance.result, bytes):
+            instance.result = instance.serializer.loads(instance.result)
+        elif isinstance(instance.result, str):
+            instance.result = instance.serializer.loads(instance.result.encode())
+        return instance
+
+    @classmethod
+    def deserialize_from_result_and_metadata(
+        cls, result: bytes, metadata: bytes
+    ) -> "ResultRecord[R]":
+        """
+        Deserialize a record from separate result and metadata bytes.
+
+        Args:
+            result: the result
+            metadata: the serialized metadata
+
+        Returns:
+            ResultRecord: the deserialized record
+        """
+        result_record_metadata = ResultRecordMetadata.load_bytes(metadata)
+        return cls(
+            metadata=result_record_metadata,
+            result=result_record_metadata.serializer.loads(result),
+        )
 
 
 @register_base_type
@@ -432,40 +608,12 @@ class BaseResult(BaseModel, abc.ABC, Generic[R]):
         return cls.__name__ if isinstance(default, PydanticUndefinedType) else default
 
 
-class UnpersistedResult(BaseResult):
-    """
-    Result type for results that are not persisted outside of local memory.
-    """
-
-    type: str = "unpersisted"
-
-    @sync_compatible
-    async def get(self) -> R:
-        if self.has_cached_object():
-            return self._cache
-
-        raise MissingResult("The result was not persisted and is no longer available.")
-
-    @classmethod
-    @sync_compatible
-    async def create(
-        cls: "Type[UnpersistedResult]",
-        obj: R,
-        cache_object: bool = True,
-    ) -> "UnpersistedResult[R]":
-        result = cls()
-        # Only store the object in local memory, it will not be sent to the API
-        if cache_object:
-            result._cache_object(obj)
-        return result
-
-
 class PersistedResult(BaseResult):
     """
     Result type which stores a reference to a persisted result.
 
     When created, the user's object is serialized and stored. The format for the content
-    is defined by `PersistedResultBlob`. This reference contains metadata necessary for retrieval
+    is defined by `ResultRecord`. This reference contains metadata necessary for retrieval
     of the object, such as a reference to the storage block and the key where the
     content was written.
     """
@@ -476,12 +624,19 @@ class PersistedResult(BaseResult):
     storage_key: str
     storage_block_id: Optional[uuid.UUID] = None
     expiration: Optional[DateTime] = None
+    serialize_to_none: bool = False
 
-    _should_cache_object: bool = PrivateAttr(default=True)
     _persisted: bool = PrivateAttr(default=False)
+    _should_cache_object: bool = PrivateAttr(default=True)
     _storage_block: WritableFileSystem = PrivateAttr(default=None)
     _serializer: Serializer = PrivateAttr(default=None)
 
+    @model_serializer(mode="wrap")
+    def serialize_model(self, handler, info):
+        if self.serialize_to_none:
+            return None
+        return handler(self, info)
+
     def _cache_object(
         self,
         obj: Any,
@@ -512,21 +667,20 @@ class PersistedResult(BaseResult):
         if self.has_cached_object():
             return self._cache
 
-        blob = await self._read_blob(client=client)
-        obj = blob.load()
-        self.expiration = blob.expiration
+        record = await self._read_result_record(client=client)
+        self.expiration = record.expiration
 
         if self._should_cache_object:
-            self._cache_object(obj)
+            self._cache_object(record.result)
 
-        return obj
+        return record.result
 
     @inject_client
-    async def _read_blob(self, client: "PrefectClient") -> "PersistedResultBlob":
+    async def _read_result_record(self, client: "PrefectClient") -> "ResultRecord":
         block = await self._get_storage_block(client=client)
         content = await block.read_path(self.storage_key)
-        blob = PersistedResultBlob.model_validate_json(content)
-        return blob
+        record = ResultRecord.deserialize(content)
+        return record
 
     @staticmethod
     def _infer_path(storage_block, key) -> str:
@@ -547,7 +701,7 @@ class PersistedResult(BaseResult):
         Write the result to the storage block.
         """
 
-        if self._persisted:
+        if self._persisted or self.serialize_to_none:
             # don't double write or overwrite
             return
 
@@ -567,50 +721,15 @@ class PersistedResult(BaseResult):
             # this could error if the serializer requires kwargs
             serializer = Serializer(type=self.serializer_type)
 
-        try:
-            data = serializer.dumps(obj)
-        except Exception as exc:
-            extra_info = (
-                'You can try a different serializer (e.g. result_serializer="json") '
-                "or disabling persistence (persist_result=False) for this flow or task."
-            )
-            # check if this is a known issue with cloudpickle and pydantic
-            # and add extra information to help the user recover
-
-            if (
-                isinstance(exc, TypeError)
-                and isinstance(obj, BaseModel)
-                and str(exc).startswith("cannot pickle")
-            ):
-                try:
-                    from IPython import get_ipython
-
-                    if get_ipython() is not None:
-                        extra_info = inspect.cleandoc(
-                            """
-                            This is a known issue in Pydantic that prevents
-                            locally-defined (non-imported) models from being
-                            serialized by cloudpickle in IPython/Jupyter
-                            environments. Please see
-                            https://github.com/pydantic/pydantic/issues/8232 for
-                            more information. To fix the issue, either: (1) move
-                            your Pydantic class definition to an importable
-                            location, (2) use the JSON serializer for your flow
-                            or task (`result_serializer="json"`), or (3)
-                            disable result persistence for your flow or task
-                            (`persist_result=False`).
-                            """
-                        ).replace("\n", " ")
-                except ImportError:
-                    pass
-            raise ValueError(
-                f"Failed to serialize object of type {type(obj).__name__!r} with "
-                f"serializer {serializer.type!r}. {extra_info}"
-            ) from exc
-        blob = PersistedResultBlob(
-            serializer=serializer, data=data, expiration=self.expiration
+        record = ResultRecord(
+            result=obj,
+            metadata=ResultRecordMetadata(
+                storage_key=self.storage_key,
+                expiration=self.expiration,
+                serializer=serializer,
+            ),
         )
-        await storage_block.write_path(self.storage_key, content=blob.to_bytes())
+        await storage_block.write_path(self.storage_key, content=record.serialize())
         self._persisted = True
 
         if not self._should_cache_object:
@@ -627,7 +746,7 @@ class PersistedResult(BaseResult):
         storage_block_id: Optional[uuid.UUID] = None,
         cache_object: bool = True,
         expiration: Optional[DateTime] = None,
-        defer_persistence: bool = False,
+        serialize_to_none: bool = False,
     ) -> "PersistedResult[R]":
         """
         Create a new result reference from a user's object.
@@ -651,79 +770,23 @@ class PersistedResult(BaseResult):
             storage_block_id=storage_block_id,
             storage_key=key,
             expiration=expiration,
+            serialize_to_none=serialize_to_none,
         )
 
-        if cache_object and not defer_persistence:
-            # Attach the object to the result so it's available without deserialization
-            result._cache_object(
-                obj, storage_block=storage_block, serializer=serializer
-            )
-
         object.__setattr__(result, "_should_cache_object", cache_object)
-
-        if not defer_persistence:
-            await result.write(obj=obj)
-        else:
-            # we must cache temporarily to allow for writing later
-            # the cache will be removed on write
-            result._cache_object(
-                obj, storage_block=storage_block, serializer=serializer
-            )
+        # we must cache temporarily to allow for writing later
+        # the cache will be removed on write
+        result._cache_object(obj, storage_block=storage_block, serializer=serializer)
 
         return result
 
-
-class PersistedResultBlob(BaseModel):
-    """
-    The format of the content stored by a persisted result.
-
-    Typically, this is written to a file as bytes.
-    """
-
-    serializer: Serializer
-    data: bytes
-    prefect_version: str = Field(default=prefect.__version__)
-    expiration: Optional[DateTime] = None
-
-    def load(self) -> Any:
-        return self.serializer.loads(self.data)
-
-    def to_bytes(self) -> bytes:
-        return self.model_dump_json(serialize_as_any=True).encode()
-
-
-class UnknownResult(BaseResult):
-    """
-    Result type for unknown results. Typically used to represent the result
-    of tasks that were forced from a failure state into a completed state.
-
-    The value for this result is always None and is not persisted to external
-    result storage, but orchestration treats the result the same as persisted
-    results when determining orchestration rules, such as whether to rerun a
-    completed task.
-    """
-
-    type: str = "unknown"
-    value: None
-
-    def has_cached_object(self) -> bool:
-        # This result type always has the object cached in memory
-        return True
-
-    @sync_compatible
-    async def get(self) -> R:
-        return self.value
-
-    @classmethod
-    @sync_compatible
-    async def create(
-        cls: "Type[UnknownResult]",
-        obj: R = None,
-    ) -> "UnknownResult[R]":
-        if obj is not None:
-            raise TypeError(
-                f"Unsupported type {type(obj).__name__!r} for unknown result. "
-                "Only None is supported."
-            )
-
-        return cls(value=obj)
+    def __eq__(self, other):
+        if not isinstance(other, PersistedResult):
+            return False
+        return (
+            self.type == other.type
+            and self.serializer_type == other.serializer_type
+            and self.storage_key == other.storage_key
+            and self.storage_block_id == other.storage_block_id
+            and self.expiration == other.expiration
+        )

prefect/runner/runner.py

@@ -35,7 +35,6 @@ import datetime
 import inspect
 import logging
 import os
-import shlex
 import shutil
 import signal
 import subprocess
@@ -90,7 +89,11 @@ from prefect.utilities.asyncutils import (
     sync_compatible,
 )
 from prefect.utilities.engine import propose_state
-from prefect.utilities.processutils import _register_signal, run_process
+from prefect.utilities.processutils import (
+    _register_signal,
+    get_sys_executable,
+    run_process,
+)
 from prefect.utilities.services import (
     critical_service_loop,
     start_client_metrics_server,
@@ -527,7 +530,7 @@ class Runner:
             task_status: anyio task status used to send a message to the caller
                 than the flow run process has started.
         """
-        command = f"{shlex.quote(sys.executable)} -m prefect.engine"
+        command = [get_sys_executable(), "-m", "prefect.engine"]
 
         flow_run_logger = self._get_flow_run_logger(flow_run)
 
@@ -574,7 +577,7 @@ class Runner:
                 setattr(storage, "last_adhoc_pull", datetime.datetime.now())
 
         process = await run_process(
-            shlex.split(command),
+            command=command,
             stream_output=True,
             task_status=task_status,
             env=env,

prefect/settings.py

@@ -425,13 +425,20 @@ def default_database_connection_url(settings: "Settings", value: Optional[str]):
                 f"Missing required database connection settings: {', '.join(missing)}"
             )
 
-        host = PREFECT_API_DATABASE_HOST.value_from(settings)
-        port = PREFECT_API_DATABASE_PORT.value_from(settings) or 5432
-        user = PREFECT_API_DATABASE_USER.value_from(settings)
-        name = PREFECT_API_DATABASE_NAME.value_from(settings)
-        password = PREFECT_API_DATABASE_PASSWORD.value_from(settings)
-
-        return f"{driver}://{user}:{password}@{host}:{port}/{name}"
+        # We only need SQLAlchemy here if we're parsing a remote database connection
+        # string.  Import it here so that we don't require the prefect-client package
+        # to have SQLAlchemy installed.
+        from sqlalchemy import URL
+
+        return URL(
+            drivername=driver,
+            host=PREFECT_API_DATABASE_HOST.value_from(settings),
+            port=PREFECT_API_DATABASE_PORT.value_from(settings) or 5432,
+            username=PREFECT_API_DATABASE_USER.value_from(settings),
+            password=PREFECT_API_DATABASE_PASSWORD.value_from(settings),
+            database=PREFECT_API_DATABASE_NAME.value_from(settings),
+            query=[],
+        ).render_as_string(hide_password=False)
 
     elif driver == "sqlite+aiosqlite":
         path = PREFECT_API_DATABASE_NAME.value_from(settings)
@@ -1379,14 +1386,6 @@ PREFECT_API_MAX_FLOW_RUN_GRAPH_ARTIFACTS = Setting(int, default=10000)
 The maximum number of artifacts to show on a flow run graph on the v2 API
 """
 
-
-PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION = Setting(
-    bool, default=False
-)
-"""
-Whether or not to enable experimental client side task run orchestration.
-"""
-
 # Prefect Events feature flags
 
 PREFECT_RUNNER_PROCESS_LIMIT = Setting(int, default=5)