prefect-client 3.0.0rc4__py3-none-any.whl → 3.0.0rc5__py3-none-any.whl

prefect/docker/docker_image.py

@@ -0,0 +1,82 @@
+from pathlib import Path
+from typing import Optional
+
+from pendulum import now as pendulum_now
+
+from prefect.settings import (
+    PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE,
+)
+from prefect.utilities.dockerutils import (
+    PushError,
+    build_image,
+    docker_client,
+    generate_default_dockerfile,
+    parse_image_tag,
+    split_repository_path,
+)
+from prefect.utilities.slugify import slugify
+
+
+class DockerImage:
+    """
+    Configuration used to build and push a Docker image for a deployment.
+
+    Attributes:
+        name: The name of the Docker image to build, including the registry and
+            repository.
+        tag: The tag to apply to the built image.
+        dockerfile: The path to the Dockerfile to use for building the image. If
+            not provided, a default Dockerfile will be generated.
+        **build_kwargs: Additional keyword arguments to pass to the Docker build request.
+            See the [`docker-py` documentation](https://docker-py.readthedocs.io/en/stable/images.html#docker.models.images.ImageCollection.build)
+            for more information.
+
+    """
+
+    def __init__(
+        self, name: str, tag: Optional[str] = None, dockerfile="auto", **build_kwargs
+    ):
+        image_name, image_tag = parse_image_tag(name)
+        if tag and image_tag:
+            raise ValueError(
+                f"Only one tag can be provided - both {image_tag!r} and {tag!r} were"
+                " provided as tags."
+            )
+        namespace, repository = split_repository_path(image_name)
+        # if the provided image name does not include a namespace (registry URL or user/org name),
+        # use the default namespace
+        if not namespace:
+            namespace = PREFECT_DEFAULT_DOCKER_BUILD_NAMESPACE.value()
+        # join the namespace and repository to create the full image name
+        # ignore namespace if it is None
+        self.name = "/".join(filter(None, [namespace, repository]))
+        self.tag = tag or image_tag or slugify(pendulum_now("utc").isoformat())
+        self.dockerfile = dockerfile
+        self.build_kwargs = build_kwargs
+
+    @property
+    def reference(self):
+        return f"{self.name}:{self.tag}"
+
+    def build(self):
+        full_image_name = self.reference
+        build_kwargs = self.build_kwargs.copy()
+        build_kwargs["context"] = Path.cwd()
+        build_kwargs["tag"] = full_image_name
+        build_kwargs["pull"] = build_kwargs.get("pull", True)
+
+        if self.dockerfile == "auto":
+            with generate_default_dockerfile():
+                build_image(**build_kwargs)
+        else:
+            build_kwargs["dockerfile"] = self.dockerfile
+            build_image(**build_kwargs)
+
+    def push(self):
+        with docker_client() as client:
+            events = client.api.push(
+                repository=self.name, tag=self.tag, stream=True, decode=True
+            )
+            for event in events:
+                if "error" in event:
+                    raise PushError(event["error"])

prefect/flow_engine.py

@@ -168,6 +168,20 @@ class FlowRunEngine(Generic[P, R]):
             )
             return state
 
+        # validate prior to context so that context receives validated params
+        if self.flow.should_validate_parameters:
+            try:
+                self.parameters = self.flow.validate_parameters(self.parameters or {})
+            except Exception as exc:
+                message = "Validation of flow parameters failed with error:"
+                self.logger.error("%s %s", message, exc)
+                self.handle_exception(
+                    exc,
+                    msg=message,
+                    result_factory=run_coro_as_sync(ResultFactory.from_flow(self.flow)),
+                )
+                self.short_circuit = True
+
         new_state = Running()
         state = self.set_state(new_state)
         while state.is_pending():
@@ -484,24 +498,6 @@ class FlowRunEngine(Generic[P, R]):
                     flow_version=self.flow.version,
                     empirical_policy=self.flow_run.empirical_policy,
                 )
-
-            # validate prior to context so that context receives validated params
-            if self.flow.should_validate_parameters:
-                try:
-                    self.parameters = self.flow.validate_parameters(
-                        self.parameters or {}
-                    )
-                except Exception as exc:
-                    message = "Validation of flow parameters failed with error:"
-                    self.logger.error("%s %s", message, exc)
-                    self.handle_exception(
-                        exc,
-                        msg=message,
-                        result_factory=run_coro_as_sync(
-                            ResultFactory.from_flow(self.flow)
-                        ),
-                    )
-                    self.short_circuit = True
             try:
                 yield self
             except Exception:

prefect/flows.py

@@ -17,11 +17,9 @@ import warnings
 from copy import copy
 from functools import partial, update_wrapper
 from pathlib import Path
-from tempfile import NamedTemporaryFile
 from typing import (
     TYPE_CHECKING,
     Any,
-    AnyStr,
     Awaitable,
     Callable,
     Coroutine,
@@ -56,9 +54,9 @@ from prefect.client.schemas.objects import Flow as FlowSchema
 from prefect.client.schemas.objects import FlowRun
 from prefect.client.schemas.schedules import SCHEDULE_TYPES
 from prefect.client.utilities import client_injector
-from prefect.context import PrefectObjectRegistry, registry_from_script
-from prefect.deployments.runner import DeploymentImage, EntrypointType, deploy
+from prefect.deployments.runner import deploy
 from prefect.deployments.steps.core import run_steps
+from prefect.docker.docker_image import DockerImage
 from prefect.events import DeploymentTriggerTypes, TriggerTypes
 from prefect.exceptions import (
     InvalidNameError,
@@ -87,6 +85,7 @@ from prefect.settings import (
 from prefect.states import State
 from prefect.task_runners import TaskRunner, ThreadPoolTaskRunner
 from prefect.types import BANNED_CHARACTERS, WITHOUT_BANNED_CHARACTERS
+from prefect.types.entrypoint import EntrypointType
 from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import (
     run_sync_in_worker_thread,
@@ -118,11 +117,11 @@ logger = get_logger("flows")
 
 if TYPE_CHECKING:
     from prefect.client.orchestration import PrefectClient
-    from prefect.deployments.runner import FlexibleScheduleList, RunnerDeployment
+    from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+    from prefect.deployments.runner import RunnerDeployment
     from prefect.flows import FlowRun
 
 
-@PrefectObjectRegistry.register_instances
 class Flow(Generic[P, R]):
     """
     A Prefect workflow definition.
@@ -145,7 +144,7 @@ class Flow(Generic[P, R]):
             be provided as a string template with the flow's parameters as variables,
             or a function that returns a string.
         task_runner: An optional task runner to use for task execution within the flow;
-            if not provided, a `ConcurrentTaskRunner` will be used.
+            if not provided, a `ThreadPoolTaskRunner` will be used.
         description: An optional string description for the flow; if not provided, the
             description will be pulled from the docstring for the decorated function.
         timeout_seconds: An optional number of seconds indicating a maximum runtime for
@@ -998,7 +997,7 @@ class Flow(Generic[P, R]):
         self,
         name: str,
         work_pool_name: Optional[str] = None,
-        image: Optional[Union[str, DeploymentImage]] = None,
+        image: Optional[Union[str, DockerImage]] = None,
         build: bool = True,
         push: bool = True,
         work_queue_name: Optional[str] = None,
@@ -1034,7 +1033,7 @@ class Flow(Generic[P, R]):
             work_pool_name: The name of the work pool to use for this deployment. Defaults to
                 the value of `PREFECT_DEFAULT_WORK_POOL_NAME`.
             image: The name of the Docker image to build, including the registry and
-                repository. Pass a DeploymentImage instance to customize the Dockerfile used
+                repository. Pass a DockerImage instance to customize the Dockerfile used
                 and build arguments.
             build: Whether or not to build a new image for the flow. If False, the provided
                 image will be used as-is and pulled at runtime.
@@ -1638,47 +1637,6 @@ def select_flow(
         return list(flows_dict.values())[0]
 
 
-def load_flows_from_script(path: str) -> List[Flow]:
-    """
-    Load all flow objects from the given python script. All of the code in the file
-    will be executed.
-
-    Returns:
-        A list of flows
-
-    Raises:
-        FlowScriptError: If an exception is encountered while running the script
-    """
-    return registry_from_script(path).get_instances(Flow)
-
-
-def load_flow_from_script(path: str, flow_name: Optional[str] = None) -> Flow:
-    """
-    Extract a flow object from a script by running all of the code in the file.
-
-    If the script has multiple flows in it, a flow name must be provided to specify
-    the flow to return.
-
-    Args:
-        path: A path to a Python script containing flows
-        flow_name: An optional flow name to look for in the script
-
-    Returns:
-        The flow object from the script
-
-    Raises:
-        FlowScriptError: If an exception is encountered while running the script
-        MissingFlowError: If no flows exist in the iterable
-        MissingFlowError: If a flow name is provided and that flow does not exist
-        UnspecifiedFlowError: If multiple flows exist but no flow name was provided
-    """
-    return select_flow(
-        load_flows_from_script(path),
-        flow_name=flow_name,
-        from_message=f"in script '{path}'",
-    )
-
-
 def load_flow_from_entrypoint(
     entrypoint: str,
 ) -> Flow:
@@ -1696,52 +1654,25 @@ def load_flow_from_entrypoint(
         FlowScriptError: If an exception is encountered while running the script
         MissingFlowError: If the flow function specified in the entrypoint does not exist
     """
-    with PrefectObjectRegistry(  # type: ignore
-        block_code_execution=True,
-        capture_failures=True,
-    ):
-        if ":" in entrypoint:
-            # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
-            path, func_name = entrypoint.rsplit(":", maxsplit=1)
-        else:
-            path, func_name = entrypoint.rsplit(".", maxsplit=1)
-        try:
-            flow = import_object(entrypoint)
-        except AttributeError as exc:
-            raise MissingFlowError(
-                f"Flow function with name {func_name!r} not found in {path!r}. "
-            ) from exc
-
-        if not isinstance(flow, Flow):
-            raise MissingFlowError(
-                f"Function with name {func_name!r} is not a flow. Make sure that it is "
-                "decorated with '@flow'."
-            )
-
-        return flow
 
+    if ":" in entrypoint:
+        # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
+        path, func_name = entrypoint.rsplit(":", maxsplit=1)
+    else:
+        path, func_name = entrypoint.rsplit(".", maxsplit=1)
+    try:
+        flow = import_object(entrypoint)
+    except AttributeError as exc:
+        raise MissingFlowError(
+            f"Flow function with name {func_name!r} not found in {path!r}. "
+        ) from exc
 
-def load_flow_from_text(script_contents: AnyStr, flow_name: str) -> Flow:
-    """
-    Load a flow from a text script.
+    if not isinstance(flow, Flow):
+        raise MissingFlowError(
+            f"Function with name {func_name!r} is not a flow. Make sure that it is "
+            "decorated with '@flow'."
+        )
 
-    The script will be written to a temporary local file path so errors can refer
-    to line numbers and contextual tracebacks can be provided.
-    """
-    with NamedTemporaryFile(
-        mode="wt" if isinstance(script_contents, str) else "wb",
-        prefix=f"flow-script-{flow_name}",
-        suffix=".py",
-        delete=False,
-    ) as tmpfile:
-        tmpfile.write(script_contents)
-        tmpfile.flush()
-    try:
-        flow = load_flow_from_script(tmpfile.name, flow_name=flow_name)
-    finally:
-        # windows compat
-        tmpfile.close()
-        os.remove(tmpfile.name)
     return flow
 
 

prefect/futures.py

@@ -10,7 +10,7 @@ from typing_extensions import TypeVar
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.objects import TaskRun
 from prefect.exceptions import ObjectNotFound
-from prefect.logging.loggers import get_logger
+from prefect.logging.loggers import get_logger, get_run_logger
 from prefect.states import Pending, State
 from prefect.task_runs import TaskRunWaiter
 from prefect.utilities.annotations import quote
@@ -143,6 +143,18 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[concurrent.futures.Future]):
             _result = run_coro_as_sync(_result)
         return _result
 
+    def __del__(self):
+        if self._final_state or self._wrapped_future.done():
+            return
+        try:
+            local_logger = get_run_logger()
+        except Exception:
+            local_logger = logger
+        local_logger.warning(
+            "A future was garbage collected before it resolved."
+            " Please call `.wait()` or `.result()` on futures to ensure they resolve.",
+        )
+
 
 class PrefectDistributedFuture(PrefectFuture):
     """

prefect/infrastructure/provisioners/cloud_run.py

@@ -404,7 +404,7 @@ class CloudRunPushProvisioner:
                         dedent(
                             f"""\
                         from prefect import flow
-                        from prefect.deployments import DeploymentImage
+                        from prefect.docker import DockerImage
 
 
                         @flow(log_prints=True)
@@ -416,7 +416,7 @@ class CloudRunPushProvisioner:
                             my_flow.deploy(
                                 name="my-deployment",
                                 work_pool_name="{work_pool_name}",
-                                image=DeploymentImage(
+                                image=DockerImage(
                                     name="my-image:latest",
                                     platform="linux/amd64",
                                 )

prefect/infrastructure/provisioners/container_instance.py

@@ -1042,7 +1042,7 @@ class ContainerInstancePushProvisioner:
                     dedent(
                         f"""\
                         from prefect import flow
-                        from prefect.deployments import DeploymentImage
+                        from prefect.docker import DockerImage
 
 
                         @flow(log_prints=True)
@@ -1054,7 +1054,7 @@ class ContainerInstancePushProvisioner:
                             my_flow.deploy(
                                 name="my-deployment",
                                 work_pool_name="{work_pool_name}",
-                                image=DeploymentImage(
+                                image=DockerImage(
                                     name="my-image:latest",
                                     platform="linux/amd64",
                                 )

prefect/infrastructure/provisioners/ecs.py

@@ -950,7 +950,7 @@ class ContainerRepositoryResource:
                         dedent(
                             f"""\
                                 from prefect import flow
-                                from prefect.deployments import DeploymentImage
+                                from prefect.docker import DockerImage
 
 
                                 @flow(log_prints=True)
@@ -962,7 +962,7 @@ class ContainerRepositoryResource:
                                     my_flow.deploy(
                                         name="my-deployment",
                                         work_pool_name="{self._work_pool_name}",
-                                        image=DeploymentImage(
+                                        image=DockerImage(
                                             name="{self._repository_name}:latest",
                                             platform="linux/amd64",
                                         )

prefect/records/result_store.py

@@ -44,6 +44,10 @@ class ResultFactoryStore(RecordStore):
             raise ValueError("Result could not be read")
 
     def write(self, key: str, value: Any) -> BaseResult:
-        if isinstance(value, BaseResult):
+        if isinstance(value, PersistedResult):
+            # if the value is already a persisted result, write it
+            value.write(_sync=True)
+            return value
+        elif isinstance(value, BaseResult):
             return value
         return run_coro_as_sync(self.result_factory.create_result(obj=value, key=key))

prefect/results.py

@@ -431,7 +431,11 @@ class ResultFactory(BaseModel):
 
     @sync_compatible
     async def create_result(
-        self, obj: R, key: Optional[str] = None, expiration: Optional[DateTime] = None
+        self,
+        obj: R,
+        key: Optional[str] = None,
+        expiration: Optional[DateTime] = None,
+        defer_persistence: bool = False,
     ) -> Union[R, "BaseResult[R]"]:
         """
         Create a result type for the given object.
@@ -464,6 +468,7 @@ class ResultFactory(BaseModel):
             serializer=self.serializer,
             cache_object=should_cache_object,
             expiration=expiration,
+            defer_persistence=defer_persistence,
         )
 
     @sync_compatible
@@ -589,6 +594,19 @@ class PersistedResult(BaseResult):
     expiration: Optional[DateTime] = None
 
     _should_cache_object: bool = PrivateAttr(default=True)
+    _persisted: bool = PrivateAttr(default=False)
+    _storage_block: WritableFileSystem = PrivateAttr(default=None)
+    _serializer: Serializer = PrivateAttr(default=None)
+
+    def _cache_object(
+        self,
+        obj: Any,
+        storage_block: WritableFileSystem = None,
+        serializer: Serializer = None,
+    ) -> None:
+        self._cache = obj
+        self._storage_block = storage_block
+        self._serializer = serializer
 
     @sync_compatible
     @inject_client
@@ -601,7 +619,7 @@ class PersistedResult(BaseResult):
             return self._cache
 
         blob = await self._read_blob(client=client)
-        obj = blob.serializer.loads(blob.data)
+        obj = blob.load()
         self.expiration = blob.expiration
 
         if self._should_cache_object:
@@ -632,6 +650,46 @@ class PersistedResult(BaseResult):
         if hasattr(storage_block, "_remote_file_system"):
             return storage_block._remote_file_system._resolve_path(key)
 
+    @sync_compatible
+    @inject_client
+    async def write(self, obj: R = NotSet, client: "PrefectClient" = None) -> None:
+        """
+        Write the result to the storage block.
+        """
+
+        if self._persisted:
+            # don't double write or overwrite
+            return
+
+        # load objects from a cache
+
+        # first the object itself
+        if obj is NotSet and not self.has_cached_object():
+            raise ValueError("Cannot write a result that has no object cached.")
+        obj = obj if obj is not NotSet else self._cache
+
+        # next, the storage block
+        storage_block = self._storage_block
+        if storage_block is None:
+            block_document = await client.read_block_document(self.storage_block_id)
+            storage_block = Block._from_block_document(block_document)
+
+        # finally, the serializer
+        serializer = self._serializer
+        if serializer is None:
+            # this could error if the serializer requires kwargs
+            serializer = Serializer(type=self.serializer_type)
+
+        data = serializer.dumps(obj)
+        blob = PersistedResultBlob(
+            serializer=serializer, data=data, expiration=self.expiration
+        )
+        await storage_block.write_path(self.storage_key, content=blob.to_bytes())
+        self._persisted = True
+
+        if not self._should_cache_object:
+            self._cache = NotSet
+
     @classmethod
     @sync_compatible
     async def create(
@@ -643,6 +701,7 @@ class PersistedResult(BaseResult):
         serializer: Serializer,
         cache_object: bool = True,
         expiration: Optional[DateTime] = None,
+        defer_persistence: bool = False,
     ) -> "PersistedResult[R]":
         """
         Create a new result reference from a user's object.
@@ -652,19 +711,13 @@ class PersistedResult(BaseResult):
         """
         assert (
             storage_block_id is not None
-        ), "Unexpected storage block ID. Was it persisted?"
-        data = serializer.dumps(obj)
-        blob = PersistedResultBlob(
-            serializer=serializer, data=data, expiration=expiration
-        )
+        ), "Unexpected storage block ID. Was it saved?"
 
         key = storage_key_fn()
         if not isinstance(key, str):
             raise TypeError(
                 f"Expected type 'str' for result storage key; got value {key!r}"
             )
-        await storage_block.write_path(key, content=blob.to_bytes())
-
         description = f"Result of type `{type(obj).__name__}`"
         uri = cls._infer_path(storage_block, key)
         if uri:
@@ -684,12 +737,23 @@ class PersistedResult(BaseResult):
             expiration=expiration,
         )
 
-        if cache_object:
+        if cache_object and not defer_persistence:
             # Attach the object to the result so it's available without deserialization
-            result._cache_object(obj)
+            result._cache_object(
+                obj, storage_block=storage_block, serializer=serializer
+            )
 
         object.__setattr__(result, "_should_cache_object", cache_object)
 
+        if not defer_persistence:
+            await result.write(obj=obj)
+        else:
+            # we must cache temporarily to allow for writing later
+            # the cache will be removed on write
+            result._cache_object(
+                obj, storage_block=storage_block, serializer=serializer
+            )
+
         return result
 
 
@@ -705,6 +769,9 @@ class PersistedResultBlob(BaseModel):
     prefect_version: str = Field(default=prefect.__version__)
     expiration: Optional[DateTime] = None
 
+    def load(self) -> Any:
+        return self.serializer.loads(self.data)
+
     def to_bytes(self) -> bytes:
         return self.model_dump_json(serialize_as_any=True).encode()
 

prefect/runner/runner.py

@@ -45,7 +45,7 @@ import threading
 from copy import deepcopy
 from functools import partial
 from pathlib import Path
-from typing import Callable, Dict, Iterable, List, Optional, Set, Union
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Set, Union
 from uuid import UUID, uuid4
 
 import anyio
@@ -75,7 +75,6 @@ from prefect.deployments.runner import (
     EntrypointType,
     RunnerDeployment,
 )
-from prefect.deployments.schedules import FlexibleScheduleList
 from prefect.events import DeploymentTriggerTypes, TriggerTypes
 from prefect.exceptions import Abort, ObjectNotFound
 from prefect.flows import Flow, load_flow_from_flow_run
@@ -98,6 +97,9 @@ from prefect.utilities.engine import propose_state
 from prefect.utilities.processutils import _register_signal, run_process
 from prefect.utilities.services import critical_service_loop
 
+if TYPE_CHECKING:
+    from prefect.client.types.flexible_schedule_list import FlexibleScheduleList
+
 __all__ = ["Runner"]
 
 
@@ -221,7 +223,7 @@ class Runner:
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[FlexibleScheduleList] = None,
+        schedules: Optional["FlexibleScheduleList"] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,

prefect/runner/server.py

@@ -10,7 +10,7 @@ from typing_extensions import Literal
 from prefect._internal.schemas.validators import validate_values_conform_to_schema
 from prefect.client.orchestration import get_client
 from prefect.exceptions import MissingFlowError, ScriptError
-from prefect.flows import Flow, load_flow_from_entrypoint, load_flows_from_script
+from prefect.flows import Flow, load_flow_from_entrypoint
 from prefect.logging import get_logger
 from prefect.runner.utils import (
     inject_schemas_into_openapi,
@@ -24,6 +24,7 @@ from prefect.settings import (
     PREFECT_RUNNER_SERVER_PORT,
 )
 from prefect.utilities.asyncutils import sync_compatible
+from prefect.utilities.importtools import load_script_as_module
 
 if TYPE_CHECKING:
     from prefect.client.schemas.responses import DeploymentResponse
@@ -155,7 +156,10 @@ async def get_subflow_schemas(runner: "Runner") -> Dict[str, Dict]:
                 continue
 
             script = deployment.entrypoint.split(":")[0]
-            subflows = load_flows_from_script(script)
+            module = load_script_as_module(script)
+            subflows = [
+                obj for obj in module.__dict__.values() if isinstance(obj, Flow)
+            ]
             for flow in subflows:
                 schemas[flow.name] = flow.parameters.model_dump()
 

prefect/states.py

@@ -209,6 +209,7 @@ async def return_value_to_state(
     result_factory: ResultFactory,
     key: Optional[str] = None,
     expiration: Optional[datetime.datetime] = None,
+    defer_persistence: bool = False,
 ) -> State[R]:
     """
     Given a return value from a user's function, create a `State` the run should
@@ -242,7 +243,10 @@ async def return_value_to_state(
         # to update the data to the correct type
         if not isinstance(state.data, BaseResult):
             state.data = await result_factory.create_result(
-                state.data, key=key, expiration=expiration
+                state.data,
+                key=key,
+                expiration=expiration,
+                defer_persistence=defer_persistence,
             )
 
         return state
@@ -284,7 +288,10 @@ async def return_value_to_state(
             type=new_state_type,
             message=message,
             data=await result_factory.create_result(
-                retval, key=key, expiration=expiration
+                retval,
+                key=key,
+                expiration=expiration,
+                defer_persistence=defer_persistence,
             ),
         )
 
@@ -300,7 +307,10 @@ async def return_value_to_state(
     else:
         return Completed(
             data=await result_factory.create_result(
-                data, key=key, expiration=expiration
+                data,
+                key=key,
+                expiration=expiration,
+                defer_persistence=defer_persistence,
             )
         )