prefect-client 2.16.1__py3-none-any.whl → 2.16.3__py3-none-any.whl

prefect/_internal/concurrency/services.py

@@ -91,6 +91,11 @@ class QueueService(abc.ABC, Generic[T]):
             self._remove_instance()
             self._stopped = True
 
+            # Allow asyncio task to be garbage-collected. Its context may contain
+            # references to all Prefect Task calls made during a flow run, through
+            # EngineContext. Issue #10338.
+            self._task = None
+
             # Signal completion to the loop
             self._queue.put_nowait(None)
 

prefect/_internal/concurrency/threads.py

@@ -214,6 +214,9 @@ class EventLoopThread(Portal):
         for call in self._on_shutdown:
             await self._run_call(call)
 
+        # Empty the list to allow calls to be garbage collected. Issue #10338.
+        self._on_shutdown = []
+
     async def _run_call(self, call: Call) -> None:
         task = call.run()
         if task is not None:

prefect/client/orchestration.py

@@ -2215,6 +2215,24 @@ class PrefectClient:
         response = await self._client.post("/task_runs/filter", json=body)
         return pydantic.parse_obj_as(List[TaskRun], response.json())
 
+    async def delete_task_run(self, task_run_id: UUID) -> None:
+        """
+        Delete a task run by id.
+
+        Args:
+            task_run_id: the task run ID of interest
+        Raises:
+            prefect.exceptions.ObjectNotFound: If request returns 404
+            httpx.RequestError: If requests fails
+        """
+        try:
+            await self._client.delete(f"/task_runs/{task_run_id}")
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 404:
+                raise prefect.exceptions.ObjectNotFound(http_exc=e) from e
+            else:
+                raise
+
     async def set_task_run_state(
         self,
         task_run_id: UUID,

prefect/deployments/__init__.py

@@ -12,4 +12,9 @@ from prefect.deployments.deployments import (
     load_deployments_from_yaml,
     Deployment,
 )
-from prefect.deployments.runner import RunnerDeployment, deploy, DeploymentImage
+from prefect.deployments.runner import (
+    RunnerDeployment,
+    deploy,
+    DeploymentImage,
+    EntrypointType,
+)

prefect/deployments/deployments.py

@@ -36,6 +36,10 @@ from prefect.client.schemas.objects import (
 from prefect.client.schemas.schedules import SCHEDULE_TYPES
 from prefect.client.utilities import inject_client
 from prefect.context import FlowRunContext, PrefectObjectRegistry, TaskRunContext
+from prefect.deployments.schedules import (
+    FlexibleScheduleList,
+    normalize_to_minimal_deployment_schedules,
+)
 from prefect.deployments.steps.core import run_steps
 from prefect.events.schemas import DeploymentTrigger
 from prefect.exceptions import (
@@ -222,12 +226,28 @@ async def load_flow_from_flow_run(
     is largely for testing, and assumes the flow is already available locally.
     """
     deployment = await client.read_deployment(flow_run.deployment_id)
+
+    if deployment.entrypoint is None:
+        raise ValueError(
+            f"Deployment {deployment.id} does not have an entrypoint and can not be run."
+        )
+
     run_logger = flow_run_logger(flow_run)
 
     runner_storage_base_path = storage_base_path or os.environ.get(
         "PREFECT__STORAGE_BASE_PATH"
     )
 
+    # If there's no colon, assume it's a module path
+    if ":" not in deployment.entrypoint:
+        run_logger.debug(
+            f"Importing flow code from module path {deployment.entrypoint}"
+        )
+        flow = await run_sync_in_worker_thread(
+            load_flow_from_entrypoint, deployment.entrypoint
+        )
+        return flow
+
     if not ignore_storage and not deployment.pull_steps:
         sys.path.insert(0, ".")
         if deployment.storage_document_id:
@@ -259,8 +279,6 @@ async def load_flow_from_flow_run(
             os.chdir(output["directory"])
 
     import_path = relative_path_to_current_platform(deployment.entrypoint)
-    run_logger.debug(f"Importing flow code from '{import_path}'")
-
     # for backwards compat
     if deployment.manifest_path:
         with open(deployment.manifest_path, "r") as f:
@@ -268,7 +286,10 @@ async def load_flow_from_flow_run(
             import_path = (
                 Path(deployment.manifest_path).parent / import_path
             ).absolute()
+    run_logger.debug(f"Importing flow code from '{import_path}'")
+
     flow = await run_sync_in_worker_thread(load_flow_from_entrypoint, str(import_path))
+
     return flow
 
 
@@ -632,11 +653,17 @@ class Deployment(BaseModel):
             cls._validate_schedule(value)
         return value
 
-    @validator("schedules")
-    def validate_schedules(cls, value):
-        for schedule in value:
-            cls._validate_schedule(schedule.schedule)
-        return value
+    @root_validator(pre=True)
+    def validate_schedules(cls, values):
+        if "schedules" in values:
+            values["schedules"] = normalize_to_minimal_deployment_schedules(
+                values["schedules"]
+            )
+
+            for schedule in values["schedules"]:
+                cls._validate_schedule(schedule.schedule)
+
+        return values
 
     @classmethod
     @sync_compatible
@@ -919,6 +946,7 @@ class Deployment(BaseModel):
         ignore_file: str = ".prefectignore",
         apply: bool = False,
         load_existing: bool = True,
+        schedules: Optional[FlexibleScheduleList] = None,
         **kwargs,
     ) -> "Deployment":
         """
@@ -938,6 +966,14 @@ class Deployment(BaseModel):
             load_existing: if True, load any settings that may already be configured for
                 the named deployment server-side (e.g., schedules, default parameter
                 values, etc.)
+            schedules: An optional list of schedules. Each item in the list can be:
+                  - An instance of `MinimalDeploymentSchedule`.
+                  - A dictionary with a `schedule` key, and optionally, an
+                    `active` key. The `schedule` key should correspond to a
+                    schedule type, and `active` is a boolean indicating whether
+                    the schedule is active or not.
+                  - An instance of one of the predefined schedule types:
+                    `IntervalSchedule`, `CronSchedule`, or `RRuleSchedule`.
             **kwargs: other keyword arguments to pass to the constructor for the
                 `Deployment` class
         """
@@ -946,7 +982,11 @@ class Deployment(BaseModel):
 
         # note that `deployment.load` only updates settings that were *not*
         # provided at initialization
-        deployment = cls(name=name, **kwargs)
+        deployment = cls(
+            name=name,
+            schedules=schedules,
+            **kwargs,
+        )
         deployment.flow_name = flow.name
         if not deployment.entrypoint:
             ## first see if an entrypoint can be determined

prefect/deployments/runner.py

@@ -29,11 +29,12 @@ Example:
 
 """
 
+import enum
 import importlib
 import tempfile
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, get_args
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 from uuid import UUID
 
 import pendulum
@@ -62,6 +63,11 @@ from prefect.client.schemas.schedules import (
     SCHEDULE_TYPES,
     construct_schedule,
 )
+from prefect.deployments.schedules import (
+    FlexibleScheduleList,
+    create_minimal_deployment_schedule,
+    normalize_to_minimal_deployment_schedules,
+)
 from prefect.events.schemas import DeploymentTrigger
 from prefect.exceptions import (
     ObjectNotFound,
@@ -82,24 +88,27 @@ from prefect.utilities.slugify import slugify
 if TYPE_CHECKING:
     from prefect.flows import Flow
 
-FlexibleScheduleList = Union[MinimalDeploymentSchedule, dict, SCHEDULE_TYPES]
-
 __all__ = ["RunnerDeployment"]
 
 
-def _to_deployment_schedule(
-    schedule: Optional[SCHEDULE_TYPES] = None,
-    active: Optional[bool] = True,
-) -> MinimalDeploymentSchedule:
-    return MinimalDeploymentSchedule(schedule=schedule, active=active)
-
-
 class DeploymentApplyError(RuntimeError):
     """
     Raised when an error occurs while applying a deployment.
     """
 
 
+class EntrypointType(enum.Enum):
+    """
+    Enum representing a entrypoint type.
+
+    File path entrypoints are in the format: `path/to/file.py:function_name`.
+    Module path entrypoints are in the format: `path.to.module.function_name`.
+    """
+
+    FILE_PATH = "file_path"
+    MODULE_PATH = "module_path"
+
+
 class RunnerDeployment(BaseModel):
     """
     A Prefect RunnerDeployment definition, used for specifying and building deployments.
@@ -205,6 +214,9 @@ class RunnerDeployment(BaseModel):
             " a built runner."
         ),
     )
+    _entrypoint_type: EntrypointType = PrivateAttr(
+        default=EntrypointType.FILE_PATH,
+    )
     _path: Optional[str] = PrivateAttr(
         default=None,
     )
@@ -212,6 +224,10 @@ class RunnerDeployment(BaseModel):
         default_factory=ParameterSchema,
     )
 
+    @property
+    def entrypoint_type(self) -> EntrypointType:
+        return self._entrypoint_type
+
     @validator("triggers", allow_reuse=True)
     def validate_automation_names(cls, field_value, values, field, config):
         """Ensure that each trigger has a name for its automation if none is provided."""
@@ -244,23 +260,9 @@ class RunnerDeployment(BaseModel):
         schedules = values.get("schedules")
 
         if schedules is None and schedule is not None:
-            values["schedules"] = [_to_deployment_schedule(schedule)]
+            values["schedules"] = [create_minimal_deployment_schedule(schedule)]
         elif schedules is not None and len(schedules) > 0:
-            reconciled = []
-            for obj in schedules:
-                if isinstance(obj, get_args(SCHEDULE_TYPES)):
-                    reconciled.append(_to_deployment_schedule(obj))
-                elif isinstance(obj, dict):
-                    reconciled.append(_to_deployment_schedule(**obj))
-                elif isinstance(obj, MinimalDeploymentSchedule):
-                    reconciled.append(obj)
-                else:
-                    raise ValueError(
-                        "Invalid schedule provided. Must be a schedule object, a dict,"
-                        " or a MinimalDeploymentSchedule."
-                    )
-
-            values["schedules"] = reconciled
+            values["schedules"] = normalize_to_minimal_deployment_schedules(schedules)
 
         return values
 
@@ -369,8 +371,8 @@ class RunnerDeployment(BaseModel):
         rrule: Optional[Union[Iterable[str], str]] = None,
         timezone: Optional[str] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
-    ) -> Union[List[MinimalDeploymentSchedule], List[FlexibleScheduleList]]:
+        schedules: Optional[FlexibleScheduleList] = None,
+    ) -> Union[List[MinimalDeploymentSchedule], FlexibleScheduleList]:
         """
         Construct a schedule or schedules from the provided arguments.
 
@@ -428,7 +430,7 @@ class RunnerDeployment(BaseModel):
                 value = [value]
 
             return [
-                _to_deployment_schedule(
+                create_minimal_deployment_schedule(
                     construct_schedule(
                         **{
                             schedule_type: v,
@@ -440,7 +442,7 @@ class RunnerDeployment(BaseModel):
                 for v in value
             ]
         else:
-            return [_to_deployment_schedule(schedule)]
+            return [create_minimal_deployment_schedule(schedule)]
 
     def _set_defaults_from_flow(self, flow: "Flow"):
         self._parameter_openapi_schema = parameter_schema(flow)
@@ -461,7 +463,7 @@ class RunnerDeployment(BaseModel):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,
@@ -473,6 +475,7 @@ class RunnerDeployment(BaseModel):
         work_pool_name: Optional[str] = None,
         work_queue_name: Optional[str] = None,
         job_variables: Optional[Dict[str, Any]] = None,
+        entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
     ) -> "RunnerDeployment":
         """
         Configure a deployment for a given flow.
@@ -545,30 +548,42 @@ class RunnerDeployment(BaseModel):
             ## first see if an entrypoint can be determined
             flow_file = getattr(flow, "__globals__", {}).get("__file__")
             mod_name = getattr(flow, "__module__", None)
-            if not flow_file:
-                if not mod_name:
-                    raise ValueError(no_file_location_error)
-                try:
-                    module = importlib.import_module(mod_name)
-                    flow_file = getattr(module, "__file__", None)
-                except ModuleNotFoundError as exc:
-                    if "__prefect_loader__" in str(exc):
-                        raise ValueError(
-                            "Cannot create a RunnerDeployment from a flow that has been"
-                            " loaded from an entrypoint. To deploy a flow via"
-                            " entrypoint, use RunnerDeployment.from_entrypoint instead."
-                        )
-                    raise ValueError(no_file_location_error)
+            if entrypoint_type == EntrypointType.MODULE_PATH:
+                if mod_name:
+                    deployment.entrypoint = f"{mod_name}.{flow.__name__}"
+                else:
+                    raise ValueError(
+                        "Unable to determine module path for provided flow."
+                    )
+            else:
                 if not flow_file:
-                    raise ValueError(no_file_location_error)
-
-            # set entrypoint
-            entry_path = Path(flow_file).absolute().relative_to(Path.cwd().absolute())
-            deployment.entrypoint = f"{entry_path}:{flow.fn.__name__}"
+                    if not mod_name:
+                        raise ValueError(no_file_location_error)
+                    try:
+                        module = importlib.import_module(mod_name)
+                        flow_file = getattr(module, "__file__", None)
+                    except ModuleNotFoundError as exc:
+                        if "__prefect_loader__" in str(exc):
+                            raise ValueError(
+                                "Cannot create a RunnerDeployment from a flow that has been"
+                                " loaded from an entrypoint. To deploy a flow via"
+                                " entrypoint, use RunnerDeployment.from_entrypoint instead."
+                            )
+                        raise ValueError(no_file_location_error)
+                    if not flow_file:
+                        raise ValueError(no_file_location_error)
+
+                # set entrypoint
+                entry_path = (
+                    Path(flow_file).absolute().relative_to(Path.cwd().absolute())
+                )
+                deployment.entrypoint = f"{entry_path}:{flow.fn.__name__}"
 
-        if not deployment._path:
+        if entrypoint_type == EntrypointType.FILE_PATH and not deployment._path:
             deployment._path = "."
 
+        deployment._entrypoint_type = entrypoint_type
+
         cls._set_defaults_from_flow(deployment, flow)
 
         return deployment
@@ -584,7 +599,7 @@ class RunnerDeployment(BaseModel):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,
@@ -682,7 +697,7 @@ class RunnerDeployment(BaseModel):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,
@@ -905,7 +920,10 @@ async def deploy(
     """
     work_pool_name = work_pool_name or PREFECT_DEFAULT_WORK_POOL_NAME.value()
 
-    if not image and not all(d.storage for d in deployments):
+    if not image and not all(
+        d.storage or d.entrypoint_type == EntrypointType.MODULE_PATH
+        for d in deployments
+    ):
         raise ValueError(
             "Either an image or remote storage location must be provided when deploying"
             " a deployment."

prefect/deployments/schedules.py

@@ -0,0 +1,37 @@
+from typing import List, Optional, Sequence, Union, get_args
+
+from prefect.client.schemas.objects import MinimalDeploymentSchedule
+from prefect.client.schemas.schedules import SCHEDULE_TYPES
+
+FlexibleScheduleList = Sequence[Union[MinimalDeploymentSchedule, dict, SCHEDULE_TYPES]]
+
+
+def create_minimal_deployment_schedule(
+    schedule: SCHEDULE_TYPES,
+    active: Optional[bool] = True,
+) -> MinimalDeploymentSchedule:
+    return MinimalDeploymentSchedule(
+        schedule=schedule,
+        active=active if active is not None else True,
+    )
+
+
+def normalize_to_minimal_deployment_schedules(
+    schedules: Optional[FlexibleScheduleList],
+) -> List[MinimalDeploymentSchedule]:
+    normalized = []
+    if schedules is not None:
+        for obj in schedules:
+            if isinstance(obj, get_args(SCHEDULE_TYPES)):
+                normalized.append(create_minimal_deployment_schedule(obj))
+            elif isinstance(obj, dict):
+                normalized.append(create_minimal_deployment_schedule(**obj))
+            elif isinstance(obj, MinimalDeploymentSchedule):
+                normalized.append(obj)
+            else:
+                raise ValueError(
+                    "Invalid schedule provided. Must be a schedule object, a dict,"
+                    " or a MinimalDeploymentSchedule."
+                )
+
+    return normalized

prefect/engine.py

@@ -80,6 +80,7 @@ Client-side execution and orchestration of flows and tasks.
     _Ideally, for local and sequential task runners we would send the task run to the
     user thread as we do for flows. See [#9855](https://github.com/PrefectHQ/prefect/pull/9855).
 """
+
 import asyncio
 import contextlib
 import logging
@@ -169,6 +170,7 @@ from prefect.logging.loggers import (
 from prefect.results import BaseResult, ResultFactory, UnknownResult
 from prefect.settings import (
     PREFECT_DEBUG_MODE,
+    PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING,
     PREFECT_LOGGING_LOG_PRINTS,
     PREFECT_TASK_INTROSPECTION_WARN_THRESHOLD,
     PREFECT_TASKS_REFRESH_CACHE,
@@ -799,6 +801,8 @@ async def orchestrate_flow_run(
     # flag to ensure we only update the flow run name once
     run_name_set = False
 
+    await _run_flow_hooks(flow=flow, flow_run=flow_run, state=state)
+
     while state.is_running():
         waited_for_task_runs = False
 
@@ -954,7 +958,6 @@ async def orchestrate_flow_run(
                     f"Received non-final state {state.name!r} when proposing final"
                     f" state {terminal_state.name!r} and will attempt to run again..."
                 ),
-                extra={"send_to_api": False},
             )
             # Attempt to enter a running state again
             state = await propose_state(client, Running(), flow_run_id=flow_run.id)
@@ -1368,10 +1371,28 @@ def enter_task_run_engine(
     flow_run_context = FlowRunContext.get()
 
     if not flow_run_context:
-        raise RuntimeError(
-            "Tasks cannot be run outside of a flow"
-            " - if you meant to submit an autonomous task, you need to set"
-            " `prefect config set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING=true`"
+        if (
+            not PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value()
+            or return_type == "future"
+            or mapped
+        ):
+            raise RuntimeError(
+                "Tasks cannot be run outside of a flow by default."
+                " If you meant to submit an autonomous task, you need to set"
+                " `prefect config set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING=true`"
+                " and use `your_task.submit()` instead of `your_task()`."
+                " Mapping autonomous tasks is not yet supported."
+            )
+        from prefect.task_engine import submit_autonomous_task_run_to_engine
+
+        return submit_autonomous_task_run_to_engine(
+            task=task,
+            task_run=None,
+            parameters=parameters,
+            task_runner=task_runner,
+            wait_for=wait_for,
+            return_type=return_type,
+            client=get_client(),
         )
 
     if TaskRunContext.get():
@@ -1402,12 +1423,13 @@ def enter_task_run_engine(
 
 async def begin_task_map(
     task: Task,
-    flow_run_context: FlowRunContext,
+    flow_run_context: Optional[FlowRunContext],
     parameters: Dict[str, Any],
     wait_for: Optional[Iterable[PrefectFuture]],
     return_type: EngineReturnType,
     task_runner: Optional[BaseTaskRunner],
-) -> List[Union[PrefectFuture, Awaitable[PrefectFuture]]]:
+    autonomous: bool = False,
+) -> List[Union[PrefectFuture, Awaitable[PrefectFuture], TaskRun]]:
     """Async entrypoint for task mapping"""
     # We need to resolve some futures to map over their data, collect the upstream
     # links beforehand to retain relationship tracking.
@@ -1474,18 +1496,29 @@ async def begin_task_map(
         # Collapse any previously exploded kwargs
         call_parameters = collapse_variadic_parameters(task.fn, call_parameters)
 
-        task_runs.append(
-            partial(
-                get_task_call_return_value,
-                task=task,
-                flow_run_context=flow_run_context,
-                parameters=call_parameters,
-                wait_for=wait_for,
-                return_type=return_type,
-                task_runner=task_runner,
-                extra_task_inputs=task_inputs,
+        if autonomous:
+            task_runs.append(
+                await create_autonomous_task_run(
+                    task=task,
+                    parameters=call_parameters,
+                )
+            )
+        else:
+            task_runs.append(
+                partial(
+                    get_task_call_return_value,
+                    task=task,
+                    flow_run_context=flow_run_context,
+                    parameters=call_parameters,
+                    wait_for=wait_for,
+                    return_type=return_type,
+                    task_runner=task_runner,
+                    extra_task_inputs=task_inputs,
+                )
             )
-        )
+
+    if autonomous:
+        return task_runs
 
     # Maintain the order of the task runs when using the sequential task runner
     runner = task_runner if task_runner else flow_run_context.task_runner
@@ -1697,7 +1730,10 @@ async def create_task_run(
         task_inputs=task_inputs,
     )
 
-    logger.info(f"Created task run {task_run.name!r} for task {task.name!r}")
+    if flow_run_context.flow_run:
+        logger.info(f"Created task run {task_run.name!r} for task {task.name!r}")
+    else:
+        engine_logger.info(f"Created task run {task_run.name!r} for task {task.name!r}")
 
     return task_run
 
@@ -1715,7 +1751,7 @@ async def submit_task_run(
 
     if (
         task_runner.concurrency_type == TaskConcurrencyType.SEQUENTIAL
-        and not flow_run_context.autonomous_task_run
+        and flow_run_context.flow_run
     ):
         logger.info(f"Executing {task_run.name!r} immediately...")
 
@@ -2181,7 +2217,6 @@ async def orchestrate_task_run(
                         f" state {terminal_state.name!r} and will attempt to run"
                         " again..."
                     ),
-                    extra={"send_to_api": False},
                 )
                 # Attempt to enter a running state again
                 state = await propose_state(client, Running(), task_run_id=task_run.id)
@@ -2801,7 +2836,10 @@ async def _run_flow_hooks(flow: Flow, flow_run: FlowRun, state: State) -> None:
         os.environ.get("PREFECT__ENABLE_CANCELLATION_AND_CRASHED_HOOKS", "true").lower()
         == "true"
     )
-    if state.is_failed() and flow.on_failure:
+
+    if state.is_running() and flow.on_running:
+        hooks = flow.on_running
+    elif state.is_failed() and flow.on_failure:
         hooks = flow.on_failure
     elif state.is_completed() and flow.on_completion:
         hooks = flow.on_completion
@@ -2924,7 +2962,8 @@ async def create_autonomous_task_run(task: Task, parameters: Dict[str, Any]) ->
 
             # TODO: Improve use of result storage for parameter storage / reference
             task.persist_result = True
-            factory = await ResultFactory.from_task(task, client=client)
+
+            factory = await ResultFactory.from_autonomous_task(task, client=client)
             await factory.store_parameters(parameters_id, parameters)
 
         task_run = await client.create_task_run(