prefect-client 2.16.2__py3-none-any.whl → 2.16.3__py3-none-any.whl

prefect/_internal/concurrency/services.py

@@ -91,6 +91,11 @@ class QueueService(abc.ABC, Generic[T]):
             self._remove_instance()
             self._stopped = True
 
+            # Allow asyncio task to be garbage-collected. Its context may contain
+            # references to all Prefect Task calls made during a flow run, through
+            # EngineContext. Issue #10338.
+            self._task = None
+
             # Signal completion to the loop
             self._queue.put_nowait(None)
 

prefect/_internal/concurrency/threads.py

@@ -214,6 +214,9 @@ class EventLoopThread(Portal):
         for call in self._on_shutdown:
             await self._run_call(call)
 
+        # Empty the list to allow calls to be garbage collected. Issue #10338.
+        self._on_shutdown = []
+
     async def _run_call(self, call: Call) -> None:
         task = call.run()
         if task is not None:

prefect/deployments/deployments.py

@@ -36,6 +36,10 @@ from prefect.client.schemas.objects import (
 from prefect.client.schemas.schedules import SCHEDULE_TYPES
 from prefect.client.utilities import inject_client
 from prefect.context import FlowRunContext, PrefectObjectRegistry, TaskRunContext
+from prefect.deployments.schedules import (
+    FlexibleScheduleList,
+    normalize_to_minimal_deployment_schedules,
+)
 from prefect.deployments.steps.core import run_steps
 from prefect.events.schemas import DeploymentTrigger
 from prefect.exceptions import (
@@ -649,11 +653,17 @@ class Deployment(BaseModel):
             cls._validate_schedule(value)
         return value
 
-    @validator("schedules")
-    def validate_schedules(cls, value):
-        for schedule in value:
-            cls._validate_schedule(schedule.schedule)
-        return value
+    @root_validator(pre=True)
+    def validate_schedules(cls, values):
+        if "schedules" in values:
+            values["schedules"] = normalize_to_minimal_deployment_schedules(
+                values["schedules"]
+            )
+
+            for schedule in values["schedules"]:
+                cls._validate_schedule(schedule.schedule)
+
+        return values
 
     @classmethod
     @sync_compatible
@@ -936,6 +946,7 @@ class Deployment(BaseModel):
         ignore_file: str = ".prefectignore",
         apply: bool = False,
         load_existing: bool = True,
+        schedules: Optional[FlexibleScheduleList] = None,
         **kwargs,
     ) -> "Deployment":
         """
@@ -955,6 +966,14 @@ class Deployment(BaseModel):
             load_existing: if True, load any settings that may already be configured for
                 the named deployment server-side (e.g., schedules, default parameter
                 values, etc.)
+            schedules: An optional list of schedules. Each item in the list can be:
+                  - An instance of `MinimalDeploymentSchedule`.
+                  - A dictionary with a `schedule` key, and optionally, an
+                    `active` key. The `schedule` key should correspond to a
+                    schedule type, and `active` is a boolean indicating whether
+                    the schedule is active or not.
+                  - An instance of one of the predefined schedule types:
+                    `IntervalSchedule`, `CronSchedule`, or `RRuleSchedule`.
             **kwargs: other keyword arguments to pass to the constructor for the
                 `Deployment` class
         """
@@ -963,7 +982,11 @@ class Deployment(BaseModel):
 
         # note that `deployment.load` only updates settings that were *not*
         # provided at initialization
-        deployment = cls(name=name, **kwargs)
+        deployment = cls(
+            name=name,
+            schedules=schedules,
+            **kwargs,
+        )
         deployment.flow_name = flow.name
         if not deployment.entrypoint:
             ## first see if an entrypoint can be determined

prefect/deployments/runner.py

@@ -34,7 +34,7 @@ import importlib
 import tempfile
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, get_args
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union
 from uuid import UUID
 
 import pendulum
@@ -63,6 +63,11 @@ from prefect.client.schemas.schedules import (
     SCHEDULE_TYPES,
     construct_schedule,
 )
+from prefect.deployments.schedules import (
+    FlexibleScheduleList,
+    create_minimal_deployment_schedule,
+    normalize_to_minimal_deployment_schedules,
+)
 from prefect.events.schemas import DeploymentTrigger
 from prefect.exceptions import (
     ObjectNotFound,
@@ -83,18 +88,9 @@ from prefect.utilities.slugify import slugify
 if TYPE_CHECKING:
     from prefect.flows import Flow
 
-FlexibleScheduleList = Union[MinimalDeploymentSchedule, dict, SCHEDULE_TYPES]
-
 __all__ = ["RunnerDeployment"]
 
 
-def _to_deployment_schedule(
-    schedule: Optional[SCHEDULE_TYPES] = None,
-    active: Optional[bool] = True,
-) -> MinimalDeploymentSchedule:
-    return MinimalDeploymentSchedule(schedule=schedule, active=active)
-
-
 class DeploymentApplyError(RuntimeError):
     """
     Raised when an error occurs while applying a deployment.
@@ -264,23 +260,9 @@ class RunnerDeployment(BaseModel):
         schedules = values.get("schedules")
 
         if schedules is None and schedule is not None:
-            values["schedules"] = [_to_deployment_schedule(schedule)]
+            values["schedules"] = [create_minimal_deployment_schedule(schedule)]
         elif schedules is not None and len(schedules) > 0:
-            reconciled = []
-            for obj in schedules:
-                if isinstance(obj, get_args(SCHEDULE_TYPES)):
-                    reconciled.append(_to_deployment_schedule(obj))
-                elif isinstance(obj, dict):
-                    reconciled.append(_to_deployment_schedule(**obj))
-                elif isinstance(obj, MinimalDeploymentSchedule):
-                    reconciled.append(obj)
-                else:
-                    raise ValueError(
-                        "Invalid schedule provided. Must be a schedule object, a dict,"
-                        " or a MinimalDeploymentSchedule."
-                    )
-
-            values["schedules"] = reconciled
+            values["schedules"] = normalize_to_minimal_deployment_schedules(schedules)
 
         return values
 
@@ -389,8 +371,8 @@ class RunnerDeployment(BaseModel):
         rrule: Optional[Union[Iterable[str], str]] = None,
         timezone: Optional[str] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
-    ) -> Union[List[MinimalDeploymentSchedule], List[FlexibleScheduleList]]:
+        schedules: Optional[FlexibleScheduleList] = None,
+    ) -> Union[List[MinimalDeploymentSchedule], FlexibleScheduleList]:
         """
         Construct a schedule or schedules from the provided arguments.
 
@@ -448,7 +430,7 @@ class RunnerDeployment(BaseModel):
                 value = [value]
 
             return [
-                _to_deployment_schedule(
+                create_minimal_deployment_schedule(
                     construct_schedule(
                         **{
                             schedule_type: v,
@@ -460,7 +442,7 @@ class RunnerDeployment(BaseModel):
                 for v in value
             ]
         else:
-            return [_to_deployment_schedule(schedule)]
+            return [create_minimal_deployment_schedule(schedule)]
 
     def _set_defaults_from_flow(self, flow: "Flow"):
         self._parameter_openapi_schema = parameter_schema(flow)
@@ -481,7 +463,7 @@ class RunnerDeployment(BaseModel):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,
@@ -617,7 +599,7 @@ class RunnerDeployment(BaseModel):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,
@@ -715,7 +697,7 @@ class RunnerDeployment(BaseModel):
         cron: Optional[Union[Iterable[str], str]] = None,
         rrule: Optional[Union[Iterable[str], str]] = None,
         paused: Optional[bool] = None,
-        schedules: Optional[List[FlexibleScheduleList]] = None,
+        schedules: Optional[FlexibleScheduleList] = None,
         schedule: Optional[SCHEDULE_TYPES] = None,
         is_schedule_active: Optional[bool] = None,
         parameters: Optional[dict] = None,

prefect/deployments/schedules.py

@@ -0,0 +1,37 @@
+from typing import List, Optional, Sequence, Union, get_args
+
+from prefect.client.schemas.objects import MinimalDeploymentSchedule
+from prefect.client.schemas.schedules import SCHEDULE_TYPES
+
+FlexibleScheduleList = Sequence[Union[MinimalDeploymentSchedule, dict, SCHEDULE_TYPES]]
+
+
+def create_minimal_deployment_schedule(
+    schedule: SCHEDULE_TYPES,
+    active: Optional[bool] = True,
+) -> MinimalDeploymentSchedule:
+    return MinimalDeploymentSchedule(
+        schedule=schedule,
+        active=active if active is not None else True,
+    )
+
+
+def normalize_to_minimal_deployment_schedules(
+    schedules: Optional[FlexibleScheduleList],
+) -> List[MinimalDeploymentSchedule]:
+    normalized = []
+    if schedules is not None:
+        for obj in schedules:
+            if isinstance(obj, get_args(SCHEDULE_TYPES)):
+                normalized.append(create_minimal_deployment_schedule(obj))
+            elif isinstance(obj, dict):
+                normalized.append(create_minimal_deployment_schedule(**obj))
+            elif isinstance(obj, MinimalDeploymentSchedule):
+                normalized.append(obj)
+            else:
+                raise ValueError(
+                    "Invalid schedule provided. Must be a schedule object, a dict,"
+                    " or a MinimalDeploymentSchedule."
+                )
+
+    return normalized

prefect/engine.py

@@ -170,6 +170,7 @@ from prefect.logging.loggers import (
 from prefect.results import BaseResult, ResultFactory, UnknownResult
 from prefect.settings import (
     PREFECT_DEBUG_MODE,
+    PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING,
     PREFECT_LOGGING_LOG_PRINTS,
     PREFECT_TASK_INTROSPECTION_WARN_THRESHOLD,
     PREFECT_TASKS_REFRESH_CACHE,
@@ -800,6 +801,8 @@ async def orchestrate_flow_run(
     # flag to ensure we only update the flow run name once
     run_name_set = False
 
+    await _run_flow_hooks(flow=flow, flow_run=flow_run, state=state)
+
     while state.is_running():
         waited_for_task_runs = False
 
@@ -955,7 +958,6 @@ async def orchestrate_flow_run(
                     f"Received non-final state {state.name!r} when proposing final"
                     f" state {terminal_state.name!r} and will attempt to run again..."
                 ),
-                extra={"send_to_api": False},
             )
             # Attempt to enter a running state again
             state = await propose_state(client, Running(), flow_run_id=flow_run.id)
@@ -1369,10 +1371,28 @@ def enter_task_run_engine(
     flow_run_context = FlowRunContext.get()
 
     if not flow_run_context:
-        raise RuntimeError(
-            "Tasks cannot be run outside of a flow"
-            " - if you meant to submit an autonomous task, you need to set"
-            " `prefect config set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING=true`"
+        if (
+            not PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value()
+            or return_type == "future"
+            or mapped
+        ):
+            raise RuntimeError(
+                "Tasks cannot be run outside of a flow by default."
+                " If you meant to submit an autonomous task, you need to set"
+                " `prefect config set PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING=true`"
+                " and use `your_task.submit()` instead of `your_task()`."
+                " Mapping autonomous tasks is not yet supported."
+            )
+        from prefect.task_engine import submit_autonomous_task_run_to_engine
+
+        return submit_autonomous_task_run_to_engine(
+            task=task,
+            task_run=None,
+            parameters=parameters,
+            task_runner=task_runner,
+            wait_for=wait_for,
+            return_type=return_type,
+            client=get_client(),
         )
 
     if TaskRunContext.get():
@@ -1403,12 +1423,13 @@ def enter_task_run_engine(
 
 async def begin_task_map(
     task: Task,
-    flow_run_context: FlowRunContext,
+    flow_run_context: Optional[FlowRunContext],
     parameters: Dict[str, Any],
     wait_for: Optional[Iterable[PrefectFuture]],
     return_type: EngineReturnType,
     task_runner: Optional[BaseTaskRunner],
-) -> List[Union[PrefectFuture, Awaitable[PrefectFuture]]]:
+    autonomous: bool = False,
+) -> List[Union[PrefectFuture, Awaitable[PrefectFuture], TaskRun]]:
     """Async entrypoint for task mapping"""
     # We need to resolve some futures to map over their data, collect the upstream
     # links beforehand to retain relationship tracking.
@@ -1475,18 +1496,29 @@ async def begin_task_map(
         # Collapse any previously exploded kwargs
         call_parameters = collapse_variadic_parameters(task.fn, call_parameters)
 
-        task_runs.append(
-            partial(
-                get_task_call_return_value,
-                task=task,
-                flow_run_context=flow_run_context,
-                parameters=call_parameters,
-                wait_for=wait_for,
-                return_type=return_type,
-                task_runner=task_runner,
-                extra_task_inputs=task_inputs,
+        if autonomous:
+            task_runs.append(
+                await create_autonomous_task_run(
+                    task=task,
+                    parameters=call_parameters,
+                )
             )
-        )
+        else:
+            task_runs.append(
+                partial(
+                    get_task_call_return_value,
+                    task=task,
+                    flow_run_context=flow_run_context,
+                    parameters=call_parameters,
+                    wait_for=wait_for,
+                    return_type=return_type,
+                    task_runner=task_runner,
+                    extra_task_inputs=task_inputs,
+                )
+            )
+
+    if autonomous:
+        return task_runs
 
     # Maintain the order of the task runs when using the sequential task runner
     runner = task_runner if task_runner else flow_run_context.task_runner
@@ -1698,7 +1730,10 @@ async def create_task_run(
         task_inputs=task_inputs,
     )
 
-    logger.info(f"Created task run {task_run.name!r} for task {task.name!r}")
+    if flow_run_context.flow_run:
+        logger.info(f"Created task run {task_run.name!r} for task {task.name!r}")
+    else:
+        engine_logger.info(f"Created task run {task_run.name!r} for task {task.name!r}")
 
     return task_run
 
@@ -1716,7 +1751,7 @@ async def submit_task_run(
 
     if (
         task_runner.concurrency_type == TaskConcurrencyType.SEQUENTIAL
-        and not flow_run_context.autonomous_task_run
+        and flow_run_context.flow_run
     ):
         logger.info(f"Executing {task_run.name!r} immediately...")
 
@@ -2182,7 +2217,6 @@ async def orchestrate_task_run(
                         f" state {terminal_state.name!r} and will attempt to run"
                         " again..."
                     ),
-                    extra={"send_to_api": False},
                 )
                 # Attempt to enter a running state again
                 state = await propose_state(client, Running(), task_run_id=task_run.id)
@@ -2802,7 +2836,10 @@ async def _run_flow_hooks(flow: Flow, flow_run: FlowRun, state: State) -> None:
         os.environ.get("PREFECT__ENABLE_CANCELLATION_AND_CRASHED_HOOKS", "true").lower()
         == "true"
     )
-    if state.is_failed() and flow.on_failure:
+
+    if state.is_running() and flow.on_running:
+        hooks = flow.on_running
+    elif state.is_failed() and flow.on_failure:
         hooks = flow.on_failure
     elif state.is_completed() and flow.on_completion:
         hooks = flow.on_completion