prefect-client 2.18.3__py3-none-any.whl → 2.19.1__py3-none-any.whl

prefect/runner/runner.py

@@ -51,9 +51,6 @@ from uuid import UUID, uuid4
 import anyio
 import anyio.abc
 import pendulum
-import sniffio
-from rich.console import Console, Group
-from rich.table import Table
 
 from prefect._internal.concurrency.api import (
     create_call,
@@ -80,7 +77,6 @@ from prefect.deployments.runner import (
     RunnerDeployment,
 )
 from prefect.deployments.schedules import FlexibleScheduleList
-from prefect.engine import propose_state
 from prefect.events import DeploymentTriggerTypes, TriggerTypes
 from prefect.exceptions import (
     Abort,
@@ -94,7 +90,6 @@ from prefect.settings import (
     PREFECT_RUNNER_POLL_FREQUENCY,
     PREFECT_RUNNER_PROCESS_LIMIT,
     PREFECT_RUNNER_SERVER_ENABLE,
-    PREFECT_UI_URL,
     get_current_settings,
 )
 from prefect.states import Crashed, Pending, exception_to_failed_state
@@ -103,10 +98,11 @@ from prefect.utilities.asyncutils import (
     is_async_fn,
     sync_compatible,
 )
+from prefect.utilities.engine import propose_state
 from prefect.utilities.processutils import _register_signal, run_process
 from prefect.utilities.services import critical_service_loop
 
-__all__ = ["Runner", "serve"]
+__all__ = ["Runner"]
 
 
 class Runner:
@@ -550,7 +546,6 @@ class Runner:
         if sys.platform == "win32":
             kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
 
-        _use_threaded_child_watcher()
         flow_run_logger.info("Opening process...")
 
         env = get_current_settings().to_environment_variables(exclude_unset=True)
@@ -1190,106 +1185,6 @@ if sys.platform == "win32":
     STATUS_CONTROL_C_EXIT = 0xC000013A
 
 
-def _use_threaded_child_watcher():
-    if (
-        sys.version_info < (3, 8)
-        and sniffio.current_async_library() == "asyncio"
-        and sys.platform != "win32"
-    ):
-        from prefect.utilities.compat import ThreadedChildWatcher
-
-        # Python < 3.8 does not use a `ThreadedChildWatcher` by default which can
-        # lead to errors in tests on unix as the previous default `SafeChildWatcher`
-        # is not compatible with threaded event loops.
-        asyncio.get_event_loop_policy().set_child_watcher(ThreadedChildWatcher())
-
-
-@sync_compatible
-async def serve(
-    *args: RunnerDeployment,
-    pause_on_shutdown: bool = True,
-    print_starting_message: bool = True,
-    limit: Optional[int] = None,
-    **kwargs,
-):
-    """
-    Serve the provided list of deployments.
-
-    Args:
-        *args: A list of deployments to serve.
-        pause_on_shutdown: A boolean for whether or not to automatically pause
-            deployment schedules on shutdown.
-        print_starting_message: Whether or not to print message to the console
-            on startup.
-        limit: The maximum number of runs that can be executed concurrently.
-        **kwargs: Additional keyword arguments to pass to the runner.
-
-    Examples:
-        Prepare two deployments and serve them:
-
-        ```python
-        import datetime
-
-        from prefect import flow, serve
-
-        @flow
-        def my_flow(name):
-            print(f"hello {name}")
-
-        @flow
-        def my_other_flow(name):
-            print(f"goodbye {name}")
-
-        if __name__ == "__main__":
-            # Run once a day
-            hello_deploy = my_flow.to_deployment(
-                "hello", tags=["dev"], interval=datetime.timedelta(days=1)
-            )
-
-            # Run every Sunday at 4:00 AM
-            bye_deploy = my_other_flow.to_deployment(
-                "goodbye", tags=["dev"], cron="0 4 * * sun"
-            )
-
-            serve(hello_deploy, bye_deploy)
-        ```
-    """
-    runner = Runner(pause_on_shutdown=pause_on_shutdown, limit=limit, **kwargs)
-    for deployment in args:
-        await runner.add_deployment(deployment)
-
-    if print_starting_message:
-        help_message_top = (
-            "[green]Your deployments are being served and polling for"
-            " scheduled runs!\n[/]"
-        )
-
-        table = Table(title="Deployments", show_header=False)
-
-        table.add_column(style="blue", no_wrap=True)
-
-        for deployment in args:
-            table.add_row(f"{deployment.flow_name}/{deployment.name}")
-
-        help_message_bottom = (
-            "\nTo trigger any of these deployments, use the"
-            " following command:\n[blue]\n\t$ prefect deployment run"
-            " [DEPLOYMENT_NAME]\n[/]"
-        )
-        if PREFECT_UI_URL:
-            help_message_bottom += (
-                "\nYou can also trigger your deployments via the Prefect UI:"
-                f" [blue]{PREFECT_UI_URL.value()}/deployments[/]\n"
-            )
-
-        console = Console()
-        console.print(
-            Group(help_message_top, table, help_message_bottom), soft_wrap=True
-        )
-
-    await runner.start()
-
-
 async def _run_hooks(
     hooks: List[Callable[[Flow, "FlowRun", State], None]], flow_run, flow, state
 ):

prefect/settings.py

@@ -1501,6 +1501,11 @@ PREFECT_RUNNER_SERVER_ENABLE = Setting(bool, default=False)
 Whether or not to enable the runner's webserver.
 """
 
+PREFECT_DEPLOYMENT_SCHEDULE_MAX_SCHEDULED_RUNS = Setting(int, default=50)
+"""
+The maximum number of scheduled runs to create for a deployment.
+"""
+
 PREFECT_WORKER_HEARTBEAT_SECONDS = Setting(float, default=30)
 """
 Number of seconds a worker should wait between sending a heartbeat.
@@ -1614,6 +1619,11 @@ PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE = Setting(bool, default=False)
 Whether or not to enable experimental new engine.
 """
 
+PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT = Setting(bool, default=False)
+"""
+Whether or not to disable the sync_compatible decorator utility.
+"""
+
 
 # Defaults -----------------------------------------------------------------------------
 
@@ -1745,6 +1755,17 @@ PREFECT_API_EVENTS_RELATED_RESOURCE_CACHE_TTL = Setting(
 How long to cache related resource data for emitting server-side vents
 """
 
+
+def automation_settings_enabled() -> bool:
+    """
+    Whether or not automations are enabled.
+    """
+    return (
+        PREFECT_EXPERIMENTAL_EVENTS.value()
+        and PREFECT_API_SERVICES_TRIGGERS_ENABLED.value()
+    )
+
+
 # Deprecated settings ------------------------------------------------------------------
 
 

prefect/tasks.py

@@ -7,7 +7,6 @@ Module containing the base workflow task class and decorator - for most use case
 import datetime
 import inspect
 import os
-import warnings
 from copy import copy
 from functools import partial, update_wrapper
 from typing import (
@@ -33,9 +32,15 @@ from uuid import uuid4
 from typing_extensions import Literal, ParamSpec
 
 from prefect._internal.concurrency.api import create_call, from_async, from_sync
+from prefect.client.orchestration import PrefectClient, SyncPrefectClient
 from prefect.client.schemas import TaskRun
-from prefect.client.schemas.objects import TaskRunInput
-from prefect.context import FlowRunContext, PrefectObjectRegistry, TagsContext
+from prefect.client.schemas.objects import TaskRunInput, TaskRunResult
+from prefect.context import (
+    FlowRunContext,
+    PrefectObjectRegistry,
+    TagsContext,
+    TaskRunContext,
+)
 from prefect.futures import PrefectFuture
 from prefect.logging.loggers import get_logger, get_run_logger
 from prefect.results import ResultSerializer, ResultStorage
@@ -332,33 +337,7 @@ class Task(Generic[P, R]):
         self.result_serializer = result_serializer
         self.result_storage_key = result_storage_key
         self.cache_result_in_memory = cache_result_in_memory
-
         self.timeout_seconds = float(timeout_seconds) if timeout_seconds else None
-        # Warn if this task's `name` conflicts with another task while having a
-        # different function. This is to detect the case where two or more tasks
-        # share a name or are lambdas, which should result in a warning, and to
-        # differentiate it from the case where the task was 'copied' via
-        # `with_options`, which should not result in a warning.
-        registry = PrefectObjectRegistry.get()
-
-        if registry and any(
-            other
-            for other in registry.get_instances(Task)
-            if other.name == self.name and id(other.fn) != id(self.fn)
-        ):
-            try:
-                file = inspect.getsourcefile(self.fn)
-                line_number = inspect.getsourcelines(self.fn)[1]
-            except TypeError:
-                file = "unknown"
-                line_number = "unknown"
-
-            warnings.warn(
-                f"A task named {self.name!r} and defined at '{file}:{line_number}' "
-                "conflicts with another task. Consider specifying a unique `name` "
-                "parameter in the task definition:\n\n "
-                "`@task(name='my_unique_name', ...)`"
-            )
         self.on_completion = on_completion
         self.on_failure = on_failure
 
@@ -539,48 +518,93 @@ class Task(Generic[P, R]):
 
     async def create_run(
         self,
-        flow_run_context: FlowRunContext,
-        parameters: Dict[str, Any],
-        wait_for: Optional[Iterable[PrefectFuture]],
+        client: Optional[Union[PrefectClient, SyncPrefectClient]],
+        parameters: Dict[str, Any] = None,
+        flow_run_context: Optional[FlowRunContext] = None,
+        parent_task_run_context: Optional[TaskRunContext] = None,
+        wait_for: Optional[Iterable[PrefectFuture]] = None,
         extra_task_inputs: Optional[Dict[str, Set[TaskRunInput]]] = None,
     ) -> TaskRun:
-        # TODO: Investigate if we can replace create_task_run on the task run engine
-        # with this method. Would require updating to work without the flow run context.
         from prefect.utilities.engine import (
             _dynamic_key_for_task_run,
+            _resolve_custom_task_run_name,
             collect_task_run_inputs,
         )
 
-        dynamic_key = _dynamic_key_for_task_run(flow_run_context, self)
+        if flow_run_context is None:
+            flow_run_context = FlowRunContext.get()
+        if parent_task_run_context is None:
+            parent_task_run_context = TaskRunContext.get()
+        if parameters is None:
+            parameters = {}
+
+        try:
+            task_run_name = _resolve_custom_task_run_name(self, parameters)
+        except TypeError:
+            task_run_name = None
+
+        if flow_run_context:
+            dynamic_key = _dynamic_key_for_task_run(context=flow_run_context, task=self)
+        else:
+            dynamic_key = uuid4().hex
+
+        # collect task inputs
         task_inputs = {
             k: await collect_task_run_inputs(v) for k, v in parameters.items()
         }
+
+        # check if this task has a parent task run based on running in another
+        # task run's existing context. A task run is only considered a parent if
+        # it is in the same flow run (because otherwise presumably the child is
+        # in a subflow, so the subflow serves as the parent) or if there is no
+        # flow run
+        if parent_task_run_context:
+            # there is no flow run
+            if not flow_run_context:
+                task_inputs["__parents__"] = [
+                    TaskRunResult(id=parent_task_run_context.task_run.id)
+                ]
+            # there is a flow run and the task run is in the same flow run
+            elif (
+                flow_run_context
+                and parent_task_run_context.task_run.flow_run_id
+                == flow_run_context.flow_run.id
+            ):
+                task_inputs["__parents__"] = [
+                    TaskRunResult(id=parent_task_run_context.task_run.id)
+                ]
+
         if wait_for:
             task_inputs["wait_for"] = await collect_task_run_inputs(wait_for)
 
         # Join extra task inputs
-        extra_task_inputs = extra_task_inputs or {}
-        for k, extras in extra_task_inputs.items():
+        for k, extras in (extra_task_inputs or {}).items():
             task_inputs[k] = task_inputs[k].union(extras)
 
-        flow_run_logger = get_run_logger(flow_run_context)
-
-        task_run = await flow_run_context.client.create_task_run(
+        # create the task run
+        task_run = client.create_task_run(
             task=self,
-            name=f"{self.name} - {dynamic_key}",
-            flow_run_id=flow_run_context.flow_run.id,
-            dynamic_key=dynamic_key,
+            name=task_run_name,
+            flow_run_id=(
+                getattr(flow_run_context.flow_run, "id", None)
+                if flow_run_context and flow_run_context.flow_run
+                else None
+            ),
+            dynamic_key=str(dynamic_key),
             state=Pending(),
-            extra_tags=TagsContext.get().current_tags,
             task_inputs=task_inputs,
+            extra_tags=TagsContext.get().current_tags,
         )
+        # the new engine uses sync clients but old engines use async clients
+        if inspect.isawaitable(task_run):
+            task_run = await task_run
 
-        if flow_run_context.flow_run:
-            flow_run_logger.info(
+        if flow_run_context and flow_run_context.flow_run:
+            get_run_logger(flow_run_context).debug(
                 f"Created task run {task_run.name!r} for task {self.name!r}"
             )
         else:
-            logger.info(f"Created task run {task_run.name!r} for task {self.name!r}")
+            logger.debug(f"Created task run {task_run.name!r} for task {self.name!r}")
 
         return task_run
 
@@ -637,21 +661,15 @@ class Task(Generic[P, R]):
                 self.isasync, self.name, parameters, self.viz_return_value
             )
 
-        # new engine currently only compatible with async tasks
         if PREFECT_EXPERIMENTAL_ENABLE_NEW_ENGINE.value():
-            from prefect.new_task_engine import run_task, run_task_sync
+            from prefect.new_task_engine import run_task
 
-            run_kwargs = dict(
+            return run_task(
                 task=self,
                 parameters=parameters,
                 wait_for=wait_for,
                 return_type=return_type,
             )
-            if self.isasync:
-                # this returns an awaitable coroutine
-                return run_task(**run_kwargs)
-            else:
-                return run_task_sync(**run_kwargs)
 
         if (
             PREFECT_EXPERIMENTAL_ENABLE_TASK_SCHEDULING.value()
@@ -931,11 +949,12 @@ class Task(Generic[P, R]):
         wait_for: Optional[Iterable[PrefectFuture]],
         return_state: bool,
     ):
-        from prefect.new_task_engine import run_task
+        from prefect.new_task_engine import run_task_async
 
         task_runner = flow_run_context.task_runner
 
         task_run = await self.create_run(
+            client=flow_run_context.client,
             flow_run_context=flow_run_context,
             parameters=parameters,
             wait_for=wait_for,
@@ -952,7 +971,7 @@ class Task(Generic[P, R]):
         await task_runner.submit(
             key=future.key,
             call=partial(
-                run_task,
+                run_task_async,
                 task=self,
                 task_run=task_run,
                 parameters=parameters,

prefect/types/__init__.py

@@ -1,4 +1,3 @@
-from dataclasses import dataclass
 from typing import Any, Callable, ClassVar, Generator
 
 from pydantic_core import core_schema, CoreSchema, SchemaValidator
@@ -6,8 +5,9 @@ from typing_extensions import Self
 from datetime import timedelta
 
 
-@dataclass
 class NonNegativeInteger(int):
+    """An integer that must be greater than or equal to 0."""
+
     schema: ClassVar[CoreSchema] = core_schema.int_schema(ge=0)
 
     @classmethod
@@ -25,8 +25,9 @@ class NonNegativeInteger(int):
         return SchemaValidator(schema=cls.schema).validate_python(v)
 
 
-@dataclass
 class PositiveInteger(int):
+    """An integer that must be greater than 0."""
+
     schema: ClassVar[CoreSchema] = core_schema.int_schema(gt=0)
 
     @classmethod
@@ -44,8 +45,27 @@ class PositiveInteger(int):
         return SchemaValidator(schema=cls.schema).validate_python(v)
 
 
-@dataclass
+class NonNegativeFloat(float):
+    schema: ClassVar[CoreSchema] = core_schema.float_schema(ge=0)
+
+    @classmethod
+    def __get_validators__(cls) -> Generator[Callable[..., Any], None, None]:
+        yield cls.validate
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, source_type: Any, handler: Callable[..., Any]
+    ) -> CoreSchema:
+        return cls.schema
+
+    @classmethod
+    def validate(cls, v: Any) -> Self:
+        return SchemaValidator(schema=cls.schema).validate_python(v)
+
+
 class NonNegativeDuration(timedelta):
+    """A timedelta that must be greater than or equal to 0."""
+
     schema: ClassVar = core_schema.timedelta_schema(ge=timedelta(seconds=0))
 
     @classmethod
@@ -63,8 +83,9 @@ class NonNegativeDuration(timedelta):
         return SchemaValidator(schema=cls.schema).validate_python(v)
 
 
-@dataclass
 class PositiveDuration(timedelta):
+    """A timedelta that must be greater than 0."""
+
     schema: ClassVar = core_schema.timedelta_schema(gt=timedelta(seconds=0))
 
     @classmethod
@@ -85,6 +106,7 @@ class PositiveDuration(timedelta):
 __all__ = [
     "NonNegativeInteger",
     "PositiveInteger",
+    "NonNegativeFloat",
     "NonNegativeDuration",
     "PositiveDuration",
 ]

prefect/utilities/annotations.py

@@ -1,4 +1,3 @@
-import sys
 import warnings
 from abc import ABC
 from collections import namedtuple
@@ -17,13 +16,7 @@ class BaseAnnotation(
     """
 
     def unwrap(self) -> T:
-        if sys.version_info < (3, 8):
-            # cannot simply return self.value due to recursion error in Python 3.7
-            # also _asdict does not follow convention; it's not an internal method
-            # https://stackoverflow.com/a/26180604
-            return self._asdict()["value"]
-        else:
-            return self.value
+        return self.value
 
     def rewrap(self, value: T) -> "BaseAnnotation[T]":
         return type(self)(value)

prefect/utilities/asyncutils.py

@@ -266,6 +266,10 @@ def sync_compatible(async_fn: T) -> T:
         from prefect._internal.concurrency.calls import get_current_call, logger
         from prefect._internal.concurrency.event_loop import get_running_loop
         from prefect._internal.concurrency.threads import get_global_loop
+        from prefect.settings import PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT
+
+        if PREFECT_EXPERIMENTAL_DISABLE_SYNC_COMPAT:
+            return async_fn(*args, **kwargs)
 
         global_thread_portal = get_global_loop()
         current_thread = threading.current_thread()

prefect/utilities/engine.py

@@ -1,5 +1,6 @@
 import asyncio
 import contextlib
+import inspect
 import os
 import signal
 import time
@@ -23,7 +24,7 @@ import prefect
 import prefect.context
 import prefect.plugins
 from prefect._internal.concurrency.cancellation import get_deadline
-from prefect.client.orchestration import PrefectClient
+from prefect.client.orchestration import PrefectClient, SyncPrefectClient
 from prefect.client.schemas import OrchestrationResult, TaskRun
 from prefect.client.schemas.objects import (
     StateType,
@@ -60,6 +61,7 @@ from prefect.tasks import Task
 from prefect.utilities.annotations import allow_failure, quote
 from prefect.utilities.asyncutils import (
     gather,
+    run_sync,
 )
 from prefect.utilities.collections import StopVisiting, visit_collection
 from prefect.utilities.text import truncated_to
@@ -409,6 +411,109 @@ async def propose_state(
         )
 
 
+def propose_state_sync(
+    client: SyncPrefectClient,
+    state: State[object],
+    force: bool = False,
+    task_run_id: Optional[UUID] = None,
+    flow_run_id: Optional[UUID] = None,
+) -> State[object]:
+    """
+    Propose a new state for a flow run or task run, invoking Prefect orchestration logic.
+
+    If the proposed state is accepted, the provided `state` will be augmented with
+     details and returned.
+
+    If the proposed state is rejected, a new state returned by the Prefect API will be
+    returned.
+
+    If the proposed state results in a WAIT instruction from the Prefect API, the
+    function will sleep and attempt to propose the state again.
+
+    If the proposed state results in an ABORT instruction from the Prefect API, an
+    error will be raised.
+
+    Args:
+        state: a new state for the task or flow run
+        task_run_id: an optional task run id, used when proposing task run states
+        flow_run_id: an optional flow run id, used when proposing flow run states
+
+    Returns:
+        a [State model][prefect.client.schemas.objects.State] representation of the
+            flow or task run state
+
+    Raises:
+        ValueError: if neither task_run_id or flow_run_id is provided
+        prefect.exceptions.Abort: if an ABORT instruction is received from
+            the Prefect API
+    """
+
+    # Determine if working with a task run or flow run
+    if not task_run_id and not flow_run_id:
+        raise ValueError("You must provide either a `task_run_id` or `flow_run_id`")
+
+    # Handle task and sub-flow tracing
+    if state.is_final():
+        if isinstance(state.data, BaseResult) and state.data.has_cached_object():
+            # Avoid fetching the result unless it is cached, otherwise we defeat
+            # the purpose of disabling `cache_result_in_memory`
+            result = state.result(raise_on_failure=False, fetch=True)
+            if inspect.isawaitable(result):
+                result = run_sync(result)
+        else:
+            result = state.data
+
+        link_state_to_result(state, result)
+
+    # Handle repeated WAITs in a loop instead of recursively, to avoid
+    # reaching max recursion depth in extreme cases.
+    def set_state_and_handle_waits(set_state_func) -> OrchestrationResult:
+        response = set_state_func()
+        while response.status == SetStateStatus.WAIT:
+            engine_logger.debug(
+                f"Received wait instruction for {response.details.delay_seconds}s: "
+                f"{response.details.reason}"
+            )
+            time.sleep(response.details.delay_seconds)
+            response = set_state_func()
+        return response
+
+    # Attempt to set the state
+    if task_run_id:
+        set_state = partial(client.set_task_run_state, task_run_id, state, force=force)
+        response = set_state_and_handle_waits(set_state)
+    elif flow_run_id:
+        set_state = partial(client.set_flow_run_state, flow_run_id, state, force=force)
+        response = set_state_and_handle_waits(set_state)
+    else:
+        raise ValueError(
+            "Neither flow run id or task run id were provided. At least one must "
+            "be given."
+        )
+
+    # Parse the response to return the new state
+    if response.status == SetStateStatus.ACCEPT:
+        # Update the state with the details if provided
+        state.id = response.state.id
+        state.timestamp = response.state.timestamp
+        if response.state.state_details:
+            state.state_details = response.state.state_details
+        return state
+
+    elif response.status == SetStateStatus.ABORT:
+        raise prefect.exceptions.Abort(response.details.reason)
+
+    elif response.status == SetStateStatus.REJECT:
+        if response.state.is_paused():
+            raise Pause(response.details.reason, state=response.state)
+        return response.state
+
+    else:
+        raise ValueError(
+            f"Received unexpected `SetStateStatus` from server: {response.status!r}"
+        )
+
+
 def _dynamic_key_for_task_run(context: FlowRunContext, task: Task) -> int:
     if context.flow_run is None:  # this is an autonomous task run
         context.task_run_dynamic_keys[task.task_key] = getattr(

prefect/utilities/schema_tools/__init__.py

@@ -1,5 +1,10 @@
 from .hydration import HydrationContext, HydrationError, hydrate
-from .validation import CircularSchemaRefError, ValidationError, validate
+from .validation import (
+    CircularSchemaRefError,
+    ValidationError,
+    validate,
+    is_valid_schema,
+)
 
 __all__ = [
     "CircularSchemaRefError",