prefect-client 3.0.0rc20__py3-none-any.whl → 3.0.2__py3-none-any.whl

prefect/concurrency/asyncio.py

@@ -6,6 +6,8 @@ import anyio
 import httpx
 import pendulum
 
+from prefect._internal.compatibility.deprecated import deprecated_parameter
+
 try:
     from pendulum import Interval
 except ImportError:
@@ -14,6 +16,8 @@ except ImportError:
 
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
+from prefect.logging.loggers import get_run_logger
+from prefect.utilities.asyncutils import sync_compatible
 
 from .context import ConcurrencyContext
 from .events import (
@@ -36,8 +40,9 @@ async def concurrency(
     names: Union[str, List[str]],
     occupy: int = 1,
     timeout_seconds: Optional[float] = None,
-    create_if_missing: bool = True,
     max_retries: Optional[int] = None,
+    create_if_missing: Optional[bool] = None,
+    strict: bool = False,
 ) -> AsyncGenerator[None, None]:
     """A context manager that acquires and releases concurrency slots from the
     given concurrency limits.
@@ -47,11 +52,13 @@ async def concurrency(
         occupy: The number of slots to acquire and hold from each limit.
         timeout_seconds: The number of seconds to wait for the slots to be acquired before
             raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
-        create_if_missing: Whether to create the concurrency limits if they do not exist.
         max_retries: The maximum number of retries to acquire the concurrency slots.
+        strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
+            Defaults to `False`.
 
     Raises:
         TimeoutError: If the slots are not acquired within the given timeout.
+        ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
 
     Example:
     A simple example of using the async `concurrency` context manager:
@@ -78,6 +85,7 @@ async def concurrency(
         timeout_seconds=timeout_seconds,
         create_if_missing=create_if_missing,
         max_retries=max_retries,
+        strict=strict,
     )
     acquisition_time = pendulum.now("UTC")
     emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
@@ -106,7 +114,8 @@ async def rate_limit(
     names: Union[str, List[str]],
     occupy: int = 1,
     timeout_seconds: Optional[float] = None,
-    create_if_missing: Optional[bool] = True,
+    create_if_missing: Optional[bool] = None,
+    strict: bool = False,
 ) -> None:
     """Block execution until an `occupy` number of slots of the concurrency
     limits given in `names` are acquired. Requires that all given concurrency
@@ -117,7 +126,12 @@ async def rate_limit(
         occupy: The number of slots to acquire and hold from each limit.
         timeout_seconds: The number of seconds to wait for the slots to be acquired before
             raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
-        create_if_missing: Whether to create the concurrency limits if they do not exist.
+        strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
+            Defaults to `False`.
+
+    Raises:
+        TimeoutError: If the slots are not acquired within the given timeout.
+        ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
     """
     if not names:
         return
@@ -130,17 +144,27 @@ async def rate_limit(
         mode="rate_limit",
         timeout_seconds=timeout_seconds,
         create_if_missing=create_if_missing,
+        strict=strict,
     )
     _emit_concurrency_acquisition_events(limits, occupy)
 
 
+@sync_compatible
+@deprecated_parameter(
+    name="create_if_missing",
+    start_date="Sep 2024",
+    end_date="Oct 2024",
+    when=lambda x: x is not None,
+    help="Limits must be explicitly created before acquiring concurrency slots; see `strict` if you want to enforce this behavior.",
+)
 async def _acquire_concurrency_slots(
     names: List[str],
     slots: int,
     mode: Union[Literal["concurrency"], Literal["rate_limit"]] = "concurrency",
     timeout_seconds: Optional[float] = None,
-    create_if_missing: Optional[bool] = True,
+    create_if_missing: Optional[bool] = None,
     max_retries: Optional[int] = None,
+    strict: bool = False,
 ) -> List[MinimalConcurrencyLimitResponse]:
     service = ConcurrencySlotAcquisitionService.instance(frozenset(names))
     future = service.send(
@@ -158,9 +182,24 @@ async def _acquire_concurrency_slots(
             f"Unable to acquire concurrency slots on {names!r}"
         ) from response_or_exception
 
-    return _response_to_minimal_concurrency_limit_response(response_or_exception)
+    retval = _response_to_minimal_concurrency_limit_response(response_or_exception)
+
+    if strict and not retval:
+        raise ConcurrencySlotAcquisitionError(
+            f"Concurrency limits {names!r} must be created before acquiring slots"
+        )
+    elif not retval:
+        try:
+            logger = get_run_logger()
+            logger.warning(
+                f"Concurrency limits {names!r} do not exist - skipping acquisition."
+            )
+        except Exception:
+            pass
+    return retval
 
 
+@sync_compatible
 async def _release_concurrency_slots(
     names: List[str], slots: int, occupancy_seconds: float
 ) -> List[MinimalConcurrencyLimitResponse]:

prefect/concurrency/services.py

@@ -63,7 +63,7 @@ class ConcurrencySlotAcquisitionService(QueueService):
         slots: int,
         mode: str,
         timeout_seconds: Optional[float] = None,
-        create_if_missing: Optional[bool] = False,
+        create_if_missing: Optional[bool] = None,
         max_retries: Optional[int] = None,
     ) -> httpx.Response:
         with timeout_async(seconds=timeout_seconds):

prefect/concurrency/sync.py

@@ -1,8 +1,5 @@
 from contextlib import contextmanager
 from typing import (
-    Any,
-    Awaitable,
-    Callable,
     Generator,
     List,
     Optional,
@@ -19,8 +16,6 @@ except ImportError:
     # pendulum < 3
     from pendulum.period import Period as Interval  # type: ignore
 
-from prefect._internal.concurrency.api import create_call, from_sync
-from prefect._internal.concurrency.event_loop import get_running_loop
 from prefect.client.schemas.responses import MinimalConcurrencyLimitResponse
 
 from .asyncio import (
@@ -40,8 +35,9 @@ def concurrency(
     names: Union[str, List[str]],
     occupy: int = 1,
     timeout_seconds: Optional[float] = None,
-    create_if_missing: bool = True,
     max_retries: Optional[int] = None,
+    strict: bool = False,
+    create_if_missing: Optional[bool] = None,
 ) -> Generator[None, None, None]:
     """A context manager that acquires and releases concurrency slots from the
     given concurrency limits.
@@ -51,11 +47,13 @@ def concurrency(
         occupy: The number of slots to acquire and hold from each limit.
         timeout_seconds: The number of seconds to wait for the slots to be acquired before
             raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
-        create_if_missing: Whether to create the concurrency limits if they do not exist.
         max_retries: The maximum number of retries to acquire the concurrency slots.
+        strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
+            Defaults to `False`.
 
     Raises:
         TimeoutError: If the slots are not acquired within the given timeout.
+        ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
 
     Example:
     A simple example of using the sync `concurrency` context manager:
@@ -76,13 +74,14 @@ def concurrency(
 
     names = names if isinstance(names, list) else [names]
 
-    limits: List[MinimalConcurrencyLimitResponse] = _call_async_function_from_sync(
-        _acquire_concurrency_slots,
+    limits: List[MinimalConcurrencyLimitResponse] = _acquire_concurrency_slots(
         names,
         occupy,
         timeout_seconds=timeout_seconds,
         create_if_missing=create_if_missing,
+        strict=strict,
         max_retries=max_retries,
+        _sync=True,
     )
     acquisition_time = pendulum.now("UTC")
     emitted_events = _emit_concurrency_acquisition_events(limits, occupy)
@@ -91,11 +90,11 @@ def concurrency(
         yield
     finally:
         occupancy_period = cast(Interval, pendulum.now("UTC") - acquisition_time)
-        _call_async_function_from_sync(
-            _release_concurrency_slots,
+        _release_concurrency_slots(
             names,
             occupy,
             occupancy_period.total_seconds(),
+            _sync=True,
         )
         _emit_concurrency_release_events(limits, occupy, emitted_events)
 
@@ -104,7 +103,8 @@ def rate_limit(
     names: Union[str, List[str]],
     occupy: int = 1,
     timeout_seconds: Optional[float] = None,
-    create_if_missing: Optional[bool] = True,
+    create_if_missing: Optional[bool] = None,
+    strict: bool = False,
 ) -> None:
     """Block execution until an `occupy` number of slots of the concurrency
     limits given in `names` are acquired. Requires that all given concurrency
@@ -115,31 +115,25 @@ def rate_limit(
         occupy: The number of slots to acquire and hold from each limit.
         timeout_seconds: The number of seconds to wait for the slots to be acquired before
             raising a `TimeoutError`. A timeout of `None` will wait indefinitely.
-        create_if_missing: Whether to create the concurrency limits if they do not exist.
+        strict: A boolean specifying whether to raise an error if the concurrency limit does not exist.
+            Defaults to `False`.
+
+    Raises:
+        TimeoutError: If the slots are not acquired within the given timeout.
+        ConcurrencySlotAcquisitionError: If the concurrency limit does not exist and `strict` is `True`.
     """
     if not names:
         return
 
     names = names if isinstance(names, list) else [names]
 
-    limits = _call_async_function_from_sync(
-        _acquire_concurrency_slots,
+    limits = _acquire_concurrency_slots(
         names,
         occupy,
         mode="rate_limit",
         timeout_seconds=timeout_seconds,
         create_if_missing=create_if_missing,
+        strict=strict,
+        _sync=True,
     )
     _emit_concurrency_acquisition_events(limits, occupy)
-
-
-def _call_async_function_from_sync(
-    fn: Callable[..., Awaitable[T]], *args: Any, **kwargs: Any
-) -> T:
-    loop = get_running_loop()
-    call = create_call(fn, *args, **kwargs)
-
-    if loop is not None:
-        return from_sync.call_soon_in_loop_thread(call).result()
-    else:
-        return call()  # type: ignore [return-value]

prefect/concurrency/v1/asyncio.py

@@ -16,6 +16,7 @@ except ImportError:
     from pendulum.period import Period as Interval  # type: ignore
 
 from prefect.client.orchestration import get_client
+from prefect.utilities.asyncutils import sync_compatible
 
 from .context import ConcurrencyContext
 from .events import (
@@ -98,6 +99,7 @@ async def concurrency(
         _emit_concurrency_release_events(limits, emitted_events, task_run_id)
 
 
+@sync_compatible
 async def _acquire_concurrency_slots(
     names: List[str],
     task_run_id: UUID,
@@ -120,6 +122,7 @@ async def _acquire_concurrency_slots(
     return _response_to_concurrency_limit_response(response_or_exception)
 
 
+@sync_compatible
 async def _release_concurrency_slots(
     names: List[str],
     task_run_id: UUID,

prefect/concurrency/v1/sync.py

@@ -12,7 +12,6 @@ from uuid import UUID
 import pendulum
 
 from ...client.schemas.responses import MinimalConcurrencyLimitResponse
-from ..sync import _call_async_function_from_sync
 
 try:
     from pendulum import Interval
@@ -70,11 +69,11 @@ def concurrency(
 
     names = names if isinstance(names, list) else [names]
 
-    limits: List[MinimalConcurrencyLimitResponse] = _call_async_function_from_sync(
-        _acquire_concurrency_slots,
+    limits: List[MinimalConcurrencyLimitResponse] = _acquire_concurrency_slots(
         names,
         timeout_seconds=timeout_seconds,
         task_run_id=task_run_id,
+        _sync=True,
     )
     acquisition_time = pendulum.now("UTC")
     emitted_events = _emit_concurrency_acquisition_events(limits, task_run_id)
@@ -83,10 +82,10 @@ def concurrency(
         yield
     finally:
         occupancy_period = cast(Interval, pendulum.now("UTC") - acquisition_time)
-        _call_async_function_from_sync(
-            _release_concurrency_slots,
+        _release_concurrency_slots(
             names,
             task_run_id,
             occupancy_period.total_seconds(),
+            _sync=True,
         )
         _emit_concurrency_release_events(limits, emitted_events, task_run_id)

prefect/context.py

@@ -40,11 +40,10 @@ from prefect.client.orchestration import PrefectClient, SyncPrefectClient, get_c
 from prefect.client.schemas import FlowRun, TaskRun
 from prefect.events.worker import EventsWorker
 from prefect.exceptions import MissingContextError
-from prefect.results import ResultFactory
+from prefect.results import ResultStore, get_default_persist_setting
 from prefect.settings import PREFECT_HOME, Profile, Settings
 from prefect.states import State
 from prefect.task_runners import TaskRunner
-from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.services import start_client_metrics_server
 
 T = TypeVar("T")
@@ -95,20 +94,15 @@ def hydrated_context(
                 flow_run_context = FlowRunContext(
                     **flow_run_context,
                     client=client,
-                    result_factory=run_coro_as_sync(ResultFactory.from_flow(flow)),
                     task_runner=task_runner,
                     detached=True,
                 )
                 stack.enter_context(flow_run_context)
             # Set up parent task run context
             if parent_task_run_context := serialized_context.get("task_run_context"):
-                parent_task = parent_task_run_context["task"]
                 task_run_context = TaskRunContext(
                     **parent_task_run_context,
                     client=client,
-                    result_factory=run_coro_as_sync(
-                        ResultFactory.from_autonomous_task(parent_task)
-                    ),
                 )
                 stack.enter_context(task_run_context)
             # Set up tags context
@@ -216,6 +210,7 @@ class SyncClientContext(ContextModel):
         self._context_stack += 1
         if self._context_stack == 1:
             self.client.__enter__()
+            self.client.raise_for_api_version_mismatch()
             return super().__enter__()
         else:
             return self
@@ -273,6 +268,7 @@ class AsyncClientContext(ContextModel):
         self._context_stack += 1
         if self._context_stack == 1:
             await self.client.__aenter__()
+            await self.client.raise_for_api_version_mismatch()
             return super().__enter__()
         else:
             return self
@@ -346,7 +342,8 @@ class EngineContext(RunContext):
     detached: bool = False
 
     # Result handling
-    result_factory: ResultFactory
+    result_store: ResultStore
+    persist_result: bool = Field(default_factory=get_default_persist_setting)
 
     # Counter for task calls allowing unique
     task_run_dynamic_keys: Dict[str, int] = Field(default_factory=dict)
@@ -375,6 +372,8 @@ class EngineContext(RunContext):
                 "log_prints",
                 "start_time",
                 "input_keyset",
+                "result_store",
+                "persist_result",
             },
             exclude_unset=True,
         )
@@ -399,7 +398,8 @@ class TaskRunContext(RunContext):
     parameters: Dict[str, Any]
 
     # Result handling
-    result_factory: ResultFactory
+    result_store: ResultStore
+    persist_result: bool = Field(default_factory=get_default_persist_setting)
 
     __var__ = ContextVar("task_run")
 
@@ -412,6 +412,8 @@ class TaskRunContext(RunContext):
                 "log_prints",
                 "start_time",
                 "input_keyset",
+                "result_store",
+                "persist_result",
             },
             exclude_unset=True,
         )

prefect/deployments/runner.py

@@ -462,6 +462,7 @@ class RunnerDeployment(BaseModel):
             paused: Whether or not to set this deployment as paused.
             schedules: A list of schedule objects defining when to execute runs of this deployment.
                 Used to define multiple schedules or additional scheduling options like `timezone`.
+            concurrency_limit: The maximum number of concurrent runs this deployment will allow.
             triggers: A list of triggers that should kick of a run of this flow.
             parameters: A dictionary of default parameter values to pass to runs of this flow.
             description: A description for the created deployment. Defaults to the flow's
@@ -508,7 +509,7 @@ class RunnerDeployment(BaseModel):
             no_file_location_error = (
                 "Flows defined interactively cannot be deployed. Check out the"
                 " quickstart guide for help getting started:"
-                " https://docs.prefect.io/latest/getting-started/quickstart"
+                " https://docs.prefect.io/latest/get-started/quickstart"
             )
             ## first see if an entrypoint can be determined
             flow_file = getattr(flow, "__globals__", {}).get("__file__")
@@ -851,14 +852,14 @@ async def deploy(
                 " or specify a remote storage location for the flow with `.from_source`."
                 " If you are attempting to deploy a flow to a local process work pool,"
                 " consider using `flow.serve` instead. See the documentation for more"
-                " information: https://docs.prefect.io/latest/concepts/flows/#serving-a-flow"
+                " information: https://docs.prefect.io/latest/deploy/run-flows-in-local-processes"
             )
         elif work_pool.type == "process" and not ignore_warnings:
             console.print(
                 "Looks like you're deploying to a process work pool. If you're creating a"
                 " deployment for local development, calling `.serve` on your flow is a great"
                 " way to get started. See the documentation for more information:"
-                " https://docs.prefect.io/latest/concepts/flows/#serving-a-flow. "
+                " https://docs.prefect.io/latest/deploy/run-flows-in-local-processes "
                 " Set `ignore_warnings=True` to suppress this message.",
                 style="yellow",
             )

prefect/events/actions.py

@@ -113,6 +113,12 @@ class CancelFlowRun(Action):
     type: Literal["cancel-flow-run"] = "cancel-flow-run"
 
 
+class ResumeFlowRun(Action):
+    """Resumes a flow run associated with the trigger"""
+
+    type: Literal["resume-flow-run"] = "resume-flow-run"
+
+
 class SuspendFlowRun(Action):
     """Suspends a flow run associated with the trigger"""
 

prefect/exceptions.py

@@ -418,3 +418,9 @@ class SerializationError(PrefectException):
     """
     Raised when an object cannot be serialized.
     """
+
+
+class ConfigurationError(PrefectException):
+    """
+    Raised when a configuration is invalid.
+    """

prefect/filesystems.py

@@ -84,7 +84,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
     _block_type_name = "Local File System"
     _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/ad39089fa66d273b943394a68f003f7a19aa850e-48x48.png"
     _documentation_url = (
-        "https://docs.prefect.io/concepts/filesystems/#local-filesystem"
+        "https://docs.prefect.io/latest/develop/results#specifying-a-default-filesystem"
     )
 
     basepath: Optional[str] = Field(
@@ -260,7 +260,7 @@ class RemoteFileSystem(WritableFileSystem, WritableDeploymentStorage):
     _block_type_name = "Remote File System"
     _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/e86b41bc0f9c99ba9489abeee83433b43d5c9365-48x48.png"
     _documentation_url = (
-        "https://docs.prefect.io/concepts/filesystems/#remote-file-system"
+        "https://docs.prefect.io/latest/develop/results#specifying-a-default-filesystem"
     )
 
     basepath: str = Field(
@@ -433,7 +433,9 @@ class SMB(WritableFileSystem, WritableDeploymentStorage):
 
     _block_type_name = "SMB"
     _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/3f624663f7beb97d011d011bffd51ecf6c499efc-195x195.png"
-    _documentation_url = "https://docs.prefect.io/concepts/filesystems/#smb"
+    _documentation_url = (
+        "https://docs.prefect.io/latest/develop/results#specifying-a-default-filesystem"
+    )
 
     share_path: str = Field(
         default=...,

prefect/flow_engine.py

@@ -47,7 +47,12 @@ from prefect.logging.loggers import (
     get_run_logger,
     patch_print,
 )
-from prefect.results import BaseResult, ResultFactory
+from prefect.results import (
+    BaseResult,
+    ResultStore,
+    get_result_store,
+    should_persist_result,
+)
 from prefect.settings import PREFECT_DEBUG_MODE
 from prefect.states import (
     Failed,
@@ -202,7 +207,9 @@ class FlowRunEngine(Generic[P, R]):
                 self.handle_exception(
                     exc,
                     msg=message,
-                    result_factory=run_coro_as_sync(ResultFactory.from_flow(self.flow)),
+                    result_store=get_result_store().update_for_flow(
+                        self.flow, _sync=True
+                    ),
                 )
                 self.short_circuit = True
                 self.call_hooks()
@@ -261,15 +268,15 @@ class FlowRunEngine(Generic[P, R]):
         return _result
 
     def handle_success(self, result: R) -> R:
-        result_factory = getattr(FlowRunContext.get(), "result_factory", None)
-        if result_factory is None:
-            raise ValueError("Result factory is not set")
+        result_store = getattr(FlowRunContext.get(), "result_store", None)
+        if result_store is None:
+            raise ValueError("Result store is not set")
         resolved_result = resolve_futures_to_states(result)
         terminal_state = run_coro_as_sync(
             return_value_to_state(
                 resolved_result,
-                result_factory=result_factory,
-                write_result=True,
+                result_store=result_store,
+                write_result=should_persist_result(),
             )
         )
         self.set_state(terminal_state)
@@ -280,15 +287,14 @@ class FlowRunEngine(Generic[P, R]):
         self,
         exc: Exception,
         msg: Optional[str] = None,
-        result_factory: Optional[ResultFactory] = None,
+        result_store: Optional[ResultStore] = None,
     ) -> State:
         context = FlowRunContext.get()
         terminal_state = run_coro_as_sync(
             exception_to_failed_state(
                 exc,
                 message=msg or "Flow run encountered an exception:",
-                result_factory=result_factory
-                or getattr(context, "result_factory", None),
+                result_store=result_store or getattr(context, "result_store", None),
                 write_result=True,
             )
         )
@@ -506,8 +512,13 @@ class FlowRunEngine(Generic[P, R]):
                     flow_run=self.flow_run,
                     parameters=self.parameters,
                     client=client,
-                    result_factory=run_coro_as_sync(ResultFactory.from_flow(self.flow)),
+                    result_store=get_result_store().update_for_flow(
+                        self.flow, _sync=True
+                    ),
                     task_runner=task_runner,
+                    persist_result=self.flow.persist_result
+                    if self.flow.persist_result is not None
+                    else should_persist_result(),
                 )
             )
             stack.enter_context(ConcurrencyContextV1())

prefect/flows.py

@@ -1032,8 +1032,6 @@ class Flow(Generic[P, R]):
             if not isinstance(storage, LocalStorage):
                 storage.set_base_path(Path(tmpdir))
                 await storage.pull_code()
-            storage.set_base_path(Path(tmpdir))
-            await storage.pull_code()
 
             full_entrypoint = str(storage.destination / entrypoint)
             flow: Flow = await from_async.wait_for_call_in_new_thread(

prefect/futures.py

@@ -179,7 +179,8 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
             local_logger = logger
         local_logger.warning(
             "A future was garbage collected before it resolved."
-            " Please call `.wait()` or `.result()` on futures to ensure they resolve.",
+            " Please call `.wait()` or `.result()` on futures to ensure they resolve."
+            "\nSee https://docs.prefect.io/latest/develop/task-runners for more details.",
         )