prefect-client 3.1.14__py3-none-any.whl → 3.1.15__py3-none-any.whl

prefect/blocks/core.py

@@ -669,7 +669,7 @@ class Block(BaseModel, ABC):
         module_str = ".".join(qualified_name.split(".")[:-1])
         origin = cls.__pydantic_generic_metadata__.get("origin") or cls
         class_name = origin.__name__
-        block_variable_name = f'{cls.get_block_type_slug().replace("-", "_")}_block'
+        block_variable_name = f"{cls.get_block_type_slug().replace('-', '_')}_block"
 
         return dedent(
             f"""\

prefect/blocks/notifications.py

@@ -570,6 +570,10 @@ class MattermostWebhook(AbstractAppriseNotificationBlock):
         description="The hostname of your Mattermost server.",
         examples=["Mattermost.example.com"],
     )
+    secure: bool = Field(
+        default=False,
+        description="Whether to use secure https connection.",
+    )
 
     token: SecretStr = Field(
         default=...,
@@ -621,6 +625,7 @@ class MattermostWebhook(AbstractAppriseNotificationBlock):
                 channels=self.channels,
                 include_image=self.include_image,
                 port=self.port,
+                secure=self.secure,
             ).url()  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] incomplete type hints in apprise
         )
         self._start_apprise_client(url)

prefect/client/schemas/actions.py

@@ -1,5 +1,5 @@
 from copy import deepcopy
-from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union
 from uuid import UUID, uuid4
 
 import jsonschema
@@ -93,6 +93,16 @@ class DeploymentScheduleCreate(ActionBaseModel):
         description="The maximum number of scheduled runs for the schedule.",
     )
 
+    @field_validator("active", mode="wrap")
+    @classmethod
+    def validate_active(cls, v: Any, handler: Callable[[Any], Any]) -> bool:
+        try:
+            return handler(v)
+        except Exception:
+            raise ValueError(
+                f"active must be able to be parsed as a boolean, got {v!r} of type {type(v)}"
+            )
+
     @field_validator("max_scheduled_runs")
     @classmethod
     def validate_max_scheduled_runs(cls, v: Optional[int]) -> Optional[int]:

prefect/client/schemas/schedules.py

@@ -145,7 +145,7 @@ class CronSchedule(PrefectBaseModel):
 
     @field_validator("timezone")
     @classmethod
-    def valid_timezone(cls, v: Optional[str]) -> str:
+    def valid_timezone(cls, v: Optional[str]) -> Optional[str]:
         return default_timezone(v)
 
     @field_validator("cron")
@@ -276,7 +276,7 @@ class RRuleSchedule(PrefectBaseModel):
                 kwargs.update(
                     until=until.replace(tzinfo=timezone),
                 )
-            return rrule.replace(**kwargs)
+            return rrule.replace(**kwargs)  # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType] missing type hints
 
         # update rrules
         localized_rrules: list[dateutil.rrule.rrule] = []
@@ -286,7 +286,7 @@ class RRuleSchedule(PrefectBaseModel):
             kwargs: dict[str, Any] = dict(dtstart=dtstart.replace(tzinfo=timezone))
             if until := _rrule_dt(rr, "_until"):
                 kwargs.update(until=until.replace(tzinfo=timezone))
-            localized_rrules.append(rr.replace(**kwargs))
+            localized_rrules.append(rr.replace(**kwargs))  # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] missing type hints
         setattr(rrule, "_rrule", localized_rrules)
 
         # update exrules
@@ -297,7 +297,7 @@ class RRuleSchedule(PrefectBaseModel):
             kwargs = dict(dtstart=dtstart.replace(tzinfo=timezone))
             if until := _rrule_dt(exr, "_until"):
                 kwargs.update(until=until.replace(tzinfo=timezone))
-            localized_exrules.append(exr.replace(**kwargs))
+            localized_exrules.append(exr.replace(**kwargs))  # pyright: ignore[reportUnknownArgumentType, reportUnknownMemberType] missing type hints
         setattr(rrule, "_exrule", localized_exrules)
 
         # update rdates

prefect/deployments/base.py

@@ -10,12 +10,11 @@ from __future__ import annotations
 import os
 from copy import deepcopy
 from pathlib import Path
-from typing import Any, Dict, List, Optional, cast
+from typing import Any, Dict, List, Optional
 
 import yaml
 from ruamel.yaml import YAML
 
-from prefect.client.schemas.actions import DeploymentScheduleCreate
 from prefect.client.schemas.objects import ConcurrencyLimitStrategy
 from prefect.client.schemas.schedules import IntervalSchedule
 from prefect.utilities._git import get_git_branch, get_git_remote_origin_url
@@ -207,8 +206,8 @@ def initialize_project(
 
 
 def _format_deployment_for_saving_to_prefect_file(
-    deployment: Dict,
-) -> Dict:
+    deployment: dict[str, Any],
+) -> dict[str, Any]:
     """
     Formats a deployment into a templated deploy config for saving to prefect.yaml.
 
@@ -227,10 +226,8 @@ def _format_deployment_for_saving_to_prefect_file(
     deployment.pop("flow_name", None)
 
     if deployment.get("schedules"):
-        schedules = []
-        for deployment_schedule in cast(
-            List[DeploymentScheduleCreate], deployment["schedules"]
-        ):
+        schedules: list[dict[str, Any]] = []
+        for deployment_schedule in deployment["schedules"]:
             if isinstance(deployment_schedule.schedule, IntervalSchedule):
                 schedule_config = _interval_schedule_to_dict(
                     deployment_schedule.schedule
@@ -257,7 +254,7 @@ def _format_deployment_for_saving_to_prefect_file(
     return deployment
 
 
-def _interval_schedule_to_dict(schedule: IntervalSchedule) -> Dict:
+def _interval_schedule_to_dict(schedule: IntervalSchedule) -> dict[str, Any]:
     """
     Converts an IntervalSchedule to a dictionary.
 
@@ -265,7 +262,7 @@ def _interval_schedule_to_dict(schedule: IntervalSchedule) -> Dict:
         - schedule (IntervalSchedule): the schedule to convert
 
     Returns:
-        - Dict: the schedule as a dictionary
+        - dict[str, Any]: the schedule as a dictionary
     """
     schedule_config = schedule.model_dump()
     schedule_config["interval"] = schedule_config["interval"].total_seconds()
@@ -275,12 +272,12 @@ def _interval_schedule_to_dict(schedule: IntervalSchedule) -> Dict:
 
 
 def _save_deployment_to_prefect_file(
-    deployment: Dict,
-    build_steps: Optional[List[Dict]] = None,
-    push_steps: Optional[List[Dict]] = None,
-    pull_steps: Optional[List[Dict]] = None,
-    triggers: Optional[List[Dict]] = None,
-    sla: Optional[list[dict]] = None,
+    deployment: dict[str, Any],
+    build_steps: list[dict[str, Any]] | None = None,
+    push_steps: list[dict[str, Any]] | None = None,
+    pull_steps: list[dict[str, Any]] | None = None,
+    triggers: list[dict[str, Any]] | None = None,
+    sla: list[dict[str, Any]] | None = None,
     prefect_file: Path = Path("prefect.yaml"),
 ):
     """

prefect/deployments/runner.py

@@ -49,6 +49,7 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, track
 from rich.table import Table
 
 from prefect._experimental.sla.objects import SlaTypes
+from prefect._internal.compatibility.async_dispatch import async_dispatch
 from prefect._internal.concurrency.api import create_call, from_async
 from prefect._internal.schemas.validators import (
     reconcile_paused_deployment,
@@ -82,7 +83,7 @@ from prefect.settings import (
 )
 from prefect.types import ListOfNonEmptyStrings
 from prefect.types.entrypoint import EntrypointType
-from prefect.utilities.asyncutils import sync_compatible
+from prefect.utilities.asyncutils import run_coro_as_sync, sync_compatible
 from prefect.utilities.callables import ParameterSchema, parameter_schema
 from prefect.utilities.collections import get_from_dict, isiterable
 from prefect.utilities.dockerutils import (
@@ -717,8 +718,7 @@ class RunnerDeployment(BaseModel):
         return deployment
 
     @classmethod
-    @sync_compatible
-    async def from_storage(
+    async def afrom_storage(
         cls,
         storage: RunnerStorage,
         entrypoint: str,
@@ -742,7 +742,7 @@ class RunnerDeployment(BaseModel):
         work_queue_name: Optional[str] = None,
         job_variables: Optional[dict[str, Any]] = None,
         _sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None,  # experimental
-    ):
+    ) -> "RunnerDeployment":
         """
         Create a RunnerDeployment from a flow located at a given entrypoint and stored in a
         local storage location.
@@ -831,6 +831,119 @@ class RunnerDeployment(BaseModel):
 
         return deployment
 
+    @classmethod
+    @async_dispatch(afrom_storage)
+    def from_storage(
+        cls,
+        storage: RunnerStorage,
+        entrypoint: str,
+        name: str,
+        flow_name: Optional[str] = None,
+        interval: Optional[
+            Union[Iterable[Union[int, float, timedelta]], int, float, timedelta]
+        ] = None,
+        cron: Optional[Union[Iterable[str], str]] = None,
+        rrule: Optional[Union[Iterable[str], str]] = None,
+        paused: Optional[bool] = None,
+        schedules: Optional["FlexibleScheduleList"] = None,
+        concurrency_limit: Optional[Union[int, ConcurrencyLimitConfig, None]] = None,
+        parameters: Optional[dict[str, Any]] = None,
+        triggers: Optional[List[Union[DeploymentTriggerTypes, TriggerTypes]]] = None,
+        description: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        version: Optional[str] = None,
+        enforce_parameter_schema: bool = True,
+        work_pool_name: Optional[str] = None,
+        work_queue_name: Optional[str] = None,
+        job_variables: Optional[dict[str, Any]] = None,
+        _sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None,  # experimental
+    ) -> "RunnerDeployment":
+        """
+        Create a RunnerDeployment from a flow located at a given entrypoint and stored in a
+        local storage location.
+
+        Args:
+            entrypoint:  The path to a file containing a flow and the name of the flow function in
+                the format `./path/to/file.py:flow_func_name`.
+            name: A name for the deployment
+            flow_name: The name of the flow to deploy
+            storage: A storage object to use for retrieving flow code. If not provided, a
+                URL must be provided.
+            interval: An interval on which to execute the current flow. Accepts either a number
+                or a timedelta object. If a number is given, it will be interpreted as seconds.
+            cron: A cron schedule of when to execute runs of this flow.
+            rrule: An rrule schedule of when to execute runs of this flow.
+            triggers: A list of triggers that should kick of a run of this flow.
+            parameters: A dictionary of default parameter values to pass to runs of this flow.
+            description: A description for the created deployment. Defaults to the flow's
+                description if not provided.
+            tags: A list of tags to associate with the created deployment for organizational
+                purposes.
+            version: A version for the created deployment. Defaults to the flow's version.
+            enforce_parameter_schema: Whether or not the Prefect API should enforce the
+                parameter schema for this deployment.
+            work_pool_name: The name of the work pool to use for this deployment.
+            work_queue_name: The name of the work queue to use for this deployment's scheduled runs.
+                If not provided the default work queue for the work pool will be used.
+            job_variables: Settings used to override the values specified default base job template
+                of the chosen work pool. Refer to the base job template of the chosen work pool for
+                available settings.
+            _sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
+        """
+        from prefect.flows import load_flow_from_entrypoint
+
+        constructed_schedules = cls._construct_deployment_schedules(
+            interval=interval,
+            cron=cron,
+            rrule=rrule,
+            schedules=schedules,
+        )
+
+        if isinstance(concurrency_limit, ConcurrencyLimitConfig):
+            concurrency_options = {
+                "collision_strategy": concurrency_limit.collision_strategy
+            }
+            concurrency_limit = concurrency_limit.limit
+        else:
+            concurrency_options = None
+
+        job_variables = job_variables or {}
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            storage.set_base_path(Path(tmpdir))
+            run_coro_as_sync(storage.pull_code())
+
+            full_entrypoint = str(storage.destination / entrypoint)
+            flow = load_flow_from_entrypoint(full_entrypoint)
+
+        deployment = cls(
+            name=Path(name).stem,
+            flow_name=flow_name or flow.name,
+            schedules=constructed_schedules,
+            concurrency_limit=concurrency_limit,
+            concurrency_options=concurrency_options,
+            paused=paused,
+            tags=tags or [],
+            triggers=triggers or [],
+            parameters=parameters or {},
+            description=description,
+            version=version,
+            entrypoint=entrypoint,
+            enforce_parameter_schema=enforce_parameter_schema,
+            storage=storage,
+            work_pool_name=work_pool_name,
+            work_queue_name=work_queue_name,
+            job_variables=job_variables,
+        )
+        deployment._sla = _sla
+        deployment._path = str(storage.destination).replace(
+            tmpdir, "$STORAGE_BASE_PATH"
+        )
+
+        cls._set_defaults_from_flow(deployment, flow)
+
+        return deployment
+
 
 @sync_compatible
 async def deploy(

prefect/events/clients.py

@@ -1,6 +1,7 @@
 import abc
 import asyncio
 import os
+import ssl
 from types import TracebackType
 from typing import (
     TYPE_CHECKING,
@@ -37,6 +38,7 @@ from prefect.events import Event
 from prefect.logging import get_logger
 from prefect.settings import (
     PREFECT_API_KEY,
+    PREFECT_API_TLS_INSECURE_SKIP_VERIFY,
     PREFECT_API_URL,
     PREFECT_CLOUD_API_URL,
     PREFECT_DEBUG_MODE,
@@ -120,6 +122,13 @@ class WebsocketProxyConnect(Connect):
         self._host = host
         self._port = port
 
+        if PREFECT_API_TLS_INSECURE_SKIP_VERIFY:
+            # Create an unverified context for insecure connections
+            ctx = ssl.create_default_context()
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+            self._kwargs.setdefault("ssl", ctx)
+
     async def _proxy_connect(self: Self) -> WebSocketClientProtocol:
         if self._proxy:
             sock = await self._proxy.connect(
@@ -348,16 +357,26 @@ class PrefectEventsClient(EventsClient):
         await self._connect.__aexit__(exc_type, exc_val, exc_tb)
         return await super().__aexit__(exc_type, exc_val, exc_tb)
 
+    def _log_debug(self, message: str, *args, **kwargs) -> None:
+        message = f"EventsClient(id={id(self)}): " + message
+        logger.debug(message, *args, **kwargs)
+
     async def _reconnect(self) -> None:
+        logger.debug("Reconnecting websocket connection.")
+
         if self._websocket:
             self._websocket = None
             await self._connect.__aexit__(None, None, None)
+            logger.debug("Cleared existing websocket connection.")
 
         try:
+            logger.debug("Opening websocket connection.")
             self._websocket = await self._connect.__aenter__()
             # make sure we have actually connected
+            logger.debug("Pinging to ensure websocket connected.")
             pong = await self._websocket.ping()
             await pong
+            logger.debug("Pong received. Websocket connected.")
         except Exception as e:
             # The client is frequently run in a background thread
             # so we log an additional warning to ensure
@@ -375,11 +394,13 @@ class PrefectEventsClient(EventsClient):
             raise
 
         events_to_resend = self._unconfirmed_events
+        logger.debug("Resending %s unconfirmed events.", len(events_to_resend))
         # Clear the unconfirmed events here, because they are going back through emit
         # and will be added again through the normal checkpointing process
         self._unconfirmed_events = []
         for event in events_to_resend:
             await self.emit(event)
+        logger.debug("Finished resending unconfirmed events.")
 
     async def _checkpoint(self, event: Event) -> None:
         assert self._websocket
@@ -387,11 +408,20 @@ class PrefectEventsClient(EventsClient):
         self._unconfirmed_events.append(event)
 
         unconfirmed_count = len(self._unconfirmed_events)
+
+        logger.debug(
+            "Added event id=%s to unconfirmed events list. "
+            "There are now %s unconfirmed events.",
+            event.id,
+            unconfirmed_count,
+        )
         if unconfirmed_count < self._checkpoint_every:
             return
 
+        logger.debug("Pinging to checkpoint unconfirmed events.")
         pong = await self._websocket.ping()
         await pong
+        self._log_debug("Pong received. Events checkpointed.")
 
         # once the pong returns, we know for sure that we've sent all the messages
         # we had enqueued prior to that.  There could be more that came in after, so
@@ -401,7 +431,9 @@ class PrefectEventsClient(EventsClient):
         EVENT_WEBSOCKET_CHECKPOINTS.labels(self.client_name).inc()
 
     async def _emit(self, event: Event) -> None:
+        self._log_debug("Emitting event id=%s.", event.id)
         for i in range(self._reconnection_attempts + 1):
+            self._log_debug("Emit reconnection attempt %s.", i)
             try:
                 # If we're here and the websocket is None, then we've had a failure in a
                 # previous reconnection attempt.
@@ -410,14 +442,18 @@ class PrefectEventsClient(EventsClient):
                 # from a ConnectionClosed, so reconnect now, resending any unconfirmed
                 # events before we send this one.
                 if not self._websocket or i > 0:
+                    self._log_debug("Attempting websocket reconnection.")
                     await self._reconnect()
                     assert self._websocket
 
+                self._log_debug("Sending event id=%s.", event.id)
                 await self._websocket.send(event.model_dump_json())
+                self._log_debug("Checkpointing event id=%s.", event.id)
                 await self._checkpoint(event)
 
                 return
             except ConnectionClosed:
+                self._log_debug("Got ConnectionClosed error.")
                 if i == self._reconnection_attempts:
                     # this was our final chance, raise the most recent error
                     raise
@@ -426,6 +462,9 @@ class PrefectEventsClient(EventsClient):
                     # let the first two attempts happen quickly in case this is just
                     # a standard load balancer timeout, but after that, just take a
                     # beat to let things come back around.
+                    logger.debug(
+                        "Sleeping for 1 second before next reconnection attempt."
+                    )
                     await asyncio.sleep(1)
 
 

prefect/flow_engine.py

@@ -2,10 +2,13 @@ from __future__ import annotations
 
 import asyncio
 import logging
+import multiprocessing
+import multiprocessing.context
 import os
 import time
 from contextlib import ExitStack, asynccontextmanager, contextmanager, nullcontext
 from dataclasses import dataclass, field
+from functools import wraps
 from typing import (
     Any,
     AsyncGenerator,
@@ -37,9 +40,12 @@ from prefect.concurrency.v1.context import ConcurrencyContext as ConcurrencyCont
 from prefect.context import (
     AsyncClientContext,
     FlowRunContext,
+    SettingsContext,
     SyncClientContext,
     TagsContext,
+    get_settings_context,
     hydrated_context,
+    serialize_context,
 )
 from prefect.exceptions import (
     Abort,
@@ -62,6 +68,8 @@ from prefect.results import (
     should_persist_result,
 )
 from prefect.settings import PREFECT_DEBUG_MODE
+from prefect.settings.context import get_current_settings
+from prefect.settings.models.root import Settings
 from prefect.states import (
     Failed,
     Pending,
@@ -83,6 +91,7 @@ from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.callables import (
     call_with_parameters,
+    cloudpickle_wrapped_call,
     get_call_parameters,
     parameters_to_args_kwargs,
 )
@@ -1533,3 +1542,113 @@ def _flow_parameters(
     parameters = flow_run.parameters if flow_run else {}
     call_args, call_kwargs = parameters_to_args_kwargs(flow.fn, parameters)
     return get_call_parameters(flow.fn, call_args, call_kwargs)
+
+
+def run_flow_in_subprocess(
+    flow: "Flow[..., Any]",
+    flow_run: "FlowRun | None" = None,
+    parameters: dict[str, Any] | None = None,
+    wait_for: Iterable[PrefectFuture[Any]] | None = None,
+    context: dict[str, Any] | None = None,
+) -> multiprocessing.context.SpawnProcess:
+    """
+    Run a flow in a subprocess.
+
+    Note the result of the flow will only be accessible if the flow is configured to
+    persist its result.
+
+    Args:
+        flow: The flow to run.
+        flow_run: The flow run object containing run metadata.
+        parameters: The parameters to use when invoking the flow.
+        wait_for: The futures to wait for before starting the flow.
+        context: A serialized context to hydrate before running the flow. If not provided,
+            the current context will be used. A serialized context should be provided if
+            this function is called in a separate memory space from the parent run (e.g.
+            in a subprocess or on another machine).
+
+    Returns:
+        A multiprocessing.context.SpawnProcess representing the process that is running the flow.
+    """
+    from prefect.flow_engine import run_flow
+
+    @wraps(run_flow)
+    def run_flow_with_env(
+        *args: Any,
+        env: dict[str, str] | None = None,
+        **kwargs: Any,
+    ):
+        """
+        Wrapper function to update environment variables and settings before running the flow.
+        """
+        engine_logger = logging.getLogger("prefect.engine")
+
+        os.environ.update(env or {})
+        settings_context = get_settings_context()
+        # Create a new settings context with a new settings object to pick up the updated
+        # environment variables
+        with SettingsContext(
+            profile=settings_context.profile,
+            settings=Settings(),
+        ):
+            try:
+                maybe_coro = run_flow(*args, **kwargs)
+                if asyncio.iscoroutine(maybe_coro):
+                    # This is running in a brand new process, so there won't be an existing
+                    # event loop.
+                    asyncio.run(maybe_coro)
+            except Abort as abort_signal:
+                abort_signal: Abort
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' aborted by orchestrator: {abort_signal}"
+                else:
+                    msg = f"Execution aborted by orchestrator: {abort_signal}"
+                engine_logger.info(msg)
+                exit(0)
+            except Pause as pause_signal:
+                pause_signal: Pause
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' is paused: {pause_signal}"
+                else:
+                    msg = f"Execution is paused: {pause_signal}"
+                engine_logger.info(msg)
+                exit(0)
+            except Exception:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' exited with unexpected exception"
+                else:
+                    msg = "Execution exited with unexpected exception"
+                engine_logger.error(msg, exc_info=True)
+                exit(1)
+            except BaseException:
+                if flow_run:
+                    msg = f"Execution of flow run '{flow_run.id}' interrupted by base exception"
+                else:
+                    msg = "Execution interrupted by base exception"
+                engine_logger.error(msg, exc_info=True)
+                # Let the exit code be determined by the base exception type
+                raise
+
+    ctx = multiprocessing.get_context("spawn")
+
+    context = context or serialize_context()
+
+    process = ctx.Process(
+        target=cloudpickle_wrapped_call(
+            run_flow_with_env,
+            env=get_current_settings().to_environment_variables(exclude_unset=True)
+            | os.environ
+            | {
+                # TODO: make this a thing we can pass into the engine
+                "PREFECT__ENABLE_CANCELLATION_AND_CRASHED_HOOKS": "false",
+            },
+            flow=flow,
+            flow_run=flow_run,
+            parameters=parameters,
+            wait_for=wait_for,
+            context=context,
+        ),
+    )
+    process.start()
+
+    return process