prefect-client 3.1.10__py3-none-any.whl → 3.1.12__py3-none-any.whl

prefect/deployments/base.py

@@ -5,27 +5,21 @@ build system for managing flows and deployments.
 To get started, follow along with [the deloyments tutorial](/tutorials/deployments/).
 """
 
-import ast
-import asyncio
-import math
+from __future__ import annotations
+
 import os
-import subprocess
-import sys
 from copy import deepcopy
 from pathlib import Path
 from typing import Any, Dict, List, Optional, cast
 
-import anyio
 import yaml
 from ruamel.yaml import YAML
 
 from prefect.client.schemas.actions import DeploymentScheduleCreate
 from prefect.client.schemas.objects import ConcurrencyLimitStrategy
 from prefect.client.schemas.schedules import IntervalSchedule
-from prefect.logging import get_logger
-from prefect.settings import PREFECT_DEBUG_MODE
-from prefect.utilities.asyncutils import LazySemaphore
-from prefect.utilities.filesystem import create_default_ignore_file, get_open_file_limit
+from prefect.utilities._git import get_git_branch, get_git_remote_origin_url
+from prefect.utilities.filesystem import create_default_ignore_file
 from prefect.utilities.templating import apply_values
 
 
@@ -146,36 +140,6 @@ def configure_project_by_recipe(recipe: str, **formatting_kwargs) -> dict:
     return config
 
 
-def _get_git_remote_origin_url() -> Optional[str]:
-    """
-    Returns the git remote origin URL for the current directory.
-    """
-    try:
-        origin_url = subprocess.check_output(
-            ["git", "config", "--get", "remote.origin.url"],
-            shell=sys.platform == "win32",
-            stderr=subprocess.DEVNULL,
-        )
-        origin_url = origin_url.decode().strip()
-    except subprocess.CalledProcessError:
-        return None
-
-    return origin_url
-
-
-def _get_git_branch() -> Optional[str]:
-    """
-    Returns the git branch for the current directory.
-    """
-    try:
-        branch = subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"])
-        branch = branch.decode().strip()
-    except subprocess.CalledProcessError:
-        return None
-
-    return branch
-
-
 def initialize_project(
     name: Optional[str] = None,
     recipe: Optional[str] = None,
@@ -198,11 +162,11 @@ def initialize_project(
     formatting_kwargs = {"directory": str(Path(".").absolute().resolve())}
     dir_name = os.path.basename(os.getcwd())
 
-    remote_url = _get_git_remote_origin_url()
+    remote_url = get_git_remote_origin_url()
     if remote_url:
         formatting_kwargs["repository"] = remote_url
         is_git_based = True
-        branch = _get_git_branch()
+        branch = get_git_branch()
         formatting_kwargs["branch"] = branch or "main"
 
     formatting_kwargs["name"] = dir_name
@@ -313,6 +277,7 @@ def _save_deployment_to_prefect_file(
     push_steps: Optional[List[Dict]] = None,
     pull_steps: Optional[List[Dict]] = None,
     triggers: Optional[List[Dict]] = None,
+    sla: Optional[list[dict]] = None,
     prefect_file: Path = Path("prefect.yaml"),
 ):
     """
@@ -357,6 +322,9 @@ def _save_deployment_to_prefect_file(
         if triggers and triggers != parsed_prefect_file_contents.get("triggers"):
             deployment["triggers"] = triggers
 
+        if sla and sla != parsed_prefect_file_contents.get("sla"):
+            deployment["sla"] = sla
+
         deployments = parsed_prefect_file_contents.get("deployments")
         if deployments is None:
             parsed_prefect_file_contents["deployments"] = [deployment]
@@ -373,105 +341,3 @@ def _save_deployment_to_prefect_file(
 
         with prefect_file.open(mode="w") as f:
             ryaml.dump(parsed_prefect_file_contents, f)
-
-
-# Only allow half of the open file limit to be open at once to allow for other
-# actors to open files.
-OPEN_FILE_SEMAPHORE = LazySemaphore(lambda: math.floor(get_open_file_limit() * 0.5))
-
-
-async def _find_flow_functions_in_file(filename: str) -> List[Dict]:
-    decorator_name = "flow"
-    decorator_module = "prefect"
-    decorated_functions = []
-    async with OPEN_FILE_SEMAPHORE:
-        try:
-            async with await anyio.open_file(filename) as f:
-                try:
-                    tree = ast.parse(await f.read())
-                except SyntaxError:
-                    if PREFECT_DEBUG_MODE:
-                        get_logger().debug(
-                            f"Could not parse {filename} as a Python file. Skipping."
-                        )
-                    return decorated_functions
-        except Exception as exc:
-            if PREFECT_DEBUG_MODE:
-                get_logger().debug(f"Could not open {filename}: {exc}. Skipping.")
-            return decorated_functions
-
-    for node in ast.walk(tree):
-        if isinstance(
-            node,
-            (
-                ast.FunctionDef,
-                ast.AsyncFunctionDef,
-            ),
-        ):
-            for decorator in node.decorator_list:
-                # handles @flow
-                is_name_match = (
-                    isinstance(decorator, ast.Name) and decorator.id == decorator_name
-                )
-                # handles @flow()
-                is_func_name_match = (
-                    isinstance(decorator, ast.Call)
-                    and isinstance(decorator.func, ast.Name)
-                    and decorator.func.id == decorator_name
-                )
-                # handles @prefect.flow
-                is_module_attribute_match = (
-                    isinstance(decorator, ast.Attribute)
-                    and isinstance(decorator.value, ast.Name)
-                    and decorator.value.id == decorator_module
-                    and decorator.attr == decorator_name
-                )
-                # handles @prefect.flow()
-                is_module_attribute_func_match = (
-                    isinstance(decorator, ast.Call)
-                    and isinstance(decorator.func, ast.Attribute)
-                    and decorator.func.attr == decorator_name
-                    and isinstance(decorator.func.value, ast.Name)
-                    and decorator.func.value.id == decorator_module
-                )
-                if is_name_match or is_module_attribute_match:
-                    decorated_functions.append(
-                        {
-                            "flow_name": node.name,
-                            "function_name": node.name,
-                            "filepath": str(filename),
-                        }
-                    )
-                if is_func_name_match or is_module_attribute_func_match:
-                    name_kwarg_node = next(
-                        (kw for kw in decorator.keywords if kw.arg == "name"), None
-                    )
-                    flow_name = (
-                        name_kwarg_node.value.value
-                        if isinstance(name_kwarg_node, ast.Constant)
-                        else node.name
-                    )
-                    decorated_functions.append(
-                        {
-                            "flow_name": flow_name,
-                            "function_name": node.name,
-                            "filepath": str(filename),
-                        }
-                    )
-    return decorated_functions
-
-
-async def _search_for_flow_functions(directory: str = ".") -> List[Dict]:
-    """
-    Search for flow functions in the provided directory. If no directory is provided,
-    the current working directory is used.
-
-    Returns:
-        List[Dict]: the flow name, function name, and filepath of all flow functions found
-    """
-    path = anyio.Path(directory)
-    coros = []
-    async for file in path.rglob("*.py"):
-        coros.append(_find_flow_functions_in_file(file))
-
-    return [fn for file_fns in await asyncio.gather(*coros) for fn in file_fns]

prefect/deployments/flow_runs.py

@@ -10,9 +10,12 @@ from prefect.client.schemas import FlowRun
 from prefect.client.utilities import inject_client
 from prefect.context import FlowRunContext, TaskRunContext
 from prefect.logging import get_logger
-from prefect.results import BaseResult, ResultRecordMetadata
+from prefect.results import ResultRecordMetadata
 from prefect.states import Pending, Scheduled
 from prefect.tasks import Task
+from prefect.telemetry.run_telemetry import (
+    LABELS_TRACEPARENT_KEY,
+)
 from prefect.utilities.asyncutils import sync_compatible
 from prefect.utilities.slugify import slugify
 
@@ -22,7 +25,6 @@ if TYPE_CHECKING:
 
 prefect.client.schemas.StateCreate.model_rebuild(
     _types_namespace={
-        "BaseResult": BaseResult,
         "ResultRecordMetadata": ResultRecordMetadata,
     }
 )
@@ -156,6 +158,13 @@ async def run_deployment(
     else:
         parent_task_run_id = None
 
+    if flow_run_ctx and flow_run_ctx.flow_run:
+        traceparent = flow_run_ctx.flow_run.labels.get(LABELS_TRACEPARENT_KEY)
+    else:
+        traceparent = None
+
+    trace_labels = {LABELS_TRACEPARENT_KEY: traceparent} if traceparent else {}
+
     flow_run = await client.create_flow_run_from_deployment(
         deployment.id,
         parameters=parameters,
@@ -166,6 +175,7 @@ async def run_deployment(
         parent_task_run_id=parent_task_run_id,
         work_queue_name=work_queue_name,
         job_variables=job_variables,
+        labels=trace_labels,
     )
 
     flow_run_id = flow_run.id

prefect/deployments/runner.py

@@ -33,26 +33,30 @@ import importlib
 import tempfile
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Iterable, List, Optional, Union
 from uuid import UUID
 
+from exceptiongroup import ExceptionGroup  # novermin
 from pydantic import (
     BaseModel,
     ConfigDict,
     Field,
     PrivateAttr,
+    field_validator,
     model_validator,
 )
 from rich.console import Console
 from rich.progress import Progress, SpinnerColumn, TextColumn, track
 from rich.table import Table
 
+from prefect._experimental.sla.objects import SlaTypes
 from prefect._internal.concurrency.api import create_call, from_async
 from prefect._internal.schemas.validators import (
     reconcile_paused_deployment,
     reconcile_schedules_runner,
 )
-from prefect.client.orchestration import get_client
+from prefect.client.base import ServerType
+from prefect.client.orchestration import PrefectClient, get_client
 from prefect.client.schemas.actions import DeploymentScheduleCreate
 from prefect.client.schemas.filters import WorkerFilter, WorkerFilterStatus
 from prefect.client.schemas.objects import (
@@ -127,9 +131,10 @@ class RunnerDeployment(BaseModel):
         job_variables: Settings used to override the values specified default base job template
             of the chosen work pool. Refer to the base job template of the chosen work pool for
             available settings.
+        _sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
     """
 
-    model_config = ConfigDict(arbitrary_types_allowed=True)
+    model_config: ClassVar[ConfigDict] = ConfigDict(arbitrary_types_allowed=True)
 
     name: str = Field(..., description="The name of the deployment.")
     flow_name: Optional[str] = Field(
@@ -206,6 +211,10 @@ class RunnerDeployment(BaseModel):
             " a built runner."
         ),
     )
+    # (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
+    _sla: Optional[Union[SlaTypes, list[SlaTypes]]] = PrivateAttr(
+        default=None,
+    )
     _entrypoint_type: EntrypointType = PrivateAttr(
         default=EntrypointType.FILE_PATH,
     )
@@ -220,6 +229,13 @@ class RunnerDeployment(BaseModel):
     def entrypoint_type(self) -> EntrypointType:
         return self._entrypoint_type
 
+    @field_validator("name", mode="before")
+    @classmethod
+    def validate_name(cls, value: str) -> str:
+        if value.endswith(".py"):
+            return Path(value).stem
+        return value
+
     @model_validator(mode="after")
     def validate_automation_names(self):
         """Ensure that each trigger has a name for its automation if none is provided."""
@@ -343,8 +359,32 @@ class RunnerDeployment(BaseModel):
                 trigger.set_deployment_id(deployment_id)
                 await client.create_automation(trigger.as_automation())
 
+            # We plan to support SLA configuration on the Prefect Server in the future.
+            # For now, we only support it on Prefect Cloud.
+            if self._sla:
+                await self._create_slas(deployment_id, client)
+
             return deployment_id
 
+    async def _create_slas(self, deployment_id: UUID, client: PrefectClient):
+        if not isinstance(self._sla, list):
+            self._sla = [self._sla]
+
+        if client.server_type == ServerType.CLOUD:
+            exceptions = []
+            for sla in self._sla:
+                try:
+                    sla.set_deployment_id(deployment_id)
+                    await client.create_sla(sla)
+                except Exception as e:
+                    exceptions.append(e)
+            if exceptions:
+                raise ExceptionGroup("Failed to create SLAs", exceptions)  # novermin
+        else:
+            raise ValueError(
+                "SLA configuration is currently only supported on Prefect Cloud."
+            )
+
     @staticmethod
     def _construct_deployment_schedules(
         interval: Optional[
@@ -459,6 +499,7 @@ class RunnerDeployment(BaseModel):
         work_queue_name: Optional[str] = None,
         job_variables: Optional[dict[str, Any]] = None,
         entrypoint_type: EntrypointType = EntrypointType.FILE_PATH,
+        _sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None,  # experimental
     ) -> "RunnerDeployment":
         """
         Configure a deployment for a given flow.
@@ -489,6 +530,7 @@ class RunnerDeployment(BaseModel):
             job_variables: Settings used to override the values specified default base job template
                 of the chosen work pool. Refer to the base job template of the chosen work pool for
                 available settings.
+            _sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
         """
         constructed_schedules = cls._construct_deployment_schedules(
             interval=interval,
@@ -508,7 +550,7 @@ class RunnerDeployment(BaseModel):
             concurrency_options = None
 
         deployment = cls(
-            name=Path(name).stem,
+            name=name,
             flow_name=flow.name,
             schedules=constructed_schedules,
             concurrency_limit=concurrency_limit,
@@ -524,6 +566,7 @@ class RunnerDeployment(BaseModel):
             work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        deployment._sla = _sla
 
         if not deployment.entrypoint:
             no_file_location_error = (
@@ -599,6 +642,7 @@ class RunnerDeployment(BaseModel):
         work_pool_name: Optional[str] = None,
         work_queue_name: Optional[str] = None,
         job_variables: Optional[dict[str, Any]] = None,
+        _sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None,  # experimental
     ) -> "RunnerDeployment":
         """
         Configure a deployment for a given flow located at a given entrypoint.
@@ -630,6 +674,7 @@ class RunnerDeployment(BaseModel):
             job_variables: Settings used to override the values specified default base job template
                 of the chosen work pool. Refer to the base job template of the chosen work pool for
                 available settings.
+            _sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
         """
         from prefect.flows import load_flow_from_entrypoint
 
@@ -669,6 +714,7 @@ class RunnerDeployment(BaseModel):
             work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        deployment._sla = _sla
         deployment._path = str(Path.cwd())
 
         cls._set_defaults_from_flow(deployment, flow)
@@ -700,6 +746,7 @@ class RunnerDeployment(BaseModel):
         work_pool_name: Optional[str] = None,
         work_queue_name: Optional[str] = None,
         job_variables: Optional[dict[str, Any]] = None,
+        _sla: Optional[Union[SlaTypes, list[SlaTypes]]] = None,  # experimental
     ):
         """
         Create a RunnerDeployment from a flow located at a given entrypoint and stored in a
@@ -731,6 +778,7 @@ class RunnerDeployment(BaseModel):
             job_variables: Settings used to override the values specified default base job template
                 of the chosen work pool. Refer to the base job template of the chosen work pool for
                 available settings.
+            _sla: (Experimental) SLA configuration for the deployment. May be removed or modified at any time. Currently only supported on Prefect Cloud.
         """
         from prefect.flows import load_flow_from_entrypoint
 
@@ -779,6 +827,7 @@ class RunnerDeployment(BaseModel):
             work_queue_name=work_queue_name,
             job_variables=job_variables,
         )
+        deployment._sla = _sla
         deployment._path = str(storage.destination).replace(
             tmpdir, "$STORAGE_BASE_PATH"
         )

prefect/deployments/steps/pull.py

@@ -50,6 +50,7 @@ async def agit_clone(
     include_submodules: bool = False,
     access_token: Optional[str] = None,
     credentials: Optional["Block"] = None,
+    directories: Optional[list[str]] = None,
 ) -> dict[str, str]:
     """
     Asynchronously clones a git repository into the current working directory.
@@ -81,6 +82,7 @@ async def agit_clone(
         credentials=_credentials,
         branch=branch,
         include_submodules=include_submodules,
+        directories=directories,
     )
 
     await _pull_git_repository_with_retries(storage)
@@ -95,6 +97,7 @@ def git_clone(
     include_submodules: bool = False,
     access_token: Optional[str] = None,
     credentials: Optional["Block"] = None,
+    directories: Optional[list[str]] = None,
 ) -> dict[str, str]:
     """
     Clones a git repository into the current working directory.
@@ -107,6 +110,7 @@ def git_clone(
             the repository will be cloned using the default git credentials
         credentials: a GitHubCredentials, GitLabCredentials, or BitBucketCredentials block can be used to specify the
             credentials to use for cloning the repository.
+        directories: Specify directories you want to be included (uses git sparse-checkout)
 
     Returns:
         dict: a dictionary containing a `directory` key of the new directory that was created
@@ -164,6 +168,14 @@ def git_clone(
             - prefect.deployments.steps.git_clone:
                 repository: git@github.com:org/repo.git
         ```
+
+        Clone a repository using sparse-checkout (allows specific folders of the repository to be checked out)
+        ```yaml
+        pull:
+            - prefect.deployments.steps.git_clone:
+                repository: https://github.com/org/repo.git
+                directories: ["dir_1", "dir_2", "prefect"]
+        ```
     """
     if access_token and credentials:
         raise ValueError(
@@ -177,6 +189,7 @@ def git_clone(
         credentials=_credentials,
         branch=branch,
         include_submodules=include_submodules,
+        directories=directories,
     )
 
     run_coro_as_sync(_pull_git_repository_with_retries(storage))

prefect/engine.py

@@ -31,16 +31,29 @@ if __name__ == "__main__":
 
     try:
         from prefect.flow_engine import (
-            load_flow_and_flow_run,
+            flow_run_logger,
+            load_flow,
+            load_flow_run,
             run_flow,
         )
 
-        flow_run, flow = load_flow_and_flow_run(flow_run_id=flow_run_id)
+        flow_run = load_flow_run(flow_run_id=flow_run_id)
+        run_logger = flow_run_logger(flow_run=flow_run)
+
+        try:
+            flow = load_flow(flow_run)
+        except Exception:
+            run_logger.error(
+                "Unexpected exception encountered when trying to load flow",
+                exc_info=True,
+            )
+            raise
+
         # run the flow
         if flow.isasync:
-            run_coro_as_sync(run_flow(flow, flow_run=flow_run))
+            run_coro_as_sync(run_flow(flow, flow_run=flow_run, error_logger=run_logger))
         else:
-            run_flow(flow, flow_run=flow_run)
+            run_flow(flow, flow_run=flow_run, error_logger=run_logger)
 
     except Abort as exc:
         engine_logger.info(

prefect/events/clients.py

@@ -16,6 +16,7 @@ from typing import (
     cast,
 )
 from urllib.parse import urlparse
+from urllib.request import proxy_bypass
 from uuid import UUID
 
 import orjson
@@ -95,6 +96,9 @@ class WebsocketProxyConnect(Connect):
         u = urlparse(uri)
         host = u.hostname
 
+        if not host:
+            raise ValueError(f"Invalid URI {uri}, no hostname found")
+
         if u.scheme == "ws":
             port = u.port or 80
             proxy_url = os.environ.get("HTTP_PROXY")
@@ -107,7 +111,9 @@ class WebsocketProxyConnect(Connect):
                 "Unsupported scheme %s. Expected 'ws' or 'wss'. " % u.scheme
             )
 
-        self._proxy = Proxy.from_url(proxy_url) if proxy_url else None
+        self._proxy = (
+            Proxy.from_url(proxy_url) if proxy_url and not proxy_bypass(host) else None
+        )
         self._host = host
         self._port = port
 

prefect/events/schemas/events.py

@@ -2,6 +2,7 @@ import copy
 from collections import defaultdict
 from typing import (
     Any,
+    ClassVar,
     Dict,
     Iterable,
     List,
@@ -108,7 +109,7 @@ def _validate_related_resources(value) -> List:
 class Event(PrefectBaseModel):
     """The client-side view of an event that has happened to a Resource"""
 
-    model_config = ConfigDict(extra="ignore")
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="ignore")
 
     occurred: DateTime = Field(
         default_factory=lambda: DateTime.now("UTC"),
@@ -177,7 +178,7 @@ class ReceivedEvent(Event):
     """The server-side view of an event that has happened to a Resource after it has
     been received by the server"""
 
-    model_config = ConfigDict(from_attributes=True)
+    model_config: ClassVar[ConfigDict] = ConfigDict(from_attributes=True)
 
     received: DateTime = Field(
         ...,

prefect/filesystems.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import abc
 import urllib.parse
 from pathlib import Path
@@ -92,7 +94,9 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
     )
 
     @field_validator("basepath", mode="before")
-    def cast_pathlib(cls, value):
+    def cast_pathlib(cls, value: str | Path | None) -> str | None:
+        if value is None:
+            return value
         return stringify_path(value)
 
     def _resolve_path(self, path: str, validate: bool = False) -> Path:
@@ -132,7 +136,7 @@ class LocalFileSystem(WritableFileSystem, WritableDeploymentStorage):
         Defaults to copying the entire contents of the block's basepath to the current working directory.
         """
         if not from_path:
-            from_path = Path(self.basepath).expanduser().resolve()
+            from_path = Path(self.basepath or ".").expanduser().resolve()
         else:
             from_path = self._resolve_path(from_path)