prefect 3.3.5.dev3__tar.gz → 3.3.6.dev1__tar.gz

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.3.5.dev3
+Version: 3.3.6.dev1
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -56,7 +56,7 @@ Requires-Dist: prometheus-client>=0.20.0
 Requires-Dist: pydantic!=2.10.0,<3.0.0,>=2.9
 Requires-Dist: pydantic-core<3.0.0,>=2.12.0
 Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
-Requires-Dist: pydantic-settings>2.2.1
+Requires-Dist: pydantic-settings!=2.9.0,<3.0.0,>2.2.1
 Requires-Dist: python-dateutil<3.0.0,>=2.8.2
 Requires-Dist: python-slugify<9.0,>=5.0
 Requires-Dist: python-socks[asyncio]<3.0,>=2.5.3

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/pyproject.toml

@@ -67,7 +67,7 @@ dependencies = [
     "pydantic>=2.9,<3.0.0,!=2.10.0",
     "pydantic_core>=2.12.0,<3.0.0",
     "pydantic_extra_types>=2.8.2,<3.0.0",
-    "pydantic_settings>2.2.1",
+    "pydantic_settings>2.2.1,<3.0.0,!=2.9.0",
     "python-dateutil>=2.8.2,<3.0.0",
     "python-slugify>=5.0,<9.0",
     "python-socks[asyncio]>=2.5.3,<3.0",

prefect-3.3.6.dev1/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.3.6.dev1"
+__build_date__ = "2025-04-19 08:08:00.544911+00:00"
+__git_commit__ = "c9c018d468694ea61ce3a482948bf4d03bb67227"
+__dirty__ = False

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/src/prefect/_experimental/bundles.py

@@ -8,6 +8,7 @@ import multiprocessing.context
 import os
 import subprocess
 import sys
+from pathlib import Path
 from typing import Any, TypedDict
 
 import cloudpickle
@@ -99,7 +100,9 @@ def extract_flow_from_bundle(bundle: SerializedBundle) -> Flow[Any, Any]:
 
 
 def _extract_and_run_flow(
-    bundle: SerializedBundle, env: dict[str, Any] | None = None
+    bundle: SerializedBundle,
+    cwd: Path | str | None = None,
+    env: dict[str, Any] | None = None,
 ) -> None:
     """
     Extracts a flow from a bundle and runs it.
@@ -108,6 +111,7 @@ def _extract_and_run_flow(
 
     Args:
         bundle: The bundle to extract and run.
+        cwd: The working directory to use when running the flow.
         env: The environment to use when running the flow.
     """
 
@@ -120,6 +124,9 @@ def _extract_and_run_flow(
     context = _deserialize_bundle_object(bundle["context"])
     flow_run = FlowRun.model_validate(bundle["flow_run"])
 
+    if cwd:
+        os.chdir(cwd)
+
     with SettingsContext(
         profile=settings_context.profile,
         settings=Settings(),
@@ -138,6 +145,8 @@ def _extract_and_run_flow(
 
 def execute_bundle_in_subprocess(
     bundle: SerializedBundle,
+    env: dict[str, Any] | None = None,
+    cwd: Path | str | None = None,
 ) -> multiprocessing.context.SpawnProcess:
     """
     Executes a bundle in a subprocess.
@@ -150,6 +159,7 @@ def execute_bundle_in_subprocess(
     """
 
     ctx = multiprocessing.get_context("spawn")
+    env = env or {}
 
     # Install dependencies if necessary
     if dependencies := bundle.get("dependencies"):
@@ -164,7 +174,9 @@ def execute_bundle_in_subprocess(
         kwargs={
             "bundle": bundle,
             "env": get_current_settings().to_environment_variables(exclude_unset=True)
-            | os.environ,
+            | os.environ
+            | env,
+            "cwd": cwd,
         },
     )
 

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/src/prefect/cache_policies.py

@@ -33,9 +33,9 @@ def _register_stable_transforms() -> None:
     so that cache keys that utilize them are deterministic across invocations.
     """
     try:
-        import pandas as pd
+        import pandas as pd  # pyright: ignore
 
-        STABLE_TRANSFORMS[pd.DataFrame] = lambda df: [
+        STABLE_TRANSFORMS[pd.DataFrame] = lambda df: [  # pyright: ignore
             df[col] for col in sorted(df.columns)
         ]
     except (ImportError, ModuleNotFoundError):
@@ -183,7 +183,7 @@ class CompoundCachePolicy(CachePolicy):
     Any keys that return `None` will be ignored.
     """
 
-    policies: list[CachePolicy] = field(default_factory=list)
+    policies: list[CachePolicy] = field(default_factory=lambda: [])
 
     def __post_init__(self) -> None:
         # flatten any CompoundCachePolicies
@@ -349,7 +349,7 @@ class Inputs(CachePolicy):
     Policy that computes a cache key based on a hash of the runtime inputs provided to the task..
     """
 
-    exclude: list[str] = field(default_factory=list)
+    exclude: list[str] = field(default_factory=lambda: [])
 
     def compute_key(
         self,

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/src/prefect/cli/_prompts.py

@@ -4,9 +4,7 @@ Utilities for prompting the user for input
 
 from __future__ import annotations
 
-import ast
 import asyncio
-import math
 import os
 import shutil
 from datetime import timedelta
@@ -46,11 +44,10 @@ from prefect.client.utilities import client_injector
 from prefect.exceptions import ObjectAlreadyExists, ObjectNotFound
 from prefect.flows import load_flow_from_entrypoint
 from prefect.logging.loggers import get_logger
-from prefect.settings import PREFECT_DEBUG_MODE
 from prefect.utilities import urls
+from prefect.utilities._ast import find_flow_functions_in_file
 from prefect.utilities._git import get_git_remote_origin_url
-from prefect.utilities.asyncutils import LazySemaphore
-from prefect.utilities.filesystem import filter_files, get_open_file_limit
+from prefect.utilities.filesystem import filter_files
 from prefect.utilities.processutils import get_sys_executable, run_process
 from prefect.utilities.slugify import slugify
 
@@ -71,99 +68,10 @@ REQUIRED_FIELDS_FOR_CREDS_BLOCK = {
     "azure-blob-storage-credentials": ["account_url", "connection_string"],
 }
 
-# Only allow half of the open file limit to be open at once to allow for other
-# actors to open files.
-OPEN_FILE_SEMAPHORE = LazySemaphore(lambda: math.floor(get_open_file_limit() * 0.5))
 
 logger = get_logger(__name__)
 
 
-async def find_flow_functions_in_file(path: anyio.Path) -> list[dict[str, str]]:
-    decorator_name = "flow"
-    decorator_module = "prefect"
-    decorated_functions: list[dict[str, str]] = []
-    async with OPEN_FILE_SEMAPHORE:
-        try:
-            async with await anyio.open_file(path) as f:
-                try:
-                    tree = ast.parse(await f.read())
-                except SyntaxError:
-                    if PREFECT_DEBUG_MODE:
-                        get_logger().debug(
-                            f"Could not parse {path} as a Python file. Skipping."
-                        )
-                    return decorated_functions
-        except Exception as exc:
-            if PREFECT_DEBUG_MODE:
-                get_logger().debug(f"Could not open {path}: {exc}. Skipping.")
-            return decorated_functions
-
-    for node in ast.walk(tree):
-        if isinstance(
-            node,
-            (
-                ast.FunctionDef,
-                ast.AsyncFunctionDef,
-            ),
-        ):
-            for decorator in node.decorator_list:
-                # handles @flow
-                is_name_match = (
-                    isinstance(decorator, ast.Name) and decorator.id == decorator_name
-                )
-                # handles @flow()
-                is_func_name_match = (
-                    isinstance(decorator, ast.Call)
-                    and isinstance(decorator.func, ast.Name)
-                    and decorator.func.id == decorator_name
-                )
-                # handles @prefect.flow
-                is_module_attribute_match = (
-                    isinstance(decorator, ast.Attribute)
-                    and isinstance(decorator.value, ast.Name)
-                    and decorator.value.id == decorator_module
-                    and decorator.attr == decorator_name
-                )
-                # handles @prefect.flow()
-                is_module_attribute_func_match = (
-                    isinstance(decorator, ast.Call)
-                    and isinstance(decorator.func, ast.Attribute)
-                    and decorator.func.attr == decorator_name
-                    and isinstance(decorator.func.value, ast.Name)
-                    and decorator.func.value.id == decorator_module
-                )
-                if is_name_match or is_module_attribute_match:
-                    decorated_functions.append(
-                        {
-                            "flow_name": node.name,
-                            "function_name": node.name,
-                            "filepath": str(path),
-                        }
-                    )
-                if is_func_name_match or is_module_attribute_func_match:
-                    name_kwarg_node = None
-                    if TYPE_CHECKING:
-                        assert isinstance(decorator, ast.Call)
-                    for kw in decorator.keywords:
-                        if kw.arg == "name":
-                            name_kwarg_node = kw
-                            break
-                    if name_kwarg_node is not None and isinstance(
-                        name_kwarg_node.value, ast.Constant
-                    ):
-                        flow_name = name_kwarg_node.value.value
-                    else:
-                        flow_name = node.name
-                    decorated_functions.append(
-                        {
-                            "flow_name": flow_name,
-                            "function_name": node.name,
-                            "filepath": str(path),
-                        }
-                    )
-    return decorated_functions
-
-
 async def search_for_flow_functions(
     directory: str = ".", exclude_patterns: list[str] | None = None
 ) -> list[dict[str, str]]:

prefect-3.3.6.dev1/src/prefect/cli/task.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import inspect
+import logging
+from typing import Any, Optional
+
+import typer
+
+from prefect.cli._types import PrefectTyper
+from prefect.cli._utilities import exit_with_error
+from prefect.cli.root import app
+from prefect.logging import get_logger
+from prefect.task_worker import serve as task_serve
+from prefect.tasks import Task
+from prefect.utilities.importtools import import_object, load_module
+
+task_app: PrefectTyper = PrefectTyper(name="task", help="Work with task scheduling.")
+app.add_typer(task_app, aliases=["task"])
+
+logger: logging.Logger = get_logger("prefect.cli.task")
+
+
+def _import_tasks_from_module(module: str) -> list[Task[Any, Any]]:
+    mod = load_module(module)
+    return [
+        obj
+        for _, obj in inspect.getmembers(mod)
+        if isinstance(obj, Task) and not inspect.ismodule(obj)
+    ]
+
+
+@task_app.command()
+async def serve(
+    entrypoints: Optional[list[str]] = typer.Argument(
+        None,
+        help="The paths to one or more tasks, in the form of `./path/to/file.py:task_func_name`.",
+    ),
+    module: Optional[list[str]] = typer.Option(
+        None,
+        "--module",
+        "-m",
+        help="The module(s) to import the tasks from.",
+    ),
+    limit: int = typer.Option(
+        10,
+        help="The maximum number of tasks that can be run concurrently. Defaults to 10.",
+    ),
+):
+    """
+    Serve the provided tasks so that their runs may be submitted to and
+    executed in the engine.
+
+    Args:
+        entrypoints: List of strings representing the paths to one or more
+            tasks. Each path should be in the format
+            `./path/to/file.py:task_func_name`.
+        module: The module(s) to import the task definitions from.
+        limit: The maximum number of tasks that can be run concurrently. Defaults to 10.
+    """
+    if (entrypoints and any(entrypoints)) and (module and any(module)):
+        exit_with_error(
+            "You may provide entrypoints or modules, but not both at the same time."
+        )
+
+    if not entrypoints and not module:
+        exit_with_error(
+            "You must provide either `path/to/file.py:task_func` or `--module module_name`."
+        )
+
+    tasks: list[Task[Any, Any]] = []
+
+    if entrypoints:
+        for entrypoint in entrypoints:
+            if ".py:" not in entrypoint:
+                exit_with_error(
+                    (
+                        f"Error: Invalid entrypoint format {entrypoint!r}. It "
+                        "must be of the form `./path/to/file.py:task_func_name`."
+                    )
+                )
+
+            try:
+                tasks.append(import_object(entrypoint))
+            except Exception:
+                mod, task_name = entrypoint.split(":")
+                exit_with_error(
+                    f"Error: {mod!r} has no function {task_name!r}.", style="red"
+                )
+
+    elif module:
+        for mod in module:
+            try:
+                module_tasks = _import_tasks_from_module(mod)
+            except Exception as e:
+                exit_with_error(
+                    f"Module '{mod}' could not be imported. Please check the module name and try again.\n\n{e.__class__.__name__}: {e}"
+                )
+            plural = "s" if len(module_tasks) != 1 else ""
+            logger.debug(f"Found {len(module_tasks)} task{plural} in {mod!r}.")
+            tasks.extend(module_tasks)
+
+    if not tasks:
+        sources = (
+            f"entrypoints: {entrypoints!r}" if entrypoints else f"modules: {module!r}"
+        )
+        exit_with_error(f"No tasks found to serve in {sources}.")
+
+    await task_serve(*tasks, limit=limit)

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/src/prefect/cli/work_pool.py

@@ -706,6 +706,14 @@ def _determine_storage_type(storage_config: WorkPoolStorageConfiguration) -> str
         "prefect_aws" in step for step in storage_config.bundle_upload_step.keys()
     ):
         return "S3"
+    if storage_config.bundle_upload_step and any(
+        "prefect_gcp" in step for step in storage_config.bundle_upload_step.keys()
+    ):
+        return "GCS"
+    if storage_config.bundle_upload_step and any(
+        "prefect_azure" in step for step in storage_config.bundle_upload_step.keys()
+    ):
+        return "Azure Blob Storage"
     return "Unknown"
 
 
@@ -771,14 +779,11 @@ work_pool_storage_app.add_typer(work_pool_storage_configure_app)
 
 @work_pool_storage_configure_app.command()
 async def s3(
-    work_pool_name: Annotated[
-        str,
-        typer.Argument(
-            ...,
-            help="The name of the work pool to configure storage for.",
-            show_default=False,
-        ),
-    ],
+    work_pool_name: str = typer.Argument(
+        ...,
+        help="The name of the work pool to configure storage for.",
+        show_default=False,
+    ),
     bucket: str = typer.Option(
         ...,
         "--bucket",
@@ -789,7 +794,7 @@ async def s3(
     credentials_block_name: str = typer.Option(
         ...,
         "--aws-credentials-block-name",
-        help="The name of the credentials block to use.",
+        help="The name of the AWS credentials block to use.",
         show_default=False,
         prompt="Enter the name of the AWS credentials block to use",
     ),
@@ -799,7 +804,7 @@ async def s3(
 
     \b
     Examples:
-        $ prefect work-pool storage configure s3 "my-pool" --bucket my-bucket --credentials-block-name my-credentials
+        $ prefect work-pool storage configure s3 "my-pool" --bucket my-bucket --aws-credentials-block-name my-credentials
     """
     # TODO: Allow passing in AWS keys and creating a block for the user.
     async with get_client() as client:
@@ -838,3 +843,140 @@ async def s3(
             exit_with_error(f"Work pool {work_pool_name!r} does not exist.")
 
         exit_with_success(f"Configured S3 storage for work pool {work_pool_name!r}")
+
+
+@work_pool_storage_configure_app.command()
+async def gcs(
+    work_pool_name: str = typer.Argument(
+        ...,
+        help="The name of the work pool to configure storage for.",
+        show_default=False,
+    ),
+    bucket: str = typer.Option(
+        ...,
+        "--bucket",
+        help="The name of the Google Cloud Storage bucket to use.",
+        show_default=False,
+        prompt="Enter the name of the Google Cloud Storage bucket to use",
+    ),
+    credentials_block_name: str = typer.Option(
+        ...,
+        "--gcp-credentials-block-name",
+        help="The name of the Google Cloud credentials block to use.",
+        show_default=False,
+        prompt="Enter the name of the Google Cloud credentials block to use",
+    ),
+):
+    """
+    EXPERIMENTAL: Configure Google Cloud storage for a work pool.
+
+    \b
+    Examples:
+        $ prefect work-pool storage configure gcs "my-pool" --bucket my-bucket --gcp-credentials-block-name my-credentials
+    """
+    async with get_client() as client:
+        try:
+            await client.read_block_document_by_name(
+                name=credentials_block_name, block_type_slug="gcp-credentials"
+            )
+        except ObjectNotFound:
+            exit_with_error(
+                f"GCS credentials block {credentials_block_name!r} does not exist. Please create one using `prefect block create gcp-credentials`."
+            )
+
+        try:
+            await client.update_work_pool(
+                work_pool_name=work_pool_name,
+                work_pool=WorkPoolUpdate(
+                    storage_configuration=WorkPoolStorageConfiguration(
+                        bundle_upload_step={
+                            "prefect_gcp.experimental.bundles.upload": {
+                                "requires": "prefect-gcp",
+                                "bucket": bucket,
+                                "credentials_block_name": credentials_block_name,
+                            }
+                        },
+                        bundle_execution_step={
+                            "prefect_gcp.experimental.bundles.execute": {
+                                "requires": "prefect-gcp",
+                                "bucket": bucket,
+                                "credentials_block_name": credentials_block_name,
+                            }
+                        },
+                    ),
+                ),
+            )
+        except ObjectNotFound:
+            exit_with_error(f"Work pool {work_pool_name!r} does not exist.")
+
+        exit_with_success(f"Configured GCS storage for work pool {work_pool_name!r}")
+
+
+@work_pool_storage_configure_app.command()
+async def azure_blob_storage(
+    work_pool_name: str = typer.Argument(
+        ...,
+        help="The name of the work pool to configure storage for.",
+        show_default=False,
+    ),
+    container: str = typer.Option(
+        ...,
+        "--container",
+        help="The name of the Azure Blob Storage container to use.",
+        show_default=False,
+        prompt="Enter the name of the Azure Blob Storage container to use",
+    ),
+    credentials_block_name: str = typer.Option(
+        ...,
+        "--azure-blob-storage-credentials-block-name",
+        help="The name of the Azure Blob Storage credentials block to use.",
+        show_default=False,
+        prompt="Enter the name of the Azure Blob Storage credentials block to use",
+    ),
+):
+    """
+    EXPERIMENTAL: Configure Azure Blob Storage for a work pool.
+
+    \b
+    Examples:
+        $ prefect work-pool storage configure azure-blob-storage "my-pool" --container my-container --azure-blob-storage-credentials-block-name my-credentials
+    """
+    async with get_client() as client:
+        try:
+            await client.read_block_document_by_name(
+                name=credentials_block_name,
+                block_type_slug="azure-blob-storage-credentials",
+            )
+        except ObjectNotFound:
+            exit_with_error(
+                f"Azure Blob Storage credentials block {credentials_block_name!r} does not exist. Please create one using `prefect block create azure-blob-storage-credentials`."
+            )
+
+        try:
+            await client.update_work_pool(
+                work_pool_name=work_pool_name,
+                work_pool=WorkPoolUpdate(
+                    storage_configuration=WorkPoolStorageConfiguration(
+                        bundle_upload_step={
+                            "prefect_azure.experimental.bundles.upload": {
+                                "requires": "prefect-azure",
+                                "container": container,
+                                "azure_blob_storage_credentials_block_name": credentials_block_name,
+                            }
+                        },
+                        bundle_execution_step={
+                            "prefect_azure.experimental.bundles.execute": {
+                                "requires": "prefect-azure",
+                                "container": container,
+                                "azure_blob_storage_credentials_block_name": credentials_block_name,
+                            }
+                        },
+                    ),
+                ),
+            )
+        except ObjectNotFound:
+            exit_with_error(f"Work pool {work_pool_name!r} does not exist.")
+
+        exit_with_success(
+            f"Configured Azure Blob Storage for work pool {work_pool_name!r}"
+        )

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/src/prefect/concurrency/context.py

@@ -13,7 +13,9 @@ class ConcurrencyContext(ContextModel):
     # Track the slots that have been acquired but were not able to be released
     # due to cancellation or some other error. These slots are released when
     # the context manager exits.
-    cleanup_slots: list[tuple[list[str], int, float]] = Field(default_factory=list)
+    cleanup_slots: list[tuple[list[str], int, float]] = Field(
+        default_factory=lambda: []
+    )
 
     def __exit__(self, *exc_info: Any) -> None:
         if self.cleanup_slots:

{prefect-3.3.5.dev3 → prefect-3.3.6.dev1}/src/prefect/context.py

@@ -18,8 +18,6 @@ from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, TypeVar, Un
 from pydantic import BaseModel, ConfigDict, Field, PrivateAttr
 from typing_extensions import Self
 
-import prefect.logging
-import prefect.logging.configuration
 import prefect.settings
 import prefect.types._datetime
 from prefect._internal.compatibility.migration import getattr_migration
@@ -128,7 +126,7 @@ class ContextModel(BaseModel):
         def __init__(self, **kwargs: Any) -> None: ...
 
     # The context variable for storing data must be defined by the child class
-    __var__: ClassVar[ContextVar[Self]]
+    __var__: ClassVar[ContextVar[Any]]
     _token: Optional[Token[Self]] = PrivateAttr(None)
     model_config: ClassVar[ConfigDict] = ConfigDict(
         arbitrary_types_allowed=True,