prefect 3.2.14.dev2__tar.gz → 3.2.15.dev1__tar.gz

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.2.14.dev2
+Version: 3.2.15.dev1
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io

prefect-3.2.15.dev1/src/prefect/_build_info.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "3.2.15.dev1"
+__build_date__ = "2025-03-22 08:08:06.987916+00:00"
+__git_commit__ = "007ef069892d244888264400c878755a0b5feec6"
+__dirty__ = False

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/cli/work_pool.py

@@ -2,8 +2,11 @@
 Command line interface for working with work queues.
 """
 
+from __future__ import annotations
+
 import json
 import textwrap
+from typing import Annotated
 
 import typer
 from rich.pretty import Pretty
@@ -19,7 +22,11 @@ from prefect.cli.root import app, is_interactive
 from prefect.client.collections import get_collections_metadata_client
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.actions import WorkPoolCreate, WorkPoolUpdate
-from prefect.client.schemas.objects import FlowRun, WorkPool
+from prefect.client.schemas.objects import (
+    FlowRun,
+    WorkPool,
+    WorkPoolStorageConfiguration,
+)
 from prefect.exceptions import ObjectAlreadyExists, ObjectNotFound
 from prefect.infrastructure.provisioners import (
     _provisioners,
@@ -681,3 +688,154 @@ async def preview(
             ),
             style="yellow",
         )
+
+
+# --------------------------------------------------------------------------
+# Work Pool Storage Configuration
+# --------------------------------------------------------------------------
+
+work_pool_storage_app: PrefectTyper = PrefectTyper(
+    name="storage", help="EXPERIMENTAL: Manage work pool storage."
+)
+work_pool_app.add_typer(work_pool_storage_app)
+
+
+def _determine_storage_type(storage_config: WorkPoolStorageConfiguration) -> str | None:
+    if storage_config.bundle_upload_step is None:
+        return None
+    if storage_config.bundle_upload_step and any(
+        "prefect_aws" in step for step in storage_config.bundle_upload_step.keys()
+    ):
+        return "S3"
+    return "Unknown"
+
+
+@work_pool_storage_app.command(name="inspect")
+async def storage_inspect(
+    work_pool_name: Annotated[
+        str,
+        typer.Argument(
+            ..., help="The name of the work pool to display storage configuration for."
+        ),
+    ],
+):
+    """
+    EXPERIMENTAL: Inspect the storage configuration for a work pool.
+
+    Examples:
+        $ prefect work-pool storage inspect "my-pool"
+    """
+    async with get_client() as client:
+        try:
+            work_pool = await client.read_work_pool(work_pool_name=work_pool_name)
+            from rich.panel import Panel
+            from rich.table import Table
+
+            storage_table = Table(show_header=True, header_style="bold")
+            storage_table.add_column("Setting", style="cyan")
+            storage_table.add_column("Value")
+
+            storage_type = _determine_storage_type(work_pool.storage_configuration)
+            if not storage_type:
+                app.console.print(
+                    f"No storage configuration found for work pool {work_pool_name!r}",
+                    style="yellow",
+                )
+                return
+
+            storage_table.add_row("type", storage_type)
+
+            # Add other storage settings, filtering out None values
+            if work_pool.storage_configuration.bundle_upload_step is not None:
+                fqn = list(work_pool.storage_configuration.bundle_upload_step.keys())[0]
+                config_values = work_pool.storage_configuration.bundle_upload_step[fqn]
+                for key, value in config_values.items():
+                    storage_table.add_row(key, str(value))
+
+            panel = Panel(
+                storage_table,
+                title=f"[bold]Storage Configuration for {work_pool_name}[/bold]",
+                expand=False,
+            )
+
+            app.console.print(panel)
+
+        except ObjectNotFound:
+            exit_with_error(f"Work pool {work_pool_name!r} does not exist.")
+
+
+work_pool_storage_configure_app: PrefectTyper = PrefectTyper(
+    name="configure", help="EXPERIMENTAL: Configure work pool storage."
+)
+work_pool_storage_app.add_typer(work_pool_storage_configure_app)
+
+
+@work_pool_storage_configure_app.command()
+async def s3(
+    work_pool_name: Annotated[
+        str,
+        typer.Argument(
+            ...,
+            help="The name of the work pool to configure storage for.",
+            show_default=False,
+        ),
+    ],
+    bucket: str = typer.Option(
+        ...,
+        "--bucket",
+        help="The name of the S3 bucket to use.",
+        show_default=False,
+        prompt="Enter the name of the S3 bucket to use",
+    ),
+    credentials_block_name: str = typer.Option(
+        ...,
+        "--aws-credentials-block-name",
+        help="The name of the credentials block to use.",
+        show_default=False,
+        prompt="Enter the name of the AWS credentials block to use",
+    ),
+):
+    """
+    EXPERIMENTAL: Configure AWS S3 storage for a work pool.
+
+    \b
+    Examples:
+        $ prefect work-pool storage configure s3 "my-pool" --bucket my-bucket --credentials-block-name my-credentials
+    """
+    # TODO: Allow passing in AWS keys and creating a block for the user.
+    async with get_client() as client:
+        try:
+            await client.read_block_document_by_name(
+                name=credentials_block_name, block_type_slug="aws-credentials"
+            )
+        except ObjectNotFound:
+            exit_with_error(
+                f"AWS credentials block {credentials_block_name!r} does not exist. Please create one using `prefect block create aws-credentials`."
+            )
+
+        try:
+            await client.update_work_pool(
+                work_pool_name=work_pool_name,
+                work_pool=WorkPoolUpdate(
+                    storage_configuration=WorkPoolStorageConfiguration(
+                        bundle_upload_step={
+                            "prefect_aws.experimental.bundles.upload": {
+                                "requires": "prefect-aws",
+                                "bucket": bucket,
+                                "aws_credentials_block_name": credentials_block_name,
+                            }
+                        },
+                        bundle_execution_step={
+                            "prefect_aws.experimental.bundles.execute": {
+                                "requires": "prefect-aws",
+                                "bucket": bucket,
+                                "aws_credentials_block_name": credentials_block_name,
+                            }
+                        },
+                    ),
+                ),
+            )
+        except ObjectNotFound:
+            exit_with_error(f"Work pool {work_pool_name!r} does not exist.")
+
+        exit_with_success(f"Configured S3 storage for work pool {work_pool_name!r}")

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/client/orchestration/_flow_runs/client.py

@@ -46,6 +46,9 @@ class FlowRunClient(BaseClient):
         tags: "Iterable[str] | None" = None,
         parent_task_run_id: "UUID | None" = None,
         state: "State[R] | None" = None,
+        work_pool_name: str | None = None,
+        work_queue_name: str | None = None,
+        job_variables: dict[str, Any] | None = None,
     ) -> "FlowRun":
         """
         Create a flow run for a flow.
@@ -59,7 +62,10 @@ class FlowRunClient(BaseClient):
             parent_task_run_id: if a subflow run is being created, the placeholder task
                 run identifier in the parent flow
             state: The initial state for the run. If not provided, defaults to
-                `Scheduled` for now. Should always be a `Scheduled` type.
+                `Pending`.
+            work_pool_name: The name of the work pool to run the flow run in.
+            work_queue_name: The name of the work queue to place the flow run in.
+            job_variables: The job variables to use when setting up flow run infrastructure.
 
         Raises:
             httpx.RequestError: if the Prefect API does not successfully create a run for any reason
@@ -100,7 +106,16 @@ class FlowRunClient(BaseClient):
             ),
         )
 
-        flow_run_create_json = flow_run_create.model_dump(mode="json")
+        if work_pool_name is not None:
+            flow_run_create.work_pool_name = work_pool_name
+        if work_queue_name is not None:
+            flow_run_create.work_queue_name = work_queue_name
+        if job_variables is not None:
+            flow_run_create.job_variables = job_variables
+
+        flow_run_create_json = flow_run_create.model_dump(
+            mode="json", exclude_unset=True
+        )
         response = self.request("POST", "/flow_runs/", json=flow_run_create_json)
 
         flow_run = FlowRun.model_validate(response.json())
@@ -480,6 +495,9 @@ class FlowRunAsyncClient(BaseAsyncClient):
         tags: "Iterable[str] | None" = None,
         parent_task_run_id: "UUID | None" = None,
         state: "State[R] | None" = None,
+        work_pool_name: str | None = None,
+        work_queue_name: str | None = None,
+        job_variables: dict[str, Any] | None = None,
     ) -> "FlowRun":
         """
         Create a flow run for a flow.
@@ -493,7 +511,10 @@ class FlowRunAsyncClient(BaseAsyncClient):
             parent_task_run_id: if a subflow run is being created, the placeholder task
                 run identifier in the parent flow
             state: The initial state for the run. If not provided, defaults to
-                `Scheduled` for now. Should always be a `Scheduled` type.
+                `Pending`.
+            work_pool_name: The name of the work pool to run the flow run in.
+            work_queue_name: The name of the work queue to place the flow run in.
+            job_variables: The job variables to use when setting up flow run infrastructure.
 
         Raises:
             httpx.RequestError: if the Prefect API does not successfully create a run for any reason
@@ -534,7 +555,16 @@ class FlowRunAsyncClient(BaseAsyncClient):
             ),
         )
 
-        flow_run_create_json = flow_run_create.model_dump(mode="json")
+        if work_pool_name is not None:
+            flow_run_create.work_pool_name = work_pool_name
+        if work_queue_name is not None:
+            flow_run_create.work_queue_name = work_queue_name
+        if job_variables is not None:
+            flow_run_create.job_variables = job_variables
+
+        flow_run_create_json = flow_run_create.model_dump(
+            mode="json", exclude_unset=True
+        )
         response = await self.request("POST", "/flow_runs/", json=flow_run_create_json)
 
         flow_run = FlowRun.model_validate(response.json())

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/client/schemas/actions.py

@@ -23,6 +23,7 @@ from prefect._internal.schemas.validators import (
 from prefect.client.schemas.objects import (
     StateDetails,
     StateType,
+    WorkPoolStorageConfiguration,
 )
 from prefect.client.schemas.schedules import (
     SCHEDULE_TYPES,
@@ -443,6 +444,9 @@ class FlowRunCreate(ActionBaseModel):
     idempotency_key: Optional[str] = Field(default=None)
 
     labels: KeyValueLabelsField = Field(default_factory=dict)
+    work_pool_name: Optional[str] = Field(default=None)
+    work_queue_name: Optional[str] = Field(default=None)
+    job_variables: Optional[dict[str, Any]] = Field(default=None)
 
 
 class DeploymentFlowRunCreate(ActionBaseModel):
@@ -684,6 +688,10 @@ class WorkPoolCreate(ActionBaseModel):
     concurrency_limit: Optional[NonNegativeInteger] = Field(
         default=None, description="A concurrency limit for the work pool."
     )
+    storage_configuration: WorkPoolStorageConfiguration = Field(
+        default_factory=WorkPoolStorageConfiguration,
+        description="A storage configuration for the work pool.",
+    )
 
 
 class WorkPoolUpdate(ActionBaseModel):
@@ -693,6 +701,10 @@ class WorkPoolUpdate(ActionBaseModel):
     is_paused: Optional[bool] = Field(default=None)
     base_job_template: Optional[dict[str, Any]] = Field(default=None)
     concurrency_limit: Optional[int] = Field(default=None)
+    storage_configuration: Optional[WorkPoolStorageConfiguration] = Field(
+        default=None,
+        description="A storage configuration for the work pool.",
+    )
 
 
 class WorkQueueCreate(ActionBaseModel):

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/client/schemas/objects.py

@@ -1445,6 +1445,19 @@ class Agent(ObjectBaseModel):
     )
 
 
+class WorkPoolStorageConfiguration(PrefectBaseModel):
+    """A work pool storage configuration"""
+
+    model_config: ClassVar[ConfigDict] = ConfigDict(extra="forbid")
+
+    bundle_upload_step: Optional[dict[str, Any]] = Field(
+        default=None, description="The bundle upload step for the work pool."
+    )
+    bundle_execution_step: Optional[dict[str, Any]] = Field(
+        default=None, description="The bundle execution step for the work pool."
+    )
+
+
 class WorkPool(ObjectBaseModel):
     """An ORM representation of a work pool"""
 
@@ -1469,6 +1482,11 @@ class WorkPool(ObjectBaseModel):
         default=None, description="The current status of the work pool."
     )
 
+    storage_configuration: WorkPoolStorageConfiguration = Field(
+        default_factory=WorkPoolStorageConfiguration,
+        description="The storage configuration for the work pool.",
+    )
+
     # this required field has a default of None so that the custom validator
     # below will be called and produce a more helpful error message. Because
     # the field metadata is attached via an annotation, the default is hidden

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/deployments/runner.py

@@ -676,15 +676,7 @@ class RunnerDeployment(BaseModel):
                     try:
                         module = importlib.import_module(mod_name)
                         flow_file = getattr(module, "__file__", None)
-                    except ModuleNotFoundError as exc:
-                        # 16458 adds an identifier to the module name, so checking
-                        # for "__prefect_loader__" (2 underscores) will not match
-                        if "__prefect_loader_" in str(exc):
-                            raise ValueError(
-                                "Cannot create a RunnerDeployment from a flow that has been"
-                                " loaded from an entrypoint. To deploy a flow via"
-                                " entrypoint, use RunnerDeployment.from_entrypoint instead."
-                            )
+                    except ModuleNotFoundError:
                         raise ValueError(no_file_location_error)
                     if not flow_file:
                         raise ValueError(no_file_location_error)

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/docker/docker_image.py

@@ -64,7 +64,8 @@ class DockerImage:
     def build(self) -> None:
         full_image_name = self.reference
         build_kwargs = self.build_kwargs.copy()
-        build_kwargs["context"] = Path.cwd()
+        if "context" not in build_kwargs:
+            build_kwargs["context"] = Path.cwd()
         build_kwargs["tag"] = full_image_name
         build_kwargs["pull"] = build_kwargs.get("pull", True)
 

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/results.py

@@ -552,7 +552,11 @@ class ResultStore(BaseModel):
         if self.result_storage_block_id is None and (
             _resolve_path := getattr(self.result_storage, "_resolve_path", None)
         ):
-            return str(_resolve_path(key))
+            path_key = _resolve_path(key)
+            if path_key is not None:
+                return str(_resolve_path(key))
+            else:
+                return key
         return key
 
     @sync_compatible
@@ -684,7 +688,9 @@ class ResultStore(BaseModel):
 
         if self.result_storage_block_id is None:
             if _resolve_path := getattr(self.result_storage, "_resolve_path", None):
-                key = str(_resolve_path(key))
+                path_key = _resolve_path(key)
+                if path_key is not None:
+                    key = str(_resolve_path(key))
 
         return ResultRecord(
             result=obj,
@@ -773,7 +779,7 @@ class ResultStore(BaseModel):
                 )
                 else Path(".").resolve()
             )
-            base_key = str(Path(key).relative_to(basepath))
+            base_key = key if basepath is None else str(Path(key).relative_to(basepath))
         else:
             base_key = key
         if (

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/server/api/collections_data/views/aggregate-worker-metadata.json

@@ -559,7 +559,7 @@
               "description": "To use any private container registry with a username and password, choose DockerRegistry. To use a private Azure Container Registry with a managed identity, choose ACRManagedIdentity.",
               "anyOf": [
                 {
-                  "$ref": "#/definitions/DockerRegistry"
+                  "$ref": "#/definitions/DockerRegistryCredentials"
                 },
                 {
                   "$ref": "#/definitions/ACRManagedIdentity"
@@ -637,8 +637,8 @@
             "subscription_id"
           ],
           "definitions": {
-            "DockerRegistry": {
-              "title": "DockerRegistry",
+            "DockerRegistryCredentials": {
+              "title": "DockerRegistryCredentials",
               "description": "Connects to a Docker registry.\n\nRequires a Docker Engine to be connectable.",
               "type": "object",
               "properties": {
@@ -671,7 +671,7 @@
                 "password",
                 "registry_url"
               ],
-              "block_type_slug": "docker-registry",
+              "block_type_slug": "docker-registry-credentials",
               "secret_fields": [
                 "password"
               ],

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/server/api/flow_runs.py

@@ -29,6 +29,7 @@ import prefect.server.schemas as schemas
 from prefect.logging import get_logger
 from prefect.server.api.run_history import run_history
 from prefect.server.api.validation import validate_job_variables_for_deployment_flow_run
+from prefect.server.api.workers import WorkerLookups
 from prefect.server.database import PrefectDBInterface, provide_database_interface
 from prefect.server.exceptions import FlowRunGraphTooLarge
 from prefect.server.models.flow_runs import (
@@ -68,6 +69,7 @@ async def create_flow_run(
         orchestration_dependencies.provide_flow_orchestration_parameters
     ),
     api_version: str = Depends(dependencies.provide_request_api_version),
+    worker_lookups: WorkerLookups = Depends(WorkerLookups),
 ) -> schemas.responses.FlowRunResponse:
     """
     Create a flow run. If a flow run with the same flow_id and
@@ -91,6 +93,25 @@ async def create_flow_run(
     right_now = now("UTC")
 
     async with db.session_context(begin_transaction=True) as session:
+        if flow_run.work_pool_name:
+            if flow_run.work_queue_name:
+                work_queue_id = await worker_lookups._get_work_queue_id_from_name(
+                    session=session,
+                    work_pool_name=flow_run.work_pool_name,
+                    work_queue_name=flow_run.work_queue_name,
+                )
+            else:
+                work_queue_id = (
+                    await worker_lookups._get_default_work_queue_id_from_work_pool_name(
+                        session=session,
+                        work_pool_name=flow_run.work_pool_name,
+                    )
+                )
+        else:
+            work_queue_id = None
+
+        flow_run_object.work_queue_id = work_queue_id
+
         model = await models.flow_runs.create_flow_run(
             session=session,
             flow_run=flow_run_object,

{prefect-3.2.14.dev2 → prefect-3.2.15.dev1}/src/prefect/server/database/_migrations/MIGRATION-NOTES.md

@@ -8,6 +8,14 @@ Each time a database migration is written, an entry is included here with:
 
 This gives us a history of changes and will create merge conflicts if two migrations are made at once, flagging situations where a branch needs to be updated before merging.
 
+# Add `deployment_version` table
+SQLite: `bbca16f6f218`
+Postgres: `06b7c293bc69`
+
+# Add `storage_configuration` to work pool
+SQLite: `3457c6ca2360`
+Postgres: `b5f5644500d2`
+
 # Add parameters column to deployment schedules
 SQLite: `67f886da208e`
 Postgres: `c163acd7e8e3`

prefect-3.2.15.dev1/src/prefect/server/database/_migrations/versions/postgresql/2025_03_19_143502_b5f5644500d2_add_storage_configuration_to_work_pool.py

@@ -0,0 +1,36 @@
+"""add storage configuration to work pool
+
+Revision ID: b5f5644500d2
+Revises: a03d00b8e275
+Create Date: 2025-03-19 14:35:02.405170
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+import prefect
+
+# revision identifiers, used by Alembic.
+revision = "b5f5644500d2"
+down_revision = "a03d00b8e275"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "work_pool",
+        sa.Column(
+            "storage_configuration",
+            prefect.server.utilities.database.Pydantic(
+                prefect.server.schemas.core.WorkPoolStorageConfiguration
+            ),
+            server_default="{}",
+            nullable=False,
+        ),
+    )
+
+
+def downgrade():
+    op.drop_column("work_pool", "storage_configuration")