prefect 3.1.7.dev1__tar.gz → 3.1.7.dev3__tar.gz

{prefect-3.1.7.dev1/src/prefect.egg-info → prefect-3.1.7.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect
-Version: 3.1.7.dev1
+Version: 3.1.7.dev3
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.
@@ -66,6 +66,7 @@ Requires-Dist: pydantic_extra_types<3.0.0,>=2.8.2
 Requires-Dist: pydantic_settings>2.2.1
 Requires-Dist: python_dateutil<3.0.0,>=2.8.2
 Requires-Dist: python-slugify<9.0,>=5.0
+Requires-Dist: python-socks[asyncio]<3.0,>=2.5.3
 Requires-Dist: pyyaml<7.0.0,>=5.4.1
 Requires-Dist: rfc3339-validator<0.2.0,>=0.1.4
 Requires-Dist: rich<14.0,>=11.0

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/requirements-client.txt

@@ -26,6 +26,7 @@ pydantic_extra_types >= 2.8.2, < 3.0.0
 pydantic_settings > 2.2.1
 python_dateutil >= 2.8.2, < 3.0.0
 python-slugify >= 5.0, < 9.0
+python-socks[asyncio] >= 2.5.3, < 3.0
 pyyaml >= 5.4.1, < 7.0.0
 rfc3339-validator >= 0.1.4, < 0.2.0
 rich >= 11.0, < 14.0

prefect-3.1.7.dev3/src/prefect/_experimental/lineage.py

@@ -0,0 +1,181 @@
+from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Sequence, Union
+
+from prefect.events.related import related_resources_from_run_context
+from prefect.events.schemas.events import RelatedResource, Resource
+from prefect.events.utilities import emit_event
+from prefect.settings import get_current_settings
+
+if TYPE_CHECKING:
+    from prefect.results import ResultStore
+
+UpstreamResources = Sequence[Union[RelatedResource, dict[str, str]]]
+DownstreamResources = Sequence[Union[Resource, dict[str, str]]]
+
+# Map block types to their URI schemes
+STORAGE_URI_SCHEMES = {
+    "local-file-system": "file://{path}",
+    "s3-bucket": "s3://{storage.bucket_name}/{path}",
+    "gcs-bucket": "gs://{storage.bucket}/{path}",
+    "azure-blob-storage": "azure-blob://{storage.container_name}/{path}",
+}
+
+
+def get_result_resource_uri(
+    store: "ResultStore",
+    key: str,
+) -> Optional[str]:
+    """
+    Generate a URI for a result based on its storage backend.
+
+    Args:
+        store: A `ResultStore` instance.
+        key: The key of the result to generate a URI for.
+    """
+    storage = store.result_storage
+    if storage is None:
+        return
+
+    path = store._resolved_key_path(key)
+
+    block_type = storage.get_block_type_slug()
+    if block_type and block_type in STORAGE_URI_SCHEMES:
+        return STORAGE_URI_SCHEMES[block_type].format(storage=storage, path=path)
+
+    # Generic fallback
+    return f"prefect://{block_type}/{path}"
+
+
+async def emit_lineage_event(
+    event_name: str,
+    upstream_resources: Optional[UpstreamResources] = None,
+    downstream_resources: Optional[DownstreamResources] = None,
+    direction_of_run_from_event: Literal["upstream", "downstream"] = "downstream",
+) -> None:
+    """Emit lineage events showing relationships between resources.
+
+    Args:
+        event_name: The name of the event to emit
+        upstream_resources: Optional list of RelatedResources that were upstream of
+            the event
+        downstream_resources: Optional list of Resources that were downstream
+            of the event
+        direction_of_run_from_event: The direction of the current run from
+            the event. E.g., if we're in a flow run and
+            `direction_of_run_from_event` is "downstream", then the flow run is
+            considered downstream of the resource's event.
+    """
+    from prefect.client.orchestration import get_client  # Avoid a circular import
+
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    upstream_resources = list(upstream_resources) if upstream_resources else []
+    downstream_resources = list(downstream_resources) if downstream_resources else []
+
+    async with get_client() as client:
+        related_resources = await related_resources_from_run_context(client)
+
+    # NOTE: We handle adding run-related resources to the event here instead of in
+    # the EventsWorker because not all run-related resources are upstream from
+    # every lineage event (they might be downstream). The EventsWorker only adds
+    # related resources to the "related" field in the event, which, for
+    # lineage-related events, tracks upstream resources only. For downstream
+    # resources, we need to emit an event for each downstream resource.
+    if direction_of_run_from_event == "downstream":
+        downstream_resources.extend(related_resources)
+    else:
+        upstream_resources.extend(related_resources)
+
+    # Emit an event for each downstream resource. This is necessary because
+    # our event schema allows one primary resource and many related resources,
+    # and for the purposes of lineage, related resources can only represent
+    # upstream resources.
+    for resource in downstream_resources:
+        # Downstream lineage resources need to have the
+        # prefect.resource.lineage-group label. All upstram resources from a
+        # downstream resource with this label will be considered lineage-related
+        # resources.
+        if "prefect.resource.lineage-group" not in resource:
+            resource["prefect.resource.lineage-group"] = "global"
+
+        emit_kwargs: Dict[str, Any] = {
+            "event": event_name,
+            "resource": resource,
+            "related": upstream_resources,
+        }
+
+        emit_event(**emit_kwargs)
+
+
+async def emit_result_read_event(
+    store: "ResultStore",
+    result_key: str,
+    downstream_resources: Optional[DownstreamResources] = None,
+    cached: bool = False,
+) -> None:
+    """
+    Emit a lineage event showing a task or flow result was read.
+
+    Args:
+        store: A `ResultStore` instance.
+        result_key: The key of the result to generate a URI for.
+        downstream_resources: List of resources that were
+            downstream of the event's resource.
+    """
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    result_resource_uri = get_result_resource_uri(store, result_key)
+    if result_resource_uri:
+        upstream_resources = [
+            RelatedResource(
+                root={
+                    "prefect.resource.id": result_resource_uri,
+                    "prefect.resource.role": "result",
+                }
+            )
+        ]
+        event_name = "prefect.result.read"
+        if cached:
+            event_name += ".cached"
+
+        await emit_lineage_event(
+            event_name=event_name,
+            upstream_resources=upstream_resources,
+            downstream_resources=downstream_resources,
+            direction_of_run_from_event="downstream",
+        )
+
+
+async def emit_result_write_event(
+    store: "ResultStore",
+    result_key: str,
+    upstream_resources: Optional[UpstreamResources] = None,
+) -> None:
+    """
+    Emit a lineage event showing a task or flow result was written.
+
+    Args:
+        store: A `ResultStore` instance.
+        result_key: The key of the result to generate a URI for.
+        upstream_resources: Optional list of resources that were
+            upstream of the event's resource.
+    """
+    if not get_current_settings().experiments.lineage_events_enabled:
+        return
+
+    result_resource_uri = get_result_resource_uri(store, result_key)
+    if result_resource_uri:
+        downstream_resources = [
+            {
+                "prefect.resource.id": result_resource_uri,
+                "prefect.resource.role": "result",
+                "prefect.resource.lineage-group": "global",
+            }
+        ]
+        await emit_lineage_event(
+            event_name="prefect.result.write",
+            upstream_resources=upstream_resources,
+            downstream_resources=downstream_resources,
+            direction_of_run_from_event="upstream",
+        )

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/_internal/compatibility/async_dispatch.py

@@ -40,7 +40,9 @@ def is_in_async_context() -> bool:
             return False
 
 
-def _is_acceptable_callable(obj: Union[Callable, "Task", classmethod]) -> bool:
+def _is_acceptable_callable(
+    obj: Union[Callable[P, R], "Task[P, R]", classmethod],
+) -> bool:
     if inspect.iscoroutinefunction(obj):
         return True
 

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/_internal/pydantic/v2_validated_func.py

@@ -6,16 +6,17 @@ Specifically it allows for us to validate v2 models used as flow/task
 arguments.
 """
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union
+from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
 
 # importing directly from v2 to be able to create a v2 model
 from pydantic import BaseModel, ConfigDict, create_model, field_validator
 from pydantic.v1.decorator import ValidatedFunction
 from pydantic.v1.errors import ConfigError
 from pydantic.v1.utils import to_camel
+from typing_extensions import TypeAlias
 
 if TYPE_CHECKING:
-    ConfigType = Union[None, Type[Any], Dict[str, Any]]
+    ConfigType: TypeAlias = Union[None, type[Any], dict[str, Any]]
 
 V_POSITIONAL_ONLY_NAME = "v__positional_only"
 V_DUPLICATE_KWARGS = "v__duplicate_kwargs"
@@ -24,13 +25,17 @@ V_DUPLICATE_KWARGS = "v__duplicate_kwargs"
 class V2ValidatedFunction(ValidatedFunction):
     def create_model(
         self,
-        fields: Dict[str, Any],
+        fields: dict[str, Any],
         takes_args: bool,
         takes_kwargs: bool,
-        config: ConfigDict,
+        config: "ConfigType",
     ) -> None:
         pos_args = len(self.arg_mapping)
 
+        config = {} if config is None else config
+        if not isinstance(config, dict):
+            raise TypeError(f"config must be None or a dict, got {type(config)}")
+
         if config.get("fields") or config.get("alias_generator"):
             raise ConfigError(
                 'Setting the "fields" and "alias_generator" property on custom Config'
@@ -42,11 +47,11 @@ class V2ValidatedFunction(ValidatedFunction):
 
         # This is the key change -- inheriting the BaseModel class from v2
         class DecoratorBaseModel(BaseModel):
-            model_config = config
+            model_config: ClassVar[ConfigDict] = ConfigDict(**config)
 
             @field_validator(self.v_args_name, check_fields=False)
             @classmethod
-            def check_args(cls, v: Optional[List[Any]]) -> Optional[List[Any]]:
+            def check_args(cls, v: Optional[list[Any]]) -> Optional[list[Any]]:
                 if takes_args or v is None:
                     return v
 
@@ -58,8 +63,8 @@ class V2ValidatedFunction(ValidatedFunction):
             @field_validator(self.v_kwargs_name, check_fields=False)
             @classmethod
             def check_kwargs(
-                cls, v: Optional[Dict[str, Any]]
-            ) -> Optional[Dict[str, Any]]:
+                cls, v: Optional[dict[str, Any]]
+            ) -> Optional[dict[str, Any]]:
                 if takes_kwargs or v is None:
                     return v
 
@@ -69,7 +74,7 @@ class V2ValidatedFunction(ValidatedFunction):
 
             @field_validator(V_POSITIONAL_ONLY_NAME, check_fields=False)
             @classmethod
-            def check_positional_only(cls, v: Optional[List[str]]) -> None:
+            def check_positional_only(cls, v: Optional[list[str]]) -> None:
                 if v is None:
                     return
 
@@ -82,7 +87,7 @@ class V2ValidatedFunction(ValidatedFunction):
 
             @field_validator(V_DUPLICATE_KWARGS, check_fields=False)
             @classmethod
-            def check_duplicate_kwargs(cls, v: Optional[List[str]]) -> None:
+            def check_duplicate_kwargs(cls, v: Optional[list[str]]) -> None:
                 if v is None:
                     return
 

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/_internal/schemas/bases.py

@@ -13,9 +13,10 @@ from pydantic import (
     ConfigDict,
     Field,
 )
-from pydantic_extra_types.pendulum_dt import DateTime
 from typing_extensions import Self
 
+from prefect.types import DateTime
+
 T = TypeVar("T")
 
 

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/_internal/schemas/validators.py

@@ -18,9 +18,9 @@ from uuid import UUID
 import jsonschema
 import pendulum
 import yaml
-from pydantic_extra_types.pendulum_dt import DateTime
 
 from prefect.exceptions import InvalidRepositoryURLError
+from prefect.types import DateTime
 from prefect.utilities.collections import isiterable
 from prefect.utilities.dockerutils import get_prefect_image_name
 from prefect.utilities.filesystem import relative_path_to_current_platform

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-12-11T18:49:03-0500",
+ "date": "2024-12-13T20:55:38-0600",
  "dirty": false,
  "error": null,
- "full-revisionid": "089bcec85e3ec5b17fbdd0602b9ce50ec127e482",
- "version": "3.1.7.dev1"
+ "full-revisionid": "df9d64f2d2f08597d7ff00700b6bfd233634b273",
+ "version": "3.1.7.dev3"
 }
 '''  # END VERSION_JSON
 

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/blocks/core.py

@@ -281,7 +281,7 @@ class Block(BaseModel, ABC):
         json_schema_extra=schema_extra,
     )
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
         self.block_initialization()
 
@@ -629,7 +629,8 @@ class Block(BaseModel, ABC):
         """Generates a default code example for the current class"""
         qualified_name = to_qualified_name(cls)
         module_str = ".".join(qualified_name.split(".")[:-1])
-        class_name = cls.__name__
+        origin = cls.__pydantic_generic_metadata__.get("origin") or cls
+        class_name = origin.__name__
         block_variable_name = f'{cls.get_block_type_slug().replace("-", "_")}_block'
 
         return dedent(

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/blocks/system.py

@@ -9,10 +9,10 @@ from pydantic import (
     field_validator,
 )
 from pydantic import Secret as PydanticSecret
-from pydantic_extra_types.pendulum_dt import DateTime as PydanticDateTime
 
 from prefect._internal.compatibility.deprecated import deprecated_class
 from prefect.blocks.core import Block
+from prefect.types import DateTime as PydanticDateTime
 
 _SecretValueType = Union[
     Annotated[StrictStr, Field(title="string")],
@@ -130,6 +130,7 @@ class Secret(Block, Generic[T]):
 
     _logo_url = "https://cdn.sanity.io/images/3ugk85nk/production/c6f20e556dd16effda9df16551feecfb5822092b-48x48.png"
     _documentation_url = "https://docs.prefect.io/latest/develop/blocks"
+    _description = "A block that represents a secret value. The value stored in this block will be obfuscated when this block is viewed or edited in the UI."
 
     value: Union[SecretStr, PydanticSecret[T]] = Field(
         default=...,

{prefect-3.1.7.dev1 → prefect-3.1.7.dev3}/src/prefect/cli/global_concurrency_limit.py

@@ -150,14 +150,16 @@ async def delete_global_concurrency_limit(
     Arguments:
         name (str): The name of the global concurrency limit to delete.
     """
-    if is_interactive() and not typer.confirm(
-        f"Are you sure you want to delete global concurrency limit with name {name!r}?",
-        default=False,
-    ):
-        exit_with_error("Deletion aborted.")
-
     async with get_client() as client:
         try:
+            gcl_limit = await client.read_global_concurrency_limit_by_name(name=name)
+
+            if is_interactive() and not typer.confirm(
+                f"Are you sure you want to delete global concurrency limit with name {gcl_limit.name!r}?",
+                default=False,
+            ):
+                exit_with_error("Deletion aborted.")
+
             await client.delete_global_concurrency_limit_by_name(name=name)
         except ObjectNotFound:
             exit_with_error(f"Global concurrency limit {name!r} not found.")

prefect-3.1.7.dev3/src/prefect/client/_adapters.py

@@ -0,0 +1,123 @@
+"""Creating type adapters to avoid recreation of the pydantic core schemas when possible."""
+
+from typing import TYPE_CHECKING
+
+from pydantic import ConfigDict, TypeAdapter
+
+if TYPE_CHECKING:
+    from prefect.client.schemas.objects import (
+        Artifact,
+        ArtifactCollection,
+        BlockDocument,
+        BlockSchema,
+        BlockType,
+        ConcurrencyLimit,
+        DeploymentSchedule,
+        Flow,
+        FlowRun,
+        FlowRunInput,
+        FlowRunNotificationPolicy,
+        GlobalConcurrencyLimit,
+        Log,
+        State,
+        TaskRun,
+        Variable,
+        Worker,
+        WorkPool,
+        WorkQueue,
+    )
+    from prefect.client.schemas.responses import (
+        DeploymentResponse,
+        FlowRunResponse,
+        GlobalConcurrencyLimitResponse,
+        WorkerFlowRunResponse,
+    )
+    from prefect.events.schemas.automations import Automation
+
+defer_build_cfg = ConfigDict(defer_build=True)
+
+# Create the adapters with forward refs
+BlockTypeAdapter: TypeAdapter["BlockType"] = TypeAdapter(
+    "BlockType", config=defer_build_cfg
+)
+BlockSchemaAdapter: TypeAdapter[list["BlockSchema"]] = TypeAdapter(
+    list["BlockSchema"], config=defer_build_cfg
+)
+ConcurrencyLimitAdapter: TypeAdapter["ConcurrencyLimit"] = TypeAdapter(
+    "ConcurrencyLimit", config=defer_build_cfg
+)
+ConcurrencyLimitListAdapter: TypeAdapter[list["ConcurrencyLimit"]] = TypeAdapter(
+    list["ConcurrencyLimit"], config=defer_build_cfg
+)
+FlowAdapter: TypeAdapter["Flow"] = TypeAdapter("Flow", config=defer_build_cfg)
+FlowRunAdapter: TypeAdapter["FlowRun"] = TypeAdapter("FlowRun", config=defer_build_cfg)
+ArtifactListAdapter: TypeAdapter[list["Artifact"]] = TypeAdapter(
+    list["Artifact"], config=defer_build_cfg
+)
+ArtifactCollectionListAdapter: TypeAdapter[list["ArtifactCollection"]] = TypeAdapter(
+    list["ArtifactCollection"], config=defer_build_cfg
+)
+AutomationListAdapter: TypeAdapter[list["Automation"]] = TypeAdapter(
+    list["Automation"], config=defer_build_cfg
+)
+BlockDocumentListAdapter: TypeAdapter[list["BlockDocument"]] = TypeAdapter(
+    list["BlockDocument"], config=defer_build_cfg
+)
+BlockSchemaListAdapter: TypeAdapter[list["BlockSchema"]] = TypeAdapter(
+    list["BlockSchema"], config=defer_build_cfg
+)
+BlockTypeListAdapter: TypeAdapter[list["BlockType"]] = TypeAdapter(
+    list["BlockType"], config=defer_build_cfg
+)
+DeploymentResponseListAdapter: TypeAdapter[list["DeploymentResponse"]] = TypeAdapter(
+    list["DeploymentResponse"], config=defer_build_cfg
+)
+DeploymentScheduleListAdapter: TypeAdapter[list["DeploymentSchedule"]] = TypeAdapter(
+    list["DeploymentSchedule"], config=defer_build_cfg
+)
+FlowListAdapter: TypeAdapter[list["Flow"]] = TypeAdapter(
+    list["Flow"], config=defer_build_cfg
+)
+FlowRunListAdapter: TypeAdapter[list["FlowRun"]] = TypeAdapter(
+    list["FlowRun"], config=defer_build_cfg
+)
+FlowRunInputListAdapter: TypeAdapter[list["FlowRunInput"]] = TypeAdapter(
+    list["FlowRunInput"], config=defer_build_cfg
+)
+FlowRunNotificationPolicyListAdapter: TypeAdapter[
+    list["FlowRunNotificationPolicy"]
+] = TypeAdapter(list["FlowRunNotificationPolicy"], config=defer_build_cfg)
+FlowRunResponseListAdapter: TypeAdapter[list["FlowRunResponse"]] = TypeAdapter(
+    list["FlowRunResponse"], config=defer_build_cfg
+)
+GlobalConcurrencyLimitListAdapter = TypeAdapter(
+    list["GlobalConcurrencyLimit"], config=defer_build_cfg
+)
+GlobalConcurrencyLimitResponseListAdapter: TypeAdapter[
+    list["GlobalConcurrencyLimitResponse"]
+] = TypeAdapter(list["GlobalConcurrencyLimitResponse"], config=defer_build_cfg)
+LogListAdapter: TypeAdapter[list["Log"]] = TypeAdapter(
+    list["Log"], config=defer_build_cfg
+)
+StateListAdapter: TypeAdapter[list["State"]] = TypeAdapter(
+    list["State"], config=defer_build_cfg
+)
+TaskRunListAdapter: TypeAdapter[list["TaskRun"]] = TypeAdapter(
+    list["TaskRun"], config=defer_build_cfg
+)
+WorkerListAdapter: TypeAdapter[list["Worker"]] = TypeAdapter(
+    list["Worker"], config=defer_build_cfg
+)
+WorkPoolListAdapter: TypeAdapter[list["WorkPool"]] = TypeAdapter(
+    list["WorkPool"], config=defer_build_cfg
+)
+WorkQueueListAdapter: TypeAdapter[list["WorkQueue"]] = TypeAdapter(
+    list["WorkQueue"], config=defer_build_cfg
+)
+WorkerFlowRunResponseListAdapter: TypeAdapter[
+    list["WorkerFlowRunResponse"]
+] = TypeAdapter(list["WorkerFlowRunResponse"], config=defer_build_cfg)
+VariableListAdapter: TypeAdapter[list["Variable"]] = TypeAdapter(
+    list["Variable"], config=defer_build_cfg
+)
+StateAdapter: TypeAdapter["State"] = TypeAdapter("State", config=defer_build_cfg)