prefect-client 3.0.0rc13__py3-none-any.whl → 3.0.0rc15__py3-none-any.whl

prefect/task_runs.py

@@ -2,7 +2,7 @@ import asyncio
 import atexit
 import threading
 import uuid
-from typing import Dict, Optional
+from typing import Callable, Dict, Optional
 
 import anyio
 from cachetools import TTLCache
@@ -74,6 +74,7 @@ class TaskRunWaiter:
             maxsize=10000, ttl=600
         )
         self._completion_events: Dict[uuid.UUID, asyncio.Event] = {}
+        self._completion_callbacks: Dict[uuid.UUID, Callable] = {}
         self._loop: Optional[asyncio.AbstractEventLoop] = None
         self._observed_completed_task_runs_lock = threading.Lock()
         self._completion_events_lock = threading.Lock()
@@ -135,6 +136,8 @@ class TaskRunWaiter:
                         # so the waiter can wake up the waiting coroutine
                         if task_run_id in self._completion_events:
                             self._completion_events[task_run_id].set()
+                        if task_run_id in self._completion_callbacks:
+                            self._completion_callbacks[task_run_id]()
                 except Exception as exc:
                     self.logger.error(f"Error processing event: {exc}")
 
@@ -195,6 +198,26 @@ class TaskRunWaiter:
                 # Remove the event from the cache after it has been waited on
                 instance._completion_events.pop(task_run_id, None)
 
+    @classmethod
+    def add_done_callback(cls, task_run_id: uuid.UUID, callback):
+        """
+        Add a callback to be called when a task run finishes.
+
+        Args:
+            task_run_id: The ID of the task run to wait for.
+            callback: The callback to call when the task run finishes.
+        """
+        instance = cls.instance()
+        with instance._observed_completed_task_runs_lock:
+            if task_run_id in instance._observed_completed_task_runs:
+                callback()
+                return
+
+        with instance._completion_events_lock:
+            # Cache the event for the task run ID so the consumer can set it
+            # when the event is received
+            instance._completion_callbacks[task_run_id] = callback
+
     @classmethod
     def instance(cls):
         """

prefect/task_worker.py

@@ -38,6 +38,7 @@ from prefect.utilities.annotations import NotSet
 from prefect.utilities.asyncutils import asyncnullcontext, sync_compatible
 from prefect.utilities.engine import emit_task_run_state_change_event, propose_state
 from prefect.utilities.processutils import _register_signal
+from prefect.utilities.services import start_client_metrics_server
 from prefect.utilities.urls import url_for
 
 logger = get_logger("task_worker")
@@ -158,6 +159,8 @@ class TaskWorker:
         """
         _register_signal(signal.SIGTERM, self.handle_sigterm)
 
+        start_client_metrics_server()
+
         async with asyncnullcontext() if self.started else self:
             logger.info("Starting task worker...")
             try:
@@ -290,12 +293,15 @@ class TaskWorker:
                     await self._client._client.delete(f"/task_runs/{task_run.id}")
                 return
 
+        initial_state = task_run.state
+
         if PREFECT_EXPERIMENTAL_ENABLE_CLIENT_SIDE_TASK_ORCHESTRATION:
             new_state = Pending()
             new_state.state_details.deferred = True
             new_state.state_details.task_run_id = task_run.id
             new_state.state_details.flow_run_id = task_run.flow_run_id
             state = new_state
+            task_run.state = state
         else:
             try:
                 new_state = Pending()
@@ -327,7 +333,7 @@ class TaskWorker:
 
         emit_task_run_state_change_event(
             task_run=task_run,
-            initial_state=task_run.state,
+            initial_state=initial_state,
             validated_state=state,
         )
 

prefect/tasks.py

@@ -33,9 +33,6 @@ from uuid import UUID, uuid4
 from typing_extensions import Literal, ParamSpec
 
 import prefect.states
-from prefect._internal.compatibility.deprecated import (
-    deprecated_async_method,
-)
 from prefect.cache_policies import DEFAULT, NONE, CachePolicy
 from prefect.client.orchestration import get_client
 from prefect.client.schemas import TaskRun
@@ -1038,7 +1035,6 @@ class Task(Generic[P, R]):
     ) -> State[T]:
         ...
 
-    @deprecated_async_method
     def submit(
         self,
         *args: Any,
@@ -1203,7 +1199,6 @@ class Task(Generic[P, R]):
     ) -> PrefectFutureList[State[T]]:
         ...
 
-    @deprecated_async_method
     def map(
         self,
         *args: Any,
@@ -1455,6 +1450,15 @@ class Task(Generic[P, R]):
             )
         )  # type: ignore
 
+        from prefect.utilities.engine import emit_task_run_state_change_event
+
+        # emit a `SCHEDULED` event for the task run
+        emit_task_run_state_change_event(
+            task_run=task_run,
+            initial_state=None,
+            validated_state=task_run.state,
+        )
+
         if task_run_url := url_for(task_run):
             logger.info(
                 f"Created task run {task_run.name!r}. View it in the UI at {task_run_url!r}"

prefect/utilities/asyncutils.py

@@ -30,7 +30,6 @@ import anyio.abc
 import anyio.from_thread
 import anyio.to_thread
 import sniffio
-import wrapt
 from typing_extensions import Literal, ParamSpec, TypeGuard
 
 from prefect._internal.concurrency.api import _cast_to_call, from_sync
@@ -210,11 +209,6 @@ def run_coro_as_sync(
     Returns:
         The result of the coroutine if wait_for_result is True, otherwise None.
     """
-    if not asyncio.iscoroutine(coroutine):
-        if isinstance(coroutine, wrapt.ObjectProxy):
-            return coroutine.__wrapped__
-        else:
-            raise TypeError("`coroutine` must be a coroutine object")
 
     async def coroutine_wrapper() -> Union[R, None]:
         """

prefect/utilities/callables.py

@@ -364,17 +364,19 @@ def parameter_schema_from_entrypoint(entrypoint: str) -> ParameterSchema:
     Returns:
         ParameterSchema: The parameter schema for the function.
     """
+    filepath = None
     if ":" in entrypoint:
         # split by the last colon once to handle Windows paths with drive letters i.e C:\path\to\file.py:do_stuff
         path, func_name = entrypoint.rsplit(":", maxsplit=1)
         source_code = Path(path).read_text()
+        filepath = path
     else:
         path, func_name = entrypoint.rsplit(".", maxsplit=1)
         spec = importlib.util.find_spec(path)
         if not spec or not spec.origin:
             raise ValueError(f"Could not find module {path!r}")
         source_code = Path(spec.origin).read_text()
-    signature = _generate_signature_from_source(source_code, func_name)
+    signature = _generate_signature_from_source(source_code, func_name, filepath)
     docstring = _get_docstring_from_source(source_code, func_name)
     return generate_parameter_schema(signature, parameter_docstrings(docstring))
 
@@ -444,7 +446,7 @@ def raise_for_reserved_arguments(fn: Callable, reserved_arguments: Iterable[str]
 
 
 def _generate_signature_from_source(
-    source_code: str, func_name: str
+    source_code: str, func_name: str, filepath: Optional[str] = None
 ) -> inspect.Signature:
     """
     Extract the signature of a function from its source code.
@@ -460,7 +462,7 @@ def _generate_signature_from_source(
     """
     # Load the namespace from the source code. Missing imports and exceptions while
     # loading local class definitions are ignored.
-    namespace = safe_load_namespace(source_code)
+    namespace = safe_load_namespace(source_code, filepath=filepath)
     # Parse the source code into an AST
     parsed_code = ast.parse(source_code)
 

prefect/utilities/engine.py

@@ -783,6 +783,9 @@ def emit_task_run_state_change_event(
                     "state_type",
                     "state_name",
                     "state",
+                    # server materialized fields
+                    "estimated_start_time_delta",
+                    "estimated_run_time",
                 },
             ),
         },

prefect/utilities/importtools.py

@@ -361,79 +361,159 @@ class AliasedModuleLoader(Loader):
         sys.modules[self.alias] = root_module
 
 
-def safe_load_namespace(source_code: str):
+def safe_load_namespace(
+    source_code: str, filepath: Optional[str] = None
+) -> Dict[str, Any]:
     """
-    Safely load a namespace from source code.
+    Safely load a namespace from source code, optionally handling relative imports.
 
-    This function will attempt to import all modules and classes defined in the source
-    code. If an import fails, the error is caught and the import is skipped. This function
-    will also attempt to compile and evaluate class and function definitions locally.
+    If a `filepath` is provided, `sys.path` is modified to support relative imports.
+    Changes to `sys.path` are reverted after completion, but this function is not thread safe
+    and use of it in threaded contexts may result in undesirable behavior.
 
     Args:
         source_code: The source code to load
+        filepath: Optional file path of the source code. If provided, enables relative imports.
 
     Returns:
-        The namespace loaded from the source code. Can be used when evaluating source
-        code.
+        The namespace loaded from the source code.
     """
     parsed_code = ast.parse(source_code)
 
-    namespace = {"__name__": "prefect_safe_namespace_loader"}
+    namespace: Dict[str, Any] = {"__name__": "prefect_safe_namespace_loader"}
 
-    # Remove the body of the if __name__ == "__main__": block from the AST to prevent
-    # execution of guarded code
-    new_body = []
-    for node in parsed_code.body:
-        if _is_main_block(node):
-            continue
-        new_body.append(node)
+    # Remove the body of the if __name__ == "__main__": block
+    new_body = [node for node in parsed_code.body if not _is_main_block(node)]
     parsed_code.body = new_body
 
-    # Walk through the AST and find all import statements
-    for node in ast.walk(parsed_code):
-        if isinstance(node, ast.Import):
-            for alias in node.names:
-                module_name = alias.name
-                as_name = alias.asname if alias.asname else module_name
-                try:
-                    # Attempt to import the module
-                    namespace[as_name] = importlib.import_module(module_name)
-                    logger.debug("Successfully imported %s", module_name)
-                except ImportError as e:
-                    logger.debug(f"Failed to import {module_name}: {e}")
-        elif isinstance(node, ast.ImportFrom):
-            module_name = node.module
-            if module_name is None:
-                continue
-            try:
-                module = importlib.import_module(module_name)
+    temp_module = None
+    original_sys_path = None
+
+    if filepath:
+        # Setup for relative imports
+        file_dir = os.path.dirname(os.path.abspath(filepath))
+        package_name = os.path.basename(file_dir)
+        parent_dir = os.path.dirname(file_dir)
+
+        # Save original sys.path and modify it
+        original_sys_path = sys.path.copy()
+        sys.path.insert(0, parent_dir)
+
+        # Create a temporary module for import context
+        temp_module = ModuleType(package_name)
+        temp_module.__file__ = filepath
+        temp_module.__package__ = package_name
+
+        # Create a spec for the module
+        temp_module.__spec__ = ModuleSpec(package_name, None)
+        temp_module.__spec__.loader = None
+        temp_module.__spec__.submodule_search_locations = [file_dir]
+
+    try:
+        for node in parsed_code.body:
+            if isinstance(node, ast.Import):
                 for alias in node.names:
-                    name = alias.name
-                    asname = alias.asname if alias.asname else name
+                    module_name = alias.name
+                    as_name = alias.asname or module_name
                     try:
-                        # Get the specific attribute from the module
-                        attribute = getattr(module, name)
-                        namespace[asname] = attribute
-                    except AttributeError as e:
-                        logger.debug(
-                            "Failed to retrieve %s from %s: %s", name, module_name, e
-                        )
-            except ImportError as e:
-                logger.debug("Failed to import from %s: %s", node.module, e)
-
-    # Handle local definitions
-    for node in parsed_code.body:
-        if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.Assign)):
-            try:
-                # Compile and execute each class and function definition and assignment
-                code = compile(
-                    ast.Module(body=[node], type_ignores=[]),
-                    filename="<ast>",
-                    mode="exec",
-                )
-                exec(code, namespace)
-            except Exception as e:
-                logger.debug("Failed to compile: %s", e)
+                        namespace[as_name] = importlib.import_module(module_name)
+                        logger.debug("Successfully imported %s", module_name)
+                    except ImportError as e:
+                        logger.debug(f"Failed to import {module_name}: {e}")
+            elif isinstance(node, ast.ImportFrom):
+                module_name = node.module or ""
+                if filepath:
+                    try:
+                        if node.level > 0:
+                            # For relative imports, use the parent package to inform the import
+                            package_parts = temp_module.__package__.split(".")
+                            if len(package_parts) < node.level:
+                                raise ImportError(
+                                    "Attempted relative import beyond top-level package"
+                                )
+                            parent_package = ".".join(
+                                package_parts[: (1 - node.level)]
+                                if node.level > 1
+                                else package_parts
+                            )
+                            module = importlib.import_module(
+                                f".{module_name}" if module_name else "",
+                                package=parent_package,
+                            )
+                        else:
+                            # Absolute imports are handled as normal
+                            module = importlib.import_module(module_name)
+
+                        for alias in node.names:
+                            name = alias.name
+                            asname = alias.asname or name
+                            if name == "*":
+                                # Handle 'from module import *'
+                                module_dict = {
+                                    k: v
+                                    for k, v in module.__dict__.items()
+                                    if not k.startswith("_")
+                                }
+                                namespace.update(module_dict)
+                            else:
+                                try:
+                                    attribute = getattr(module, name)
+                                    namespace[asname] = attribute
+                                except AttributeError as e:
+                                    logger.debug(
+                                        "Failed to retrieve %s from %s: %s",
+                                        name,
+                                        module_name,
+                                        e,
+                                    )
+                    except ImportError as e:
+                        logger.debug("Failed to import from %s: %s", module_name, e)
+                else:
+                    # Handle as absolute import when no filepath is provided
+                    try:
+                        module = importlib.import_module(module_name)
+                        for alias in node.names:
+                            name = alias.name
+                            asname = alias.asname or name
+                            if name == "*":
+                                # Handle 'from module import *'
+                                module_dict = {
+                                    k: v
+                                    for k, v in module.__dict__.items()
+                                    if not k.startswith("_")
+                                }
+                                namespace.update(module_dict)
+                            else:
+                                try:
+                                    attribute = getattr(module, name)
+                                    namespace[asname] = attribute
+                                except AttributeError as e:
+                                    logger.debug(
+                                        "Failed to retrieve %s from %s: %s",
+                                        name,
+                                        module_name,
+                                        e,
+                                    )
+                    except ImportError as e:
+                        logger.debug("Failed to import from %s: %s", module_name, e)
+        # Handle local definitions
+        for node in parsed_code.body:
+            if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.Assign)):
+                try:
+                    code = compile(
+                        ast.Module(body=[node], type_ignores=[]),
+                        filename="<ast>",
+                        mode="exec",
+                    )
+                    exec(code, namespace)
+                except Exception as e:
+                    logger.debug("Failed to compile: %s", e)
+
+    finally:
+        # Restore original sys.path if it was modified
+        if original_sys_path:
+            sys.path[:] = original_sys_path
+
     return namespace
 
 

prefect/utilities/schema_tools/validation.py

@@ -253,5 +253,35 @@ def preprocess_schema(
                 process_properties(
                     definition["properties"], required_fields, allow_none_with_default
                 )
+            # Allow block types to be referenced by their id
+            if "block_type_slug" in definition:
+                schema["definitions"][definition["title"]] = {
+                    "oneOf": [
+                        definition,
+                        {
+                            "type": "object",
+                            "properties": {
+                                "$ref": {
+                                    "oneOf": [
+                                        {
+                                            "type": "string",
+                                            "format": "uuid",
+                                        },
+                                        {
+                                            "type": "object",
+                                            "additionalProperties": {
+                                                "type": "string",
+                                            },
+                                            "minProperties": 1,
+                                        },
+                                    ]
+                                }
+                            },
+                            "required": [
+                                "$ref",
+                            ],
+                        },
+                    ]
+                }
 
     return schema

prefect/utilities/services.py

@@ -1,13 +1,16 @@
 import sys
+import threading
 from collections import deque
 from traceback import format_exception
 from types import TracebackType
 from typing import Callable, Coroutine, Deque, Optional, Tuple
+from wsgiref.simple_server import WSGIServer
 
 import anyio
 import httpx
 
 from prefect.logging.loggers import get_logger
+from prefect.settings import PREFECT_CLIENT_ENABLE_METRICS, PREFECT_CLIENT_METRICS_PORT
 from prefect.utilities.collections import distinct
 from prefect.utilities.math import clamped_poisson_interval
 
@@ -150,3 +153,32 @@ async def critical_service_loop(
             sleep = interval * 2**backoff_count
 
         await anyio.sleep(sleep)
+
+
+_metrics_server: Optional[Tuple[WSGIServer, threading.Thread]] = None
+
+
+def start_client_metrics_server():
+    """Start the process-wide Prometheus metrics server for client metrics (if enabled
+    with `PREFECT_CLIENT_ENABLE_METRICS`) on the port `PREFECT_CLIENT_METRICS_PORT`."""
+    if not PREFECT_CLIENT_ENABLE_METRICS:
+        return
+
+    global _metrics_server
+    if _metrics_server:
+        return
+
+    from prometheus_client import start_http_server
+
+    _metrics_server = start_http_server(port=PREFECT_CLIENT_METRICS_PORT.value())
+
+
+def stop_client_metrics_server():
+    """Start the process-wide Prometheus metrics server for client metrics, if it has
+    previously been started"""
+    global _metrics_server
+    if _metrics_server:
+        server, thread = _metrics_server
+        server.shutdown()
+        thread.join()
+        _metrics_server = None

{prefect_client-3.0.0rc13.dist-info → prefect_client-3.0.0rc15.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: prefect-client
-Version: 3.0.0rc13
+Version: 3.0.0rc15
 Summary: Workflow orchestration and management.
 Home-page: https://www.prefect.io
 Author: Prefect Technologies, Inc.
@@ -24,46 +24,46 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: anyio <5.0.0,>=4.4.0
-Requires-Dist: asgi-lifespan <3.0,>=1.0
-Requires-Dist: cachetools <6.0,>=5.3
-Requires-Dist: cloudpickle <4.0,>=2.0
-Requires-Dist: coolname <3.0.0,>=1.0.4
-Requires-Dist: croniter <4.0.0,>=1.0.12
-Requires-Dist: exceptiongroup >=1.0.0
-Requires-Dist: fastapi <1.0.0,>=0.111.0
-Requires-Dist: fsspec >=2022.5.0
-Requires-Dist: graphviz >=0.20.1
-Requires-Dist: griffe <0.48.0,>=0.20.0
-Requires-Dist: httpcore <2.0.0,>=1.0.5
-Requires-Dist: httpx[http2] !=0.23.2,>=0.23
-Requires-Dist: importlib-resources <6.2.0,>=6.1.3
-Requires-Dist: jsonpatch <2.0,>=1.32
-Requires-Dist: jsonschema <5.0.0,>=4.0.0
-Requires-Dist: orjson <4.0,>=3.7
-Requires-Dist: packaging <24.3,>=21.3
-Requires-Dist: pathspec >=0.8.0
-Requires-Dist: pendulum <4,>=3.0.0
-Requires-Dist: pydantic <3.0.0,>=2.7
-Requires-Dist: pydantic-core <3.0.0,>=2.12.0
-Requires-Dist: pydantic-extra-types <3.0.0,>=2.8.2
+Requires-Dist: anyio<5.0.0,>=4.4.0
+Requires-Dist: asgi-lifespan<3.0,>=1.0
+Requires-Dist: cachetools<6.0,>=5.3
+Requires-Dist: cloudpickle<4.0,>=2.0
+Requires-Dist: coolname<3.0.0,>=1.0.4
+Requires-Dist: croniter<4.0.0,>=1.0.12
+Requires-Dist: exceptiongroup>=1.0.0
+Requires-Dist: fastapi<1.0.0,>=0.111.0
+Requires-Dist: fsspec>=2022.5.0
+Requires-Dist: graphviz>=0.20.1
+Requires-Dist: griffe<0.48.0,>=0.20.0
+Requires-Dist: httpcore<2.0.0,>=1.0.5
+Requires-Dist: httpx[http2]!=0.23.2,>=0.23
+Requires-Dist: importlib-resources<6.2.0,>=6.1.3
+Requires-Dist: jsonpatch<2.0,>=1.32
+Requires-Dist: jsonschema<5.0.0,>=4.0.0
+Requires-Dist: orjson<4.0,>=3.7
+Requires-Dist: packaging<24.3,>=21.3
+Requires-Dist: pathspec>=0.8.0
+Requires-Dist: pendulum<4,>=3.0.0
+Requires-Dist: prometheus-client>=0.20.0
+Requires-Dist: pydantic<3.0.0,>=2.7
+Requires-Dist: pydantic-core<3.0.0,>=2.12.0
+Requires-Dist: pydantic-extra-types<3.0.0,>=2.8.2
 Requires-Dist: pydantic-settings
-Requires-Dist: python-dateutil <3.0.0,>=2.8.2
-Requires-Dist: python-slugify <9.0,>=5.0
-Requires-Dist: pyyaml <7.0.0,>=5.4.1
-Requires-Dist: rfc3339-validator <0.2.0,>=0.1.4
-Requires-Dist: rich <14.0,>=11.0
-Requires-Dist: ruamel.yaml >=0.17.0
-Requires-Dist: sniffio <2.0.0,>=1.3.0
-Requires-Dist: toml >=0.10.0
-Requires-Dist: typing-extensions <5.0.0,>=4.5.0
-Requires-Dist: ujson <6.0.0,>=5.8.0
-Requires-Dist: uvicorn !=0.29.0,>=0.14.0
-Requires-Dist: websockets <13.0,>=10.4
-Requires-Dist: wrapt >=1.16.0
-Requires-Dist: importlib-metadata >=4.4 ; python_version < "3.10"
+Requires-Dist: python-dateutil<3.0.0,>=2.8.2
+Requires-Dist: python-slugify<9.0,>=5.0
+Requires-Dist: pyyaml<7.0.0,>=5.4.1
+Requires-Dist: rfc3339-validator<0.2.0,>=0.1.4
+Requires-Dist: rich<14.0,>=11.0
+Requires-Dist: ruamel.yaml>=0.17.0
+Requires-Dist: sniffio<2.0.0,>=1.3.0
+Requires-Dist: toml>=0.10.0
+Requires-Dist: typing-extensions<5.0.0,>=4.5.0
+Requires-Dist: ujson<6.0.0,>=5.8.0
+Requires-Dist: uvicorn!=0.29.0,>=0.14.0
+Requires-Dist: websockets<13.0,>=10.4
+Requires-Dist: importlib-metadata>=4.4; python_version < "3.10"
 Provides-Extra: notifications
-Requires-Dist: apprise <2.0.0,>=1.1.0 ; extra == 'notifications'
+Requires-Dist: apprise<2.0.0,>=1.1.0; extra == "notifications"
 
 <p align="center"><img src="https://github.com/PrefectHQ/prefect/assets/3407835/c654cbc6-63e8-4ada-a92a-efd2f8f24b85" width=1000></p>