prefect-client 3.0.0rc13__py3-none-any.whl → 3.0.0rc14__py3-none-any.whl

prefect/flows.py

@@ -95,7 +95,7 @@ from prefect.utilities.callables import (
     parameters_to_args_kwargs,
     raise_for_reserved_arguments,
 )
-from prefect.utilities.collections import listrepr
+from prefect.utilities.collections import listrepr, visit_collection
 from prefect.utilities.filesystem import relative_path_to_current_platform
 from prefect.utilities.hashing import file_hash
 from prefect.utilities.importtools import import_object, safe_load_namespace
@@ -535,6 +535,21 @@ class Flow(Generic[P, R]):
         Raises:
             ParameterTypeError: if the provided parameters are not valid
         """
+
+        def resolve_block_reference(data: Any) -> Any:
+            if isinstance(data, dict) and "$ref" in data:
+                return Block.load_from_ref(data["$ref"])
+            return data
+
+        try:
+            parameters = visit_collection(
+                parameters, resolve_block_reference, return_data=True
+            )
+        except (ValueError, RuntimeError) as exc:
+            raise ParameterTypeError(
+                "Failed to resolve block references in parameters."
+            ) from exc
+
         args, kwargs = parameters_to_args_kwargs(self.fn, parameters)
 
         with warnings.catch_warnings():
@@ -1734,14 +1749,13 @@ def load_flow_from_entrypoint(
         raise MissingFlowError(
             f"Flow function with name {func_name!r} not found in {path!r}. "
         ) from exc
-    except ScriptError as exc:
+    except ScriptError:
         # If the flow has dependencies that are not installed in the current
-        # environment, fallback to loading the flow via AST parsing. The
-        # drawback of this approach is that we're unable to actually load the
-        # function, so we create a placeholder flow that will re-raise this
-        # exception when called.
+        # environment, fallback to loading the flow via AST parsing.
         if use_placeholder_flow:
-            flow = load_placeholder_flow(entrypoint=entrypoint, raises=exc)
+            flow = safe_load_flow_from_entrypoint(entrypoint)
+            if flow is None:
+                raise
         else:
             raise
 
@@ -1976,6 +1990,147 @@ def load_placeholder_flow(entrypoint: str, raises: Exception):
     return Flow(**arguments)
 
 
+def safe_load_flow_from_entrypoint(entrypoint: str) -> Optional[Flow]:
+    """
+    Load a flow from an entrypoint and return None if an exception is raised.
+
+    Args:
+        entrypoint: a string in the format `<path_to_script>:<flow_func_name>`
+          or a module path to a flow function
+    """
+    func_def, source_code = _entrypoint_definition_and_source(entrypoint)
+    path = None
+    if ":" in entrypoint:
+        path = entrypoint.rsplit(":")[0]
+    namespace = safe_load_namespace(source_code, filepath=path)
+    if func_def.name in namespace:
+        return namespace[func_def.name]
+    else:
+        # If the function is not in the namespace, if may be due to missing dependencies
+        # for the function. We will attempt to compile each annotation and default value
+        # and remove them from the function definition to see if the function can be
+        # compiled without them.
+
+        return _sanitize_and_load_flow(func_def, namespace)
+
+
+def _sanitize_and_load_flow(
+    func_def: Union[ast.FunctionDef, ast.AsyncFunctionDef], namespace: Dict[str, Any]
+) -> Optional[Flow]:
+    """
+    Attempt to load a flow from the function definition after sanitizing the annotations
+    and defaults that can't be compiled.
+
+    Args:
+        func_def: the function definition
+        namespace: the namespace to load the function into
+
+    Returns:
+        The loaded function or None if the function can't be loaded
+        after sanitizing the annotations and defaults.
+    """
+    args = func_def.args.posonlyargs + func_def.args.args + func_def.args.kwonlyargs
+    if func_def.args.vararg:
+        args.append(func_def.args.vararg)
+    if func_def.args.kwarg:
+        args.append(func_def.args.kwarg)
+    # Remove annotations that can't be compiled
+    for arg in args:
+        if arg.annotation is not None:
+            try:
+                code = compile(
+                    ast.Expression(arg.annotation),
+                    filename="<ast>",
+                    mode="eval",
+                )
+                exec(code, namespace)
+            except Exception as e:
+                logger.debug(
+                    "Failed to evaluate annotation for argument %s due to the following error. Ignoring annotation.",
+                    arg.arg,
+                    exc_info=e,
+                )
+                arg.annotation = None
+
+    # Remove defaults that can't be compiled
+    new_defaults = []
+    for default in func_def.args.defaults:
+        try:
+            code = compile(ast.Expression(default), "<ast>", "eval")
+            exec(code, namespace)
+            new_defaults.append(default)
+        except Exception as e:
+            logger.debug(
+                "Failed to evaluate default value %s due to the following error. Ignoring default.",
+                default,
+                exc_info=e,
+            )
+            new_defaults.append(
+                ast.Constant(
+                    value=None, lineno=default.lineno, col_offset=default.col_offset
+                )
+            )
+    func_def.args.defaults = new_defaults
+
+    # Remove kw_defaults that can't be compiled
+    new_kw_defaults = []
+    for default in func_def.args.kw_defaults:
+        if default is not None:
+            try:
+                code = compile(ast.Expression(default), "<ast>", "eval")
+                exec(code, namespace)
+                new_kw_defaults.append(default)
+            except Exception as e:
+                logger.debug(
+                    "Failed to evaluate default value %s due to the following error. Ignoring default.",
+                    default,
+                    exc_info=e,
+                )
+                new_kw_defaults.append(
+                    ast.Constant(
+                        value=None,
+                        lineno=default.lineno,
+                        col_offset=default.col_offset,
+                    )
+                )
+        else:
+            new_kw_defaults.append(
+                ast.Constant(
+                    value=None,
+                    lineno=func_def.lineno,
+                    col_offset=func_def.col_offset,
+                )
+            )
+    func_def.args.kw_defaults = new_kw_defaults
+
+    if func_def.returns is not None:
+        try:
+            code = compile(
+                ast.Expression(func_def.returns), filename="<ast>", mode="eval"
+            )
+            exec(code, namespace)
+        except Exception as e:
+            logger.debug(
+                "Failed to evaluate return annotation due to the following error. Ignoring annotation.",
+                exc_info=e,
+            )
+            func_def.returns = None
+
+    # Attempt to compile the function without annotations and defaults that
+    # can't be compiled
+    try:
+        code = compile(
+            ast.Module(body=[func_def], type_ignores=[]),
+            filename="<ast>",
+            mode="exec",
+        )
+        exec(code, namespace)
+    except Exception as e:
+        logger.debug("Failed to compile: %s", e)
+    else:
+        return namespace.get(func_def.name)
+
+
 def load_flow_arguments_from_entrypoint(
     entrypoint: str, arguments: Optional[Union[List[str], Set[str]]] = None
 ) -> dict[str, Any]:
@@ -1991,6 +2146,9 @@ def load_flow_arguments_from_entrypoint(
     """
 
     func_def, source_code = _entrypoint_definition_and_source(entrypoint)
+    path = None
+    if ":" in entrypoint:
+        path = entrypoint.rsplit(":")[0]
 
     if arguments is None:
         # If no arguments are provided default to known arguments that are of
@@ -2026,7 +2184,7 @@ def load_flow_arguments_from_entrypoint(
 
                 # if the arg value is not a raw str (i.e. a variable or expression),
                 # then attempt to evaluate it
-                namespace = safe_load_namespace(source_code)
+                namespace = safe_load_namespace(source_code, filepath=path)
                 literal_arg_value = ast.get_source_segment(source_code, keyword.value)
                 cleaned_value = (
                     literal_arg_value.replace("\n", "") if literal_arg_value else ""

prefect/futures.py

@@ -2,10 +2,11 @@ import abc
 import collections
 import concurrent.futures
 import inspect
+import threading
 import uuid
-from collections.abc import Iterator
+from collections.abc import Generator, Iterator
 from functools import partial
-from typing import Any, Generic, List, Optional, Set, Union, cast
+from typing import Any, Callable, Generic, List, Optional, Set, Union, cast
 
 from typing_extensions import TypeVar
 
@@ -91,6 +92,16 @@ class PrefectFuture(abc.ABC, Generic[R]):
             The result of the task run.
         """
 
+    @abc.abstractmethod
+    def add_done_callback(self, fn):
+        """
+        Add a callback to be run when the future completes or is cancelled.
+
+        Args:
+            fn: A callable that will be called with this future as its only argument when the future completes or is cancelled.
+        """
+        ...
+
 
 class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
     """
@@ -106,6 +117,17 @@ class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
         """The underlying future object wrapped by this Prefect future"""
         return self._wrapped_future
 
+    def add_done_callback(self, fn: Callable[[PrefectFuture], None]):
+        if not self._final_state:
+
+            def call_with_self(future):
+                """Call the callback with self as the argument, this is necessary to ensure we remove the future from the pending set"""
+                fn(self)
+
+            self._wrapped_future.add_done_callback(call_with_self)
+            return
+        fn(self)
+
 
 class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]):
     """
@@ -138,6 +160,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
 
             if isinstance(future_result, State):
                 self._final_state = future_result
+
             else:
                 return future_result
 
@@ -172,6 +195,9 @@ class PrefectDistributedFuture(PrefectFuture[R]):
     any task run scheduled in Prefect's API.
     """
 
+    done_callbacks: List[Callable[[PrefectFuture], None]] = []
+    waiter = None
+
     @deprecated_async_method
     def wait(self, timeout: Optional[float] = None) -> None:
         return run_coro_as_sync(self.wait_async(timeout=timeout))
@@ -235,11 +261,27 @@ class PrefectDistributedFuture(PrefectFuture[R]):
             raise_on_failure=raise_on_failure, fetch=True
         )
 
+    def add_done_callback(self, fn: Callable[[PrefectFuture], None]):
+        if self._final_state:
+            fn(self)
+            return
+        TaskRunWaiter.instance()
+        with get_client(sync_client=True) as client:
+            task_run = client.read_task_run(task_run_id=self._task_run_id)
+            if task_run.state.is_final():
+                self._final_state = task_run.state
+                fn(self)
+                return
+            TaskRunWaiter.add_done_callback(self._task_run_id, partial(fn, self))
+
     def __eq__(self, other):
         if not isinstance(other, PrefectDistributedFuture):
             return False
         return self.task_run_id == other.task_run_id
 
+    def __hash__(self):
+        return hash(self.task_run_id)
+
 
 class PrefectFutureList(list, Iterator, Generic[F]):
     """
@@ -292,6 +334,46 @@ class PrefectFutureList(list, Iterator, Generic[F]):
             ) from exc
 
 
+def as_completed(
+    futures: List[PrefectFuture], timeout: Optional[float] = None
+) -> Generator[PrefectFuture, None]:
+    unique_futures: Set[PrefectFuture] = set(futures)
+    total_futures = len(unique_futures)
+    try:
+        with timeout_context(timeout):
+            done = {f for f in unique_futures if f._final_state}
+            pending = unique_futures - done
+            yield from done
+
+            finished_event = threading.Event()
+            finished_lock = threading.Lock()
+            finished_futures = []
+
+            def add_to_done(future):
+                with finished_lock:
+                    finished_futures.append(future)
+                    finished_event.set()
+
+            for future in pending:
+                future.add_done_callback(add_to_done)
+
+            while pending:
+                finished_event.wait()
+                with finished_lock:
+                    done = finished_futures
+                    finished_futures = []
+                    finished_event.clear()
+
+                for future in done:
+                    pending.remove(future)
+                    yield future
+
+    except TimeoutError:
+        raise TimeoutError(
+            "%d (of %d) futures unfinished" % (len(pending), total_futures)
+        )
+
+
 DoneAndNotDoneFutures = collections.namedtuple("DoneAndNotDoneFutures", "done not_done")
 
 

prefect/profiles.toml

@@ -1,3 +1,14 @@
-active = "default"
+# This is a template for profile configuration for Prefect.
+# You can modify these profiles or create new ones to suit your needs.
 
-[profiles.default]
+active = "ephemeral"
+
+[profiles.ephemeral]
+PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = "true"
+
+[profiles.local]
+# You will need to set these values appropriately for your local development environment
+PREFECT_API_URL = "http://127.0.0.1:4200/api"
+
+
+[profiles.cloud]

prefect/runner/runner.py

@@ -92,7 +92,10 @@ from prefect.utilities.asyncutils import (
 )
 from prefect.utilities.engine import propose_state
 from prefect.utilities.processutils import _register_signal, run_process
-from prefect.utilities.services import critical_service_loop
+from prefect.utilities.services import (
+    critical_service_loop,
+    start_client_metrics_server,
+)
 from prefect.utilities.slugify import slugify
 
 if TYPE_CHECKING:
@@ -380,6 +383,8 @@ class Runner:
             )
             server_thread.start()
 
+        start_client_metrics_server()
+
         async with self as runner:
             async with self._loops_task_group as tg:
                 for storage in self._storage_objs:

prefect/settings.py

@@ -1216,6 +1216,19 @@ compromise. Adjust this setting based on your specific security requirements
 and usage patterns.
 """
 
+PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = Setting(bool, default=False)
+"""
+Controls whether or not a subprocess server can be started when no API URL is provided.
+"""
+
+PREFECT_SERVER_EPHEMERAL_STARTUP_TIMEOUT_SECONDS = Setting(
+    int,
+    default=10,
+)
+"""
+The number of seconds to wait for an ephemeral server to respond on start up before erroring.
+"""
+
 PREFECT_UI_ENABLED = Setting(
     bool,
     default=True,
@@ -1561,6 +1574,26 @@ The page size for the queries to backfill events for websocket subscribers
 """
 
 
+# Metrics settings
+
+PREFECT_API_ENABLE_METRICS = Setting(bool, default=False)
+"""
+Whether or not to enable Prometheus metrics in the server application.  Metrics are
+served at the path /api/metrics on the API server.
+"""
+
+PREFECT_CLIENT_ENABLE_METRICS = Setting(bool, default=False)
+"""
+Whether or not to enable Prometheus metrics in the client SDK.  Metrics are served
+at the path /metrics.
+"""
+
+PREFECT_CLIENT_METRICS_PORT = Setting(int, default=4201)
+"""
+The port to expose the client Prometheus metrics on.
+"""
+
+
 # Deprecated settings ------------------------------------------------------------------
 
 
@@ -2125,10 +2158,10 @@ def load_current_profile():
     This will _not_ include settings from the current settings context. Only settings
     that have been persisted to the profiles file will be saved.
     """
-    from prefect.context import SettingsContext
+    import prefect.context
 
     profiles = load_profiles()
-    context = SettingsContext.get()
+    context = prefect.context.get_settings_context()
 
     if context:
         profiles.set_active(context.profile.name)