prefect-client 3.0.0rc13__py3-none-any.whl → 3.0.0rc15__py3-none-any.whl

prefect/futures.py

@@ -2,14 +2,14 @@ import abc
 import collections
 import concurrent.futures
 import inspect
+import threading
 import uuid
-from collections.abc import Iterator
+from collections.abc import Generator, Iterator
 from functools import partial
-from typing import Any, Generic, List, Optional, Set, Union, cast
+from typing import Any, Callable, Generic, List, Optional, Set, Union, cast
 
 from typing_extensions import TypeVar
 
-from prefect._internal.compatibility.deprecated import deprecated_async_method
 from prefect.client.orchestration import get_client
 from prefect.client.schemas.objects import TaskRun
 from prefect.exceptions import ObjectNotFound
@@ -91,6 +91,16 @@ class PrefectFuture(abc.ABC, Generic[R]):
             The result of the task run.
         """
 
+    @abc.abstractmethod
+    def add_done_callback(self, fn):
+        """
+        Add a callback to be run when the future completes or is cancelled.
+
+        Args:
+            fn: A callable that will be called with this future as its only argument when the future completes or is cancelled.
+        """
+        ...
+
 
 class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
     """
@@ -106,6 +116,17 @@ class PrefectWrappedFuture(PrefectFuture, abc.ABC, Generic[R, F]):
         """The underlying future object wrapped by this Prefect future"""
         return self._wrapped_future
 
+    def add_done_callback(self, fn: Callable[[PrefectFuture], None]):
+        if not self._final_state:
+
+            def call_with_self(future):
+                """Call the callback with self as the argument, this is necessary to ensure we remove the future from the pending set"""
+                fn(self)
+
+            self._wrapped_future.add_done_callback(call_with_self)
+            return
+        fn(self)
+
 
 class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]):
     """
@@ -113,7 +134,6 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
     when the task run is submitted to a ThreadPoolExecutor.
     """
 
-    @deprecated_async_method
     def wait(self, timeout: Optional[float] = None) -> None:
         try:
             result = self._wrapped_future.result(timeout=timeout)
@@ -122,7 +142,6 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
         if isinstance(result, State):
             self._final_state = result
 
-    @deprecated_async_method
     def result(
         self,
         timeout: Optional[float] = None,
@@ -138,6 +157,7 @@ class PrefectConcurrentFuture(PrefectWrappedFuture[R, concurrent.futures.Future]
 
             if isinstance(future_result, State):
                 self._final_state = future_result
+
             else:
                 return future_result
 
@@ -172,7 +192,9 @@ class PrefectDistributedFuture(PrefectFuture[R]):
     any task run scheduled in Prefect's API.
     """
 
-    @deprecated_async_method
+    done_callbacks: List[Callable[[PrefectFuture], None]] = []
+    waiter = None
+
     def wait(self, timeout: Optional[float] = None) -> None:
         return run_coro_as_sync(self.wait_async(timeout=timeout))
 
@@ -209,7 +231,6 @@ class PrefectDistributedFuture(PrefectFuture[R]):
                 self._final_state = task_run.state
             return
 
-    @deprecated_async_method
     def result(
         self,
         timeout: Optional[float] = None,
@@ -235,11 +256,27 @@ class PrefectDistributedFuture(PrefectFuture[R]):
             raise_on_failure=raise_on_failure, fetch=True
         )
 
+    def add_done_callback(self, fn: Callable[[PrefectFuture], None]):
+        if self._final_state:
+            fn(self)
+            return
+        TaskRunWaiter.instance()
+        with get_client(sync_client=True) as client:
+            task_run = client.read_task_run(task_run_id=self._task_run_id)
+            if task_run.state.is_final():
+                self._final_state = task_run.state
+                fn(self)
+                return
+            TaskRunWaiter.add_done_callback(self._task_run_id, partial(fn, self))
+
     def __eq__(self, other):
         if not isinstance(other, PrefectDistributedFuture):
             return False
         return self.task_run_id == other.task_run_id
 
+    def __hash__(self):
+        return hash(self.task_run_id)
+
 
 class PrefectFutureList(list, Iterator, Generic[F]):
     """
@@ -292,6 +329,46 @@ class PrefectFutureList(list, Iterator, Generic[F]):
             ) from exc
 
 
+def as_completed(
+    futures: List[PrefectFuture], timeout: Optional[float] = None
+) -> Generator[PrefectFuture, None]:
+    unique_futures: Set[PrefectFuture] = set(futures)
+    total_futures = len(unique_futures)
+    try:
+        with timeout_context(timeout):
+            done = {f for f in unique_futures if f._final_state}
+            pending = unique_futures - done
+            yield from done
+
+            finished_event = threading.Event()
+            finished_lock = threading.Lock()
+            finished_futures = []
+
+            def add_to_done(future):
+                with finished_lock:
+                    finished_futures.append(future)
+                    finished_event.set()
+
+            for future in pending:
+                future.add_done_callback(add_to_done)
+
+            while pending:
+                finished_event.wait()
+                with finished_lock:
+                    done = finished_futures
+                    finished_futures = []
+                    finished_event.clear()
+
+                for future in done:
+                    pending.remove(future)
+                    yield future
+
+    except TimeoutError:
+        raise TimeoutError(
+            "%d (of %d) futures unfinished" % (len(pending), total_futures)
+        )
+
+
 DoneAndNotDoneFutures = collections.namedtuple("DoneAndNotDoneFutures", "done not_done")
 
 

prefect/profiles.toml

@@ -1,3 +1,17 @@
-active = "default"
+# This is a template for profile configuration for Prefect.
+# You can modify these profiles or create new ones to suit your needs.
 
-[profiles.default]
+active = "ephemeral"
+
+[profiles.ephemeral]
+PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = "true"
+
+[profiles.local]
+# You will need to set these values appropriately for your local development environment
+PREFECT_API_URL = "http://127.0.0.1:4200/api"
+
+[profiles.test]
+PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = "true"
+PREFECT_API_DATABASE_CONNECTION_URL = "sqlite+aiosqlite:///:memory:"
+
+[profiles.cloud]

prefect/runner/runner.py

@@ -92,7 +92,10 @@ from prefect.utilities.asyncutils import (
 )
 from prefect.utilities.engine import propose_state
 from prefect.utilities.processutils import _register_signal, run_process
-from prefect.utilities.services import critical_service_loop
+from prefect.utilities.services import (
+    critical_service_loop,
+    start_client_metrics_server,
+)
 from prefect.utilities.slugify import slugify
 
 if TYPE_CHECKING:
@@ -380,6 +383,8 @@ class Runner:
             )
             server_thread.start()
 
+        start_client_metrics_server()
+
         async with self as runner:
             async with self._loops_task_group as tg:
                 for storage in self._storage_objs:

prefect/runner/storage.py

@@ -280,6 +280,10 @@ class GitRepository:
                 "branch": self._branch,
             }
         }
+        if self._include_submodules:
+            pull_step["prefect.deployments.steps.git_clone"][
+                "include_submodules"
+            ] = self._include_submodules
         if isinstance(self._credentials, Block):
             pull_step["prefect.deployments.steps.git_clone"][
                 "credentials"

prefect/settings.py

@@ -351,6 +351,7 @@ def warn_on_database_password_value_without_usage(values):
     if (
         value
         and not value.startswith(OBFUSCATED_PREFIX)
+        and values["PREFECT_API_DATABASE_CONNECTION_URL"] is not None
         and (
             "PREFECT_API_DATABASE_PASSWORD"
             not in values["PREFECT_API_DATABASE_CONNECTION_URL"]
@@ -403,7 +404,38 @@ def warn_on_misconfigured_api_url(values):
     return values
 
 
-def default_database_connection_url(settings, value):
+def default_database_connection_url(settings: "Settings", value: Optional[str]):
+    driver = PREFECT_API_DATABASE_DRIVER.value_from(settings)
+    if driver == "postgresql+asyncpg":
+        required = [
+            PREFECT_API_DATABASE_HOST,
+            PREFECT_API_DATABASE_USER,
+            PREFECT_API_DATABASE_NAME,
+            PREFECT_API_DATABASE_PASSWORD,
+        ]
+        missing = [
+            setting.name for setting in required if not setting.value_from(settings)
+        ]
+        if missing:
+            raise ValueError(
+                f"Missing required database connection settings: {', '.join(missing)}"
+            )
+
+        host = PREFECT_API_DATABASE_HOST.value_from(settings)
+        port = PREFECT_API_DATABASE_PORT.value_from(settings) or 5432
+        user = PREFECT_API_DATABASE_USER.value_from(settings)
+        name = PREFECT_API_DATABASE_NAME.value_from(settings)
+        password = PREFECT_API_DATABASE_PASSWORD.value_from(settings)
+
+        return f"{driver}://{user}:{password}@{host}:{port}/{name}"
+
+    elif driver == "sqlite+aiosqlite":
+        path = PREFECT_API_DATABASE_NAME.value_from(settings)
+        if path:
+            return f"{driver}:///{path}"
+    elif driver:
+        raise ValueError(f"Unsupported database driver: {driver}")
+
     templater = template_with_settings(PREFECT_HOME, PREFECT_API_DATABASE_PASSWORD)
 
     # If the user has provided a value, use it
@@ -941,17 +973,6 @@ backend on application startup. If not set, block types must be manually
 registered.
 """
 
-PREFECT_API_DATABASE_PASSWORD = Setting(
-    Optional[str],
-    default=None,
-    is_secret=True,
-)
-"""
-Password to template into the `PREFECT_API_DATABASE_CONNECTION_URL`.
-This is useful if the password must be provided separately from the connection URL.
-To use this setting, you must include it in your connection URL.
-"""
-
 PREFECT_API_DATABASE_CONNECTION_URL = Setting(
     Optional[str],
     default=None,
@@ -981,6 +1002,46 @@ PREFECT_API_DATABASE_CONNECTION_URL='postgresql+asyncpg://postgres:${PREFECT_API
 ```
 """
 
+PREFECT_API_DATABASE_DRIVER = Setting(
+    Optional[Literal["postgresql+asyncpg", "sqlite+aiosqlite"]],
+    default=None,
+)
+"""
+The database driver to use when connecting to the database.
+"""
+
+PREFECT_API_DATABASE_HOST = Setting(Optional[str], default=None)
+"""
+The database server host.
+"""
+
+PREFECT_API_DATABASE_PORT = Setting(Optional[int], default=None)
+"""
+The database server port.
+"""
+
+PREFECT_API_DATABASE_USER = Setting(Optional[str], default=None)
+"""
+The user to use when connecting to the database.
+"""
+
+PREFECT_API_DATABASE_NAME = Setting(Optional[str], default=None)
+"""
+The name of the Prefect database on the remote server, or the path to the database file
+for SQLite.
+"""
+
+PREFECT_API_DATABASE_PASSWORD = Setting(
+    Optional[str],
+    default=None,
+    is_secret=True,
+)
+"""
+Password to template into the `PREFECT_API_DATABASE_CONNECTION_URL`.
+This is useful if the password must be provided separately from the connection URL.
+To use this setting, you must include it in your connection URL.
+"""
+
 PREFECT_API_DATABASE_ECHO = Setting(
     bool,
     default=False,
@@ -1216,6 +1277,19 @@ compromise. Adjust this setting based on your specific security requirements
 and usage patterns.
 """
 
+PREFECT_SERVER_ALLOW_EPHEMERAL_MODE = Setting(bool, default=False)
+"""
+Controls whether or not a subprocess server can be started when no API URL is provided.
+"""
+
+PREFECT_SERVER_EPHEMERAL_STARTUP_TIMEOUT_SECONDS = Setting(
+    int,
+    default=10,
+)
+"""
+The number of seconds to wait for an ephemeral server to respond on start up before erroring.
+"""
+
 PREFECT_UI_ENABLED = Setting(
     bool,
     default=True,
@@ -1561,6 +1635,26 @@ The page size for the queries to backfill events for websocket subscribers
 """
 
 
+# Metrics settings
+
+PREFECT_API_ENABLE_METRICS = Setting(bool, default=False)
+"""
+Whether or not to enable Prometheus metrics in the server application.  Metrics are
+served at the path /api/metrics on the API server.
+"""
+
+PREFECT_CLIENT_ENABLE_METRICS = Setting(bool, default=False)
+"""
+Whether or not to enable Prometheus metrics in the client SDK.  Metrics are served
+at the path /metrics.
+"""
+
+PREFECT_CLIENT_METRICS_PORT = Setting(int, default=4201)
+"""
+The port to expose the client Prometheus metrics on.
+"""
+
+
 # Deprecated settings ------------------------------------------------------------------
 
 
@@ -2125,10 +2219,10 @@ def load_current_profile():
     This will _not_ include settings from the current settings context. Only settings
     that have been persisted to the profiles file will be saved.
     """
-    from prefect.context import SettingsContext
+    import prefect.context
 
     profiles = load_profiles()
-    context = SettingsContext.get()
+    context = prefect.context.get_settings_context()
 
     if context:
         profiles.set_active(context.profile.name)