prefect 3.6.6__py3-none-any.whl → 3.6.7__py3-none-any.whl

prefect/server/services/task_run_recorder.py

@@ -43,6 +43,8 @@ if TYPE_CHECKING:
 
 logger: "logging.Logger" = get_logger(__name__)
 
+DEFAULT_PERSIST_MAX_RETRIES = 5
+
 
 @db_injector
 async def _insert_task_run_states(
@@ -300,7 +302,9 @@ class RetryableEvent(BaseModel):
 
 @asynccontextmanager
 async def consumer(
-    write_batch_size: int, flush_every: int
+    write_batch_size: int,
+    flush_every: int,
+    max_persist_retries: int = DEFAULT_PERSIST_MAX_RETRIES,
 ) -> AsyncGenerator[MessageHandler, None]:
     logger.info(
         f"Creating TaskRunRecorder consumer with batch size {write_batch_size} and flush every {flush_every} seconds"
@@ -323,8 +327,25 @@ async def consumer(
         try:
             await handle_task_run_events([batch.event for batch in batch])
         except Exception:
-            logger.error(f"Error flushing {len(batch)} events", exc_info=True)
-            raise
+            dropped = 0
+            to_retry = 0
+            for item in batch:
+                item.persist_attempts += 1
+                if item.persist_attempts <= max_persist_retries:
+                    to_retry += 1
+                    await queue.put(item)
+                else:
+                    dropped += 1
+                    logger.error(
+                        f"Dropping event {item.event.id} after {item.persist_attempts} failed attempts"
+                    )
+            logger.error(
+                f"Error flushing {len(batch)} events ({to_retry} to retry, {dropped} dropped)",
+                exc_info=True,
+            )
+
+            if dropped > 0:
+                raise
 
     async def flush_periodically():
         try:
@@ -396,7 +417,9 @@ class TaskRunRecorder(RunInEphemeralServers, Service):
     def started_event(self, value: asyncio.Event) -> None:
         self._started_event = value
 
-    async def start(self) -> NoReturn:
+    async def start(
+        self, max_persist_retries: int = DEFAULT_PERSIST_MAX_RETRIES
+    ) -> NoReturn:
         assert self.consumer_task is None, "TaskRunRecorder already started"
         self.consumer: Consumer = create_consumer(
             "events",
@@ -408,6 +431,7 @@ class TaskRunRecorder(RunInEphemeralServers, Service):
         async with consumer(
             write_batch_size=self.service_settings().batch_size,
             flush_every=int(self.service_settings().flush_interval),
+            max_persist_retries=max_persist_retries,
         ) as handler:
             self.consumer_task = asyncio.create_task(self.consumer.run(handler))
             self.metrics_task = asyncio.create_task(log_metrics_periodically())

prefect/server/services/telemetry.py

@@ -1,143 +1,114 @@
 """
-The Telemetry service.
+The Telemetry service. Sends anonymous data to Prefect to help us improve.
 """
 
-import asyncio
+import logging
 import os
 import platform
-from typing import Any, Optional
+from datetime import timedelta
 from uuid import uuid4
 
 import httpx
+from docket import Perpetual
 
 import prefect
-from prefect.server.database import PrefectDBInterface
-from prefect.server.database.dependencies import db_injector
+from prefect.logging import get_logger
+from prefect.server.database import PrefectDBInterface, provide_database_interface
 from prefect.server.models import configuration
 from prefect.server.schemas.core import Configuration
-from prefect.server.services.base import (
-    LoopService,
-    RunInEphemeralServers,
-    RunInWebservers,
-)
+from prefect.server.services.perpetual_services import perpetual_service
 from prefect.settings import PREFECT_DEBUG_MODE
 from prefect.settings.context import get_current_settings
-from prefect.settings.models.server.services import ServicesBaseSetting
 from prefect.types._datetime import now
 
+logger: logging.Logger = get_logger(__name__)
+
 
-class Telemetry(RunInEphemeralServers, RunInWebservers, LoopService):
+async def _fetch_or_set_telemetry_session(
+    db: PrefectDBInterface,
+) -> tuple[str, str]:
     """
-    Sends anonymous data to Prefect to help us improve
+    Fetch or create a telemetry session in the configuration table.
 
-    It can be toggled off with the PREFECT_SERVER_ANALYTICS_ENABLED setting.
+    Returns:
+        tuple of (session_start_timestamp, session_id)
     """
+    async with db.session_context(begin_transaction=True) as session:
+        telemetry_session = await configuration.read_configuration(
+            session, "TELEMETRY_SESSION"
+        )
 
-    loop_seconds: float = 600
+        if telemetry_session is None:
+            logger.debug("No telemetry session found, setting")
+            session_id = str(uuid4())
+            session_start_timestamp = now("UTC").isoformat()
+
+            telemetry_session = Configuration(
+                key="TELEMETRY_SESSION",
+                value={
+                    "session_id": session_id,
+                    "session_start_timestamp": session_start_timestamp,
+                },
+            )
 
-    @classmethod
-    def service_settings(cls) -> ServicesBaseSetting:
-        raise NotImplementedError("Telemetry service does not have settings")
+            await configuration.write_configuration(session, telemetry_session)
+        else:
+            logger.debug("Session information retrieved from database")
+            session_id = telemetry_session.value["session_id"]
+            session_start_timestamp = telemetry_session.value["session_start_timestamp"]
 
-    @classmethod
-    def environment_variable_name(cls) -> str:
-        return "PREFECT_SERVER_ANALYTICS_ENABLED"
+    logger.debug(f"Telemetry Session: {session_id}, {session_start_timestamp}")
+    return (session_start_timestamp, session_id)
 
-    @classmethod
-    def enabled(cls) -> bool:
-        return get_current_settings().server.analytics_enabled
 
-    def __init__(self, loop_seconds: Optional[int] = None, **kwargs: Any):
-        super().__init__(loop_seconds=loop_seconds, **kwargs)
-        self.telemetry_environment: str = os.environ.get(
-            "PREFECT_API_TELEMETRY_ENVIRONMENT", "production"
-        )
+@perpetual_service(
+    enabled_getter=lambda: get_current_settings().server.analytics_enabled,
+    run_in_ephemeral=True,
+    run_in_webserver=True,
+)
+async def send_telemetry_heartbeat(
+    perpetual: Perpetual = Perpetual(automatic=True, every=timedelta(seconds=600)),
+) -> None:
+    """
+    Sends anonymous telemetry data to Prefect to help us improve.
 
-    @db_injector
-    async def _fetch_or_set_telemetry_session(self, db: PrefectDBInterface):
-        """
-        This method looks for a telemetry session in the configuration table. If there
-        isn't one, it sets one. It then sets `self.session_id` and
-        `self.session_start_timestamp`.
-
-        Telemetry sessions last until the database is reset.
-        """
-        async with db.session_context(begin_transaction=True) as session:
-            telemetry_session = await configuration.read_configuration(
-                session, "TELEMETRY_SESSION"
+    It can be toggled off with the PREFECT_SERVER_ANALYTICS_ENABLED setting.
+    """
+    from prefect.client.constants import SERVER_API_VERSION
+
+    db = provide_database_interface()
+    session_start_timestamp, session_id = await _fetch_or_set_telemetry_session(db=db)
+    telemetry_environment = os.environ.get(
+        "PREFECT_API_TELEMETRY_ENVIRONMENT", "production"
+    )
+
+    heartbeat = {
+        "source": "prefect_server",
+        "type": "heartbeat",
+        "payload": {
+            "platform": platform.system(),
+            "architecture": platform.machine(),
+            "python_version": platform.python_version(),
+            "python_implementation": platform.python_implementation(),
+            "environment": telemetry_environment,
+            "ephemeral_server": bool(os.getenv("PREFECT__SERVER_EPHEMERAL", False)),
+            "api_version": SERVER_API_VERSION,
+            "prefect_version": prefect.__version__,
+            "session_id": session_id,
+            "session_start_timestamp": session_start_timestamp,
+        },
+    }
+
+    try:
+        async with httpx.AsyncClient() as client:
+            result = await client.post(
+                "https://sens-o-matic.prefect.io/",
+                json=heartbeat,
+                headers={"x-prefect-event": "prefect_server"},
             )
-
-            if telemetry_session is None:
-                self.logger.debug("No telemetry session found, setting")
-                session_id = str(uuid4())
-                session_start_timestamp = now("UTC").isoformat()
-
-                telemetry_session = Configuration(
-                    key="TELEMETRY_SESSION",
-                    value={
-                        "session_id": session_id,
-                        "session_start_timestamp": session_start_timestamp,
-                    },
-                )
-
-                await configuration.write_configuration(session, telemetry_session)
-
-                self.session_id = session_id
-                self.session_start_timestamp = session_start_timestamp
-            else:
-                self.logger.debug("Session information retrieved from database")
-                self.session_id: str = telemetry_session.value["session_id"]
-                self.session_start_timestamp: str = telemetry_session.value[
-                    "session_start_timestamp"
-                ]
-        self.logger.debug(
-            f"Telemetry Session: {self.session_id}, {self.session_start_timestamp}"
+        result.raise_for_status()
+    except Exception as exc:
+        logger.error(
+            f"Failed to send telemetry: {exc}",
+            exc_info=PREFECT_DEBUG_MODE.value(),
         )
-        return (self.session_start_timestamp, self.session_id)
-
-    async def run_once(self) -> None:
-        """
-        Sends a heartbeat to the sens-o-matic
-        """
-        from prefect.client.constants import SERVER_API_VERSION
-
-        if not hasattr(self, "session_id"):
-            await self._fetch_or_set_telemetry_session()
-
-        heartbeat = {
-            "source": "prefect_server",
-            "type": "heartbeat",
-            "payload": {
-                "platform": platform.system(),
-                "architecture": platform.machine(),
-                "python_version": platform.python_version(),
-                "python_implementation": platform.python_implementation(),
-                "environment": self.telemetry_environment,
-                "ephemeral_server": bool(os.getenv("PREFECT__SERVER_EPHEMERAL", False)),
-                "api_version": SERVER_API_VERSION,
-                "prefect_version": prefect.__version__,
-                "session_id": self.session_id,
-                "session_start_timestamp": self.session_start_timestamp,
-            },
-        }
-
-        try:
-            async with httpx.AsyncClient() as client:
-                result = await client.post(
-                    "https://sens-o-matic.prefect.io/",
-                    json=heartbeat,
-                    headers={"x-prefect-event": "prefect_server"},
-                )
-            result.raise_for_status()
-        except Exception as exc:
-            self.logger.error(
-                f"Failed to send telemetry: {exc}\nShutting down telemetry service...",
-                # The traceback is only needed if doing deeper debugging, otherwise
-                # this looks like an impactful server error
-                exc_info=PREFECT_DEBUG_MODE.value(),
-            )
-            await self.stop(block=False)
-
-
-if __name__ == "__main__":
-    asyncio.run(Telemetry(handle_signals=True).start())

prefect/settings/models/_defaults.py

@@ -80,7 +80,6 @@ def default_database_connection_url(settings: "Settings") -> SecretStr:
             "host",
             "user",
             "name",
-            "password",
         ]
         missing = [
             attr for attr in required if getattr(settings.server.database, attr) is None

prefect/tasks.py

@@ -1088,17 +1088,17 @@ class Task(Generic[P, R]):
     # These preserve full parameter type checking when users call tasks normally
     @overload
     def __call__(
-        self: "Task[P, Coroutine[Any, Any, R]]",
+        self: "Task[P, Coroutine[Any, Any, T]]",
         *args: P.args,
         **kwargs: P.kwargs,
-    ) -> Coroutine[Any, Any, R]: ...
+    ) -> Coroutine[Any, Any, T]: ...
 
     @overload
     def __call__(
-        self: "Task[P, R]",
+        self: "Task[P, T]",
         *args: P.args,
         **kwargs: P.kwargs,
-    ) -> R: ...
+    ) -> T: ...
 
     @overload
     def __call__(
@@ -1116,65 +1116,65 @@ class Task(Generic[P, R]):
     # are advanced use cases.
     @overload
     def __call__(
-        self: "Task[..., Coroutine[Any, Any, R]]",
+        self: "Task[..., Coroutine[Any, Any, T]]",
         *args: Any,
         return_state: Literal[False],
         wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
         **kwargs: Any,
-    ) -> Coroutine[Any, Any, R]: ...
+    ) -> Coroutine[Any, Any, T]: ...
 
     @overload
     def __call__(
-        self: "Task[..., Coroutine[Any, Any, R]]",
+        self: "Task[..., Coroutine[Any, Any, T]]",
         *args: Any,
         return_state: Literal[True],
         wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
         **kwargs: Any,
-    ) -> State[R]: ...
+    ) -> State[T]: ...
 
     @overload
     def __call__(
-        self: "Task[..., R]",
+        self: "Task[..., T]",
         *args: Any,
         return_state: Literal[False],
         wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
         **kwargs: Any,
-    ) -> R: ...
+    ) -> T: ...
 
     @overload
     def __call__(
-        self: "Task[..., R]",
+        self: "Task[..., T]",
         *args: Any,
         return_state: Literal[True],
         wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
         **kwargs: Any,
-    ) -> State[R]: ...
+    ) -> State[T]: ...
 
     @overload
     def __call__(
-        self: "Task[..., Coroutine[Any, Any, R]]",
+        self: "Task[..., Coroutine[Any, Any, T]]",
         *args: Any,
         wait_for: OneOrManyFutureOrResult[Any],
         return_state: Literal[False] = False,
         **kwargs: Any,
-    ) -> Coroutine[Any, Any, R]: ...
+    ) -> Coroutine[Any, Any, T]: ...
 
     @overload
     def __call__(
-        self: "Task[..., R]",
+        self: "Task[..., T]",
         *args: Any,
         wait_for: OneOrManyFutureOrResult[Any],
         return_state: Literal[False] = False,
         **kwargs: Any,
-    ) -> R: ...
+    ) -> T: ...
 
     def __call__(
-        self: "Union[Task[..., R], Task[..., NoReturn]]",
+        self: "Union[Task[..., T], Task[..., NoReturn]]",
         *args: Any,
         return_state: bool = False,
         wait_for: Optional[OneOrManyFutureOrResult[Any]] = None,
         **kwargs: Any,
-    ) -> Union[R, State[R], None]:
+    ) -> Union[T, State[T], None]:
         """
         Run the task and return the result. If `return_state` is True returns
         the result is wrapped in a Prefect State which provides error handling.

prefect/testing/utilities.py

@@ -5,6 +5,7 @@ Internal utilities for tests.
 from __future__ import annotations
 
 import atexit
+import inspect
 import shutil
 import warnings
 from contextlib import ExitStack, contextmanager
@@ -31,6 +32,7 @@ from prefect.results import (
 from prefect.serializers import Serializer
 from prefect.server.api.server import SubprocessASGIServer
 from prefect.states import State
+from prefect.utilities.asyncutils import run_coro_as_sync
 
 if TYPE_CHECKING:
     from prefect.client.orchestration import PrefectClient
@@ -172,9 +174,26 @@ def prefect_test_harness(server_startup_timeout: int | None = 30):
         )
         yield
         # drain the logs before stopping the server to avoid connection errors on shutdown
-        APILogWorker.instance().drain()
-        # drain events to prevent stale events from leaking into subsequent test harnesses
-        EventsWorker.drain_all()
+        # When running in an async context, drain() and drain_all() return awaitables.
+        # We use a wrapper coroutine passed to run_coro_as_sync to ensure the awaitable
+        # is created and awaited on the same loop, avoiding cross-loop issues (issue #19762)
+
+        async def drain_workers():
+            try:
+                result = APILogWorker.instance().drain()
+                if inspect.isawaitable(result):
+                    await result
+            except RuntimeError:
+                # Worker may not have been started
+                pass
+
+            # drain events to prevent stale events from leaking into subsequent test harnesses
+            result = EventsWorker.drain_all()
+            if inspect.isawaitable(result):
+                await result
+
+        run_coro_as_sync(drain_workers())
+
         test_server.stop()
 
 

prefect/variables.py

@@ -1,6 +1,7 @@
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Generic, Optional, overload
 
 from pydantic import BaseModel, Field
+from typing_extensions import TypeVar
 
 from prefect._internal.compatibility.async_dispatch import async_dispatch
 from prefect._internal.compatibility.migration import getattr_migration
@@ -10,8 +11,10 @@ from prefect.client.utilities import get_or_create_client
 from prefect.exceptions import ObjectNotFound
 from prefect.types import MAX_VARIABLE_NAME_LENGTH, StrictVariableValue
 
+T = TypeVar("T", bound=StrictVariableValue, default=StrictVariableValue)
 
-class Variable(BaseModel):
+
+class Variable(BaseModel, Generic[T]):
     """
     Variables are named, mutable JSON values that can be shared across tasks and flows.
 
@@ -135,12 +138,27 @@ class Variable(BaseModel):
 
             return cls.model_validate(var_dict)
 
+    @overload
+    @classmethod
+    async def aget(
+        cls,
+        name: str,
+    ) -> T | None: ...
+
+    @overload
     @classmethod
     async def aget(
         cls,
         name: str,
-        default: StrictVariableValue = None,
-    ) -> StrictVariableValue:
+        default: T,
+    ) -> T: ...
+
+    @classmethod
+    async def aget(
+        cls,
+        name: str,
+        default: StrictVariableValue | None = None,
+    ) -> StrictVariableValue | None:
         """
         Asynchronously get a variable's value by name.
 
@@ -160,19 +178,44 @@ class Variable(BaseModel):
             async def my_flow():
                 var = await Variable.aget("my_var")
             ```
+
+            Get a variable's value with explicit typing.
+            ```python
+            from prefect import flow
+            from prefect.variables import Variable
+
+            @flow
+            async def my_flow():
+                var = await Variable[str].aget("my_var")
+            ```
         """
         client, _ = get_or_create_client()
         variable = await client.read_variable_by_name(name)
 
         return variable.value if variable else default
 
+    @overload
+    @classmethod
+    def get(
+        cls,
+        name: str,
+    ) -> T | None: ...
+
+    @overload
+    @classmethod
+    def get(
+        cls,
+        name: str,
+        default: T,
+    ) -> T: ...
+
     @classmethod
     @async_dispatch(aget)
     def get(
         cls,
         name: str,
-        default: StrictVariableValue = None,
-    ) -> StrictVariableValue:
+        default: StrictVariableValue | None = None,
+    ) -> StrictVariableValue | None:
         """
         Get a variable's value by name.
 
@@ -192,6 +235,16 @@ class Variable(BaseModel):
             def my_flow():
                 var = Variable.get("my_var")
             ```
+
+            Get a variable's value with explicit typing.
+            ```python
+            from prefect import flow
+            from prefect.variables import Variable
+
+            @flow
+            def my_flow():
+                var = Variable[str].get("my_var")
+            ```
         """
         with get_client(sync_client=True) as client:
             variable = client.read_variable_by_name(name)

prefect/workers/base.py

@@ -536,7 +536,6 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
         self._limit = limit
         self._limiter: Optional[anyio.CapacityLimiter] = None
         self._submitting_flow_run_ids: set[UUID] = set()
-        self._cancelling_flow_run_ids: set[UUID] = set()
         self._scheduled_task_scopes: set[anyio.CancelScope] = set()
         self._worker_metadata_sent = False
 
@@ -1539,13 +1538,6 @@ class BaseWorker(abc.ABC, Generic[C, V, R]):
                 f"Flow run '{flow_run.id}' was deleted before it could be marked as cancelled"
             )
 
-        # Do not remove the flow run from the cancelling set immediately because
-        # the API caches responses for the `read_flow_runs` and we do not want to
-        # duplicate cancellations.
-        await self._schedule_task(
-            60 * 10, self._cancelling_flow_run_ids.remove, flow_run.id
-        )
-
     async def _set_work_pool_template(
         self, work_pool: "WorkPool", job_template: dict[str, Any]
     ):

{prefect-3.6.6.dist-info → prefect-3.6.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: prefect
-Version: 3.6.6
+Version: 3.6.7
 Summary: Workflow orchestration and management.
 Project-URL: Changelog, https://github.com/PrefectHQ/prefect/releases
 Project-URL: Documentation, https://docs.prefect.io
@@ -72,7 +72,6 @@ Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0
 Requires-Dist: toml>=0.10.0
 Requires-Dist: typer<0.20.0,>=0.16.0
 Requires-Dist: typing-extensions<5.0.0,>=4.10.0
-Requires-Dist: uv>=0.6.0
 Requires-Dist: uvicorn!=0.29.0,>=0.14.0
 Requires-Dist: websockets<16.0,>=15.0.1
 Requires-Dist: whenever<0.10.0,>=0.7.3; python_version >= '3.13'
@@ -82,6 +81,8 @@ Provides-Extra: azure
 Requires-Dist: prefect-azure>=0.4.0; extra == 'azure'
 Provides-Extra: bitbucket
 Requires-Dist: prefect-bitbucket>=0.3.0; extra == 'bitbucket'
+Provides-Extra: bundles
+Requires-Dist: uv>=0.6.0; extra == 'bundles'
 Provides-Extra: dask
 Requires-Dist: prefect-dask>=0.3.0; extra == 'dask'
 Provides-Extra: databricks