prefect-client 3.3.4.dev2__py3-none-any.whl → 3.3.5__py3-none-any.whl

prefect/tasks.py

@@ -764,11 +764,6 @@ class Task(Generic[P, R]):
     def on_rollback(
         self, fn: Callable[["Transaction"], None]
     ) -> Callable[["Transaction"], None]:
-        if asyncio.iscoroutinefunction(fn):
-            raise ValueError(
-                "Asynchronous rollback hooks are not yet supported. Rollback hooks must be synchronous functions."
-            )
-
         self.on_rollback_hooks.append(fn)
         return fn
 

prefect/telemetry/instrumentation.py

@@ -1,6 +1,6 @@
 import logging
-import os
 import re
+import socket
 from typing import TYPE_CHECKING
 from urllib.parse import urljoin
 from uuid import UUID
@@ -74,7 +74,7 @@ def setup_exporters(
     resource = Resource.create(
         {
             "service.name": "prefect",
-            "service.instance.id": os.uname().nodename,
+            "service.instance.id": socket.gethostname(),
             "prefect.account": str(account_id),
             "prefect.workspace": str(workspace_id),
         }

prefect/transactions.py

@@ -1,19 +1,26 @@
+from __future__ import annotations
+
+import abc
+import asyncio
 import copy
+import inspect
 import logging
-from contextlib import contextmanager
+from contextlib import asynccontextmanager, contextmanager
 from contextvars import ContextVar, Token
 from functools import partial
 from typing import (
     Any,
+    AsyncGenerator,
     Callable,
-    Dict,
+    ClassVar,
     Generator,
-    List,
+    NoReturn,
     Optional,
     Type,
     Union,
 )
 
+import anyio.to_thread
 from pydantic import Field, PrivateAttr
 from typing_extensions import Self
 
@@ -32,8 +39,11 @@ from prefect.results import (
 )
 from prefect.utilities._engine import get_hook_name
 from prefect.utilities.annotations import NotSet
+from prefect.utilities.asyncutils import run_coro_as_sync
 from prefect.utilities.collections import AutoEnum
 
+logger: logging.Logger = get_logger("transactions")
+
 
 class IsolationLevel(AutoEnum):
     READ_COMMITTED = AutoEnum.auto()
@@ -54,29 +64,27 @@ class TransactionState(AutoEnum):
     ROLLED_BACK = AutoEnum.auto()
 
 
-class Transaction(ContextModel):
+class BaseTransaction(ContextModel, abc.ABC):
     """
     A base model for transaction state.
     """
 
     store: Optional[ResultStore] = None
     key: Optional[str] = None
-    children: List["Transaction"] = Field(default_factory=list)
+    children: list[Self] = Field(default_factory=list)
     commit_mode: Optional[CommitMode] = None
     isolation_level: Optional[IsolationLevel] = IsolationLevel.READ_COMMITTED
     state: TransactionState = TransactionState.PENDING
-    on_commit_hooks: List[Callable[["Transaction"], None]] = Field(default_factory=list)
-    on_rollback_hooks: List[Callable[["Transaction"], None]] = Field(
-        default_factory=list
-    )
+    on_commit_hooks: list[Callable[[Self], None]] = Field(default_factory=list)
+    on_rollback_hooks: list[Callable[[Self], None]] = Field(default_factory=list)
     overwrite: bool = False
     logger: Union[logging.Logger, LoggingAdapter] = Field(
         default_factory=partial(get_logger, "transactions")
     )
     write_on_commit: bool = True
-    _stored_values: Dict[str, Any] = PrivateAttr(default_factory=dict)
-    _staged_value: Any = None
-    __var__: ContextVar[Self] = ContextVar("transaction")
+    _stored_values: dict[str, Any] = PrivateAttr(default_factory=dict)
+    _staged_value: ResultRecord[Any] | Any = None
+    __var__: ClassVar[ContextVar[Self]] = ContextVar("transaction")
 
     def set(self, name: str, value: Any) -> None:
         """
@@ -174,7 +182,8 @@ class Transaction(ContextModel):
     def is_active(self) -> bool:
         return self.state == TransactionState.ACTIVE
 
-    def __enter__(self) -> Self:
+    def prepare_transaction(self) -> None:
+        """Helper method to prepare transaction state and validate configuration."""
         if self._token is not None:
             raise RuntimeError(
                 "Context already entered. Context enter calls cannot be nested."
@@ -203,6 +212,51 @@ class Transaction(ContextModel):
 
         # this needs to go before begin, which could set the state to committed
         self.state = TransactionState.ACTIVE
+
+    def add_child(self, transaction: Self) -> None:
+        self.children.append(transaction)
+
+    def get_parent(self) -> Self | None:
+        parent = None
+        if self._token:
+            prev_var = self._token.old_value
+            if prev_var != Token.MISSING:
+                parent = prev_var
+        else:
+            # `_token` has been reset so we need to get the active transaction from the context var
+            parent = self.get_active()
+        return parent
+
+    def stage(
+        self,
+        value: Any,
+        on_rollback_hooks: Optional[list[Callable[..., Any]]] = None,
+        on_commit_hooks: Optional[list[Callable[..., Any]]] = None,
+    ) -> None:
+        """
+        Stage a value to be committed later.
+        """
+        on_commit_hooks = on_commit_hooks or []
+        on_rollback_hooks = on_rollback_hooks or []
+
+        if self.state != TransactionState.COMMITTED:
+            self._staged_value = value
+            self.on_rollback_hooks += on_rollback_hooks
+            self.on_commit_hooks += on_commit_hooks
+            self.state = TransactionState.STAGED
+
+    @classmethod
+    def get_active(cls: Type[Self]) -> Optional[Self]:
+        return cls.__var__.get(None)
+
+
+class Transaction(BaseTransaction):
+    """
+    A model representing the state of a transaction.
+    """
+
+    def __enter__(self) -> Self:
+        self.prepare_transaction()
         self.begin()
         self._token = self.__var__.set(self)
         return self
@@ -252,11 +306,9 @@ class Transaction(ContextModel):
         ):
             self.state = TransactionState.COMMITTED
 
-    def read(self) -> Optional[ResultRecord[Any]]:
+    def read(self) -> ResultRecord[Any] | None:
         if self.store and self.key:
-            record = self.store.read(key=self.key)
-            if isinstance(record, ResultRecord):
-                return record
+            return self.store.read(key=self.key)
         return None
 
     def reset(self) -> None:
@@ -274,20 +326,6 @@ class Transaction(ContextModel):
         if parent and self.state == TransactionState.ROLLED_BACK:
             parent.rollback()
 
-    def add_child(self, transaction: "Transaction") -> None:
-        self.children.append(transaction)
-
-    def get_parent(self) -> Optional["Transaction"]:
-        parent = None
-        if self._token:
-            prev_var = getattr(self._token, "old_value")
-            if prev_var != Token.MISSING:
-                parent = prev_var
-        else:
-            # `_token` has been reset so we need to get the active transaction from the context var
-            parent = self.get_active()
-        return parent
-
     def commit(self) -> bool:
         if self.state in [TransactionState.ROLLED_BACK, TransactionState.COMMITTED]:
             if (
@@ -302,21 +340,19 @@ class Transaction(ContextModel):
 
         try:
             for child in self.children:
-                child.commit()
+                if inspect.iscoroutinefunction(child.commit):
+                    run_coro_as_sync(child.commit())
+                else:
+                    child.commit()
 
             for hook in self.on_commit_hooks:
                 self.run_hook(hook, "commit")
 
             if self.store and self.key and self.write_on_commit:
-                if isinstance(self.store, ResultStore):
-                    if isinstance(self._staged_value, ResultRecord):
-                        self.store.persist_result_record(
-                            result_record=self._staged_value
-                        )
-                    else:
-                        self.store.write(key=self.key, obj=self._staged_value)
+                if isinstance(self._staged_value, ResultRecord):
+                    self.store.persist_result_record(result_record=self._staged_value)
                 else:
-                    self.store.write(key=self.key, result=self._staged_value)
+                    self.store.write(key=self.key, obj=self._staged_value)
 
             self.state = TransactionState.COMMITTED
             if (
@@ -353,7 +389,10 @@ class Transaction(ContextModel):
             self.logger.info(f"Running {hook_type} hook {hook_name!r}")
 
         try:
-            hook(self)
+            if asyncio.iscoroutinefunction(hook):
+                run_coro_as_sync(hook(self))
+            else:
+                hook(self)
         except Exception as exc:
             if should_log:
                 self.logger.error(
@@ -366,24 +405,6 @@ class Transaction(ContextModel):
                     f"{hook_type.capitalize()} hook {hook_name!r} finished running successfully"
                 )
 
-    def stage(
-        self,
-        value: Any,
-        on_rollback_hooks: Optional[list[Callable[..., Any]]] = None,
-        on_commit_hooks: Optional[list[Callable[..., Any]]] = None,
-    ) -> None:
-        """
-        Stage a value to be committed later.
-        """
-        on_commit_hooks = on_commit_hooks or []
-        on_rollback_hooks = on_rollback_hooks or []
-
-        if self.state != TransactionState.COMMITTED:
-            self._staged_value = value
-            self.on_rollback_hooks += on_rollback_hooks
-            self.on_commit_hooks += on_commit_hooks
-            self.state = TransactionState.STAGED
-
     def rollback(self) -> bool:
         if self.state in [TransactionState.ROLLED_BACK, TransactionState.COMMITTED]:
             return False
@@ -392,10 +413,13 @@ class Transaction(ContextModel):
             for hook in reversed(self.on_rollback_hooks):
                 self.run_hook(hook, "rollback")
 
-            self.state = TransactionState.ROLLED_BACK
+            self.state: TransactionState = TransactionState.ROLLED_BACK
 
             for child in reversed(self.children):
-                child.rollback()
+                if inspect.iscoroutinefunction(child.rollback):
+                    run_coro_as_sync(child.rollback())
+                else:
+                    child.rollback()
 
             return True
         except Exception:
@@ -414,24 +438,221 @@ class Transaction(ContextModel):
                 self.logger.debug(f"Releasing lock for transaction {self.key!r}")
                 self.store.release_lock(self.key)
 
-    @classmethod
-    def get_active(cls: Type[Self]) -> Optional[Self]:
-        return cls.__var__.get(None)
 
+class AsyncTransaction(BaseTransaction):
+    """
+    A model representing the state of an asynchronous transaction.
+    """
+
+    async def begin(self) -> None:
+        if (
+            self.store
+            and self.key
+            and self.isolation_level == IsolationLevel.SERIALIZABLE
+        ):
+            self.logger.debug(f"Acquiring lock for transaction {self.key!r}")
+            await self.store.aacquire_lock(self.key)
+        if (
+            not self.overwrite
+            and self.store
+            and self.key
+            and await self.store.aexists(key=self.key)
+        ):
+            self.state = TransactionState.COMMITTED
+
+    async def read(self) -> ResultRecord[Any] | None:
+        if self.store and self.key:
+            return await self.store.aread(key=self.key)
+        return None
 
-def get_transaction() -> Optional[Transaction]:
-    return Transaction.get_active()
+    async def reset(self) -> None:
+        parent = self.get_parent()
+
+        if parent:
+            # parent takes responsibility
+            parent.add_child(self)
+
+        if self._token:
+            self.__var__.reset(self._token)
+            self._token = None
+
+        # do this below reset so that get_transaction() returns the relevant txn
+        if parent and self.state == TransactionState.ROLLED_BACK:
+            await parent.rollback()
+
+    async def commit(self) -> bool:
+        if self.state in [TransactionState.ROLLED_BACK, TransactionState.COMMITTED]:
+            if (
+                self.store
+                and self.key
+                and self.isolation_level == IsolationLevel.SERIALIZABLE
+            ):
+                self.logger.debug(f"Releasing lock for transaction {self.key!r}")
+                self.store.release_lock(self.key)
+
+            return False
+
+        try:
+            for child in self.children:
+                if isinstance(child, AsyncTransaction):
+                    await child.commit()
+                else:
+                    child.commit()
+
+            for hook in self.on_commit_hooks:
+                await self.run_hook(hook, "commit")
+
+            if self.store and self.key and self.write_on_commit:
+                if isinstance(self._staged_value, ResultRecord):
+                    await self.store.apersist_result_record(
+                        result_record=self._staged_value
+                    )
+                else:
+                    await self.store.awrite(key=self.key, obj=self._staged_value)
+
+            self.state = TransactionState.COMMITTED
+            if (
+                self.store
+                and self.key
+                and self.isolation_level == IsolationLevel.SERIALIZABLE
+            ):
+                self.logger.debug(f"Releasing lock for transaction {self.key!r}")
+                self.store.release_lock(self.key)
+            return True
+        except SerializationError as exc:
+            if self.logger:
+                self.logger.warning(
+                    f"Encountered an error while serializing result for transaction {self.key!r}: {exc}"
+                    " Code execution will continue, but the transaction will not be committed.",
+                )
+            await self.rollback()
+            return False
+        except Exception:
+            if self.logger:
+                self.logger.exception(
+                    f"An error was encountered while committing transaction {self.key!r}",
+                    exc_info=True,
+                )
+            await self.rollback()
+            return False
+
+    async def run_hook(self, hook: Callable[..., Any], hook_type: str) -> None:
+        hook_name = get_hook_name(hook)
+        # Undocumented way to disable logging for a hook. Subject to change.
+        should_log = getattr(hook, "log_on_run", True)
+
+        if should_log:
+            self.logger.info(f"Running {hook_type} hook {hook_name!r}")
+
+        try:
+            if asyncio.iscoroutinefunction(hook):
+                await hook(self)
+            else:
+                await anyio.to_thread.run_sync(hook, self)
+        except Exception as exc:
+            if should_log:
+                self.logger.error(
+                    f"An error was encountered while running {hook_type} hook {hook_name!r}",
+                )
+            raise exc
+        else:
+            if should_log:
+                self.logger.info(
+                    f"{hook_type.capitalize()} hook {hook_name!r} finished running successfully"
+                )
+
+    async def rollback(self) -> bool:
+        if self.state in [TransactionState.ROLLED_BACK, TransactionState.COMMITTED]:
+            return False
+
+        try:
+            for hook in reversed(self.on_rollback_hooks):
+                await self.run_hook(hook, "rollback")
+
+            self.state: TransactionState = TransactionState.ROLLED_BACK
+
+            for child in reversed(self.children):
+                if isinstance(child, AsyncTransaction):
+                    await child.rollback()
+                else:
+                    child.rollback()
+
+            return True
+        except Exception:
+            if self.logger:
+                self.logger.exception(
+                    f"An error was encountered while rolling back transaction {self.key!r}",
+                    exc_info=True,
+                )
+            return False
+        finally:
+            if (
+                self.store
+                and self.key
+                and self.isolation_level == IsolationLevel.SERIALIZABLE
+            ):
+                self.logger.debug(f"Releasing lock for transaction {self.key!r}")
+                self.store.release_lock(self.key)
+
+    async def __aenter__(self) -> Self:
+        self.prepare_transaction()
+        await self.begin()
+        self._token = self.__var__.set(self)
+        return self
+
+    async def __aexit__(self, *exc_info: Any) -> None:
+        exc_type, exc_val, _ = exc_info
+        if not self._token:
+            raise RuntimeError(
+                "Asymmetric use of context. Context exit called without an enter."
+            )
+        if exc_type:
+            await self.rollback()
+            await self.reset()
+            raise exc_val
+
+        if self.commit_mode == CommitMode.EAGER:
+            await self.commit()
+
+        # if parent, let them take responsibility
+        if self.get_parent():
+            await self.reset()
+            return
+
+        if self.commit_mode == CommitMode.OFF:
+            # if no one took responsibility to commit, rolling back
+            # note that rollback returns if already committed
+            await self.rollback()
+        elif self.commit_mode == CommitMode.LAZY:
+            # no one left to take responsibility for committing
+            await self.commit()
+
+        await self.reset()
+
+    def __enter__(self) -> NoReturn:
+        raise NotImplementedError(
+            "AsyncTransaction does not support the `with` statement. Use the `async with` statement instead."
+        )
+
+    def __exit__(self, *exc_info: Any) -> NoReturn:
+        raise NotImplementedError(
+            "AsyncTransaction does not support the `with` statement. Use the `async with` statement instead."
+        )
+
+
+def get_transaction() -> BaseTransaction | None:
+    return BaseTransaction.get_active()
 
 
 @contextmanager
 def transaction(
-    key: Optional[str] = None,
-    store: Optional[ResultStore] = None,
-    commit_mode: Optional[CommitMode] = None,
-    isolation_level: Optional[IsolationLevel] = None,
+    key: str | None = None,
+    store: ResultStore | None = None,
+    commit_mode: CommitMode | None = None,
+    isolation_level: IsolationLevel | None = None,
     overwrite: bool = False,
     write_on_commit: bool = True,
-    logger: Optional[Union[logging.Logger, LoggingAdapter]] = None,
+    logger: logging.Logger | LoggingAdapter | None = None,
 ) -> Generator[Transaction, None, None]:
     """
     A context manager for opening and managing a transaction.
@@ -473,3 +694,56 @@ def transaction(
         logger=_logger,
     ) as txn:
         yield txn
+
+
+@asynccontextmanager
+async def atransaction(
+    key: str | None = None,
+    store: ResultStore | None = None,
+    commit_mode: CommitMode | None = None,
+    isolation_level: IsolationLevel | None = None,
+    overwrite: bool = False,
+    write_on_commit: bool = True,
+    logger: logging.Logger | LoggingAdapter | None = None,
+) -> AsyncGenerator[AsyncTransaction, None]:
+    """
+    An asynchronous context manager for opening and managing an asynchronous transaction.
+
+    Args:
+        - key: An identifier to use for the transaction
+        - store: The store to use for persisting the transaction result. If not provided,
+            a default store will be used based on the current run context.
+        - commit_mode: The commit mode controlling when the transaction and
+            child transactions are committed
+        - overwrite: Whether to overwrite an existing transaction record in the store
+        - write_on_commit: Whether to write the result to the store on commit. If not provided,
+            the default will be determined by the current run context. If no run context is
+            available, the value of `PREFECT_RESULTS_PERSIST_BY_DEFAULT` will be used.
+
+    Yields:
+        - AsyncTransaction: An object representing the transaction state
+    """
+
+    # if there is no key, we won't persist a record
+    if key and not store:
+        store = get_result_store()
+
+    # Avoid inheriting a NullFileSystem for metadata_storage from a flow's result store
+    if store and isinstance(store.metadata_storage, NullFileSystem):
+        store = store.model_copy(update={"metadata_storage": None})
+
+    try:
+        _logger: Union[logging.Logger, LoggingAdapter] = logger or get_run_logger()
+    except MissingContextError:
+        _logger = get_logger("transactions")
+
+    async with AsyncTransaction(
+        key=key,
+        store=store,
+        commit_mode=commit_mode,
+        isolation_level=isolation_level,
+        overwrite=overwrite,
+        write_on_commit=write_on_commit,
+        logger=_logger,
+    ) as txn:
+        yield txn