zero-server 0.0.1__py3-none-any.whl

zero_server/server/pg_types.py

@@ -0,0 +1,69 @@
+"""Postgres column types as Zero sees them.
+
+Ported from rocicorp/mono `packages/zero-cache/src/types/pg-data-type.ts` at zero/v1.10.0
+(905d08516ba0caaf314cd28e93a5a47e8acdd482). Apache-2.0, Copyright Rocicorp.
+"""
+
+from typing import Final
+
+from zero_server.schema import ValueType
+
+NUMERIC: Final = frozenset(
+    {
+        "smallint", "integer", "int", "int2", "int4", "int8", "bigint", "smallserial", "serial", "serial2",
+        "serial4", "serial8", "bigserial", "decimal", "numeric", "real", "double precision", "float", "float4",
+        "float8",
+    }
+)  # fmt: skip
+NATIVE_STRING: Final = frozenset({"bpchar", "character", "character varying", "text", "varchar"})
+TEXT_REPRESENTED: Final = frozenset(
+    {
+        "cidr", "ean13", "inet", "isbn", "isbn13", "ismn", "ismn13", "issn", "issn13", "macaddr", "macaddr8",
+        "pg_lsn", "upc", "uuid",
+    }
+)  # fmt: skip
+DATE_TIME: Final = frozenset(
+    {
+        "date", "time", "timetz", "time with time zone", "time without time zone", "timestamp", "timestamptz",
+        "timestamp with time zone", "timestamp without time zone",
+    }
+)  # fmt: skip
+
+
+def _lookup(pg_type: str) -> str:
+    """Lowercased, with any arguments (the `(32)` of `char(32)`) removed."""
+    lowered = pg_type.lower()
+    start = lowered.find("(")
+    return lowered if start == -1 else lowered[:start]
+
+
+def is_number(pg_type: str) -> bool:
+    return _lookup(pg_type) in NUMERIC
+
+
+def is_native_string(pg_type: str) -> bool:
+    return _lookup(pg_type) in NATIVE_STRING
+
+
+def is_text_represented(pg_type: str) -> bool:
+    return _lookup(pg_type) in TEXT_REPRESENTED
+
+
+def is_string(pg_type: str) -> bool:
+    return is_native_string(pg_type) or is_text_represented(pg_type)
+
+
+def zql_value_type(pg_type: str, is_enum: bool, is_array: bool) -> ValueType | None:
+    """The Zero type a Postgres column must be declared as, or `None` if Zero does not support it."""
+    if is_array:
+        return "json"
+    key = _lookup(pg_type)
+    if key in NUMERIC or key in DATE_TIME:
+        return "number"
+    if key in NATIVE_STRING or key in TEXT_REPRESENTED:
+        return "string"
+    if key in ("bool", "boolean"):
+        return "boolean"
+    if key in ("json", "jsonb"):
+        return "json"
+    return "string" if is_enum else None

zero_server/server/process_mutations.py

@@ -0,0 +1,412 @@
+"""The mutate endpoint: exactly-once processing of a push from zero-cache.
+
+Ported from rocicorp/mono `packages/zero-server/src/process-mutations.ts` at zero/v1.10.0
+(905d08516ba0caaf314cd28e93a5a47e8acdd482), with `MutationAlreadyProcessedError` from
+`packages/zero-cache/src/services/mutagen/error.ts`. Apache-2.0, Copyright Rocicorp.
+
+Each mutation runs inside the same database transaction that bumps its client's `lastMutationID`.
+The bump comes first and the comparison second, so a duplicate or a gap is found after the bump and
+undone by rolling that transaction back. A mutation whose mutator fails is retried without the
+mutator: the second transaction takes the ID and records the error in the `mutations` table, which is
+where the client's pending mutation learns it failed.
+"""
+
+import json
+import logging
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass
+from typing import Any, Literal, Protocol
+
+from pydantic import TypeAdapter, ValidationError
+
+from zero_server.protocol.application_error import ApplicationError, wrap_with_application_error
+from zero_server.protocol.error import ErrorReason, ServerPushFailed
+from zero_server.protocol.mutate_server import MutateParams, MutateSuccess
+from zero_server.protocol.mutation import (
+    CLEANUP_RESULTS_MUTATION_NAME,
+    CleanupResultsArg,
+    CleanupResultsBulkArg,
+    CustomMutation,
+    MutationAppError,
+    MutationOk,
+    MutationResponse,
+    MutationZeroError,
+)
+from zero_server.protocol.mutation_id import MutationID
+from zero_server.protocol.push import PushBody
+from zero_server.protocol.wire import parse_wire
+from zero_server.shared.asserts import check
+from zero_server.shared.error import get_error_details, get_error_message
+from zero_server.shared.js import js_number, js_truthy
+from zero_server.shared.json import JSONValue
+
+logger = logging.getLogger("zero_server.mutate")
+
+_cleanup_results_arg: TypeAdapter[CleanupResultsArg] = TypeAdapter(CleanupResultsArg)
+
+
+class TransactionProviderHooks(Protocol):
+    """The three statements against zero-cache's bookkeeping tables, run in the mutation's transaction."""
+
+    async def update_client_mutation_id(self) -> int: ...
+
+    async def write_mutation_result(self, result: MutationResponse) -> None: ...
+
+    async def delete_mutation_results(self, args: CleanupResultsArg) -> None: ...
+
+
+@dataclass(frozen=True)
+class TransactionProviderInput:
+    upstream_schema: str
+    client_group_id: str
+    client_id: str
+    mutation_id: int | float
+
+
+class Database[T](Protocol):
+    """A database `handle_mutate_request` can run transactions against.
+
+    `transaction` opens one transaction, calls `callback` inside it with the app's transaction object and
+    the bookkeeping hooks, and commits when `callback` returns.
+    """
+
+    async def transaction[R](
+        self,
+        callback: Callable[[T, TransactionProviderHooks], Awaitable[R]],
+        transaction_input: TransactionProviderInput | None = None,
+    ) -> R: ...
+
+
+type TransactFnCallback[T] = Callable[[T, str, JSONValue | None], Awaitable[None]]
+type TransactFn[T] = Callable[[TransactFnCallback[T]], Awaitable[MutationResponse]]
+type MutateRequestHandler[T] = Callable[[TransactFn[T], CustomMutation], Awaitable[MutationResponse]]
+"""Runs once per custom mutation.
+
+Call `transact` to run the mutation's database work. Raising before `transact` records an application
+error for the mutation; raising after it only reaches the log, because the mutation has committed.
+"""
+
+type MutateResponse = MutateSuccess | ServerPushFailed
+
+type _DatabaseTransactionPhase = Literal["open", "execute"]
+type _MutationPhase = Literal["preTransaction", "transactionPending", "postCommit"]
+
+
+class OutOfOrderMutation(Exception):
+    error_name = "Error"
+
+    def __init__(self, client_id: str, received_mutation_id: float, last_mutation_id: float) -> None:
+        super().__init__(
+            f"Client {client_id} sent mutation ID {js_number(received_mutation_id)} "
+            f"but expected {js_number(last_mutation_id)}"
+        )
+
+
+class MutationAlreadyProcessedError(Exception):
+    error_name = "Error"
+
+    def __init__(self, client_id: str, received: float, actual: float) -> None:
+        super().__init__(
+            f"Ignoring mutation from {client_id} with ID {js_number(received)} as it was already processed. "
+            f"Expected: {js_number(actual)}"
+        )
+        check(
+            received < actual,
+            f"MutationAlreadyProcessedError: received ({js_number(received)}) must be < actual ({js_number(actual)})",
+        )
+
+
+class DatabaseTransactionError(Exception):
+    def __init__(self, phase: _DatabaseTransactionPhase, cause: BaseException) -> None:
+        message = (
+            f"Failed to open database transaction: {get_error_message(cause)}"
+            if phase == "open"
+            else f"Database transaction failed after opening: {get_error_message(cause)}"
+        )
+        super().__init__(message)
+        self.__cause__ = cause
+
+
+_PASSES_THROUGH = (DatabaseTransactionError, OutOfOrderMutation, MutationAlreadyProcessedError, ApplicationError)
+
+
+async def _application_error_wrapper[R](fn: Callable[[], Awaitable[R]]) -> R:
+    try:
+        return await fn()
+    except _PASSES_THROUGH:
+        raise
+    except Exception as error:
+        raise wrap_with_application_error(error) from error
+
+
+def _refuse_constant(name: str) -> None:
+    raise ValueError(f"{name} is not valid JSON")
+
+
+def _parse_failure(
+    subject: str, error: BaseException, mutation_ids: list[MutationID], *, syntax: bool = False
+) -> ServerPushFailed:
+    return ServerPushFailed(
+        reason=ErrorReason.PARSE,
+        message=f"Failed to parse {subject}: {get_error_message(error)}",
+        mutationIDs=mutation_ids,
+        # Upstream's parsers throw a `TypeError`, and `JSON.parse` a `SyntaxError`.
+        details={"name": "SyntaxError" if syntax else "TypeError"},
+    )
+
+
+async def handle_mutate_request[T](
+    *,
+    db_provider: Database[T],
+    handler: MutateRequestHandler[T],
+    query: Mapping[str, str],
+    body: object,
+    user_id: str | None,
+) -> MutateResponse:
+    """Process one `/mutate` request from zero-cache.
+
+    `query` is the request's query string and `body` its JSON, either raw bytes or already parsed.
+    `user_id` is the authenticated user, `None` when logged out; reporting it back is what makes
+    zero-cache treat the connection's user as server-validated.
+    """
+    return await handle_mutate(db_provider, handler, query, body, user_id, report_user_id=True)
+
+
+async def handle_mutate[T](
+    db_provider: Database[T],
+    handler: MutateRequestHandler[T],
+    query: Mapping[str, str],
+    body: object,
+    user_id: str | None,
+    *,
+    report_user_id: bool,
+) -> MutateResponse:
+    """`handle_mutate_request`, and the legacy form that reports no user ID at all."""
+    json_body: Any = body
+    if isinstance(body, bytes):
+        try:
+            json_body = json.loads(body, parse_constant=_refuse_constant)
+        except ValueError as error:
+            logger.error("Failed to parse push body", exc_info=error)
+            return _parse_failure("push body", error, [], syntax=True)
+
+    mutation_ids: list[MutationID] = []
+    try:
+        push_body = PushBody.from_json(json_body)
+        mutation_ids = [MutationID(id=m.id, clientID=m.client_id) for m in push_body.mutations]
+    except ValidationError as error:
+        logger.error("Failed to parse push body", exc_info=error)
+        return _parse_failure("push body", error, mutation_ids)
+
+    try:
+        params = MutateParams.from_json(dict(query))
+    except ValidationError as error:
+        logger.error("Failed to parse push query parameters", exc_info=error)
+        return _parse_failure("push query parameters", error, mutation_ids)
+
+    if push_body.push_version != 1:
+        return ServerPushFailed(
+            reason=ErrorReason.UNSUPPORTED_PUSH_VERSION,
+            mutationIDs=mutation_ids,
+            message=f"Unsupported push version: {js_number(push_body.push_version)}",
+        )
+
+    responses: list[MutationResponse] = []
+    processed_count = 0
+
+    try:
+        transactor = _Transactor(db_provider, push_body, params)
+
+        for m in push_body.mutations:
+            if m.type == "custom" and m.name == CLEANUP_RESULTS_MUTATION_NAME:
+                try:
+                    await _process_cleanup_results_mutation(db_provider, m, params)
+                except Exception as error:
+                    # A failed cleanup costs disk, not correctness, so it does not fail the push.
+                    logger.warning("Failed to process cleanup mutation for client %s", m.client_id, exc_info=error)
+                processed_count += 1
+                continue
+
+            check(m.type == "custom", "Expected custom mutation")
+            assert isinstance(m, CustomMutation)
+            response = await _process_one(transactor, handler, m)
+            responses.append(response)
+            processed_count += 1
+
+        if report_user_id:
+            return MutateSuccess(mutations=responses, userID=user_id)
+        return MutateSuccess(mutations=responses)
+    except Exception as error:
+        logger.error("Failed to process push request", exc_info=error)
+        details = get_error_details(error)
+        return ServerPushFailed(
+            reason=(
+                ErrorReason.OUT_OF_ORDER_MUTATION
+                if isinstance(error, OutOfOrderMutation)
+                else ErrorReason.DATABASE
+                if isinstance(error, DatabaseTransactionError)
+                else ErrorReason.INTERNAL
+            ),
+            message=get_error_message(error),
+            # Only the mutations not yet processed.
+            mutationIDs=mutation_ids[processed_count:],
+            # Upstream drops falsy details, `0` and `""` included.
+            details=details if js_truthy(details) else None,
+        )
+
+
+async def _process_one[T](
+    transactor: "_Transactor[T]", handler: MutateRequestHandler[T], m: CustomMutation
+) -> MutationResponse:
+    phase: _MutationPhase = "preTransaction"
+
+    async def transact(callback: TransactFnCallback[T]) -> MutationResponse:
+        nonlocal phase
+        phase = "transactionPending"
+        result = await transactor.transact(m, callback)
+        phase = "postCommit"
+        return result
+
+    try:
+        return await _application_error_wrapper(lambda: handler(transact, m))
+    except ApplicationError as error:
+        if phase == "preTransaction":
+            await transactor.persist_pre_transaction_failure(m, error)
+        elif phase == "postCommit":  # pragma: no branch - `transact` never raises an ApplicationError
+            logger.error(
+                "Post-commit mutation handler failed for mutation %s for client %s", m.id, m.client_id, exc_info=error
+            )
+        logger.warning("Application error processing mutation %s for client %s", m.id, m.client_id, exc_info=error)
+        return _app_error_response(m, error)
+
+
+class _Transactor[T]:
+    def __init__(self, db_provider: Database[T], push: PushBody, params: MutateParams) -> None:
+        self._db = db_provider
+        self._push = push
+        self._params = params
+
+    async def transact(self, mutation: CustomMutation, callback: TransactFnCallback[T]) -> MutationResponse:
+        app_error: ApplicationError | None = None
+        while True:
+            try:
+                result = await self._transact_impl(mutation, callback, app_error)
+                if app_error is not None:
+                    logger.warning(
+                        "Mutation %s for client %s was retried after an error",
+                        mutation.id,
+                        mutation.client_id,
+                        exc_info=app_error,
+                    )
+                    return _app_error_response(mutation, app_error)
+                return result
+            except OutOfOrderMutation as error:
+                logger.error("%s", error)
+                raise
+            except MutationAlreadyProcessedError as error:
+                logger.warning("%s", error)
+                return MutationResponse(
+                    id=MutationID(clientID=mutation.client_id, id=mutation.id),
+                    result=MutationZeroError(error="alreadyProcessed", details=str(error)),
+                )
+            except Exception as error:
+                if app_error is not None:
+                    # The retry, which runs no mutator, failed too: nothing to skip to.
+                    logger.error(
+                        "Retry also failed for mutation %s for client %s",
+                        mutation.id,
+                        mutation.client_id,
+                        exc_info=error,
+                    )
+                    raise
+                original: BaseException = error
+                if isinstance(error, DatabaseTransactionError) and error.__cause__ is not None:
+                    original = error.__cause__
+                app_error = wrap_with_application_error(original)
+                logger.warning(
+                    "Error processing mutation %s for client %s, retrying without mutator",
+                    mutation.id,
+                    mutation.client_id,
+                    exc_info=app_error,
+                )
+
+    async def persist_pre_transaction_failure(self, mutation: CustomMutation, app_error: ApplicationError) -> None:
+        """The handler raised before calling `transact`. The ID still advances and the failure is recorded."""
+
+        await self._transact_impl(mutation, None, app_error)
+
+    async def _transact_impl(
+        self, mutation: CustomMutation, callback: TransactFnCallback[T] | None, app_error: ApplicationError | None
+    ) -> MutationResponse:
+        """Bump the ID, then run `callback`, or with `app_error` set, record that error in its place."""
+        phase: _DatabaseTransactionPhase = "open"
+
+        async def run(tx: T, hooks: TransactionProviderHooks) -> MutationResponse:
+            nonlocal phase
+            phase = "execute"
+            await self._check_and_increment_last_mutation_id(hooks, mutation.client_id, mutation.id)
+            if app_error is None:
+                assert callback is not None
+                logger.debug("Executing mutator '%s' (id=%s)", mutation.name, mutation.id)
+                await callback(tx, mutation.name, mutation.args[0] if mutation.args else None)
+            else:
+                await hooks.write_mutation_result(_app_error_response(mutation, app_error))
+            return MutationResponse(id=MutationID(clientID=mutation.client_id, id=mutation.id), result=MutationOk())
+
+        try:
+            return await self._db.transaction(
+                run,
+                TransactionProviderInput(
+                    upstream_schema=self._params.schema_,
+                    client_group_id=self._push.client_group_id,
+                    client_id=mutation.client_id,
+                    mutation_id=mutation.id,
+                ),
+            )
+        except (ApplicationError, OutOfOrderMutation, MutationAlreadyProcessedError):
+            raise
+        except Exception as error:
+            raise DatabaseTransactionError(phase, error) from error
+
+    async def _check_and_increment_last_mutation_id(
+        self, hooks: TransactionProviderHooks, client_id: str, received: float
+    ) -> None:
+        last_mutation_id = await hooks.update_client_mutation_id()
+        if received < last_mutation_id:
+            raise MutationAlreadyProcessedError(client_id, received, last_mutation_id)
+        if received > last_mutation_id:
+            raise OutOfOrderMutation(client_id, received, last_mutation_id)
+
+
+def _app_error_response(m: CustomMutation, error: ApplicationError) -> MutationResponse:
+    return MutationResponse(
+        id=MutationID(clientID=m.client_id, id=m.id),
+        result=MutationAppError(message=error.message, details=error.details if js_truthy(error.details) else None),
+    )
+
+
+async def _process_cleanup_results_mutation[T](
+    db_provider: Database[T], mutation: CustomMutation, params: MutateParams
+) -> None:
+    """Deletes acknowledged results. An internal mutation, so it never touches `lastMutationID`."""
+    try:
+        args = parse_wire(_cleanup_results_arg, mutation.args[0] if mutation.args else None)
+    except ValidationError as error:
+        logger.warning("Cleanup mutation has invalid args", exc_info=error)
+        return
+
+    client_id = args.client_ids[0] if isinstance(args, CleanupResultsBulkArg) else args.client_id
+
+    async def delete(_tx: T, hooks: TransactionProviderHooks) -> None:
+        await hooks.delete_mutation_results(args)
+
+    # Only the upstream schema matters to the delete; the rest fills the interface.
+    await db_provider.transaction(
+        delete,
+        TransactionProviderInput(
+            upstream_schema=params.schema_,
+            client_group_id=args.client_group_id,
+            client_id=client_id,
+            mutation_id=0,
+        ),
+    )

zero_server/server/process_queries.py

@@ -0,0 +1,109 @@
+"""The query endpoint: turn each named query zero-cache forwards into its AST, in server names.
+
+Ported from rocicorp/mono `packages/zero-server/src/queries/process-queries.ts` at zero/v1.10.0
+(905d08516ba0caaf314cd28e93a5a47e8acdd482). Apache-2.0, Copyright Rocicorp.
+"""
+
+import inspect
+import json
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from pydantic import TypeAdapter, ValidationError
+
+from zero_server.protocol.ast import map_ast
+from zero_server.protocol.custom_queries import (
+    AppErroredQuery,
+    ParseErroredQuery,
+    TransformedQuery,
+    TransformRequestMessage,
+)
+from zero_server.protocol.error import ErrorReason, ServerTransformFailed
+from zero_server.protocol.query_server import QuerySuccess
+from zero_server.protocol.wire import parse_wire
+from zero_server.query.builder import Query
+from zero_server.query.registry import QueryParseError
+from zero_server.schema import Schema, client_to_server
+from zero_server.shared.asserts import check
+from zero_server.shared.error import get_error_details, get_error_message
+from zero_server.shared.js import js_truthy
+from zero_server.shared.json import JSONValue
+
+logger = logging.getLogger("zero_server.query")
+
+type QueryRequestHandler = Callable[[str, JSONValue | None], Query | Awaitable[Query]]
+"""Turns a query's name and its single argument into a `Query`."""
+
+type QueryResponse = QuerySuccess | ServerTransformFailed
+
+_transform_request: TypeAdapter[Any] = TypeAdapter(TransformRequestMessage)
+
+
+async def handle_query_request(
+    *,
+    handler: QueryRequestHandler,
+    schema: Schema,
+    body: object,
+    user_id: str | None,
+) -> QueryResponse:
+    """Process one `/query` request from zero-cache.
+
+    `body` is the request's JSON, raw bytes or already parsed. `user_id` is the authenticated user,
+    `None` when logged out; reporting it is what makes zero-cache trust the connection's user.
+    """
+    query_ids: list[str] = []
+    try:
+        json_body: Any = json.loads(body) if isinstance(body, bytes) else body
+        parsed = parse_wire(_transform_request, json_body)
+        query_ids = [r.id for r in parsed[1]]
+    except (ValueError, ValidationError) as error:
+        logger.error("Failed to parse query request", exc_info=error)
+        return ServerTransformFailed(
+            reason=ErrorReason.PARSE,
+            message=f"Failed to parse query request: {get_error_message(error)}",
+            queryIDs=query_ids,
+            details={"name": "TypeError" if isinstance(error, ValidationError) else "SyntaxError"},
+        )
+
+    try:
+        mapper = client_to_server(schema.tables)
+        responses: list[Any] = []
+        for request in parsed[1]:
+            try:
+                result = request_query = handler(request.name, request.args[0] if request.args else None)
+                if inspect.isawaitable(request_query):
+                    result = await request_query
+            except Exception as error:
+                details = get_error_details(error)
+                errored = ParseErroredQuery if isinstance(error, QueryParseError) else AppErroredQuery
+                responses.append(
+                    errored(
+                        id=request.id,
+                        name=request.name,
+                        message=get_error_message(error),
+                        details=details if js_truthy(details) else None,
+                    )
+                )
+                continue
+            check(
+                isinstance(result, Query),
+                "Query does not implement QueryInternals. This likely means there are two copies of Zero in your "
+                "runtime.",
+            )
+            assert isinstance(result, Query)
+            try:
+                ast = map_ast(result.ast, mapper)
+            except Exception as error:
+                logger.error("Failed to map AST", exc_info=error)
+                raise
+            responses.append(TransformedQuery(id=request.id, name=request.name, ast=ast))
+        return QuerySuccess(queries=responses, userID=user_id)
+    except Exception as error:
+        details = get_error_details(error)
+        return ServerTransformFailed(
+            reason=ErrorReason.INTERNAL,
+            message=get_error_message(error),
+            queryIDs=query_ids,
+            details=details if js_truthy(details) else None,
+        )

zero_server/server/push_processor.py

@@ -0,0 +1,49 @@
+"""`PushProcessor`: the older entry point to the mutate endpoint, dispatching to a mutator registry.
+
+Ported from rocicorp/mono `packages/zero-server/src/push-processor.ts` at zero/v1.10.0
+(905d08516ba0caaf314cd28e93a5a47e8acdd482). Apache-2.0, Copyright Rocicorp.
+
+Unlike `handle_mutate_request`, it reports no user ID to zero-cache.
+"""
+
+import re
+from collections.abc import Mapping
+from typing import Any
+
+from zero_server.mutators import Mutator
+from zero_server.protocol.mutation import CustomMutation, MutationResponse
+from zero_server.server.process_mutations import Database, MutateResponse, TransactFn, handle_mutate
+from zero_server.shared.asserts import check
+from zero_server.shared.json import JSONValue
+from zero_server.shared.registry import Registry, value_at_path
+
+# Older mutators were named with `|`, newer ones with `.`.
+SEPARATOR = re.compile(r"[.|]")
+
+
+class PushProcessor[T]:
+    def __init__(self, db_provider: Database[T], context: Any = None) -> None:
+        self._db = db_provider
+        self._context = context
+
+    async def process(
+        self, mutators: Registry | Mapping[str, Any], query: Mapping[str, str], body: object
+    ) -> MutateResponse:
+        """Process a push. A mutator is a `Mutator` from `define_mutators`, or a plain `async (tx, args)`."""
+        check(body is not None, "body is required when using query params directly")
+
+        async def handler(transact: TransactFn[T], _mutation: CustomMutation) -> MutationResponse:
+            async def dispatch(tx: T, name: str, args: JSONValue | None) -> None:
+                await self._dispatch(mutators, tx, name, args)
+
+            return await transact(dispatch)
+
+        return await handle_mutate(self._db, handler, query, body, None, report_user_id=False)
+
+    async def _dispatch(self, mutators: Mapping[str, Any], tx: T, key: str, args: JSONValue | None) -> None:
+        mutator = value_at_path(mutators, key, SEPARATOR)
+        check(callable(mutator) and not isinstance(mutator, Mapping), f"could not find mutator {key}")
+        if isinstance(mutator, Mutator):
+            await mutator.fn(args=args, ctx=self._context, tx=tx)
+        else:
+            await mutator(tx, args)