omnibase_infra 0.2.2__py3-none-any.whl → 0.2.3__py3-none-any.whl

omnibase_infra/utils/__init__.py

@@ -4,11 +4,14 @@
 
 This package provides common utilities used across the infrastructure:
     - correlation: Correlation ID generation and propagation for distributed tracing
+    - util_atomic_file: Atomic file write primitives using temp-file-rename pattern
     - util_datetime: Datetime validation and timezone normalization
+    - util_db_transaction: Database transaction context manager for asyncpg
     - util_dsn_validation: PostgreSQL DSN validation and sanitization
     - util_env_parsing: Type-safe environment variable parsing with validation
     - util_error_sanitization: Error message sanitization for secure logging and DLQ
     - util_pydantic_validators: Shared Pydantic field validator utilities
+    - util_retry_optimistic: Optimistic locking retry helper with exponential backoff
     - util_semver: Semantic versioning validation utilities
 """
 
@@ -19,12 +22,19 @@ from omnibase_infra.utils.correlation import (
     get_correlation_id,
     set_correlation_id,
 )
+from omnibase_infra.utils.util_atomic_file import (
+    write_atomic_bytes,
+    write_atomic_bytes_async,
+)
 from omnibase_infra.utils.util_datetime import (
     ensure_timezone_aware,
     is_timezone_aware,
     validate_timezone_aware_with_context,
     warn_if_naive_datetime,
 )
+from omnibase_infra.utils.util_db_transaction import (
+    transaction_context,
+)
 from omnibase_infra.utils.util_dsn_validation import (
     parse_and_validate_dsn,
     sanitize_dsn,
@@ -50,6 +60,10 @@ from omnibase_infra.utils.util_pydantic_validators import (
     validate_timezone_aware_datetime,
     validate_timezone_aware_datetime_optional,
 )
+from omnibase_infra.utils.util_retry_optimistic import (
+    OptimisticConflictError,
+    retry_on_optimistic_conflict,
+)
 from omnibase_infra.utils.util_semver import (
     SEMVER_PATTERN,
     validate_semver,
@@ -58,6 +72,7 @@ from omnibase_infra.utils.util_semver import (
 
 __all__: list[str] = [
     "CorrelationContext",
+    "OptimisticConflictError",
     "SAFE_ERROR_PATTERNS",
     "SEMVER_PATTERN",
     "SENSITIVE_PATTERNS",
@@ -69,6 +84,7 @@ __all__: list[str] = [
     "parse_and_validate_dsn",
     "parse_env_float",
     "parse_env_int",
+    "retry_on_optimistic_conflict",
     "sanitize_backend_error",
     "sanitize_consul_key",
     "sanitize_dsn",
@@ -76,6 +92,7 @@ __all__: list[str] = [
     "sanitize_error_string",
     "sanitize_secret_path",
     "set_correlation_id",
+    "transaction_context",
     "validate_contract_type_value",
     "validate_endpoint_urls_dict",
     "validate_policy_type_value",
@@ -86,4 +103,6 @@ __all__: list[str] = [
     "validate_timezone_aware_with_context",
     "validate_version_lenient",
     "warn_if_naive_datetime",
+    "write_atomic_bytes",
+    "write_atomic_bytes_async",
 ]

omnibase_infra/utils/util_atomic_file.py

@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2025 OmniNode Team
+"""Atomic file write utilities.
+
+This module provides primitives for atomic file writes using the temp-file-rename
+pattern. Atomic writes ensure that file contents are either completely written
+or not written at all - there is no intermediate state where the file contains
+partial data.
+
+POSIX Atomicity Guarantees:
+    On POSIX systems, rename() is atomic within the same filesystem. This module
+    creates the temporary file in the same directory as the target file to ensure
+    the rename operation is atomic.
+
+    **IMPORTANT**: The temp file MUST be created in the same directory as the
+    target file (using `dir=path.parent`). If the temp file is on a different
+    filesystem, the rename becomes a copy-and-delete operation, losing atomicity.
+
+NFS Caveat:
+    NFS provides weaker atomicity guarantees. While rename() is still atomic on
+    NFSv4+, there may be brief windows where both files are visible to other
+    clients. For applications requiring strict consistency on NFS, additional
+    locking mechanisms may be needed.
+
+Windows Notes:
+    - os.replace() is atomic on Windows since Python 3.3
+    - Path.rename() on Windows will fail if the target exists; use os.replace()
+    - This module uses os.replace() for cross-platform atomic rename
+
+Durability Note:
+    This module provides atomicity (all-or-nothing) but not durability guarantees.
+    For systems requiring crash-recovery durability, consider adding fsync() after
+    write and before rename. This comes at a performance cost.
+
+Example:
+    >>> from pathlib import Path
+    >>> from omnibase_infra.utils import write_atomic_bytes
+    >>>
+    >>> # Write data atomically
+    >>> data = b"Hello, World!"
+    >>> bytes_written = write_atomic_bytes(Path("/tmp/myfile.txt"), data)
+    >>> bytes_written
+    13
+    >>>
+    >>> # Async version (uses asyncio.to_thread internally)
+    >>> import asyncio
+    >>> from omnibase_infra.utils import write_atomic_bytes_async
+    >>> bytes_written = asyncio.run(write_atomic_bytes_async(Path("/tmp/myfile.txt"), data))
+    >>> bytes_written
+    13
+
+.. versionadded:: 0.10.0
+    Created as part of OMN-1524 atomic write utilities.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+import tempfile
+from pathlib import Path
+from uuid import UUID
+
+logger = logging.getLogger(__name__)
+
+
+def write_atomic_bytes(
+    path: Path,
+    data: bytes,
+    *,
+    temp_prefix: str = "",
+    temp_suffix: str = ".tmp",
+    correlation_id: UUID | None = None,
+) -> int:
+    """Write bytes to a file atomically using temp-file-rename pattern.
+
+    This function provides atomic file writes by:
+    1. Creating a temporary file in the same directory as the target
+    2. Writing all data to the temporary file
+    3. Atomically renaming the temporary file to the target path
+
+    The rename operation is atomic on POSIX systems when both files are on the
+    same filesystem. On Windows, os.replace() provides atomic semantics since
+    Python 3.3.
+
+    Args:
+        path: Target file path. Parent directory must exist.
+        data: Bytes to write to the file.
+        temp_prefix: Optional prefix for the temporary file name. Useful for
+            debugging to identify the source of temp files.
+        temp_suffix: Suffix for the temporary file name. Defaults to ".tmp".
+        correlation_id: Optional correlation ID for ONEX logging. When provided,
+            errors are logged with correlation context before being raised.
+
+    Returns:
+        Number of bytes written to the file.
+
+    Raises:
+        InfraConnectionError: If the file cannot be written (permissions, disk full,
+            etc.). The underlying OSError is chained via ``from e``. The temporary
+            file is cleaned up before raising.
+
+    Example:
+        >>> from pathlib import Path
+        >>> from omnibase_infra.utils.util_atomic_file import write_atomic_bytes
+        >>>
+        >>> # Basic atomic write
+        >>> path = Path("/tmp/test_atomic.txt")
+        >>> bytes_written = write_atomic_bytes(path, b"test data")
+        >>> bytes_written
+        9
+        >>>
+        >>> # With debugging prefix
+        >>> bytes_written = write_atomic_bytes(
+        ...     path,
+        ...     b"test data",
+        ...     temp_prefix="manifest_",
+        ...     correlation_id=UUID("12345678-1234-5678-1234-567812345678"),
+        ... )
+
+    Warning:
+        The parent directory of ``path`` must exist. This function does not
+        create parent directories.
+
+    Warning:
+        On NFS, atomicity guarantees are weaker. See module docstring for details.
+
+    Related:
+        - handler_manifest_persistence.py: Original pattern implementation
+    """
+    temp_fd: int | None = None
+    temp_path: str | None = None
+
+    try:
+        # Create temp file in same directory for atomic rename guarantee
+        temp_fd, temp_path = tempfile.mkstemp(
+            suffix=temp_suffix,
+            prefix=temp_prefix,
+            dir=path.parent,
+        )
+
+        # Write data to temp file
+        with os.fdopen(temp_fd, "wb") as f:
+            temp_fd = None  # fdopen takes ownership of fd
+            bytes_written = f.write(data)
+
+        # Atomic rename (Path.replace is atomic on both POSIX and Windows 3.3+)
+        Path(temp_path).replace(path)
+        temp_path = None  # Rename succeeded, no cleanup needed
+
+        return bytes_written
+
+    except OSError as e:
+        # Log with correlation context if provided
+        if correlation_id is not None:
+            logger.exception(
+                "Atomic write failed for '%s'",
+                path,
+                extra={
+                    "correlation_id": str(correlation_id),
+                    "target_path": str(path),
+                    "temp_prefix": temp_prefix,
+                    "temp_suffix": temp_suffix,
+                    "error_type": type(e).__name__,
+                },
+            )
+
+        # Cleanup: close fd if fdopen didn't take ownership
+        if temp_fd is not None:
+            try:
+                os.close(temp_fd)
+            except OSError:
+                pass  # Best effort cleanup
+
+        # Cleanup: remove temp file if it exists
+        if temp_path is not None:
+            try:
+                Path(temp_path).unlink(missing_ok=True)
+            except OSError:
+                pass  # Best effort cleanup
+
+        # Wrap OSError in InfraConnectionError per ONEX error handling guidelines
+        # Deferred import to avoid circular dependency (utils -> errors -> utils)
+        from omnibase_infra.enums import EnumInfraTransportType
+        from omnibase_infra.errors import InfraConnectionError, ModelInfraErrorContext
+
+        context = ModelInfraErrorContext.with_correlation(
+            correlation_id=correlation_id,
+            transport_type=EnumInfraTransportType.FILESYSTEM,
+            operation="write_atomic_bytes",
+            target_name=str(path),
+        )
+        raise InfraConnectionError(
+            f"Atomic write failed for '{path}'",
+            context=context,
+            error_type=type(e).__name__,
+            errno=getattr(e, "errno", None),
+        ) from e
+
+
+async def write_atomic_bytes_async(
+    path: Path,
+    data: bytes,
+    *,
+    temp_prefix: str = "",
+    temp_suffix: str = ".tmp",
+    correlation_id: UUID | None = None,
+) -> int:
+    """Write bytes to a file atomically (async version).
+
+    This is a thin async wrapper around :func:`write_atomic_bytes` that uses
+    ``asyncio.to_thread()`` to run the synchronous implementation in a thread
+    pool. This prevents blocking the event loop during file I/O.
+
+    All logic is delegated to :func:`write_atomic_bytes` - this function exists
+    only to provide an async interface.
+
+    Args:
+        path: Target file path. Parent directory must exist.
+        data: Bytes to write to the file.
+        temp_prefix: Optional prefix for the temporary file name.
+        temp_suffix: Suffix for the temporary file name. Defaults to ".tmp".
+        correlation_id: Optional correlation ID for ONEX logging.
+
+    Returns:
+        Number of bytes written to the file.
+
+    Raises:
+        InfraConnectionError: If the file cannot be written (permissions, disk full,
+            etc.). The underlying OSError is chained via ``from e``.
+
+    Example:
+        >>> import asyncio
+        >>> from pathlib import Path
+        >>> from omnibase_infra.utils.util_atomic_file import write_atomic_bytes_async
+        >>>
+        >>> async def example():
+        ...     path = Path("/tmp/test_async.txt")
+        ...     return await write_atomic_bytes_async(path, b"async data")
+        >>>
+        >>> asyncio.run(example())
+        10
+
+    See Also:
+        :func:`write_atomic_bytes`: The synchronous canonical implementation.
+    """
+    return await asyncio.to_thread(
+        write_atomic_bytes,
+        path,
+        data,
+        temp_prefix=temp_prefix,
+        temp_suffix=temp_suffix,
+        correlation_id=correlation_id,
+    )
+
+
+__all__: list[str] = [
+    "write_atomic_bytes",
+    "write_atomic_bytes_async",
+]

omnibase_infra/utils/util_db_transaction.py

@@ -0,0 +1,239 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2025 OmniNode Team
+"""Database transaction context manager for asyncpg.
+
+This module provides a transaction context manager that properly wraps
+database operations in transactions with configurable isolation levels,
+readonly mode, and statement timeouts.
+
+Critical Insight - Row Locks Require Explicit Transactions:
+    When using ``SELECT ... FOR UPDATE`` or similar locking constructs,
+    the locks are **released immediately after the SELECT** unless
+    executed within an explicit transaction context.
+
+    This is a subtle but critical behavior of PostgreSQL and asyncpg:
+    without ``conn.transaction()``, each statement runs in auto-commit
+    mode, causing locks to be acquired and immediately released.
+
+    Example of INCORRECT usage (locks NOT maintained):
+        ```python
+        async with pool.acquire() as conn:
+            # Lock is acquired but immediately released!
+            rows = await conn.fetch(
+                "SELECT * FROM queue WHERE status = 'pending' FOR UPDATE SKIP LOCKED"
+            )
+            # By this point, another worker could process the same row
+            await conn.execute("UPDATE queue SET status = 'processing' WHERE id = $1", row_id)
+        ```
+
+    Example of CORRECT usage (locks maintained):
+        ```python
+        async with pool.acquire() as conn:
+            async with conn.transaction():
+                # Lock is held until transaction commits
+                rows = await conn.fetch(
+                    "SELECT * FROM queue WHERE status = 'pending' FOR UPDATE SKIP LOCKED"
+                )
+                # Lock still held - safe to update
+                await conn.execute("UPDATE queue SET status = 'processing' WHERE id = $1", row_id)
+        ```
+
+    The ``transaction_context()`` function in this module encapsulates
+    this pattern, ensuring locks are properly maintained throughout
+    the transaction scope.
+
+Related Implementations:
+    - TransitionNotificationOutbox (runtime/transition_notification_outbox.py):
+      Uses explicit transaction wrapping for SELECT FOR UPDATE SKIP LOCKED
+      to safely process pending notifications with concurrent workers.
+
+See Also:
+    - PostgreSQL locking documentation: https://www.postgresql.org/docs/current/explicit-locking.html
+    - asyncpg transaction documentation: https://magicstack.github.io/asyncpg/current/api/index.html#asyncpg.connection.Connection.transaction
+
+Example:
+    >>> import asyncpg
+    >>> from omnibase_infra.utils import transaction_context
+    >>>
+    >>> async with transaction_context(pool) as conn:
+    ...     await conn.execute("INSERT INTO logs (message) VALUES ($1)", "Hello")
+    >>>
+    >>> # With isolation level and timeout
+    >>> async with transaction_context(
+    ...     pool,
+    ...     isolation="serializable",
+    ...     timeout=5.0,
+    ... ) as conn:
+    ...     await conn.execute("UPDATE accounts SET balance = balance - 100 WHERE id = $1", account_id)
+
+.. versionadded:: 0.10.0
+    Created as part of database utility consolidation.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from uuid import UUID
+
+import asyncpg
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def transaction_context(
+    pool: asyncpg.Pool,
+    *,
+    isolation: str = "read_committed",
+    readonly: bool = False,
+    deferrable: bool = False,
+    timeout: float | None = None,
+    correlation_id: UUID | None = None,
+) -> AsyncIterator[asyncpg.Connection]:
+    """Async context manager for database transactions.
+
+    Acquires a connection from the pool and starts a transaction with the
+    specified isolation level and options. The connection is yielded for
+    use within the transaction scope.
+
+    Critical - Row Locks:
+        This context manager ensures that row locks (e.g., ``FOR UPDATE``,
+        ``FOR UPDATE SKIP LOCKED``) are maintained throughout the transaction.
+        Without explicit transaction wrapping, asyncpg operates in auto-commit
+        mode where locks are released immediately after each statement.
+
+    Isolation Levels:
+        - ``read_committed`` (default): Each statement sees a snapshot of
+          committed data as of the start of that statement.
+        - ``repeatable_read``: All statements in the transaction see a
+          snapshot of committed data as of the transaction start.
+        - ``serializable``: Strictest isolation - transactions execute as
+          if they were run serially.
+
+    Args:
+        pool: asyncpg connection pool to acquire connection from.
+        isolation: Transaction isolation level. One of "read_committed",
+            "repeatable_read", or "serializable". Defaults to "read_committed".
+        readonly: If True, the transaction is marked as read-only.
+            Attempting to modify data will raise an error. Defaults to False.
+        deferrable: If True, the transaction is deferrable. Only valid when
+            both ``isolation="serializable"`` and ``readonly=True``.
+            A deferrable transaction may block when first acquiring its
+            snapshot until it can execute without conflicting with other
+            serializable transactions. Defaults to False.
+        timeout: Statement timeout in seconds. If provided, sets
+            ``statement_timeout`` for the duration of the transaction.
+            Statements exceeding this timeout will be cancelled.
+            Defaults to None (no timeout).
+        correlation_id: Optional correlation ID for logging. When provided,
+            transaction start and commit/rollback events are logged with
+            this ID for distributed tracing.
+
+    Yields:
+        asyncpg.Connection: The acquired connection within the transaction
+        context. Use this connection for all queries within the transaction.
+
+    Raises:
+        asyncpg.PostgresError: For database-level errors.
+        TimeoutError: If a statement exceeds the configured timeout.
+
+    Example:
+        Basic usage:
+
+        >>> async with transaction_context(pool) as conn:
+        ...     await conn.execute("INSERT INTO users (name) VALUES ($1)", "Alice")
+        ...     await conn.execute("INSERT INTO audit_log (action) VALUES ($1)", "user_created")
+
+        With SELECT FOR UPDATE:
+
+        >>> async with transaction_context(pool) as conn:
+        ...     # Lock is held until transaction commits
+        ...     rows = await conn.fetch(
+        ...         "SELECT * FROM jobs WHERE status = 'pending' LIMIT 1 FOR UPDATE SKIP LOCKED"
+        ...     )
+        ...     if rows:
+        ...         await conn.execute(
+        ...             "UPDATE jobs SET status = 'processing' WHERE id = $1",
+        ...             rows[0]["id"]
+        ...         )
+
+        With isolation and timeout:
+
+        >>> async with transaction_context(
+        ...     pool,
+        ...     isolation="serializable",
+        ...     readonly=True,
+        ...     timeout=10.0,
+        ...     correlation_id=uuid4(),
+        ... ) as conn:
+        ...     totals = await conn.fetchval("SELECT SUM(amount) FROM transactions")
+
+    Note:
+        The transaction is automatically committed on successful exit from
+        the context manager, or rolled back if an exception is raised.
+
+    Warning:
+        Asyncpg exception handling: This utility lets asyncpg exceptions
+        propagate naturally without wrapping them in ONEX errors. This is
+        intentional as it keeps the utility simple and composable. Callers
+        should handle asyncpg exceptions as appropriate for their use case.
+
+    Related:
+        - OMN-1139: TransitionNotificationOutbox uses this pattern for
+          SELECT FOR UPDATE SKIP LOCKED with concurrent workers.
+    """
+    async with pool.acquire() as conn:
+        # Log transaction start if correlation_id provided
+        if correlation_id is not None:
+            logger.debug(
+                "Starting database transaction",
+                extra={
+                    "correlation_id": str(correlation_id),
+                    "isolation": isolation,
+                    "readonly": readonly,
+                    "deferrable": deferrable,
+                    "timeout": timeout,
+                },
+            )
+
+        try:
+            async with conn.transaction(
+                isolation=isolation,
+                readonly=readonly,
+                deferrable=deferrable,
+            ):
+                # Set statement timeout if provided
+                # Uses LOCAL to scope timeout to this transaction only
+                if timeout is not None:
+                    timeout_ms = int(timeout * 1000)
+                    await conn.execute("SET LOCAL statement_timeout = $1", timeout_ms)
+
+                yield conn
+
+            # Log successful commit if correlation_id provided
+            if correlation_id is not None:
+                logger.debug(
+                    "Database transaction committed",
+                    extra={
+                        "correlation_id": str(correlation_id),
+                    },
+                )
+
+        except Exception:
+            # Log rollback if correlation_id provided
+            # Transaction is automatically rolled back by asyncpg
+            if correlation_id is not None:
+                logger.debug(
+                    "Database transaction rolled back",
+                    extra={
+                        "correlation_id": str(correlation_id),
+                    },
+                )
+            raise
+
+
+__all__: list[str] = [
+    "transaction_context",
+]