omnibase_infra 0.2.1__py3-none-any.whl

omnibase_infra/idempotency/models/model_idempotency_store_metrics.py

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2025 OmniNode Team
+"""Metrics model for idempotency store observability.
+
+This module provides a Pydantic model for tracking operational metrics
+of the idempotency store, enabling monitoring of:
+- Duplicate detection rate (duplicates / total checks)
+- Store error rate (failed checks / total checks)
+- Cleanup effectiveness (records deleted per run)
+
+Example:
+    >>> from omnibase_infra.idempotency.models import ModelIdempotencyStoreMetrics
+    >>>
+    >>> metrics = ModelIdempotencyStoreMetrics(
+    ...     total_checks=1000,
+    ...     duplicate_count=50,
+    ...     error_count=5,
+    ... )
+    >>> print(f"Duplicate rate: {metrics.duplicate_rate:.2%}")
+    Duplicate rate: 5.00%
+    >>> print(f"Error rate: {metrics.error_rate:.2%}")
+    Error rate: 0.50%
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class ModelIdempotencyStoreMetrics(BaseModel):
+    """Metrics for idempotency store observability.
+
+    Tracks operational statistics for monitoring store health and performance.
+    All counters are monotonically increasing over the lifetime of the store
+    instance, except for last_cleanup_deleted which is reset each cleanup.
+
+    Mutability Note:
+        This model is intentionally mutable (frozen=False) to allow the
+        idempotency store to update metrics internally during operation.
+        External consumers should use the store's get_metrics() method,
+        which returns a copy of the metrics to prevent unintended mutation
+        from external code. This design ensures:
+
+        - Internal updates are efficient (no copy on each increment)
+        - External access is safe (isolated copies)
+        - Metrics remain consistent during reads
+
+    Attributes:
+        total_checks: Total number of check_and_record calls.
+        duplicate_count: Number of duplicates detected (check_and_record returned False).
+        error_count: Number of failed check operations (exceptions raised).
+        total_cleanup_deleted: Total records cleaned up across all cleanup runs.
+        last_cleanup_deleted: Records deleted in the most recent cleanup run.
+        last_cleanup_at: Timestamp of the most recent cleanup run.
+    """
+
+    # Explicitly mutable for internal metric updates.
+    # External consumers receive copies via get_metrics() to prevent unintended mutation.
+    model_config = ConfigDict(frozen=False, extra="forbid")
+
+    total_checks: int = Field(
+        default=0,
+        ge=0,
+        description="Total check_and_record calls",
+    )
+    duplicate_count: int = Field(
+        default=0,
+        ge=0,
+        description="Number of duplicates detected",
+    )
+    error_count: int = Field(
+        default=0,
+        ge=0,
+        description="Number of failed checks",
+    )
+    total_cleanup_deleted: int = Field(
+        default=0,
+        ge=0,
+        description="Total records cleaned up across all runs",
+    )
+    last_cleanup_deleted: int = Field(
+        default=0,
+        ge=0,
+        description="Records deleted in last cleanup",
+    )
+    last_cleanup_at: datetime | None = Field(
+        default=None,
+        description="Timestamp of last cleanup run",
+    )
+
+    @property
+    def duplicate_rate(self) -> float:
+        """Calculate duplicate detection rate.
+
+        Returns:
+            Ratio of duplicates to total checks (0.0 to 1.0).
+            Returns 0.0 if no checks have been made.
+        """
+        if self.total_checks == 0:
+            return 0.0
+        return self.duplicate_count / self.total_checks
+
+    @property
+    def error_rate(self) -> float:
+        """Calculate error rate.
+
+        Returns:
+            Ratio of errors to total checks (0.0 to 1.0).
+            Returns 0.0 if no checks have been made.
+        """
+        if self.total_checks == 0:
+            return 0.0
+        return self.error_count / self.total_checks
+
+    @property
+    def success_count(self) -> int:
+        """Calculate number of successful new message recordings.
+
+        Returns:
+            Number of check_and_record calls that returned True (new messages).
+            Clamped to 0 if the calculation would yield a negative value.
+        """
+        return max(0, self.total_checks - self.duplicate_count - self.error_count)
+
+    @property
+    def success_rate(self) -> float:
+        """Calculate success rate (new messages recorded / total checks).
+
+        Returns:
+            Ratio of successful new recordings to total checks (0.0 to 1.0).
+            Returns 0.0 if no checks have been made.
+        """
+        if self.total_checks == 0:
+            return 0.0
+        return self.success_count / self.total_checks
+
+
+__all__: list[str] = ["ModelIdempotencyStoreMetrics"]

omnibase_infra/idempotency/models/model_postgres_idempotency_store_config.py

@@ -0,0 +1,299 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2025 OmniNode Team
+"""PostgreSQL Idempotency Store Configuration Model.
+
+This module provides the Pydantic configuration model for the PostgreSQL-based
+idempotency store, including connection pooling, TTL, and cleanup settings.
+
+Security Note:
+    The dsn field may contain credentials. Use environment variables for
+    sensitive values and ensure connection strings are not logged.
+
+Environment Variables:
+    ONEX_IDEMPOTENCY_TTL_SECONDS: TTL for idempotency records (default: 86400)
+    ONEX_IDEMPOTENCY_CLEANUP_INTERVAL: Cleanup interval in seconds (default: 3600)
+    ONEX_IDEMPOTENCY_BATCH_SIZE: Records per cleanup batch (default: 10000)
+    ONEX_IDEMPOTENCY_POOL_MIN_SIZE: Minimum pool connections (default: 1)
+    ONEX_IDEMPOTENCY_POOL_MAX_SIZE: Maximum pool connections (default: 5)
+    ONEX_IDEMPOTENCY_COMMAND_TIMEOUT: Command timeout in seconds (default: 30.0)
+    ONEX_IDEMPOTENCY_CLOCK_SKEW_TOLERANCE: Clock skew tolerance in seconds (default: 60)
+    ONEX_IDEMPOTENCY_CLEANUP_MAX_ITERATIONS: Max cleanup iterations (default: 100)
+"""
+
+from __future__ import annotations
+
+from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+
+from omnibase_infra.enums import EnumInfraTransportType
+from omnibase_infra.utils import validate_pool_sizes_constraint
+from omnibase_infra.utils.util_env_parsing import parse_env_float, parse_env_int
+
+# Module-level defaults from environment variables
+# These allow runtime configuration without code changes
+
+_DEFAULT_TTL_SECONDS = parse_env_int(
+    "ONEX_IDEMPOTENCY_TTL_SECONDS",
+    86400,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=60,  # Minimum 1 minute
+    max_value=2592000,  # Maximum 30 days
+)
+_DEFAULT_CLEANUP_INTERVAL = parse_env_int(
+    "ONEX_IDEMPOTENCY_CLEANUP_INTERVAL",
+    3600,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=60,  # Minimum 1 minute
+    max_value=86400,  # Maximum 24 hours
+)
+_DEFAULT_BATCH_SIZE = parse_env_int(
+    "ONEX_IDEMPOTENCY_BATCH_SIZE",
+    10000,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=100,  # Minimum batch size
+    max_value=100000,  # Maximum batch size
+)
+_DEFAULT_POOL_MIN_SIZE = parse_env_int(
+    "ONEX_IDEMPOTENCY_POOL_MIN_SIZE",
+    1,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=1,  # Minimum 1 connection
+    max_value=100,  # Maximum pool size
+)
+_DEFAULT_POOL_MAX_SIZE = parse_env_int(
+    "ONEX_IDEMPOTENCY_POOL_MAX_SIZE",
+    5,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=1,  # Minimum 1 connection
+    max_value=100,  # Maximum pool size
+)
+_DEFAULT_COMMAND_TIMEOUT = parse_env_float(
+    "ONEX_IDEMPOTENCY_COMMAND_TIMEOUT",
+    30.0,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=1.0,  # Minimum 1 second
+    max_value=300.0,  # Maximum 5 minutes
+)
+_DEFAULT_CLOCK_SKEW_TOLERANCE = parse_env_int(
+    "ONEX_IDEMPOTENCY_CLOCK_SKEW_TOLERANCE",
+    60,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=0,  # Can disable tolerance
+    max_value=3600,  # Maximum 1 hour
+)
+_DEFAULT_CLEANUP_MAX_ITERATIONS = parse_env_int(
+    "ONEX_IDEMPOTENCY_CLEANUP_MAX_ITERATIONS",
+    100,
+    transport_type=EnumInfraTransportType.DATABASE,
+    service_name="postgres_idempotency_store",
+    min_value=1,  # Minimum 1 iteration
+    max_value=1000,  # Maximum iterations
+)
+
+
+class ModelPostgresIdempotencyStoreConfig(BaseModel):
+    """Configuration for PostgreSQL-based idempotency store.
+
+    This model defines all configuration options for the PostgreSQL
+    idempotency store, including connection settings, pooling parameters,
+    and TTL-based cleanup configuration.
+
+    Security Policy:
+        - DSN may contain credentials - use environment variables
+        - Never log the full DSN value
+        - Use SSL in production environments
+
+    Attributes:
+        dsn: PostgreSQL connection string (e.g., "postgresql://user:pass@host:5432/db").
+            Should be provided via environment variable for security.
+        table_name: Name of the idempotency records table.
+            Default: "idempotency_records".
+        pool_min_size: Minimum number of connections in the pool.
+            Default: 1.
+        pool_max_size: Maximum number of connections in the pool.
+            Default: 5.
+        command_timeout: Timeout for database commands in seconds.
+            Default: 30.0.
+        ttl_seconds: Time-to-live for idempotency records in seconds.
+            Records older than this will be cleaned up. Default: 86400 (24 hours).
+        auto_cleanup: Whether to automatically clean up expired records.
+            Default: True.
+        cleanup_interval_seconds: Interval between cleanup runs in seconds.
+            Default: 3600 (1 hour).
+        clock_skew_tolerance_seconds: Buffer added to TTL during cleanup to prevent
+            premature deletion due to clock skew between distributed nodes.
+            Default: 60 (1 minute).
+        cleanup_batch_size: Number of records to delete per batch during cleanup.
+            Batched deletion reduces lock contention. Default: 10000.
+        cleanup_max_iterations: Maximum number of batch iterations during cleanup.
+            Prevents runaway loops. Default: 100.
+
+    Example:
+        >>> config = ModelPostgresIdempotencyStoreConfig(
+        ...     dsn="postgresql://user:pass@localhost:5432/mydb",
+        ...     table_name="idempotency_records",
+        ...     pool_max_size=10,
+        ...     ttl_seconds=172800,  # 48 hours
+        ... )
+        >>> print(config.table_name)
+        idempotency_records
+    """
+
+    model_config = ConfigDict(
+        frozen=True,
+        extra="forbid",
+        from_attributes=True,
+    )
+
+    dsn: str = Field(
+        description="PostgreSQL connection string (e.g., 'postgresql://user:pass@host:5432/db')",
+        min_length=1,
+    )
+
+    @field_validator("dsn", mode="before")
+    @classmethod
+    def validate_dsn(cls, v: object) -> str:
+        """Validate PostgreSQL DSN format using robust parser.
+
+        This validator uses urllib.parse for comprehensive DSN validation,
+        handling edge cases like IPv6 addresses, URL-encoded passwords,
+        and query parameters.
+
+        Edge cases validated:
+            - IPv6 addresses: postgresql://user:pass@[::1]:5432/db
+            - URL-encoded passwords: user:p%40ssword@host (p@ssword)
+            - Query parameters: postgresql://host/db?sslmode=require
+            - Missing components: postgresql://localhost/db (no user/pass/port)
+
+        Args:
+            v: DSN value (any type before Pydantic conversion)
+
+        Returns:
+            Validated DSN string
+
+        Raises:
+            ProtocolConfigurationError: If DSN format is invalid
+        """
+        from omnibase_infra.utils.util_dsn_validation import parse_and_validate_dsn
+
+        # parse_and_validate_dsn handles all validation and error context
+        # It will raise ProtocolConfigurationError with proper context if invalid
+        parse_and_validate_dsn(v)
+
+        # If validation passes, return the stripped string
+        return v.strip() if isinstance(v, str) else str(v)
+
+    table_name: str = Field(
+        default="idempotency_records",
+        description="Name of the idempotency records table",
+        min_length=1,
+        max_length=63,  # PostgreSQL identifier limit
+        pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$",
+    )
+    pool_min_size: int = Field(
+        default=_DEFAULT_POOL_MIN_SIZE,
+        description=(
+            "Minimum number of connections in the pool "
+            "(env: ONEX_IDEMPOTENCY_POOL_MIN_SIZE)"
+        ),
+        ge=1,
+        le=100,
+    )
+    pool_max_size: int = Field(
+        default=_DEFAULT_POOL_MAX_SIZE,
+        description=(
+            "Maximum number of connections in the pool "
+            "(env: ONEX_IDEMPOTENCY_POOL_MAX_SIZE)"
+        ),
+        ge=1,
+        le=100,
+    )
+    command_timeout: float = Field(
+        default=_DEFAULT_COMMAND_TIMEOUT,
+        description=(
+            "Timeout for database commands in seconds "
+            "(env: ONEX_IDEMPOTENCY_COMMAND_TIMEOUT)"
+        ),
+        ge=1.0,
+        le=300.0,
+    )
+    ttl_seconds: int = Field(
+        default=_DEFAULT_TTL_SECONDS,
+        description=(
+            "Time-to-live for idempotency records in seconds "
+            "(env: ONEX_IDEMPOTENCY_TTL_SECONDS)"
+        ),
+        ge=60,  # Minimum 1 minute
+        le=2592000,  # Maximum 30 days
+    )
+    auto_cleanup: bool = Field(
+        default=True,
+        description="Whether to automatically clean up expired records",
+    )
+    cleanup_interval_seconds: int = Field(
+        default=_DEFAULT_CLEANUP_INTERVAL,
+        description=(
+            "Interval between cleanup runs in seconds "
+            "(env: ONEX_IDEMPOTENCY_CLEANUP_INTERVAL)"
+        ),
+        ge=60,  # Minimum 1 minute
+        le=86400,  # Maximum 24 hours
+    )
+    clock_skew_tolerance_seconds: int = Field(
+        default=_DEFAULT_CLOCK_SKEW_TOLERANCE,
+        description=(
+            "Buffer added to TTL during cleanup to prevent premature deletion due to "
+            "clock skew between distributed nodes (env: ONEX_IDEMPOTENCY_CLOCK_SKEW_TOLERANCE). "
+            "In distributed systems, nodes may have slightly different system times. "
+            "This tolerance ensures records are not cleaned up before all nodes "
+            "consider them expired. Recommended: Use NTP synchronization in production "
+            "and set this to at least the maximum expected clock drift between nodes."
+        ),
+        ge=0,  # Can be 0 to disable tolerance
+        le=3600,  # Maximum 1 hour tolerance
+    )
+    cleanup_batch_size: int = Field(
+        default=_DEFAULT_BATCH_SIZE,
+        description=(
+            "Number of records to delete per batch during cleanup (env: ONEX_IDEMPOTENCY_BATCH_SIZE). "
+            "Batched deletion reduces lock contention on high-volume tables by "
+            "breaking large deletes into smaller transactions, allowing other "
+            "operations to interleave and preventing long-running locks."
+        ),
+        ge=100,  # Minimum batch size
+        le=100000,  # Maximum batch size
+    )
+    cleanup_max_iterations: int = Field(
+        default=_DEFAULT_CLEANUP_MAX_ITERATIONS,
+        description=(
+            "Maximum number of batch iterations during cleanup "
+            "(env: ONEX_IDEMPOTENCY_CLEANUP_MAX_ITERATIONS). "
+            "Prevents runaway cleanup loops in extreme cases. "
+            "Total max records deleted = cleanup_batch_size * cleanup_max_iterations."
+        ),
+        ge=1,
+        le=1000,
+    )
+
+    @field_validator("pool_max_size", mode="after")
+    @classmethod
+    def validate_pool_sizes(cls, v: int, info: ValidationInfo) -> int:
+        """Validate that pool_max_size >= pool_min_size.
+
+        Delegates to shared utility for consistent validation across all config models.
+        """
+        if info.data:
+            pool_min_size = info.data.get("pool_min_size", 1)
+            return validate_pool_sizes_constraint(
+                v, pool_min_size, target_name="postgres_idempotency_store"
+            )
+        return v
+
+
+__all__: list[str] = ["ModelPostgresIdempotencyStoreConfig"]

omnibase_infra/idempotency/protocol_idempotency_store.py

@@ -0,0 +1,184 @@
+# SPDX-License-Identifier: MIT
+# Copyright (c) 2025 OmniNode Team
+"""Protocol definition for Idempotency Store.
+
+This module defines the ProtocolIdempotencyStore protocol that all idempotency
+store implementations must follow. The protocol defines the contract for
+message deduplication in distributed systems.
+
+Migration Note:
+    This protocol is defined locally in omnibase_infra because it is not
+    available in omnibase_spi versions 0.4.0/0.4.1. This is a TEMPORARY
+    definition that should be migrated to omnibase_spi in a future release.
+
+    Migration Path (OMN-1000):
+        1. When omnibase_spi 0.5.0+ is released with ProtocolIdempotencyStore,
+           update pyproject.toml to require the new version
+        2. Update imports in StoreIdempotencyInmemory and StoreIdempotencyPostgres
+           to use: `from omnibase_spi.protocols import ProtocolIdempotencyStore`
+        3. Remove this local protocol definition
+        4. Run tests to verify compatibility
+
+    The protocol contract is intentionally designed to match the expected
+    omnibase_spi interface to ensure a smooth migration.
+
+Protocol Methods:
+    - check_and_record: Atomically check if message was processed and record if not
+    - is_processed: Check if a message was already processed (read-only)
+    - mark_processed: Mark a message as processed (upsert)
+    - cleanup_expired: Remove entries older than TTL
+
+Implementations:
+    - StoreIdempotencyInmemory: In-memory store for testing (OMN-945)
+    - StoreIdempotencyPostgres: Production PostgreSQL store (OMN-945)
+
+Security Considerations:
+    - Concurrency Safety: All implementations MUST be safe for concurrent access.
+      Multiple coroutines may call check_and_record simultaneously with the
+      same message_id. Implementations must use appropriate synchronization
+      (e.g., asyncio.Lock for in-memory, database transactions for PostgreSQL).
+
+      Note: This is coroutine-safe for asyncio concurrent access, not thread-safe.
+      For multi-threaded access, additional synchronization would be required.
+
+    - Atomicity: The check_and_record method MUST provide atomic check-and-set
+      semantics. When multiple callers race with the same (domain, message_id),
+      exactly ONE caller must receive True. This prevents duplicate processing
+      in concurrent scenarios.
+
+    - Domain Isolation: Messages are namespaced by domain to prevent cross-tenant
+      conflicts. The (domain, message_id) tuple forms the unique key. Different
+      domains can safely use the same message_id without collision. This is
+      critical for multi-tenant deployments where tenant data must be isolated.
+
+    - Correlation ID Usage: The correlation_id parameter is used ONLY for
+      distributed tracing and observability. It is NOT used for authentication
+      or authorization. Security-sensitive operations must implement their own
+      authentication layer; do not rely on correlation_id for access control.
+
+    - Input Validation: Implementations should validate that message_id is a
+      valid UUID. Domain names should be sanitized if used in storage backends
+      (e.g., as part of database keys or Redis key prefixes).
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Protocol, runtime_checkable
+from uuid import UUID
+
+
+@runtime_checkable
+class ProtocolIdempotencyStore(Protocol):
+    """Protocol for idempotency store implementations.
+
+    Defines the contract for message deduplication stores that track processed
+    messages and prevent duplicate processing in distributed systems.
+
+    All implementations must provide atomic check-and-record semantics to
+    ensure exactly-once processing guarantees.
+
+    Key Properties:
+        - Coroutine-safe: All operations must be safe for concurrent async access
+        - Atomic: check_and_record must provide atomic check-and-set semantics
+        - Domain-isolated: Messages can be namespaced by domain for isolated deduplication
+
+    Example:
+        >>> store: ProtocolIdempotencyStore = StoreIdempotencyInmemory()
+        >>> message_id = uuid4()
+        >>> is_new = await store.check_and_record(message_id, domain="orders")
+        >>> if is_new:
+        ...     # Process the message
+        ...     pass
+    """
+
+    async def check_and_record(
+        self,
+        message_id: UUID,
+        domain: str | None = None,
+        correlation_id: UUID | None = None,
+    ) -> bool:
+        """Atomically check if message was processed and record if not.
+
+        This is the primary idempotency operation. It must be atomic to ensure
+        that when multiple coroutines call this method simultaneously with the
+        same (domain, message_id), exactly ONE caller receives True.
+
+        Args:
+            message_id: Unique identifier for the message.
+            domain: Optional domain namespace for isolated deduplication.
+                Messages with the same message_id but different domains are
+                treated as distinct messages.
+            correlation_id: Optional correlation ID for distributed tracing.
+                Stored with the record for observability purposes.
+
+        Returns:
+            True if message is new (should be processed).
+            False if message is duplicate (should be skipped).
+        """
+        ...
+
+    async def is_processed(
+        self,
+        message_id: UUID,
+        domain: str | None = None,
+    ) -> bool:
+        """Check if a message was already processed.
+
+        Read-only check that does not modify the store. Useful for querying
+        message status without affecting the idempotency state.
+
+        Args:
+            message_id: Unique identifier for the message.
+            domain: Optional domain namespace.
+
+        Returns:
+            True if the message has been processed.
+            False if the message has not been processed.
+        """
+        ...
+
+    async def mark_processed(
+        self,
+        message_id: UUID,
+        domain: str | None = None,
+        correlation_id: UUID | None = None,
+        processed_at: datetime | None = None,
+    ) -> None:
+        """Mark a message as processed.
+
+        Records a message as processed without checking if it already exists.
+        If the record already exists, updates it with the new values.
+
+        This is an upsert operation - it will create a new record if one
+        doesn't exist, or update the existing record if it does.
+
+        Args:
+            message_id: Unique identifier for the message.
+            domain: Optional domain namespace for isolated deduplication.
+            correlation_id: Optional correlation ID for tracing.
+            processed_at: Optional timestamp of when processing occurred.
+                If None, implementations should use the current UTC time.
+        """
+        ...
+
+    async def cleanup_expired(
+        self,
+        ttl_seconds: int,
+    ) -> int:
+        """Remove entries older than TTL.
+
+        Cleans up old idempotency records based on their processed_at timestamp.
+        This prevents unbounded storage growth.
+
+        Args:
+            ttl_seconds: Time-to-live in seconds. Records older than this
+                value (based on processed_at timestamp) are removed.
+
+        Returns:
+            Number of entries removed.
+        """
+        ...
+
+
+__all__ = ["ProtocolIdempotencyStore"]