maleo-database 0.0.1__py3-none-any.whl

maleo/database/config/identifier.py

@@ -0,0 +1,12 @@
+from pydantic import BaseModel, Field
+from maleo.enums.environment import Environment
+from maleo.types.base.dict import OptionalStringToStringDict
+from maleo.types.base.string import OptionalString
+
+
+class DatabaseIdentifierConfig(BaseModel):
+    enabled: bool = Field(True, description="Whether the database is enabled")
+    environment: Environment = Field(..., description="Database's environment")
+    name: str = Field(..., description="Database's name")
+    description: OptionalString = Field(None, description="Database's description")
+    tags: OptionalStringToStringDict = Field(None, description="Database's tags")

maleo/database/config/pooling.py

@@ -0,0 +1,255 @@
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing import Self, TypeVar
+from maleo.types.base.dict import OptionalStringToStringDict
+from maleo.types.base.integer import ListOfIntegers
+from maleo.types.base.string import OptionalString
+from maleo.utils.formatters.case import to_camel
+from ..enums import PoolingStrategy
+
+
+class BasePoolingConfig(BaseModel):
+    """Base configuration class for database connection pooling."""
+
+
+PoolingConfigT = TypeVar("PoolingConfigT", bound=BasePoolingConfig)
+
+
+class PostgreSQLPoolingConfig(BasePoolingConfig):
+    """PostgreSQL-specific pooling configuration."""
+
+    pool_size: int = Field(
+        default=10, ge=1, le=1000, description="Number of connections in the pool"
+    )
+    max_overflow: int = Field(
+        default=20, ge=0, le=500, description="Maximum number of overflow connections"
+    )
+    pool_timeout: float = Field(
+        default=30.0,
+        ge=1.0,
+        le=300.0,
+        description="Timeout in seconds for getting connection",
+    )
+    pool_recycle: int = Field(
+        default=3600, ge=60, le=86400, description="Connection recycle time in seconds"
+    )
+    pool_pre_ping: bool = Field(
+        default=True, description="Validate connections before use"
+    )
+    # Keep strategy and prepared_statement_cache_size as they're pooling-related
+    strategy: PoolingStrategy = Field(
+        default=PoolingStrategy.DYNAMIC, description="Pooling strategy"
+    )
+    prepared_statement_cache_size: int = Field(
+        default=100, ge=0, le=10000, description="Prepared statement cache size"
+    )
+    pool_reset_on_return: bool = Field(
+        default=True, description="Reset connection state on return to pool"
+    )
+
+    @model_validator(mode="after")
+    def validate_overflow(self) -> Self:
+        if self.max_overflow > self.pool_size * 5:
+            raise ValueError("max_overflow should not exceed 5x pool_size")
+        return self
+
+
+class MySQLPoolingConfig(BasePoolingConfig):
+    """MySQL-specific pooling configuration."""
+
+    pool_size: int = Field(
+        default=8, ge=1, le=500, description="Number of connections in the pool"
+    )
+    max_overflow: int = Field(
+        default=15, ge=0, le=200, description="Maximum number of overflow connections"
+    )
+    pool_timeout: float = Field(
+        default=20.0,
+        ge=1.0,
+        le=300.0,
+        description="Timeout in seconds for getting connection",
+    )
+    pool_recycle: int = Field(
+        default=7200, ge=60, le=86400, description="Connection recycle time in seconds"
+    )
+    pool_pre_ping: bool = Field(
+        default=True, description="Validate connections before use"
+    )
+    strategy: PoolingStrategy = Field(
+        default=PoolingStrategy.FIXED, description="Pooling strategy"
+    )
+    # Add autocommit to pooling since it affects connection behavior in the pool
+    autocommit: bool = Field(default=False, description="Enable autocommit mode")
+    # Move connect_timeout here since it's about pool connection establishment
+    connect_timeout: float = Field(
+        default=10.0, ge=1.0, le=60.0, description="Connection timeout in seconds"
+    )
+
+
+class SQLitePoolingConfig(BasePoolingConfig):
+    """SQLite-specific pooling configuration."""
+
+    pool_size: int = Field(
+        default=1, ge=1, le=10, description="Number of connections (limited for SQLite)"
+    )
+    max_overflow: int = Field(
+        default=5, ge=0, le=20, description="Maximum overflow connections"
+    )
+    pool_timeout: float = Field(
+        default=30.0, ge=1.0, le=300.0, description="Timeout in seconds"
+    )
+    # SQLite-specific pooling options
+    wal_mode: bool = Field(
+        default=True, description="Enable WAL mode for better concurrency"
+    )
+    busy_timeout: int = Field(
+        default=30000, ge=1000, le=300000, description="Busy timeout in milliseconds"
+    )
+
+
+class SQLServerPoolingConfig(BasePoolingConfig):
+    """SQL Server-specific pooling configuration."""
+
+    pool_size: int = Field(
+        default=10, ge=1, le=500, description="Number of connections in the pool"
+    )
+    max_overflow: int = Field(
+        default=20, ge=0, le=200, description="Maximum number of overflow connections"
+    )
+    pool_timeout: float = Field(
+        default=30.0,
+        ge=1.0,
+        le=300.0,
+        description="Timeout in seconds for getting connection",
+    )
+    pool_recycle: int = Field(
+        default=3600, ge=60, le=86400, description="Connection recycle time in seconds"
+    )
+    pool_pre_ping: bool = Field(
+        default=True, description="Validate connections before use"
+    )
+    strategy: PoolingStrategy = Field(
+        default=PoolingStrategy.DYNAMIC, description="Pooling strategy"
+    )
+    # SQL Server-specific pooling settings
+    connection_timeout: int = Field(
+        default=30, ge=1, le=300, description="Connection timeout in seconds"
+    )
+    command_timeout: int = Field(
+        default=30, ge=1, le=3600, description="Command timeout in seconds"
+    )
+    packet_size: int = Field(
+        default=4096, ge=512, le=32767, description="Network packet size"
+    )
+    trust_server_certificate: bool = Field(
+        default=False, description="Trust server certificate"
+    )
+    # Move encrypt here since it affects connection pool behavior
+    encrypt: bool = Field(default=True, description="Encrypt connection")
+
+    @model_validator(mode="after")
+    def validate_overflow(self) -> Self:
+        if self.max_overflow > self.pool_size * 3:
+            raise ValueError("max_overflow should not exceed 3x pool_size")
+        return self
+
+
+class MongoDBPoolingConfig(BasePoolingConfig):
+    """MongoDB-specific pooling configuration."""
+
+    model_config = ConfigDict(alias_generator=to_camel)
+
+    max_pool_size: int = Field(
+        default=100, ge=1, le=500, description="Maximum number of connections in pool"
+    )
+    min_pool_size: int = Field(
+        default=0, ge=0, le=100, description="Minimum number of connections in pool"
+    )
+    max_idle_time_ms: int = Field(
+        default=600000, ge=1000, le=3600000, description="Max idle time in milliseconds"
+    )
+    connect_timeout_ms: int = Field(
+        default=20000,
+        ge=1000,
+        le=300000,
+        description="Connection timeout in milliseconds",
+    )
+    server_selection_timeout_ms: int = Field(
+        default=30000, ge=1000, le=300000, description="Server selection timeout"
+    )
+    max_connecting: int = Field(
+        default=2,
+        ge=1,
+        le=10,
+        description="Maximum number of concurrent connection attempts",
+    )
+
+
+class RedisPoolingConfig(BasePoolingConfig):
+    """Redis-specific pooling configuration."""
+
+    max_connections: int = Field(
+        default=50, ge=1, le=1000, description="Maximum number of connections in pool"
+    )
+    retry_on_timeout: bool = Field(
+        default=True, description="Retry on connection timeout"
+    )
+    health_check_interval: int = Field(
+        default=30, ge=5, le=300, description="Health check interval in seconds"
+    )
+    connection_timeout: float = Field(
+        default=5.0, ge=1.0, le=60.0, description="Connection timeout in seconds"
+    )
+    socket_timeout: float = Field(
+        default=5.0, ge=1.0, le=60.0, description="Socket timeout in seconds"
+    )
+    socket_keepalive: bool = Field(default=True, description="Enable TCP keepalive")
+    decode_responses: bool = Field(
+        default=True, description="Decode responses to strings"
+    )
+
+
+class ElasticsearchPoolingConfig(BasePoolingConfig):
+    """Elasticsearch-specific pooling configuration."""
+
+    # Connection pool settings
+    maxsize: int = Field(
+        default=25, ge=1, le=100, description="Maximum number of connections in pool"
+    )
+    connections_per_node: int = Field(
+        default=10, ge=1, le=50, description="Connections per Elasticsearch node"
+    )
+
+    # Timeout settings
+    timeout: float = Field(
+        default=10.0, ge=1.0, le=300.0, description="Request timeout in seconds"
+    )
+    max_retries: int = Field(
+        default=3, ge=0, le=10, description="Maximum number of retries"
+    )
+    retry_on_timeout: bool = Field(default=False, description="Retry on timeout")
+    retry_on_status: ListOfIntegers = Field(
+        default_factory=lambda: [502, 503, 504],
+        description="HTTP status codes to retry on",
+    )
+
+    # Connection behavior (move from connection config)
+    http_compress: bool = Field(default=True, description="Enable HTTP compression")
+    verify_certs: bool = Field(default=True, description="Verify SSL certificates")
+    ca_certs: OptionalString = Field(
+        default=None, description="Path to CA certificates"
+    )
+
+    # Advanced pool settings
+    block: bool = Field(default=False, description="Block when pool is full")
+    headers: OptionalStringToStringDict = Field(
+        default=None, description="Default headers for requests"
+    )
+    dead_timeout: float = Field(
+        default=60.0, ge=5.0, le=600.0, description="Dead node timeout in seconds"
+    )
+
+    @model_validator(mode="after")
+    def validate_overflow(self) -> Self:
+        if self.connections_per_node > self.maxsize:
+            raise ValueError("connections_per_node must not exceed maxsize")
+        return self

maleo/database/enums.py

@@ -0,0 +1,85 @@
+from enum import StrEnum
+
+
+class Connection(StrEnum):
+    ASYNC = "async"
+    SYNC = "sync"
+
+
+class Driver(StrEnum):
+    # SQL Databases - Most Popular
+    POSTGRESQL = "postgresql"
+    MYSQL = "mysql"
+    SQLITE = "sqlite"
+
+    # SQL Databases - Enterprise
+    # ORACLE = "oracle"
+    MSSQL = "mssql"  # SQL Server
+    # MARIADB = "mariadb"
+
+    # NoSQL Document Stores
+    MONGODB = "mongodb"
+    # COUCHDB = "couchdb"
+
+    # NoSQL Key-Value
+    REDIS = "redis"
+    # DYNAMODB = "dynamodb"  # AWS
+
+    # NoSQL Column Family
+    # CASSANDRA = "cassandra"
+    # HBASE = "hbase"
+
+    # NoSQL Graph
+    # NEO4J = "neo4j"
+    # ARANGODB = "arangodb"
+
+    # Time Series
+    # INFLUXDB = "influxdb"
+    # TIMESCALEDB = "timescaledb"  # PostgreSQL extension
+
+    # In-Memory
+    # MEMCACHED = "memcached"
+
+    # Search Engines
+    ELASTICSEARCH = "elasticsearch"
+    # OPENSEARCH = "opensearch"
+
+    # Cloud Native
+    # FIRESTORE = "firestore"  # Google
+    # COSMOSDB = "cosmosdb"  # Azure
+
+
+class PostgreSQLSSLMode(StrEnum):
+    DISABLE = "disable"
+    ALLOW = "allow"
+    PREFER = "prefer"
+    REQUIRE = "require"
+    VERIFY_CA = "verify-ca"
+    VERIFY_FULL = "verify-full"
+
+
+class MySQLCharset(StrEnum):
+    UTF8 = "utf8"
+    UTF8MB4 = "utf8mb4"
+    LATIN1 = "latin1"
+    ASCII = "ascii"
+
+
+class MongoDBReadPreference(StrEnum):
+    PRIMARY = "primary"
+    PRIMARY_PREFERRED = "primaryPreferred"
+    SECONDARY = "secondary"
+    SECONDARY_PREFERRED = "secondaryPreferred"
+    NEAREST = "nearest"
+
+
+class ElasticsearchScheme(StrEnum):
+    HTTP = "http"
+    HTTPS = "https"
+
+
+class PoolingStrategy(StrEnum):
+    FIXED = "fixed"
+    DYNAMIC = "dynamic"
+    OVERFLOW = "overflow"
+    QUEUE = "queue"

maleo/database/managers/clients/elasticsearch.py

@@ -0,0 +1,66 @@
+from elasticsearch import AsyncElasticsearch, Elasticsearch
+from typing import Literal, Union, overload
+from ...config import ElasticsearchDatabaseConfig
+from ...enums import Connection
+
+
+class ElasticsearchClientManager:
+    def __init__(self, config: ElasticsearchDatabaseConfig) -> None:
+        self.config = config
+        self._async_client: AsyncElasticsearch = self._init(Connection.ASYNC)
+        self._sync_client: Elasticsearch = self._init(Connection.SYNC)
+
+    @overload
+    def _init(self, connection: Literal[Connection.ASYNC]) -> AsyncElasticsearch: ...
+    @overload
+    def _init(self, connection: Literal[Connection.SYNC]) -> Elasticsearch: ...
+    def _init(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncElasticsearch, Elasticsearch]:
+        hosts = [
+            {"host": self.config.connection.host, "port": self.config.connection.port}
+        ]
+
+        # Build auth and pooling params properly
+        client_kwargs = {}
+
+        if self.config.connection.username and self.config.connection.password:
+            client_kwargs["http_auth"] = (
+                self.config.connection.username,
+                self.config.connection.password,
+            )
+
+        # Use pooling config directly
+        pooling_kwargs = self.config.pooling.model_dump(
+            exclude={
+                "connections_per_node",
+                "block",
+                "headers",
+                "dead_timeout",
+            },  # ES-specific excludes
+            exclude_none=True,
+        )
+        client_kwargs.update(pooling_kwargs)
+
+        if connection is Connection.ASYNC:
+            self._async_client = AsyncElasticsearch(hosts, **client_kwargs)
+            return self._async_client
+        else:
+            self._sync_client = Elasticsearch(hosts, **client_kwargs)
+            return self._sync_client
+
+    @overload
+    def get(self, connection: Literal[Connection.ASYNC]) -> AsyncElasticsearch: ...
+    @overload
+    def get(self, connection: Literal[Connection.SYNC]) -> Elasticsearch: ...
+    def get(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncElasticsearch, Elasticsearch]:
+        if connection is Connection.ASYNC:
+            return self._async_client or self._init(Connection.ASYNC)
+        else:
+            return self._sync_client or self._init(Connection.SYNC)
+
+    async def dispose(self):
+        await self._async_client.close()
+        self._sync_client.close()

maleo/database/managers/clients/mongodb.py

@@ -0,0 +1,53 @@
+from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo import MongoClient
+from typing import Literal, Union, overload
+from ...config import MongoDBDatabaseConfig
+from ...enums import Connection
+
+
+class MongoDBClientManager:
+    def __init__(self, config: MongoDBDatabaseConfig) -> None:
+        self.config = config
+        self._async_client: AsyncIOMotorClient = self._init(Connection.ASYNC)
+        self._sync_client: MongoClient = self._init(Connection.SYNC)
+
+    @overload
+    def _init(self, connection: Literal[Connection.ASYNC]) -> AsyncIOMotorClient: ...
+    @overload
+    def _init(self, connection: Literal[Connection.SYNC]) -> MongoClient: ...
+    def _init(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncIOMotorClient, MongoClient]:
+        url = self.config.connection.make_url(connection)
+
+        pooling_kwargs = self.config.pooling.model_dump(
+            by_alias=True, exclude_none=True
+        )
+
+        if connection is Connection.ASYNC:
+            self._async_client = AsyncIOMotorClient(url, **pooling_kwargs)
+            return self._async_client
+        else:
+            self._sync_client = MongoClient(url, **pooling_kwargs)
+            return self._sync_client
+
+    @overload
+    def get(self, connection: Literal[Connection.ASYNC]) -> AsyncIOMotorClient: ...
+    @overload
+    def get(self, connection: Literal[Connection.SYNC]) -> MongoClient: ...
+    def get(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncIOMotorClient, MongoClient]:
+        if connection is Connection.ASYNC:
+            return self._async_client
+        elif connection is Connection.SYNC:
+            return self._sync_client
+
+    def get_database(self, connection: Connection = Connection.ASYNC):
+        """Get the specific database object."""
+        client = self.get(connection)
+        return client[self.config.connection.database]
+
+    async def dispose(self):
+        self._async_client.close()
+        self._sync_client.close()

maleo/database/managers/clients/redis.py

@@ -0,0 +1,57 @@
+from redis.asyncio import Redis as AsyncRedis
+from redis import Redis as SyncRedis
+from typing import Literal, Union, overload
+from ...config import RedisDatabaseConfig
+from ...enums import Connection
+
+
+class RedisClientManager:
+    def __init__(self, config: RedisDatabaseConfig) -> None:
+        self.config = config
+        self._async_client: AsyncRedis = self._init(Connection.ASYNC)
+        self._sync_client: SyncRedis = self._init(Connection.SYNC)
+
+    @overload
+    def _init(self, connection: Literal[Connection.ASYNC]) -> AsyncRedis: ...
+    @overload
+    def _init(self, connection: Literal[Connection.SYNC]) -> SyncRedis: ...
+    def _init(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncRedis, SyncRedis]:
+        url = self.config.connection.make_url(connection)
+
+        # Redis clients expect different parameter names
+        pooling_config = self.config.pooling.model_dump(exclude_none=True)
+        redis_kwargs = {
+            "max_connections": pooling_config.get("max_connections"),
+            "retry_on_timeout": pooling_config.get("retry_on_timeout"),
+            "connection_timeout": pooling_config.get("connection_timeout"),
+            "socket_timeout": pooling_config.get("socket_timeout"),
+            "socket_keepalive": pooling_config.get("socket_keepalive"),
+            "decode_responses": pooling_config.get("decode_responses"),
+            # health_check_interval doesn't apply to from_url method
+        }
+        redis_kwargs = {k: v for k, v in redis_kwargs.items() if v is not None}
+
+        if connection is Connection.ASYNC:
+            self._async_client = AsyncRedis.from_url(url, **redis_kwargs)
+            return self._async_client
+        else:
+            self._sync_client = SyncRedis.from_url(url, **redis_kwargs)
+            return self._sync_client
+
+    @overload
+    def get(self, connection: Literal[Connection.ASYNC]) -> AsyncRedis: ...
+    @overload
+    def get(self, connection: Literal[Connection.SYNC]) -> SyncRedis: ...
+    def get(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncRedis, SyncRedis]:
+        if connection is Connection.ASYNC:
+            return self._async_client
+        elif connection is Connection.SYNC:
+            return self._sync_client
+
+    async def dispose(self):
+        await self._async_client.close()
+        self._sync_client.close()

maleo/database/managers/engines/mysql.py

@@ -0,0 +1,56 @@
+from sqlalchemy.engine import create_engine, Engine
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine
+from typing import Literal, Tuple, Union, overload
+from ...config import MySQLDatabaseConfig
+from ...enums import Connection
+
+
+class MySQLEngineManager:
+    def __init__(self, config: MySQLDatabaseConfig) -> None:
+        self.config = config
+        self._async_engine: AsyncEngine = self._init(Connection.ASYNC)
+        self._sync_engine: Engine = self._init(Connection.SYNC)
+
+    @overload
+    def _init(self, connection: Literal[Connection.ASYNC]) -> AsyncEngine: ...
+    @overload
+    def _init(self, connection: Literal[Connection.SYNC]) -> Engine: ...
+    def _init(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncEngine, Engine]:
+        url = self.config.connection.make_url(connection)
+
+        pooling_kwargs = self.config.pooling.model_dump(
+            exclude={
+                "strategy"
+            },  # autocommit and connect_timeout are valid SQLAlchemy params
+            exclude_none=True,
+        )
+
+        engine_kwargs = {"echo": self.config.connection.echo, **pooling_kwargs}
+
+        if connection is Connection.ASYNC:
+            self._async_engine = create_async_engine(url, **engine_kwargs)
+            return self._async_engine
+        elif connection is Connection.SYNC:
+            self._sync_engine = create_engine(url, **engine_kwargs)
+            return self._sync_engine
+
+    @overload
+    def get(self, connection: Literal[Connection.ASYNC]) -> AsyncEngine: ...
+    @overload
+    def get(self, connection: Literal[Connection.SYNC]) -> Engine: ...
+    def get(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncEngine, Engine]:
+        if connection is Connection.ASYNC:
+            return self._async_engine
+        elif connection is Connection.SYNC:
+            return self._sync_engine
+
+    def get_all(self) -> Tuple[AsyncEngine, Engine]:
+        return (self._async_engine, self._sync_engine)
+
+    async def dispose(self):
+        await self._async_engine.dispose()
+        self._sync_engine.dispose()

maleo/database/managers/engines/postgresql.py

@@ -0,0 +1,58 @@
+from sqlalchemy.engine import create_engine, Engine
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncEngine
+from typing import Literal, Tuple, Union, overload
+from ...config import PostgreSQLDatabaseConfig
+from ...enums import Connection
+
+
+class PostgreSQLEngineManager:
+    def __init__(self, config: PostgreSQLDatabaseConfig) -> None:
+        self.config = config
+        self._async_engine: AsyncEngine = self._init(Connection.ASYNC)
+        self._sync_engine: Engine = self._init(Connection.SYNC)
+
+    @overload
+    def _init(self, connection: Literal[Connection.ASYNC]) -> AsyncEngine: ...
+    @overload
+    def _init(self, connection: Literal[Connection.SYNC]) -> Engine: ...
+    def _init(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncEngine, Engine]:
+        url = self.config.connection.make_url(connection)
+
+        pooling_kwargs = self.config.pooling.model_dump(
+            exclude={
+                "strategy",
+                "prepared_statement_cache_size",
+                "pool_reset_on_return",
+            },
+            exclude_none=True,
+        )
+
+        engine_kwargs = {"echo": self.config.connection.echo, **pooling_kwargs}
+
+        if connection is Connection.ASYNC:
+            self._async_engine = create_async_engine(url, **engine_kwargs)
+            return self._async_engine
+        elif connection is Connection.SYNC:
+            self._sync_engine = create_engine(url, **engine_kwargs)
+            return self._sync_engine
+
+    @overload
+    def get(self, connection: Literal[Connection.ASYNC]) -> AsyncEngine: ...
+    @overload
+    def get(self, connection: Literal[Connection.SYNC]) -> Engine: ...
+    def get(
+        self, connection: Connection = Connection.ASYNC
+    ) -> Union[AsyncEngine, Engine]:
+        if connection is Connection.ASYNC:
+            return self._async_engine
+        elif connection is Connection.SYNC:
+            return self._sync_engine
+
+    def get_all(self) -> Tuple[AsyncEngine, Engine]:
+        return (self._async_engine, self._sync_engine)
+
+    async def dispose(self):
+        await self._async_engine.dispose()
+        self._sync_engine.dispose()