ddcdatabases 4.0.0__py3-none-any.whl

ddcdatabases/.env.example

@@ -0,0 +1,200 @@
+# ddcdatabases Environment Variables Example
+# Copy this file to .env and adjust values as needed
+
+# SQLite Database Settings
+SQLITE_FILE_PATH=sqlite.db
+SQLITE_ECHO=false
+# Connection Retry settings
+SQLITE_CONNECTION_ENABLE_RETRY=false
+SQLITE_CONNECTION_MAX_RETRIES=1
+SQLITE_CONNECTION_INITIAL_RETRY_DELAY=1.0
+SQLITE_CONNECTION_MAX_RETRY_DELAY=30.0
+# Operation Retry settings
+SQLITE_OPERATION_ENABLE_RETRY=false
+SQLITE_OPERATION_MAX_RETRIES=1
+SQLITE_OPERATION_INITIAL_RETRY_DELAY=0.5
+SQLITE_OPERATION_MAX_RETRY_DELAY=10.0
+SQLITE_OPERATION_JITTER=0.1
+
+# PostgreSQL Database Settings
+POSTGRESQL_HOST=localhost
+POSTGRESQL_PORT=5432
+POSTGRESQL_USER=postgres
+POSTGRESQL_PASSWORD=password
+POSTGRESQL_DATABASE=postgres
+# Comma-separated for multiple schemas (e.g. public,schema2)
+POSTGRESQL_SCHEMA=public
+# Session settings
+POSTGRESQL_ECHO=false
+POSTGRESQL_AUTOFLUSH=false
+POSTGRESQL_EXPIRE_ON_COMMIT=false
+POSTGRESQL_AUTOCOMMIT=false
+# Pool settings
+POSTGRESQL_CONNECTION_TIMEOUT=30
+POSTGRESQL_POOL_RECYCLE=3600
+POSTGRESQL_POOL_SIZE=25
+POSTGRESQL_MAX_OVERFLOW=50
+# SSL settings
+POSTGRESQL_SSL_MODE=disable
+POSTGRESQL_SSL_CA_CERT_PATH=
+POSTGRESQL_SSL_CLIENT_CERT_PATH=
+POSTGRESQL_SSL_CLIENT_KEY_PATH=
+# Connection Retry settings
+POSTGRESQL_CONNECTION_ENABLE_RETRY=true
+POSTGRESQL_CONNECTION_MAX_RETRIES=3
+POSTGRESQL_CONNECTION_INITIAL_RETRY_DELAY=1.0
+POSTGRESQL_CONNECTION_MAX_RETRY_DELAY=30.0
+POSTGRESQL_CONNECTION_DISCONNECT_IDLE_TIMEOUT=300
+# Operation Retry settings
+POSTGRESQL_OPERATION_ENABLE_RETRY=true
+POSTGRESQL_OPERATION_MAX_RETRIES=3
+POSTGRESQL_OPERATION_INITIAL_RETRY_DELAY=0.5
+POSTGRESQL_OPERATION_MAX_RETRY_DELAY=10.0
+POSTGRESQL_OPERATION_JITTER=0.1
+# Persistent connection settings
+POSTGRESQL_PERSISTENT_IDLE_TIMEOUT=300
+POSTGRESQL_PERSISTENT_HEALTH_CHECK_INTERVAL=30
+POSTGRESQL_PERSISTENT_AUTO_RECONNECT=true
+
+# Microsoft SQL Server Database Settings
+MSSQL_HOST=localhost
+MSSQL_PORT=1433
+MSSQL_USER=sa
+MSSQL_PASSWORD=password
+MSSQL_DATABASE=master
+MSSQL_SCHEMA=dbo
+# Session settings
+MSSQL_ECHO=false
+MSSQL_AUTOFLUSH=false
+MSSQL_EXPIRE_ON_COMMIT=false
+MSSQL_AUTOCOMMIT=false
+# Pool settings
+MSSQL_CONNECTION_TIMEOUT=30
+MSSQL_POOL_RECYCLE=3600
+MSSQL_POOL_SIZE=25
+MSSQL_MAX_OVERFLOW=50
+MSSQL_ODBCDRIVER_VERSION=18
+# SSL settings
+MSSQL_SSL_ENCRYPT=false
+MSSQL_SSL_TRUST_SERVER_CERTIFICATE=true
+MSSQL_SSL_CA_CERT_PATH=
+# Connection Retry settings
+MSSQL_CONNECTION_ENABLE_RETRY=true
+MSSQL_CONNECTION_MAX_RETRIES=3
+MSSQL_CONNECTION_INITIAL_RETRY_DELAY=1.0
+MSSQL_CONNECTION_MAX_RETRY_DELAY=30.0
+MSSQL_CONNECTION_DISCONNECT_IDLE_TIMEOUT=300
+# Operation Retry settings
+MSSQL_OPERATION_ENABLE_RETRY=true
+MSSQL_OPERATION_MAX_RETRIES=3
+MSSQL_OPERATION_INITIAL_RETRY_DELAY=0.5
+MSSQL_OPERATION_MAX_RETRY_DELAY=10.0
+MSSQL_OPERATION_JITTER=0.1
+# Persistent connection settings
+MSSQL_PERSISTENT_IDLE_TIMEOUT=300
+MSSQL_PERSISTENT_HEALTH_CHECK_INTERVAL=30
+MSSQL_PERSISTENT_AUTO_RECONNECT=true
+
+# MySQL Database Settings
+MYSQL_HOST=localhost
+MYSQL_PORT=3306
+MYSQL_USER=root
+MYSQL_PASSWORD=password
+MYSQL_DATABASE=dev
+# Session settings
+MYSQL_ECHO=false
+MYSQL_AUTOFLUSH=false
+MYSQL_EXPIRE_ON_COMMIT=false
+MYSQL_AUTOCOMMIT=true
+# Pool settings
+MYSQL_CONNECTION_TIMEOUT=30
+MYSQL_POOL_RECYCLE=3600
+MYSQL_POOL_SIZE=10
+MYSQL_MAX_OVERFLOW=20
+# SSL settings
+MYSQL_SSL_MODE=DISABLED
+MYSQL_SSL_CA_CERT_PATH=
+MYSQL_SSL_CLIENT_CERT_PATH=
+MYSQL_SSL_CLIENT_KEY_PATH=
+# Connection Retry settings
+MYSQL_CONNECTION_ENABLE_RETRY=true
+MYSQL_CONNECTION_MAX_RETRIES=3
+MYSQL_CONNECTION_INITIAL_RETRY_DELAY=1.0
+MYSQL_CONNECTION_MAX_RETRY_DELAY=30.0
+MYSQL_CONNECTION_DISCONNECT_IDLE_TIMEOUT=300
+# Operation Retry settings
+MYSQL_OPERATION_ENABLE_RETRY=true
+MYSQL_OPERATION_MAX_RETRIES=3
+MYSQL_OPERATION_INITIAL_RETRY_DELAY=0.5
+MYSQL_OPERATION_MAX_RETRY_DELAY=10.0
+MYSQL_OPERATION_JITTER=0.1
+# Persistent connection settings
+MYSQL_PERSISTENT_IDLE_TIMEOUT=300
+MYSQL_PERSISTENT_HEALTH_CHECK_INTERVAL=30
+MYSQL_PERSISTENT_AUTO_RECONNECT=true
+
+# Oracle Database Settings
+ORACLE_HOST=localhost
+ORACLE_PORT=1521
+ORACLE_USER=system
+ORACLE_PASSWORD=oracle
+ORACLE_SERVICENAME=xe
+# Session settings
+ORACLE_ECHO=false
+ORACLE_AUTOFLUSH=false
+ORACLE_EXPIRE_ON_COMMIT=false
+ORACLE_AUTOCOMMIT=false
+# Pool settings
+ORACLE_CONNECTION_TIMEOUT=30
+ORACLE_POOL_RECYCLE=3600
+ORACLE_POOL_SIZE=10
+ORACLE_MAX_OVERFLOW=20
+# SSL settings
+ORACLE_SSL_ENABLED=false
+ORACLE_SSL_WALLET_PATH=
+# Connection Retry settings
+ORACLE_CONNECTION_ENABLE_RETRY=true
+ORACLE_CONNECTION_MAX_RETRIES=3
+ORACLE_CONNECTION_INITIAL_RETRY_DELAY=1.0
+ORACLE_CONNECTION_MAX_RETRY_DELAY=30.0
+ORACLE_CONNECTION_DISCONNECT_IDLE_TIMEOUT=300
+# Operation Retry settings
+ORACLE_OPERATION_ENABLE_RETRY=true
+ORACLE_OPERATION_MAX_RETRIES=3
+ORACLE_OPERATION_INITIAL_RETRY_DELAY=0.5
+ORACLE_OPERATION_MAX_RETRY_DELAY=10.0
+ORACLE_OPERATION_JITTER=0.1
+# Persistent connection settings
+ORACLE_PERSISTENT_IDLE_TIMEOUT=300
+ORACLE_PERSISTENT_HEALTH_CHECK_INTERVAL=30
+ORACLE_PERSISTENT_AUTO_RECONNECT=true
+
+# MongoDB Database Settings
+MONGODB_HOST=localhost
+MONGODB_PORT=27017
+MONGODB_USER=admin
+MONGODB_PASSWORD=admin
+MONGODB_DATABASE=admin
+MONGODB_BATCH_SIZE=2865
+MONGODB_LIMIT=0
+# TLS settings
+MONGODB_TLS_ENABLED=false
+MONGODB_TLS_CA_CERT_PATH=
+MONGODB_TLS_CERT_KEY_PATH=
+MONGODB_TLS_ALLOW_INVALID_CERTIFICATES=false
+# Connection Retry settings
+MONGODB_CONNECTION_ENABLE_RETRY=true
+MONGODB_CONNECTION_MAX_RETRIES=3
+MONGODB_CONNECTION_INITIAL_RETRY_DELAY=1.0
+MONGODB_CONNECTION_MAX_RETRY_DELAY=30.0
+MONGODB_CONNECTION_DISCONNECT_IDLE_TIMEOUT=300
+# Operation Retry settings
+MONGODB_OPERATION_ENABLE_RETRY=true
+MONGODB_OPERATION_MAX_RETRIES=3
+MONGODB_OPERATION_INITIAL_RETRY_DELAY=0.5
+MONGODB_OPERATION_MAX_RETRY_DELAY=10.0
+MONGODB_OPERATION_JITTER=0.1
+# Persistent connection settings
+MONGODB_PERSISTENT_IDLE_TIMEOUT=300
+MONGODB_PERSISTENT_HEALTH_CHECK_INTERVAL=30
+MONGODB_PERSISTENT_AUTO_RECONNECT=true

ddcdatabases/__init__.py

@@ -0,0 +1,203 @@
+import logging
+from .core.operations import DBUtils, DBUtilsAsync
+from .core.persistent import PersistentConnectionConfig, close_all_persistent_connections
+from importlib.metadata import version
+
+__all__ = [
+    "DBUtils",
+    "DBUtilsAsync",
+    "PersistentConnectionConfig",
+    "close_all_persistent_connections",
+]
+
+# Conditional imports based on available dependencies
+try:
+    from .core.settings import clear_sqlite_settings_cache, get_sqlite_settings
+    from .sqlite import (
+        Sqlite,
+        SqliteConnectionRetryConfig,
+        SqliteOperationRetryConfig,
+        SqliteSessionConfig,
+    )
+
+    __all__ += [
+        "Sqlite",
+        "SqliteConnectionRetryConfig",
+        "SqliteOperationRetryConfig",
+        "SqliteSessionConfig",
+        "clear_sqlite_settings_cache",
+        "get_sqlite_settings",
+    ]
+except ImportError:
+    pass
+
+try:
+    from .core.persistent import MongoDBPersistent
+    from .core.settings import clear_mongodb_settings_cache, get_mongodb_settings
+    from .mongodb import (
+        MongoDB,
+        MongoDBConnectionConfig,
+        MongoDBConnectionRetryConfig,
+        MongoDBOperationRetryConfig,
+        MongoDBQueryConfig,
+        MongoDBTLSConfig,
+    )
+
+    __all__ += [
+        "MongoDB",
+        "MongoDBConnectionConfig",
+        "MongoDBConnectionRetryConfig",
+        "MongoDBOperationRetryConfig",
+        "MongoDBPersistent",
+        "MongoDBQueryConfig",
+        "MongoDBTLSConfig",
+        "clear_mongodb_settings_cache",
+        "get_mongodb_settings",
+    ]
+except ImportError:
+    pass
+
+try:
+    from .core.persistent import MSSQLPersistent
+    from .core.settings import clear_mssql_settings_cache, get_mssql_settings
+    from .mssql import (
+        MSSQL,
+        MSSQLConnectionConfig,
+        MSSQLConnectionRetryConfig,
+        MSSQLOperationRetryConfig,
+        MSSQLPoolConfig,
+        MSSQLSessionConfig,
+        MSSQLSSLConfig,
+    )
+
+    __all__ += [
+        "MSSQL",
+        "MSSQLConnectionConfig",
+        "MSSQLConnectionRetryConfig",
+        "MSSQLOperationRetryConfig",
+        "MSSQLPersistent",
+        "MSSQLPoolConfig",
+        "MSSQLSessionConfig",
+        "MSSQLSSLConfig",
+        "clear_mssql_settings_cache",
+        "get_mssql_settings",
+    ]
+except ImportError:
+    pass
+
+try:
+    from .core.persistent import MySQLPersistent
+    from .core.settings import clear_mysql_settings_cache, get_mysql_settings
+    from .mysql import (
+        MySQL,
+        MySQLConnectionConfig,
+        MySQLConnectionRetryConfig,
+        MySQLOperationRetryConfig,
+        MySQLPoolConfig,
+        MySQLSessionConfig,
+        MySQLSSLConfig,
+    )
+
+    # MariaDB aliases (MariaDB is fully compatible with MySQL driver)
+    MariaDB = MySQL
+    MariaDBConnectionConfig = MySQLConnectionConfig
+    MariaDBConnectionRetryConfig = MySQLConnectionRetryConfig
+    MariaDBOperationRetryConfig = MySQLOperationRetryConfig
+    MariaDBPersistent = MySQLPersistent
+    MariaDBPoolConfig = MySQLPoolConfig
+    MariaDBSessionConfig = MySQLSessionConfig
+    MariaDBSSLConfig = MySQLSSLConfig
+    clear_mariadb_settings_cache = clear_mysql_settings_cache
+    get_mariadb_settings = get_mysql_settings
+
+    __all__ += [
+        "MySQL",
+        "MySQLConnectionConfig",
+        "MySQLConnectionRetryConfig",
+        "MySQLOperationRetryConfig",
+        "MySQLPersistent",
+        "MySQLPoolConfig",
+        "MySQLSessionConfig",
+        "MySQLSSLConfig",
+        "clear_mysql_settings_cache",
+        "get_mysql_settings",
+        # MariaDB aliases
+        "MariaDB",
+        "MariaDBConnectionConfig",
+        "MariaDBConnectionRetryConfig",
+        "MariaDBOperationRetryConfig",
+        "MariaDBPersistent",
+        "MariaDBPoolConfig",
+        "MariaDBSessionConfig",
+        "MariaDBSSLConfig",
+        "clear_mariadb_settings_cache",
+        "get_mariadb_settings",
+    ]
+except ImportError:
+    pass
+
+try:
+    from .core.persistent import OraclePersistent
+    from .core.settings import clear_oracle_settings_cache, get_oracle_settings
+    from .oracle import (
+        Oracle,
+        OracleConnectionConfig,
+        OracleConnectionRetryConfig,
+        OracleOperationRetryConfig,
+        OraclePoolConfig,
+        OracleSessionConfig,
+        OracleSSLConfig,
+    )
+
+    __all__ += [
+        "Oracle",
+        "OracleConnectionConfig",
+        "OracleConnectionRetryConfig",
+        "OracleOperationRetryConfig",
+        "OraclePersistent",
+        "OraclePoolConfig",
+        "OracleSessionConfig",
+        "OracleSSLConfig",
+        "clear_oracle_settings_cache",
+        "get_oracle_settings",
+    ]
+except ImportError:
+    pass
+
+try:
+    from .core.persistent import PostgreSQLPersistent
+    from .core.settings import clear_postgresql_settings_cache, get_postgresql_settings
+    from .postgresql import (
+        PostgreSQL,
+        PostgreSQLConnectionConfig,
+        PostgreSQLConnectionRetryConfig,
+        PostgreSQLOperationRetryConfig,
+        PostgreSQLPoolConfig,
+        PostgreSQLSessionConfig,
+        PostgreSQLSSLConfig,
+    )
+
+    __all__ += [
+        "PostgreSQL",
+        "PostgreSQLConnectionConfig",
+        "PostgreSQLConnectionRetryConfig",
+        "PostgreSQLOperationRetryConfig",
+        "PostgreSQLPersistent",
+        "PostgreSQLPoolConfig",
+        "PostgreSQLSessionConfig",
+        "PostgreSQLSSLConfig",
+        "clear_postgresql_settings_cache",
+        "get_postgresql_settings",
+    ]
+except ImportError:
+    pass
+
+__all__ = tuple(__all__)
+__title__ = "ddcdatabases"
+__author__ = "Daniel Costa"
+__email__ = "daniel@ddcsoftwares.com"
+__license__ = "MIT"
+__copyright__ = "Copyright 2024-present DDC Softwares"
+__version__ = version(__title__)
+
+logging.getLogger(__name__).addHandler(logging.NullHandler())

ddcdatabases/core/base.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+import logging
+import sqlalchemy as sa
+from .configs import BaseOperationRetryConfig, BaseRetryConfig
+from .retry import retry_operation, retry_operation_async
+from collections.abc import AsyncGenerator, Generator
+from contextlib import asynccontextmanager, contextmanager
+from datetime import datetime
+from sqlalchemy.engine import URL, Engine, create_engine
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.orm import Session, sessionmaker
+from typing import Any
+
+_logger = logging.getLogger(__name__)
+_logger.addHandler(logging.NullHandler())
+
+
+class BaseConnection:
+    __slots__ = (
+        "connection_url",
+        "engine_args",
+        "autoflush",
+        "expire_on_commit",
+        "sync_driver",
+        "async_driver",
+        "session",
+        "is_connected",
+        "_temp_engine",
+        "connection_retry_config",
+        "operation_retry_config",
+        "logger",
+    )
+
+    def __init__(
+        self,
+        connection_url: dict,
+        engine_args: dict,
+        autoflush: bool,
+        expire_on_commit: bool,
+        sync_driver: str | None,
+        async_driver: str | None,
+        connection_retry_config: BaseRetryConfig | None = None,
+        operation_retry_config: BaseOperationRetryConfig | None = None,
+        logger: Any = None,
+    ) -> None:
+        self.connection_url = connection_url
+        self.engine_args = engine_args
+        self.autoflush = autoflush
+        self.expire_on_commit = expire_on_commit
+        self.sync_driver = sync_driver
+        self.async_driver = async_driver
+        self.session: Session | AsyncSession | None = None
+        self.is_connected = False
+        self._temp_engine: Engine | AsyncEngine | None = None
+        self.connection_retry_config = connection_retry_config or BaseRetryConfig()
+        self.operation_retry_config = operation_retry_config or BaseOperationRetryConfig()
+        self.logger = logger if logger is not None else _logger
+
+    def __enter__(self) -> Session:
+        def connect() -> Session:
+            with self._get_engine() as self._temp_engine:
+                session_maker = sessionmaker(
+                    bind=self._temp_engine,
+                    class_=Session,
+                    autoflush=self.autoflush,
+                    expire_on_commit=self.expire_on_commit,
+                )
+            with session_maker.begin() as self.session:
+                self._test_connection_sync(self.session)
+                self.is_connected = True
+                return self.session
+
+        return retry_operation(connect, self.connection_retry_config, "sync_connect", logger=self.logger)
+
+    def __exit__(self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: Any) -> None:
+        if self.session:
+            self.session.close()
+        if self._temp_engine:
+            self._temp_engine.dispose()
+        self.is_connected = False
+        self.logger.debug("Disconnected")
+
+    async def __aenter__(self) -> AsyncSession:
+        async def connect() -> AsyncSession:
+            async with self._get_async_engine() as self._temp_engine:
+                session_maker = async_sessionmaker(
+                    bind=self._temp_engine,
+                    class_=AsyncSession,
+                    autoflush=self.autoflush,
+                    expire_on_commit=self.expire_on_commit,
+                )
+            async with session_maker.begin() as self.session:
+                await self._test_connection_async(self.session)
+                self.is_connected = True
+                return self.session
+
+        return await retry_operation_async(connect, self.connection_retry_config, "async_connect", logger=self.logger)
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: Any,
+    ) -> None:
+        if self.session:
+            await self.session.close()
+        if self._temp_engine:
+            await self._temp_engine.dispose()
+        self.is_connected = False
+        self.logger.debug("Disconnected")
+
+    @contextmanager
+    def _get_engine(self) -> Generator[Engine, None, None]:
+        _connection_url = URL.create(drivername=self.sync_driver, **self.connection_url)
+        _engine = create_engine(url=_connection_url, **self.engine_args)
+        yield _engine
+        _engine.dispose()
+
+    @asynccontextmanager
+    async def _get_async_engine(self) -> AsyncGenerator[AsyncEngine, None]:
+        _connection_url = URL.create(drivername=self.async_driver, **self.connection_url)
+        _engine = create_async_engine(url=_connection_url, **self.engine_args)
+        yield _engine
+        await _engine.dispose()
+
+    def _test_connection_sync(self, session: Session) -> None:
+        _connection_url_copy = self.connection_url.copy()
+        _connection_url_copy.pop("password", None)
+        _connection_url = URL.create(
+            **_connection_url_copy,
+            drivername=self.sync_driver,
+        )
+        test_connection = ConnectionTester(
+            sync_session=session,
+            host_url=_connection_url,
+            logger=self.logger,
+        )
+        test_connection.test_connection_sync()
+
+    async def _test_connection_async(self, session: AsyncSession) -> None:
+        _connection_url_copy = self.connection_url.copy()
+        _connection_url_copy.pop("password", None)
+        _connection_url = URL.create(
+            **_connection_url_copy,
+            drivername=self.async_driver,
+        )
+        test_connection = ConnectionTester(
+            async_session=session,
+            host_url=_connection_url,
+            logger=self.logger,
+        )
+        await test_connection.test_connection_async()
+
+
+class ConnectionTester:
+    __slots__ = ("sync_session", "async_session", "host_url", "dt", "logger", "failed_msg")
+
+    def __init__(
+        self,
+        sync_session: Session | None = None,
+        async_session: AsyncSession | None = None,
+        host_url: URL | str = "",
+        logger: Any = None,
+    ) -> None:
+        self.sync_session = sync_session
+        self.async_session = async_session
+        self.host_url = host_url
+        self.dt = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
+        self.logger = logger if logger is not None else _logger
+        self.failed_msg = "Connection to database failed"
+
+    def test_connection_sync(self) -> bool:
+        try:
+            query_text = "SELECT 1 FROM dual" if "oracle" in str(self.sync_session.bind.url) else "SELECT 1"
+            self.sync_session.execute(sa.text(query_text))
+            return True
+        except Exception as e:
+            self.sync_session.close()
+            error_msg = f"[{self.dt}]:[ERROR]:{self.failed_msg} | {self.host_url} | {e!r}"
+            self.logger.error(error_msg)
+            raise ConnectionRefusedError(f"{self.failed_msg} | {e!r}") from e
+
+    async def test_connection_async(self) -> bool:
+        try:
+            query_text = "SELECT 1 FROM dual" if "oracle" in str(self.async_session.bind.url) else "SELECT 1"
+            await self.async_session.execute(sa.text(query_text))
+            return True
+        except Exception as e:
+            await self.async_session.close()
+            error_msg = f"[{self.dt}]:[ERROR]:{self.failed_msg} | {self.host_url} | {e!r}"
+            self.logger.error(error_msg)
+            raise ConnectionRefusedError(f"{self.failed_msg} | {e!r}") from e