sibi-dst 0.3.55__tar.gz → 0.3.56__tar.gz

{sibi_dst-0.3.55 → sibi_dst-0.3.56}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.55
+Version: 0.3.56
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.55 → sibi_dst-0.3.56}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.55"
+version = "0.3.56"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.55 → sibi_dst-0.3.56}/sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -45,7 +45,7 @@ class ArtifactUpdaterMultiWrapper:
         # dynamic scaling config
         self.min_workers = kwargs.get('min_workers', 1)
         self.max_workers = kwargs.get('max_workers', 8)
-        self.memory_per_worker_gb = kwargs.get('memory_per_worker_gb', 1)  # default 2GB per worker
+        self.memory_per_worker_gb = kwargs.get('memory_per_worker_gb', 1)  # default 1GB per worker
         self.monitor_interval = kwargs.get('monitor_interval', 10)  # default monitor interval in seconds
         self.retry_attempts = kwargs.get('retry_attempts', 3)
         self.update_timeout_seconds = kwargs.get('update_timeout_seconds', 600)

{sibi_dst-0.3.55 → sibi_dst-0.3.56}/sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -1,13 +1,14 @@
-from typing import Any, Optional
+from typing import Any, Optional, ClassVar
+import threading
 
 from pydantic import BaseModel, model_validator
 from sqlalchemy import create_engine
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.sql import text
+from sibi_dst.utils import Logger
 
 from ._sql_model_builder import SqlAlchemyModelBuilder
 
-
 class SqlAlchemyConnectionConfig(BaseModel):
     """
     Configuration class for managing an SQLAlchemy database connection.
@@ -30,26 +31,55 @@ class SqlAlchemyConnectionConfig(BaseModel):
     table: Optional[str] = None
     model: Any = None
     engine: Optional[Any] = None
+    logger: Optional[Any] = None
     pool_size: int = 10
     max_overflow: int = 5
     pool_timeout: int = 30
-    pool_recycle:int = 3600
+    pool_recycle:int = 300
+
+    # Class-level registry and lock for thread-safe engine reuse
+    _engine_registry: ClassVar[dict] = {}
+    _registry_lock: ClassVar[threading.Lock] = threading.Lock()
 
     @model_validator(mode="after")
     def validate_and_initialize(self):
         """
         Validate connection parameters, initialize the engine, and build the dynamic model if necessary.
         """
-        # Validate `connection_url`
+        if not self.logger:
+            self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
+
         if not self.connection_url:
             raise ValueError("`connection_url` must be provided.")
 
-        # Initialize the engine
-        self.engine = create_engine(self.connection_url,
+        # Validate `connection_url`
+        if self.engine is not None:
+            engine_url = str(self.engine.url)
+            if engine_url != self.connection_url:
+                raise ValueError(f"Engine URL '{engine_url}' does not match the provided connection URL '{self.connection_url}'.")
+        else:
+            # Generate a unique key for the engine registry based on the connection URL
+            engine_key = (
+                self.connection_url,
+                self.pool_size,
+                self.max_overflow,
+                self.pool_timeout,
+                self.pool_recycle
+            )
+            with self.__class__._registry_lock:
+                if engine_key in self.__class__._engine_registry:
+                    # Reuse the existing engine
+                    self.logger.info(f"Reusing existing engine for connection URL: {self.connection_url}")
+                    self.engine = self.__class__._engine_registry[engine_key]
+                else:
+                    # Initialize the engine
+                    self.logger.info(f"Creating new engine for connection URL: {self.connection_url}")
+                    self.engine = create_engine(self.connection_url,
                                     pool_size=self.pool_size,
                                     max_overflow=self.max_overflow,
                                     pool_timeout=self.pool_timeout,
                                     pool_recycle=self.pool_recycle)
+                    self.__class__._engine_registry[engine_key] = self.engine
 
         # Validate the connection
         self.validate_connection()
@@ -72,3 +102,8 @@ class SqlAlchemyConnectionConfig(BaseModel):
         except OperationalError as e:
             raise ValueError(f"Failed to connect to the database: {e}")
 
+    @classmethod
+    def clear_engine_registry(cls):
+        """Clear the global engine registry (useful for testing)."""
+        with cls._registry_lock:
+            cls._engine_registry.clear()

{sibi_dst-0.3.55 → sibi_dst-0.3.56}/sibi_dst/utils/data_wrapper.py

@@ -48,7 +48,7 @@ class DataWrapper:
                  timeout: float = 60,
                  reference_date: datetime.date = None,
                  custom_priority_map: Dict[str, int] = None,
-                 max_threads: int = 3):
+                 max_threads: int = 10):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)