sibi-dst 0.3.54__py3-none-any.whl → 0.3.56__py3-none-any.whl

sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -45,7 +45,7 @@ class ArtifactUpdaterMultiWrapper:
         # dynamic scaling config
         self.min_workers = kwargs.get('min_workers', 1)
         self.max_workers = kwargs.get('max_workers', 8)
-        self.memory_per_worker_gb = kwargs.get('memory_per_worker_gb', 1)  # default 2GB per worker
+        self.memory_per_worker_gb = kwargs.get('memory_per_worker_gb', 1)  # default 1GB per worker
         self.monitor_interval = kwargs.get('monitor_interval', 10)  # default monitor interval in seconds
         self.retry_attempts = kwargs.get('retry_attempts', 3)
         self.update_timeout_seconds = kwargs.get('update_timeout_seconds', 600)

sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -1,13 +1,14 @@
-from typing import Any, Optional
+from typing import Any, Optional, ClassVar
+import threading
 
 from pydantic import BaseModel, model_validator
 from sqlalchemy import create_engine
 from sqlalchemy.exc import OperationalError
 from sqlalchemy.sql import text
+from sibi_dst.utils import Logger
 
 from ._sql_model_builder import SqlAlchemyModelBuilder
 
-
 class SqlAlchemyConnectionConfig(BaseModel):
     """
     Configuration class for managing an SQLAlchemy database connection.
@@ -30,24 +31,55 @@ class SqlAlchemyConnectionConfig(BaseModel):
     table: Optional[str] = None
     model: Any = None
     engine: Optional[Any] = None
+    logger: Optional[Any] = None
     pool_size: int = 10
     max_overflow: int = 5
     pool_timeout: int = 30
+    pool_recycle:int = 300
+
+    # Class-level registry and lock for thread-safe engine reuse
+    _engine_registry: ClassVar[dict] = {}
+    _registry_lock: ClassVar[threading.Lock] = threading.Lock()
 
     @model_validator(mode="after")
     def validate_and_initialize(self):
         """
         Validate connection parameters, initialize the engine, and build the dynamic model if necessary.
         """
-        # Validate `connection_url`
+        if not self.logger:
+            self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
+
         if not self.connection_url:
             raise ValueError("`connection_url` must be provided.")
 
-        # Initialize the engine
-        self.engine = create_engine(self.connection_url,
+        # Validate `connection_url`
+        if self.engine is not None:
+            engine_url = str(self.engine.url)
+            if engine_url != self.connection_url:
+                raise ValueError(f"Engine URL '{engine_url}' does not match the provided connection URL '{self.connection_url}'.")
+        else:
+            # Generate a unique key for the engine registry based on the connection URL
+            engine_key = (
+                self.connection_url,
+                self.pool_size,
+                self.max_overflow,
+                self.pool_timeout,
+                self.pool_recycle
+            )
+            with self.__class__._registry_lock:
+                if engine_key in self.__class__._engine_registry:
+                    # Reuse the existing engine
+                    self.logger.info(f"Reusing existing engine for connection URL: {self.connection_url}")
+                    self.engine = self.__class__._engine_registry[engine_key]
+                else:
+                    # Initialize the engine
+                    self.logger.info(f"Creating new engine for connection URL: {self.connection_url}")
+                    self.engine = create_engine(self.connection_url,
                                     pool_size=self.pool_size,
                                     max_overflow=self.max_overflow,
-                                    pool_timeout=self.pool_timeout)
+                                    pool_timeout=self.pool_timeout,
+                                    pool_recycle=self.pool_recycle)
+                    self.__class__._engine_registry[engine_key] = self.engine
 
         # Validate the connection
         self.validate_connection()
@@ -70,3 +102,8 @@ class SqlAlchemyConnectionConfig(BaseModel):
         except OperationalError as e:
             raise ValueError(f"Failed to connect to the database: {e}")
 
+    @classmethod
+    def clear_engine_registry(cls):
+        """Clear the global engine registry (useful for testing)."""
+        with cls._registry_lock:
+            cls._engine_registry.clear()

sibi_dst/utils/data_wrapper.py

@@ -48,7 +48,7 @@ class DataWrapper:
                  timeout: float = 60,
                  reference_date: datetime.date = None,
                  custom_priority_map: Dict[str, int] = None,
-                 max_threads: int = 3):
+                 max_threads: int = 10):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)

{sibi_dst-0.3.54.dist-info → sibi_dst-0.3.56.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.54
+Version: 0.3.56
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -51,13 +51,6 @@ Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
 Requires-Dist: webdav4[fsspec] (>=0.10.0,<0.11.0)
 Description-Content-Type: text/markdown
 
----
-title: "Sibi Data Science Toolkit"
-description: "Data Science Toolkit built with Python, Pandas, Dask, GeoPandas, OpenStreetMaps, SQLAlchemy"
-author: "Luis Valverde"
-current_mantainer: "Luis Valverde"
----
-
 ### SIBI-DST
 
 Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, NetworkX, SQLAlchemy, GeoPandas, and Folium.

{sibi_dst-0.3.54.dist-info → sibi_dst-0.3.56.dist-info}/RECORD

@@ -1,6 +1,6 @@
 sibi_dst/__init__.py,sha256=3pbriM7Ym5f9gew7n9cO4G_p9n-0bnxdmQ0hwBdJjr4,253
 sibi_dst/df_helper/__init__.py,sha256=McYrw2N0MsMgtawLrONXTGdyHfQWVOBUvIDbklfjb54,342
-sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=toH2QvNF-CQNJ4Bc8xreytuWr37G0EWz4ciWVdFMVqU,11646
+sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=zkgF4kiP9AuGTipFpx4jE7Ys6j5KXkdFBOshScN46yM,11646
 sibi_dst/df_helper/_df_helper.py,sha256=D85n4oUdu92IN2QaPc6k9uJJ_Vm197me1aoHojuWEYs,29833
 sibi_dst/df_helper/_parquet_artifact.py,sha256=dDoR5Tq0EJViW52tD9XW_vno7hkOfA8WeAilR1mAb_g,10636
 sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
@@ -16,7 +16,7 @@ sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1u
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
 sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=TuVp8Ce49dCIIxtyrtFGRblarQUl8QGcS-TDZd515IE,348
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=bQxQAA2GNSmNsRWTZbsLmpvK6pnkbVzECBXQ9H2iqRc,2750
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=GJK_8JQJQTXL3tv2sqNq6en0iKg20_Ocl7dDDSfsqAw,4572
 sibi_dst/df_helper/backends/sqlalchemy/_filter_handler.py,sha256=58RCda1Hg_nsuJw-2V36IstsT8O84IQFgsdE7FnqvMk,4655
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=BtiRSYA4kFIM-mBCdrwE20vzByfq8_Biv_jPLUCDv58,5466
 sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=I2Us3RrxHci561yyZYBuUCrLVOhB0F3KBnae78m_ARw,6259
@@ -44,7 +44,7 @@ sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcW
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=-RqK_sU3uuc6U9dFfnICFZkaIetk3JOvJain6_lTWlo,12446
+sibi_dst/utils/data_wrapper.py,sha256=UJSFoy4qQP1bu9vze9QSBTeiVd5aIC8heWztA3zTBWk,12447
 sibi_dst/utils/date_utils.py,sha256=OCJqkWl5e8fE7z11Ufz4206DUeuLMd_Gf_JGZu914Pg,18539
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
@@ -75,6 +75,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.54.dist-info/METADATA,sha256=i2guOpX-oxuWb7-Jk2GNLQrSY83VjaAulNA_9AyuVzk,4502
-sibi_dst-0.3.54.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.54.dist-info/RECORD,,
+sibi_dst-0.3.56.dist-info/METADATA,sha256=4glUnenNOPnp57g6ztPBMp0TgLIngYnVqMhX55HzVkg,4292
+sibi_dst-0.3.56.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.56.dist-info/RECORD,,