sibi-dst 0.3.53__py3-none-any.whl → 0.3.54__py3-none-any.whl

sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -29,7 +29,10 @@ class SqlAlchemyConnectionConfig(BaseModel):
     connection_url: str
     table: Optional[str] = None
     model: Any = None
-    engine: Optional[Any] = None  # Save engine to reuse it
+    engine: Optional[Any] = None
+    pool_size: int = 10
+    max_overflow: int = 5
+    pool_timeout: int = 30
 
     @model_validator(mode="after")
     def validate_and_initialize(self):
@@ -41,7 +44,10 @@ class SqlAlchemyConnectionConfig(BaseModel):
             raise ValueError("`connection_url` must be provided.")
 
         # Initialize the engine
-        self.engine = create_engine(self.connection_url)
+        self.engine = create_engine(self.connection_url,
+                                    pool_size=self.pool_size,
+                                    max_overflow=self.max_overflow,
+                                    pool_timeout=self.pool_timeout)
 
         # Validate the connection
         self.validate_connection()

sibi_dst/utils/data_wrapper.py

@@ -78,6 +78,7 @@ class DataWrapper:
         self._lock = Lock()
         self.processed_dates = []
         self.age_checker = FileAgeChecker(logger=self.logger)
+        self.data_class_instance = self.dataclass(**self.class_params) or None
 
     def _init_filesystem(self) -> fsspec.AbstractFileSystem:
         with self._lock:
@@ -265,8 +266,9 @@ class DataWrapper:
         try:
             self.logger.debug(f"Class Params: {self.class_params}")
             self.logger.debug(f"Load Params: {self.load_params}")
-            data = self.dataclass(**self.class_params)
-            df = data.load_period(
+
+            #data = self.dataclass(**self.class_params)
+            df = self.data_class_instance.load_period(
                 dt_field=self.date_field,
                 start=date,
                 end=date,

sibi_dst-0.3.54.dist-info/METADATA

@@ -0,0 +1,97 @@
+Metadata-Version: 2.1
+Name: sibi-dst
+Version: 0.3.54
+Summary: Data Science Toolkit
+Author: Luis Valverde
+Author-email: lvalverdeb@gmail.com
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: complete
+Provides-Extra: df-helper
+Provides-Extra: geospatial
+Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
+Requires-Dist: chardet (>=5.2.0,<6.0.0)
+Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
+Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
+Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
+Requires-Dist: dask[complete] (>=2025.3.0,<2026.0.0)
+Requires-Dist: django (>=5.1.4,<6.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: djangorestframework (>=3.15.2,<4.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
+Requires-Dist: httpx (>=0.27.2,<0.28.0)
+Requires-Dist: ipython (>=8.29.0,<9.0.0)
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: mysqlclient (>=2.2.6,<3.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: nltk (>=3.9.1,<4.0.0)
+Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
+Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: paramiko (>=3.5.0,<4.0.0)
+Requires-Dist: psutil (>=6.1.0,<7.0.0)
+Requires-Dist: psycopg2 (>=2.9.10,<3.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
+Requires-Dist: pydantic (>=2.9.2,<3.0.0)
+Requires-Dist: pymysql (>=1.1.1,<2.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: pytest (>=8.3.3,<9.0.0)
+Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: s3fs (>=2024.12.0,<2025.0.0)
+Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: sqlmodel (>=0.0.22,<0.0.23) ; extra == "df-helper" or extra == "complete"
+Requires-Dist: tornado (>=6.4.1,<7.0.0)
+Requires-Dist: tqdm (>=4.67.0,<5.0.0)
+Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
+Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
+Requires-Dist: webdav4[fsspec] (>=0.10.0,<0.11.0)
+Description-Content-Type: text/markdown
+
+---
+title: "Sibi Data Science Toolkit"
+description: "Data Science Toolkit built with Python, Pandas, Dask, GeoPandas, OpenStreetMaps, SQLAlchemy"
+author: "Luis Valverde"
+current_mantainer: "Luis Valverde"
+---
+
+### SIBI-DST
+
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, NetworkX, SQLAlchemy, GeoPandas, and Folium.
+
+## Example Use Cases
+
+1. **Build DataCubes, DataSets, and DataObjects** from diverse data sources, including **relational databases, Parquet files, Excel (`.xlsx`), delimited tables (`.csv`, `.tsv`), JSON, and RESTful APIs**.
+2. **Comprehensive DataFrame Management** utilities for efficient data handling, transformation, and optimization using **Pandas** and **Dask**.
+3. **Flexible Data Sharing** with client applications by writing to **Data Warehouses in Clickhouse, local filesystems, and cloud storage platforms** such as **S3**.
+4. **Microservices for Data Access** – Build scalable **API-driven services** using **RESTful APIs (`Django REST Framework`, `FastAPI`)** for high-performance data exchange.
+5. **Geospatial Analysis** – Utilize **OpenStreetMaps** and **GeoPandas** for advanced geospatial data processing and visualization.
+
+## Supported Technologies
+
+- **Data Processing**: Pandas, Dask
+- **Databases & Storage**: SQLAlchemy, Parquet, S3, Clickhouse
+- **Mapping & Geospatial Analysis**: OpenStreetMaps, OSMnx, Geopy
+- **API Development**: Django REST Framework, FastAPI
+
+## Installation
+
+```bash
+# with pip
+
+pip install sibi-dst[complete]  # Install all dependencies
+pip install sibi-dst[df_helper]  # Install only df_helper dependencies
+pip install sibi-dst[geospatial]  # Install only geospatial dependencies
+
+# with poetry
+
+poetry add "sibi-dst[complete]"  # Install all dependencies
+poetry add "sibi-dst[df_helper]"  # Install only df_helper dependencies
+poetry add "sibi-dst[geospatial]"  # Install only geospatial dependencies
+
+
+```
+

{sibi_dst-0.3.53.dist-info → sibi_dst-0.3.54.dist-info}/RECORD

@@ -16,7 +16,7 @@ sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1u
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
 sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=TuVp8Ce49dCIIxtyrtFGRblarQUl8QGcS-TDZd515IE,348
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=Kli83IEg5SFVqkhsK4w45cV6PbZnfdGanfsyiW6Xw00,2502
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=bQxQAA2GNSmNsRWTZbsLmpvK6pnkbVzECBXQ9H2iqRc,2750
 sibi_dst/df_helper/backends/sqlalchemy/_filter_handler.py,sha256=58RCda1Hg_nsuJw-2V36IstsT8O84IQFgsdE7FnqvMk,4655
 sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=BtiRSYA4kFIM-mBCdrwE20vzByfq8_Biv_jPLUCDv58,5466
 sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=I2Us3RrxHci561yyZYBuUCrLVOhB0F3KBnae78m_ARw,6259
@@ -44,7 +44,7 @@ sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcW
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=OMmdjnUCjxDsC1T0K1JbzxKXLTiDOtxPp1ujV66bMAY,12345
+sibi_dst/utils/data_wrapper.py,sha256=-RqK_sU3uuc6U9dFfnICFZkaIetk3JOvJain6_lTWlo,12446
 sibi_dst/utils/date_utils.py,sha256=OCJqkWl5e8fE7z11Ufz4206DUeuLMd_Gf_JGZu914Pg,18539
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
@@ -75,6 +75,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.53.dist-info/METADATA,sha256=rNy87i3acBUu27BdqZ-J73e9oy6pd8YEKCX-HrE48ig,7182
-sibi_dst-0.3.53.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.53.dist-info/RECORD,,
+sibi_dst-0.3.54.dist-info/METADATA,sha256=i2guOpX-oxuWb7-Jk2GNLQrSY83VjaAulNA_9AyuVzk,4502
+sibi_dst-0.3.54.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.54.dist-info/RECORD,,

sibi_dst-0.3.53.dist-info/METADATA

@@ -1,199 +0,0 @@
-Metadata-Version: 2.1
-Name: sibi-dst
-Version: 0.3.53
-Summary: Data Science Toolkit
-Author: Luis Valverde
-Author-email: lvalverdeb@gmail.com
-Requires-Python: >=3.11,<4.0
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Provides-Extra: complete
-Provides-Extra: df-helper
-Provides-Extra: geospatial
-Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
-Requires-Dist: chardet (>=5.2.0,<6.0.0)
-Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
-Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
-Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
-Requires-Dist: dask[complete] (>=2025.3.0,<2026.0.0)
-Requires-Dist: django (>=5.1.4,<6.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: djangorestframework (>=3.15.2,<4.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: folium (>=0.19.4,<0.20.0) ; extra == "geospatial" or extra == "complete"
-Requires-Dist: geopandas (>=1.0.1,<2.0.0) ; extra == "geospatial" or extra == "complete"
-Requires-Dist: geopy (>=2.4.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
-Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
-Requires-Dist: httpx (>=0.27.2,<0.28.0)
-Requires-Dist: ipython (>=8.29.0,<9.0.0)
-Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
-Requires-Dist: mysqlclient (>=2.2.6,<3.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: nltk (>=3.9.1,<4.0.0)
-Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
-Requires-Dist: osmnx (>=2.0.1,<3.0.0) ; extra == "geospatial" or extra == "complete"
-Requires-Dist: pandas (>=2.2.3,<3.0.0)
-Requires-Dist: paramiko (>=3.5.0,<4.0.0)
-Requires-Dist: psutil (>=6.1.0,<7.0.0)
-Requires-Dist: psycopg2 (>=2.9.10,<3.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
-Requires-Dist: pydantic (>=2.9.2,<3.0.0)
-Requires-Dist: pymysql (>=1.1.1,<2.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: pytest (>=8.3.3,<9.0.0)
-Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
-Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
-Requires-Dist: s3fs (>=2024.12.0,<2025.0.0)
-Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: sqlmodel (>=0.0.22,<0.0.23) ; extra == "df-helper" or extra == "complete"
-Requires-Dist: tornado (>=6.4.1,<7.0.0)
-Requires-Dist: tqdm (>=4.67.0,<5.0.0)
-Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
-Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
-Requires-Dist: webdav4[fsspec] (>=0.10.0,<0.11.0)
-Description-Content-Type: text/markdown
-
-# sibi-dst
-
-Data Science Toolkit
----------------------
-Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, SQLAlchemy, DjangoRestFramework, FastAPI
-
-Major Functionality
---------------------
-1) **Build DataCubes, DataSets, and DataObjects** from diverse data sources, including **relational databases, Parquet files, Excel (`.xlsx`), delimited tables (`.csv`, `.tsv`), JSON, and RESTful APIs (`JSON API REST`)**.
-2) **Comprehensive DataFrame Management** utilities for efficient data handling, transformation, and optimization using **Pandas** and **Dask**.
-3) **Flexible Data Sharing** with client applications by writing to **Data Warehouses, local filesystems, and cloud storage platforms** such as **Amazon S3, Google Cloud Storage (GCS), and Azure Blob Storage**.
-4) **Microservices for Data Access** – Build scalable **API-driven services** using **RESTful APIs (`Django REST Framework`, `FastAPI`) and gRPC** for high-performance data exchange.
-
-Supported Technologies
---------------------
-- **Data Processing**: Pandas, Dask
-- **Machine Learning**: Scikit-Learn, XGBoost
-- **Databases & Storage**: SQLAlchemy, Django ORM, Parquet, Amazon S3, GCS, Azure Blob Storage
-- **Mapping & Geospatial Analysis**: OpenStreetMaps, OSMnx, Geopy
-- **API Development**: Django REST Framework, gRPC, FastAPI
-
-Installation
----------------------
-```bash
-pip install sibi-dst
-```
-
-Usage
----------------------
-### Loading Data from SQLAlchemy
-```python
-from sibi_dst.df_helper import DfHelper
-from conf.transforms.fields.crm import customer_fields
-from conf.credentials import replica_db_conf
-from conf.storage import get_fs_instance
-
-config = {
-    'backend': 'sqlalchemy',
-    'connection_url': replica_db_conf.get('db_url'),
-    'table': 'crm_clientes_archivo',
-    'field_map': customer_fields,
-    'legacy_filters': True,
-    'fs': get_fs_instance()
-}
-
-df_helper = DfHelper(**config)
-result = df_helper.load(id__gte=1)
-```
-
-### Saving Data to ClickHouse
-```python
-clk_creds = {
-    'host': '192.168.3.171',
-    'port': 18123,
-    'user': 'username',
-    'database': 'xxxxxxx',
-    'table': 'customer_file',
-    'order_by': 'id'
-}
-
-df_helper.save_to_clickhouse(**clk_creds)
-```
-
-### Saving Data to Parquet
-```python
-df_helper.save_to_parquet(
-    parquet_filename='filename.parquet',
-    parquet_storage_path='/path/to/my/files/'
-)
-```
-
-Backends Supported
----------------------
-| Backend       | Description |
-|--------------|-------------|
-| `sqlalchemy` | Load data from SQL databases using SQLAlchemy. |
-| `django_db`  | Load data from Django ORM models. |
-| `parquet`    | Load and save data from Parquet files. |
-| `http`       | Fetch data from HTTP endpoints. |
-| `osmnx`      | Geospatial mapping and routing using OpenStreetMap. |
-| `geopy`      | Geolocation services for address lookup and reverse geocoding. |
-
-Geospatial Utilities
----------------------
-### **OSMnx Helper (`sibi_dst.osmnx_helper`)
-**
-Provides **OpenStreetMap-based mapping utilities** using `osmnx` and `folium`.
-
-#### 🔹 Key Features
-- **BaseOsmMap**: Manages interactive Folium-based maps.
-- **PBFHandler**: Loads `.pbf` (Protocolbuffer Binary Format) files for network graphs.
-
-#### Example: Generating an OSM Map
-```python
-from sibi_dst.osmnx_helper import BaseOsmMap
-osm_map = BaseOsmMap(osmnx_graph=my_graph, df=my_dataframe)
-osm_map.generate_map()
-```
-
-### **Geopy Helper (`sibi_dst.geopy_helper`)
-**
-Provides **geolocation services** using `Geopy` for forward and reverse geocoding.
-
-#### 🔹 Key Features
-- **GeolocationService**: Interfaces with `Nominatim` API for geocoding.
-- **Error Handling**: Manages `GeocoderTimedOut` and `GeocoderServiceError` gracefully.
-- **Singleton Geolocator**: Efficiently reuses a global geolocator instance.
-
-#### Example: Reverse Geocoding
-```python
-from sibi_dst.geopy_helper import GeolocationService
-gs = GeolocationService()
-location = gs.reverse((9.935,-84.091))
-print(location)
-```
-
-Advanced Features
----------------------
-### Querying with Custom Filters
-Filters can be applied dynamically using Django-style syntax:
-```python
-result = df_helper.load(date__gte='2023-01-01', status='active')
-```
-
-### Parallel Processing
-Leverage Dask for parallel execution:
-```python
-result = df_helper.load_parallel(status='active')
-```
-
-Testing
----------------------
-To run unit tests, use:
-```bash
-pytest tests/
-```
-
-Contributing
----------------------
-Contributions are welcome! Please submit pull requests or open issues for discussions.
-
-License
----------------------
-sibi-dst is licensed under the MIT License.
-
-