sibi-dst 0.3.42__tar.gz → 0.3.43__tar.gz

sibi_dst-0.3.43/PKG-INFO

@@ -0,0 +1,194 @@
+Metadata-Version: 2.1
+Name: sibi-dst
+Version: 0.3.43
+Summary: Data Science Toolkit
+Author: Luis Valverde
+Author-email: lvalverdeb@gmail.com
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
+Requires-Dist: chardet (>=5.2.0,<6.0.0)
+Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
+Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
+Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
+Requires-Dist: dask-expr (>=1.1.20,<2.0.0)
+Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
+Requires-Dist: django (>=5.1.4,<6.0.0)
+Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)
+Requires-Dist: folium (>=0.19.4,<0.20.0)
+Requires-Dist: geopandas (>=1.0.1,<2.0.0)
+Requires-Dist: gunicorn (>=23.0.0,<24.0.0)
+Requires-Dist: httpx (>=0.27.2,<0.28.0)
+Requires-Dist: ipython (>=8.29.0,<9.0.0)
+Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
+Requires-Dist: mysqlclient (>=2.2.6,<3.0.0)
+Requires-Dist: nltk (>=3.9.1,<4.0.0)
+Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
+Requires-Dist: osmnx (>=2.0.1,<3.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: paramiko (>=3.5.0,<4.0.0)
+Requires-Dist: psutil (>=6.1.0,<7.0.0)
+Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
+Requires-Dist: pyarrow (>=18.0.0,<19.0.0)
+Requires-Dist: pydantic (>=2.9.2,<3.0.0)
+Requires-Dist: pymysql (>=1.1.1,<2.0.0)
+Requires-Dist: pytest (>=8.3.3,<9.0.0)
+Requires-Dist: pytest-mock (>=3.14.0,<4.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: s3fs (>=2024.12.0,<2025.0.0)
+Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
+Requires-Dist: tornado (>=6.4.1,<7.0.0)
+Requires-Dist: tqdm (>=4.67.0,<5.0.0)
+Requires-Dist: uvicorn (>=0.34.0,<0.35.0)
+Requires-Dist: uvicorn-worker (>=0.3.0,<0.4.0)
+Description-Content-Type: text/markdown
+
+# sibi-dst
+
+Data Science Toolkit
+---------------------
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, SQLAlchemy, DjangoRestFramework, FastAPI
+
+Major Functionality
+--------------------
+1) **Build DataCubes, DataSets, and DataObjects** from diverse data sources, including **relational databases, Parquet files, Excel (`.xlsx`), delimited tables (`.csv`, `.tsv`), JSON, and RESTful APIs (`JSON API REST`)**.
+2) **Comprehensive DataFrame Management** utilities for efficient data handling, transformation, and optimization using **Pandas** and **Dask**.
+3) **Flexible Data Sharing** with client applications by writing to **Data Warehouses, local filesystems, and cloud storage platforms** such as **Amazon S3, Google Cloud Storage (GCS), and Azure Blob Storage**.
+4) **Microservices for Data Access** – Build scalable **API-driven services** using **RESTful APIs (`Django REST Framework`, `FastAPI`) and gRPC** for high-performance data exchange.
+
+Supported Technologies
+--------------------
+- **Data Processing**: Pandas, Dask
+- **Machine Learning**: Scikit-Learn, XGBoost
+- **Databases & Storage**: SQLAlchemy, Django ORM, Parquet, Amazon S3, GCS, Azure Blob Storage
+- **Mapping & Geospatial Analysis**: OpenStreetMaps, OSMnx, Geopy
+- **API Development**: Django REST Framework, gRPC
+
+Installation
+---------------------
+```bash
+pip install sibi-dst
+```
+
+Usage
+---------------------
+### Loading Data from SQLAlchemy
+```python
+from sibi_dst.df_helper import DfHelper
+from conf.transforms.fields.crm import customer_fields
+from conf.credentials import replica_db_conf
+from conf.storage import get_fs_instance
+
+config = {
+    'backend': 'sqlalchemy',
+    'connection_url': replica_db_conf.get('db_url'),
+    'table': 'crm_clientes_archivo',
+    'field_map': customer_fields,
+    'legacy_filters': True,
+    'fs': get_fs_instance()
+}
+
+df_helper = DfHelper(**config)
+result = df_helper.load(id__gte=1)
+```
+
+### Saving Data to ClickHouse
+```python
+clk_creds = {
+    'host': '192.168.3.171',
+    'port': 18123,
+    'user': 'username',
+    'database': 'xxxxxxx',
+    'table': 'customer_file',
+    'order_by': 'id'
+}
+
+df_helper.save_to_clickhouse(**clk_creds)
+```
+
+### Saving Data to Parquet
+```python
+df_helper.save_to_parquet(
+    parquet_filename='filename.parquet',
+    parquet_storage_path='/path/to/my/files/'
+)
+```
+
+Backends Supported
+---------------------
+| Backend       | Description |
+|--------------|-------------|
+| `sqlalchemy` | Load data from SQL databases using SQLAlchemy. |
+| `django_db`  | Load data from Django ORM models. |
+| `parquet`    | Load and save data from Parquet files. |
+| `http`       | Fetch data from HTTP endpoints. |
+| `osmnx`      | Geospatial mapping and routing using OpenStreetMap. |
+| `geopy`      | Geolocation services for address lookup and reverse geocoding. |
+
+Geospatial Utilities
+---------------------
+### **OSMnx Helper (`sibi_dst.osmnx_helper`)
+**
+Provides **OpenStreetMap-based mapping utilities** using `osmnx` and `folium`.
+
+#### 🔹 Key Features
+- **BaseOsmMap**: Manages interactive Folium-based maps.
+- **PBFHandler**: Loads `.pbf` (Protocolbuffer Binary Format) files for network graphs.
+
+#### Example: Generating an OSM Map
+```python
+from sibi_dst.osmnx_helper import BaseOsmMap
+osm_map = BaseOsmMap(osmnx_graph=my_graph, df=my_dataframe)
+osm_map.generate_map()
+```
+
+### **Geopy Helper (`sibi_dst.geopy_helper`)
+**
+Provides **geolocation services** using `Geopy` for forward and reverse geocoding.
+
+#### 🔹 Key Features
+- **GeolocationService**: Interfaces with `Nominatim` API for geocoding.
+- **Error Handling**: Manages `GeocoderTimedOut` and `GeocoderServiceError` gracefully.
+- **Singleton Geolocator**: Efficiently reuses a global geolocator instance.
+
+#### Example: Reverse Geocoding
+```python
+from sibi_dst.geopy_helper import GeolocationService
+gs = GeolocationService()
+location = gs.reverse_geocode(lat=9.935, lon=-84.091)
+print(location)
+```
+
+Advanced Features
+---------------------
+### Querying with Custom Filters
+Filters can be applied dynamically using Django-style syntax:
+```python
+result = df_helper.load(date__gte='2023-01-01', status='active')
+```
+
+### Parallel Processing
+Leverage Dask for parallel execution:
+```python
+results = df_helper.load_parallel(status='active')
+```
+
+Testing
+---------------------
+To run unit tests, use:
+```bash
+pytest tests/
+```
+
+Contributing
+---------------------
+Contributions are welcome! Please submit pull requests or open issues for discussions.
+
+License
+---------------------
+sibi-dst is licensed under the MIT License.
+
+

sibi_dst-0.3.43/README.md

@@ -0,0 +1,145 @@
+# sibi-dst
+
+Data Science Toolkit
+---------------------
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, Scikit-Learn, XGBOOST, Django ORM, SQLAlchemy, DjangoRestFramework, FastAPI
+
+Major Functionality
+--------------------
+1) **Build DataCubes, DataSets, and DataObjects** from diverse data sources, including **relational databases, Parquet files, Excel (`.xlsx`), delimited tables (`.csv`, `.tsv`), JSON, and RESTful APIs (`JSON API REST`)**.
+2) **Comprehensive DataFrame Management** utilities for efficient data handling, transformation, and optimization using **Pandas** and **Dask**.
+3) **Flexible Data Sharing** with client applications by writing to **Data Warehouses, local filesystems, and cloud storage platforms** such as **Amazon S3, Google Cloud Storage (GCS), and Azure Blob Storage**.
+4) **Microservices for Data Access** – Build scalable **API-driven services** using **RESTful APIs (`Django REST Framework`, `FastAPI`) and gRPC** for high-performance data exchange.
+
+Supported Technologies
+--------------------
+- **Data Processing**: Pandas, Dask
+- **Machine Learning**: Scikit-Learn, XGBoost
+- **Databases & Storage**: SQLAlchemy, Django ORM, Parquet, Amazon S3, GCS, Azure Blob Storage
+- **Mapping & Geospatial Analysis**: OpenStreetMaps, OSMnx, Geopy
+- **API Development**: Django REST Framework, gRPC
+
+Installation
+---------------------
+```bash
+pip install sibi-dst
+```
+
+Usage
+---------------------
+### Loading Data from SQLAlchemy
+```python
+from sibi_dst.df_helper import DfHelper
+from conf.transforms.fields.crm import customer_fields
+from conf.credentials import replica_db_conf
+from conf.storage import get_fs_instance
+
+config = {
+    'backend': 'sqlalchemy',
+    'connection_url': replica_db_conf.get('db_url'),
+    'table': 'crm_clientes_archivo',
+    'field_map': customer_fields,
+    'legacy_filters': True,
+    'fs': get_fs_instance()
+}
+
+df_helper = DfHelper(**config)
+result = df_helper.load(id__gte=1)
+```
+
+### Saving Data to ClickHouse
+```python
+clk_creds = {
+    'host': '192.168.3.171',
+    'port': 18123,
+    'user': 'username',
+    'database': 'xxxxxxx',
+    'table': 'customer_file',
+    'order_by': 'id'
+}
+
+df_helper.save_to_clickhouse(**clk_creds)
+```
+
+### Saving Data to Parquet
+```python
+df_helper.save_to_parquet(
+    parquet_filename='filename.parquet',
+    parquet_storage_path='/path/to/my/files/'
+)
+```
+
+Backends Supported
+---------------------
+| Backend       | Description |
+|--------------|-------------|
+| `sqlalchemy` | Load data from SQL databases using SQLAlchemy. |
+| `django_db`  | Load data from Django ORM models. |
+| `parquet`    | Load and save data from Parquet files. |
+| `http`       | Fetch data from HTTP endpoints. |
+| `osmnx`      | Geospatial mapping and routing using OpenStreetMap. |
+| `geopy`      | Geolocation services for address lookup and reverse geocoding. |
+
+Geospatial Utilities
+---------------------
+### **OSMnx Helper (`sibi_dst.osmnx_helper`)
+**
+Provides **OpenStreetMap-based mapping utilities** using `osmnx` and `folium`.
+
+#### 🔹 Key Features
+- **BaseOsmMap**: Manages interactive Folium-based maps.
+- **PBFHandler**: Loads `.pbf` (Protocolbuffer Binary Format) files for network graphs.
+
+#### Example: Generating an OSM Map
+```python
+from sibi_dst.osmnx_helper import BaseOsmMap
+osm_map = BaseOsmMap(osmnx_graph=my_graph, df=my_dataframe)
+osm_map.generate_map()
+```
+
+### **Geopy Helper (`sibi_dst.geopy_helper`)
+**
+Provides **geolocation services** using `Geopy` for forward and reverse geocoding.
+
+#### 🔹 Key Features
+- **GeolocationService**: Interfaces with `Nominatim` API for geocoding.
+- **Error Handling**: Manages `GeocoderTimedOut` and `GeocoderServiceError` gracefully.
+- **Singleton Geolocator**: Efficiently reuses a global geolocator instance.
+
+#### Example: Reverse Geocoding
+```python
+from sibi_dst.geopy_helper import GeolocationService
+gs = GeolocationService()
+location = gs.reverse_geocode(lat=9.935, lon=-84.091)
+print(location)
+```
+
+Advanced Features
+---------------------
+### Querying with Custom Filters
+Filters can be applied dynamically using Django-style syntax:
+```python
+result = df_helper.load(date__gte='2023-01-01', status='active')
+```
+
+### Parallel Processing
+Leverage Dask for parallel execution:
+```python
+results = df_helper.load_parallel(status='active')
+```
+
+Testing
+---------------------
+To run unit tests, use:
+```bash
+pytest tests/
+```
+
+Contributing
+---------------------
+Contributions are welcome! Please submit pull requests or open issues for discussions.
+
+License
+---------------------
+sibi-dst is licensed under the MIT License.
+

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.42"
+version = "0.3.43"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -25,7 +25,7 @@ class ArtifactUpdaterMultiWrapper:
     def __init__(self, wrapped_classes=None, debug=False, **kwargs):
         self.wrapped_classes = wrapped_classes or {}
         self.debug = debug
-        self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger = kwargs.setdefault('logger',Logger.default_logger(logger_name=self.__class__.__name__))
         self.logger.set_level(logging.DEBUG if debug else logging.INFO)
 
         today = datetime.datetime.today()
@@ -73,7 +73,12 @@ class ArtifactUpdaterMultiWrapper:
             raise ValueError(f"Unsupported data type: {data_type}")
 
         return [
-            artifact_class(parquet_start_date=self.parquet_start_date, parquet_end_date=self.parquet_end_date)
+            artifact_class(
+                parquet_start_date=self.parquet_start_date,
+                parquet_end_date=self.parquet_end_date,
+                logger=self.logger,
+                debug=self.debug
+            )
             for artifact_class in self.wrapped_classes[data_type]
         ]
 

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/sibi_dst/df_helper/_df_helper.py

@@ -112,6 +112,7 @@ class DfHelper:
         :return: None
         """
         self.logger.debug(f"backend used: {self.backend}")
+        self.logger.debug(f"kwargs passed to backend plugins: {kwargs}")
         self._backend_query = self.__get_config(QueryConfig, kwargs)
         self._backend_params = self.__get_config(ParamsConfig, kwargs)
         if self.backend == 'django_db':
@@ -124,8 +125,8 @@ class DfHelper:
         elif self.backend == 'sqlalchemy':
             self.backend_sqlalchemy = self.__get_config(SqlAlchemyConnectionConfig, kwargs)
 
-    @staticmethod
-    def __get_config(model: [T], kwargs: Dict[str, Any]) -> Union[T]:
+
+    def __get_config(self, model: [T], kwargs: Dict[str, Any]) -> Union[T]:
         """
         Initializes a Pydantic model with the keys it recognizes from the kwargs,
         and removes those keys from the kwargs dictionary.
@@ -135,7 +136,9 @@ class DfHelper:
         """
         # Extract keys that the model can accept
         recognized_keys = set(model.__annotations__.keys())
+        self.logger.debug(f"recognized keys: {recognized_keys}")
         model_kwargs = {k: kwargs.pop(k) for k in list(kwargs.keys()) if k in recognized_keys}
+        self.logger.debug(f"model_kwargs: {model_kwargs}")
         return model(**model_kwargs)
 
     def load_parallel(self, **options):

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/sibi_dst/df_helper/_parquet_artifact.py

@@ -1,11 +1,11 @@
+import logging
 from typing import Optional, Any, Dict
 
 import dask.dataframe as dd
 import fsspec
 
 from sibi_dst.df_helper import DfHelper
-from sibi_dst.utils import DataWrapper
-from sibi_dst.utils import DateUtils
+from sibi_dst.utils import DataWrapper, DateUtils, Logger
 
 
 class ParquetArtifact(DfHelper):
@@ -82,7 +82,12 @@ class ParquetArtifact(DfHelper):
             **kwargs,
         }
         self.df: Optional[dd.DataFrame] = None
+        self.debug = self.config.setdefault('debug', False)
+        self.logger = self.config.setdefault('logger',Logger.default_logger(logger_name=f'parquet_artifact_{__class__.__name__}'))
+        self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
         self.data_wrapper_class = data_wrapper_class
+        self.class_params = self.config.setdefault('class_params', None)
+        self.load_params = self.config.setdefault('load_params', None)
         self.date_field = self.config.setdefault('date_field', None)
         if self.date_field is None:
             raise ValueError('date_field must be set')
@@ -150,6 +155,7 @@ class ParquetArtifact(DfHelper):
 
     def _prepare_params(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
         """Prepare the parameters for generating the Parquet file."""
+        kwargs = {**self.config, **kwargs}
         return {
             'class_params': kwargs.pop('class_params', None),
             'date_field': kwargs.pop('date_field', self.date_field),

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/sibi_dst/df_helper/_parquet_reader.py

@@ -1,10 +1,11 @@
+import logging
 from typing import Optional
 
 import dask.dataframe as dd
 import fsspec
 
 from sibi_dst.df_helper import DfHelper
-
+from sibi_dst.utils import Logger
 
 class ParquetReader(DfHelper):
     """
@@ -53,6 +54,9 @@ class ParquetReader(DfHelper):
             **kwargs,
         }
         self.df: Optional[dd.DataFrame] = None
+        self.debug = self.config.setdefault('debug', False)
+        self.logger = self.config.setdefault('logger', Logger.default_logger(logger_name=self.__class__.__name__))
+        self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
         self.parquet_storage_path = self.config.setdefault('parquet_storage_path', None)
         if self.parquet_storage_path is None:
             raise ValueError('parquet_storage_path must be set')

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py

@@ -63,3 +63,4 @@ class SqlAlchemyConnectionConfig(BaseModel):
                 connection.execute(text("SELECT 1"))
         except OperationalError as e:
             raise ValueError(f"Failed to connect to the database: {e}")
+

{sibi_dst-0.3.42 → sibi_dst-0.3.43}/sibi_dst/osmnx_helper/__init__.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from .base_osm_map import BaseOsmMap
-from .utils import PBFHandler
+from .v1.base_osm_map import BaseOsmMap
+from .v1.utils import PBFHandler
 
 __all__ = [
     "BaseOsmMap",