sibi-dst 0.3.17__tar.gz → 0.3.18__tar.gz

{sibi_dst-0.3.17 → sibi_dst-0.3.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.17
+Version: 0.3.18
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.17 → sibi_dst-0.3.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.17"
+version = "0.3.18"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.17 → sibi_dst-0.3.18}/sibi_dst/df_helper/_df_helper.py

@@ -6,17 +6,16 @@ from typing import Any, Dict, TypeVar
 from typing import Union, Optional
 
 import dask.dataframe as dd
-import dask_expr
 import pandas as pd
 from pydantic import BaseModel
 
 from sibi_dst.df_helper.core import QueryConfig, ParamsConfig, FilterHandler
 from sibi_dst.utils import Logger
 from sibi_dst.utils import ParquetSaver, ClickHouseWriter
-from .plugins.django import *
-from .plugins.http import HttpConfig
-from .plugins.parquet import ParquetConfig, ParquetFilterHandler
-from .plugins.sql_alchemy import *
+from .backends.django import *
+from .backends.http import HttpConfig
+from .backends.parquet import ParquetConfig
+from .backends.sql_alchemy import *
 
 # Define a generic type variable for BaseModel subclasses
 T = TypeVar("T", bound=BaseModel)
@@ -30,26 +29,25 @@ warnings.filterwarnings(
 
 class DfHelper:
     df: Union[dd.DataFrame, pd.DataFrame] = None
-    plugin_django_connection: Optional[DjangoConnectionConfig] = None
-    plugin_query: Optional[QueryConfig] = None
-    plugin_params: Optional[ParamsConfig] = None
-    plugin_parquet: Optional[ParquetConfig] = None
-    plugin_http: Optional[HttpConfig] = None
-    plugin_sqlalchemy: Optional[SqlAlchemyConnectionConfig] = None
+    backend_django: Optional[DjangoConnectionConfig] = None
+    backend_query: Optional[QueryConfig] = None
+    backend_params: Optional[ParamsConfig] = None
+    backend_parquet: Optional[ParquetConfig] = None
+    backend_http: Optional[HttpConfig] = None
+    backend_sqlalchemy: Optional[SqlAlchemyConnectionConfig] = None
     parquet_filename: str = None
     logger: Logger
     default_config: Dict = None
 
-    def __init__(self, source='django_db', **kwargs):
+    def __init__(self, backend='django_db', **kwargs):
         # Ensure default_config is not shared across instances
         self.default_config = self.default_config or {}
         kwargs = {**self.default_config.copy(), **kwargs}
-        self.source = source
+        self.backend = backend
         self.debug = kwargs.setdefault("debug", False)
-        self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger = kwargs.get("logger",Logger.default_logger(logger_name=self.__class__.__name__))
         # Configure logger level
         self.logger.setLevel(logging.DEBUG if self.debug else logging.INFO)
-        # Configure logger level
         self.logger.debug("Logger initialized in DEBUG mode.")
         self.parquet_storage_path = kwargs.setdefault("parquet_storage_path", None)
         self.dt_field = kwargs.setdefault("dt_field", None)
@@ -59,18 +57,18 @@ class DfHelper:
         self.post_init(**kwargs)
 
     def post_init(self, **kwargs):
-        self.logger.debug(f"Source used: {self.source}")
-        self.plugin_query = self.__get_config(QueryConfig, kwargs)
-        self.plugin_params = self.__get_config(ParamsConfig, kwargs)
-        if self.source == 'django_db':
-            self.plugin_django_connection = self.__get_config(DjangoConnectionConfig, kwargs)
-        elif self.source == 'parquet':
+        self.logger.debug(f"backend used: {self.backend}")
+        self.backend_query = self.__get_config(QueryConfig, kwargs)
+        self.backend_params = self.__get_config(ParamsConfig, kwargs)
+        if self.backend == 'django_db':
+            self.backend_django = self.__get_config(DjangoConnectionConfig, kwargs)
+        elif self.backend == 'parquet':
             self.parquet_filename = kwargs.setdefault("parquet_filename", None)
-            self.plugin_parquet = ParquetConfig(**kwargs)
-        elif self.source == 'http':
-            self.plugin_http = HttpConfig(**kwargs)
-        elif self.source == 'sqlalchemy':
-            self.plugin_sqlalchemy = self.__get_config(SqlAlchemyConnectionConfig, kwargs)
+            self.backend_parquet = ParquetConfig(**kwargs)
+        elif self.backend == 'http':
+            self.backend_http = HttpConfig(**kwargs)
+        elif self.backend == 'sqlalchemy':
+            self.backend_sqlalchemy = self.__get_config(SqlAlchemyConnectionConfig, kwargs)
 
     @staticmethod
     def __get_config(model: [T], kwargs: Dict[str, Any]) -> Union[T]:
@@ -87,7 +85,7 @@ class DfHelper:
         return model(**model_kwargs)
 
     def load(self, **options):
-        # this will be the universal method to load data from a df irrespective of the source
+        # this will be the universal method to load data from a df irrespective of the backend
         df = self._load(**options)
         if self.as_pandas:
             return df.compute()
@@ -95,15 +93,15 @@ class DfHelper:
 
     def _load(self, **options):
 
-        if self.source == 'django_db':
-            self.plugin_params.parse_params(options)
+        if self.backend == 'django_db':
+            self.backend_params.parse_params(options)
             return self._load_from_db(**options)
-        elif self.source == 'sqlalchemy':
-            self.plugin_params.parse_params(options)
+        elif self.backend == 'sqlalchemy':
+            self.backend_params.parse_params(options)
             return self._load_from_sqlalchemy(**options)
-        elif self.source == 'parquet':
+        elif self.backend == 'parquet':
             return self._load_from_parquet(**options)
-        elif self.source == 'http':
+        elif self.backend == 'http':
             if asyncio.get_event_loop().is_running():
                 self.logger.debug("Running as a task from an event loop")
                 return asyncio.create_task(self._load_from_http(**options))
@@ -115,9 +113,9 @@ class DfHelper:
         try:
             options.setdefault("debug", self.debug)
             db_loader = SqlAlchemyLoadFromDb(
-                self.plugin_sqlalchemy,
-                self.plugin_query,
-                self.plugin_params,
+                self.backend_sqlalchemy,
+                self.backend_query,
+                self.backend_params,
                 self.logger,
                 **options
             )
@@ -135,9 +133,9 @@ class DfHelper:
         try:
             options.setdefault("debug", self.debug)
             db_loader = DjangoLoadFromDb(
-                self.plugin_django_connection,
-                self.plugin_query,
-                self.plugin_params,
+                self.backend_django,
+                self.backend_query,
+                self.backend_params,
                 self.logger,
                 **options
             )
@@ -152,12 +150,12 @@ class DfHelper:
         return self.df
 
     async def _load_from_http(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
-        """Delegate asynchronous HTTP data loading to HttpDataSource plugin."""
-        if not self.plugin_http:
+        """Delegate asynchronous HTTP data loading to HttpDatabackend plugin."""
+        if not self.backend_http:
             self.logger.debug("HTTP plugin not configured properly.")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
         try:
-            self.df = await self.plugin_http.fetch_data(**options)
+            self.df = await self.backend_http.fetch_data(**options)
         except Exception as e:
             self.logger.debug(f"Failed to load data from http plugin: {e}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
@@ -168,7 +166,7 @@ class DfHelper:
         Efficiently process the DataFrame by filtering, renaming, and setting indices.
         Optimized for large datasets with Dask compatibility.
         """
-        df_params = self.plugin_params.df_params
+        df_params = self.backend_params.df_params
         fieldnames = df_params.get("fieldnames", None)
         index_col = df_params.get("index_col", None)
         datetime_index = df_params.get("datetime_index", False)
@@ -205,7 +203,7 @@ class DfHelper:
     def _process_loaded_data(self):
         self.logger.debug(f"Type of self.df: {type(self.df)}")
         if self.df.map_partitions(len).compute().sum() > 0:
-            field_map = self.plugin_params.field_map or {}
+            field_map = self.backend_params.field_map or {}
             if isinstance(field_map, dict):
                 rename_mapping = {k: v for k, v in field_map.items() if k in self.df.columns}
                 missing_columns = [k for k in field_map.keys() if k not in self.df.columns]
@@ -237,7 +235,7 @@ class DfHelper:
         self.logger.debug("Save to ClickHouse completed.")
 
     def _load_from_parquet(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
-        self.df = self.plugin_parquet.load_files()
+        self.df = self.backend_parquet.load_files()
         if options:
             """
             deprecated specific filter handling to a generic one
@@ -273,20 +271,20 @@ class DfHelper:
             raise ValueError("The 'start' date cannot be later than the 'end' date.")
 
         # Reverse map to original field name
-        field_map = getattr(self.plugin_params, 'field_map', {}) or {}
+        field_map = getattr(self.backend_params, 'field_map', {}) or {}
         reverse_map = {v: k for k, v in field_map.items()}
         mapped_field = reverse_map.get(dt_field, dt_field)
 
         # Common logic for Django and SQLAlchemy
-        if self.source == 'django_db':
-            model_fields = {field.name: field for field in self.plugin_django_connection.model._meta.get_fields()}
+        if self.backend == 'django_db':
+            model_fields = {field.name: field for field in self.backend_django.model._meta.get_fields()}
             if mapped_field not in model_fields:
                 raise ValueError(f"Field '{dt_field}' does not exist in the Django model.")
             field_type = type(model_fields[mapped_field]).__name__
             is_date_field = field_type == 'DateField'
             is_datetime_field = field_type == 'DateTimeField'
-        elif self.source == 'sqlalchemy':
-            model = self.plugin_sqlalchemy.model
+        elif self.backend == 'sqlalchemy':
+            model = self.backend_sqlalchemy.model
             fields = [column.name for column in model.__table__.columns]
             if mapped_field not in fields:
                 raise ValueError(f"Field '{dt_field}' does not exist in the SQLAlchemy model.")
@@ -295,7 +293,7 @@ class DfHelper:
             is_date_field = field_type == 'DATE'
             is_datetime_field = field_type == 'DATETIME'
         else:
-            raise ValueError(f"Unsupported source '{self.source}'")
+            raise ValueError(f"Unsupported backend '{self.backend}'")
             # Build query filters
         if start == end:
             if is_date_field:

{sibi_dst-0.3.17 → sibi_dst-0.3.18}/sibi_dst/df_helper/_parquet_artifact.py

@@ -8,7 +8,7 @@ from sibi_dst.utils import DateUtils
 
 class ParquetArtifact(DfHelper):
     DEFAULT_CONFIG = {
-        'source': 'parquet'
+        'backend': 'parquet'
     }
 
     def __init__(self, data_wrapper_class, filesystem_type="file", filesystem_options=None, **kwargs):

{sibi_dst-0.3.17/sibi_dst/df_helper/plugins → sibi_dst-0.3.18/sibi_dst/df_helper/backends}/django/_django_load_from_db.py

@@ -5,7 +5,7 @@ import pandas as pd
 from IPython.core.hooks import deprecated
 from django.db.models import Q
 
-from sibi_dst.df_helper.plugins.django import ReadFrameDask
+from sibi_dst.df_helper.backends.django import ReadFrameDask
 from sibi_dst.utils import Logger
 from sibi_dst.df_helper.core import django_field_conversion_map_dask
 

{sibi_dst-0.3.17/sibi_dst/df_helper/plugins → sibi_dst-0.3.18/sibi_dst/df_helper/backends}/sql_alchemy/_io_sqlalchemy_dask.py

@@ -1,14 +1,13 @@
 import itertools
 
 import dask.dataframe as dd
-import dask_expr
 import pandas as pd
 from sqlalchemy import create_engine, inspect, select
 from sqlalchemy.orm import sessionmaker
 
-from sibi_dst.df_helper.plugins.sql_alchemy._sqlachemy_filter_handler import SqlAlchemyFilterHandler
-from sibi_dst.utils import Logger
 from sibi_dst.df_helper.core import FilterHandler
+from sibi_dst.utils import Logger
+
 
 class SQLAlchemyDask:
     def __init__(self, model, filters, engine_url, chunk_size=1000, logger=None, debug=False):

{sibi_dst-0.3.17/sibi_dst/df_helper/plugins → sibi_dst-0.3.18/sibi_dst/df_helper/backends}/sql_alchemy/_sqlalchemy_load_from_db.py

@@ -29,6 +29,7 @@ class SqlAlchemyLoadFromDb:
         self.query_config = plugin_query
         self.params_config = plugin_params
         self.debug = kwargs.pop("debug", False)
+        self.chunk_size = kwargs.pop("chunk_size", 1000)
 
     def build_and_load(self) -> dd.DataFrame:
         """
@@ -45,8 +46,10 @@ class SqlAlchemyLoadFromDb:
                 filters=self.params_config.filters,
                 engine_url=self.engine.url,
                 logger=self.logger,
-                chunk_size=1000,
-                debug=self.debug).read_frame()
+                chunk_size=self.chunk_size,
+                debug=self.debug
+            ).read_frame()
+
             if self.df is None or len(self.df.head().index) == 0:
                 self.logger.debug("Query returned no results.")
                 dask_df=dd.from_pandas(pd.DataFrame(), npartitions=1)