sibi-dst 0.3.12__tar.gz → 0.3.15__tar.gz

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.12
+Version: 0.3.15
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -14,7 +14,8 @@ Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
-Requires-Dist: django (==4.1.13)
+Requires-Dist: django (>=5.1.4,<6.0.0)
+Requires-Dist: djangorestframework (>=3.15.2,<4.0.0)
 Requires-Dist: httpx (>=0.27.2,<0.28.0)
 Requires-Dist: ipython (>=8.29.0,<9.0.0)
 Requires-Dist: jinja2 (>=3.1.4,<4.0.0)
@@ -28,7 +29,7 @@ Requires-Dist: pydantic (>=2.9.2,<3.0.0)
 Requires-Dist: pymysql (>=1.1.1,<2.0.0)
 Requires-Dist: pytest (>=8.3.3,<9.0.0)
 Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
-Requires-Dist: sqlmodel (>=0.0.22,<0.0.23)
+Requires-Dist: sqlalchemy (>=2.0.36,<3.0.0)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
 Requires-Dist: uvicorn (>=0.32.1,<0.33.0)

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.12"
+version = "0.3.15"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -13,13 +13,12 @@ dask = {extras = ["complete"], version = "^2024.11.1"}
 pydantic = "^2.9.2"
 tornado = "^6.4.1"
 psutil = "^6.1.0"
-django = "4.1.13"
+django = "^5.1.4"
 pyarrow = "^18.0.0"
 mysqlclient = "^2.2.6"
 pymysql = "^1.1.1"
 httpx = "^0.27.2"
 python-dotenv = "^1.0.1"
-sqlmodel = "^0.0.22"
 tqdm = "^4.67.0"
 openpyxl = "^3.1.5"
 jinja2 = "^3.1.4"
@@ -32,6 +31,8 @@ paramiko = "^3.5.0"
 chardet = "^5.2.0"
 charset-normalizer = "^3.4.0"
 uvicorn = "^0.32.1"
+sqlalchemy = "^2.0.36"
+djangorestframework = "^3.15.2"
 
 
 [build-system]

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/df_helper/_df_helper.py

@@ -7,9 +7,9 @@ import dask.dataframe as dd
 import pandas as pd
 from pydantic import BaseModel
 
-from sibi_dst.utils import ParquetSaver, ClickHouseWriter
 from sibi_dst.df_helper.core import QueryConfig, ParamsConfig
 from sibi_dst.utils import Logger
+from sibi_dst.utils import ParquetSaver, ClickHouseWriter
 from .plugins.django import *
 from .plugins.http import HttpConfig
 from .plugins.parquet import ParquetConfig, ParquetFilterHandler
@@ -18,6 +18,7 @@ from .plugins.sql_alchemy import *
 # Define a generic type variable for BaseModel subclasses
 T = TypeVar("T", bound=BaseModel)
 
+
 class DfHelper:
     df: Union[dd.DataFrame, pd.DataFrame] = None
     plugin_django_connection: Optional[DjangoConnectionConfig] = None
@@ -39,13 +40,12 @@ class DfHelper:
         self.debug = kwargs.setdefault("debug", False)
         self.verbose_debug = kwargs.setdefault("verbose_debug", False)
         self.parquet_storage_path = kwargs.setdefault("parquet_storage_path", None)
-        self.dt_field=kwargs.setdefault("dt_field", None)
+        self.dt_field = kwargs.setdefault("dt_field", None)
         self.as_pandas = kwargs.setdefault("as_pandas", False)
         kwargs.setdefault("live", True)
         kwargs.setdefault("logger", self.logger)
         self.post_init(**kwargs)
 
-
     def post_init(self, **kwargs):
         self.logger.info(f"Source used: {self.source}")
         self.plugin_query = self.__get_config(QueryConfig, kwargs)
@@ -58,7 +58,7 @@ class DfHelper:
         elif self.source == 'http':
             self.plugin_http = HttpConfig(**kwargs)
         elif self.source == 'sqlalchemy':
-            self.plugin_sqlalchemy = self.__get_config(SqlAlchemyConnectionConfig,kwargs)
+            self.plugin_sqlalchemy = self.__get_config(SqlAlchemyConnectionConfig, kwargs)
 
     @staticmethod
     def __get_config(model: [T], kwargs: Dict[str, Any]) -> Union[T]:
@@ -99,7 +99,6 @@ class DfHelper:
                 self.logger.info("Regular asyncio run...")
                 return asyncio.run(self._load_from_http(**options))
 
-
     def _load_from_sqlalchemy(self, **options):
         try:
             options.setdefault("debug", self.debug)
@@ -116,7 +115,7 @@ class DfHelper:
             self._post_process_df()
             self.logger.info("Data successfully loaded from sqlalchemy database.")
         except Exception as e:
-            self.logger.error(f"Failed to load data from sqlqlchemy database: {e}")
+            self.logger.error(f"Failed to load data from sqlalchemy database: {e}: options: {options}")
             self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
         return self.df
@@ -138,7 +137,7 @@ class DfHelper:
             self.logger.info("Data successfully loaded from django database.")
         except Exception as e:
             self.logger.error(f"Failed to load data from django database: {e}")
-            self.df=dd.from_pandas(pd.DataFrame(), npartitions=1)
+            self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
 
         return self.df
 
@@ -151,10 +150,9 @@ class DfHelper:
             self.df = await self.plugin_http.fetch_data(**options)
         except Exception as e:
             self.logger.error(f"Failed to load data from http plugin: {e}")
-            self.df=dd.from_pandas(pd.DataFrame(), npartitions=1)
+            self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
         return self.df
 
-
     def _post_process_df(self):
         """
         Efficiently process the DataFrame by filtering, renaming, and setting indices.
@@ -195,10 +193,16 @@ class DfHelper:
         self.logger.info("Post-processing of DataFrame completed.")
 
     def _process_loaded_data(self):
-        if len(self.df.index) > 0:
-            field_map = self.plugin_params.field_map or []
-            if field_map:
+        self.logger.info(f"Type of self.df: {type(self.df)}")
+        if self.df.map_partitions(len).compute().sum() > 0:
+            field_map = self.plugin_params.field_map or {}
+            if isinstance(field_map, dict):
                 rename_mapping = {k: v for k, v in field_map.items() if k in self.df.columns}
+                missing_columns = [k for k in field_map.keys() if k not in self.df.columns]
+
+                if missing_columns:
+                    self.logger.warning(
+                        f"The following columns in field_map are not in the DataFrame: {missing_columns}")
 
                 def rename_columns(df, mapping):
                     return df.rename(columns=mapping)
@@ -214,14 +218,11 @@ class DfHelper:
         ps.save_to_parquet(parquet_filename)
         self.logger.info(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
 
-    def save_to_clickhouse(self, database, table, order_by=None, **credentials):
-        click_config ={
-            'database': database,
-            'table': table,
-            'order_by': order_by or 'id',
-        }
-        credentials = {**credentials, **click_config}
-        cs=ClickHouseWriter(logger=self.logger, **credentials)
+    def save_to_clickhouse(self, **credentials):
+        if self.df.map_partitions(len).compute().sum() == 0:
+            self.logger.info("Cannot write to clickhouse since Dataframe is empty")
+            return
+        cs = ClickHouseWriter(logger=self.logger, **credentials)
         cs.save_to_clickhouse(self.df)
         self.logger.info("Save to ClickHouse completed.")
 
@@ -295,7 +296,6 @@ class DfHelper:
                 kwargs[f"{mapped_field}__date__lte"] = end
         return self.load(**kwargs)
 
-
     @staticmethod
     def parse_date(date_str: str) -> Union[datetime.datetime, datetime.date]:
         try:

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/df_helper/plugins/django/_io_dask.py

@@ -1,6 +1,7 @@
 import itertools
 
 import dask.dataframe as dd
+import dask_expr
 import django
 import pandas as pd
 from django.core.cache import cache
@@ -239,4 +240,7 @@ class ReadFrameDask:
         if verbose:
             self.update_with_verbose(dask_df, fieldnames, fields)
 
+        if isinstance(dask_df, dask_expr._collection.DataFrame):
+            dask_df = dask_df.to_legacy_dataframe()
+
         return dask_df

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/df_helper/plugins/sql_alchemy/_io_sqlalchemy_dask.py

@@ -1,6 +1,7 @@
 import itertools
 
 import dask.dataframe as dd
+import dask_expr
 import pandas as pd
 from sqlalchemy import create_engine, inspect, select
 from sqlalchemy.orm import sessionmaker
@@ -10,7 +11,7 @@ from sibi_dst.utils import Logger
 
 
 class SQLAlchemyDask:
-    def __init__(self, model, filters, engine_url, chunk_size=1000, logger=None, verbose=True):
+    def __init__(self, model, filters, engine_url, chunk_size=1000, logger=None, debug=False):
         """
         Initialize with an SQLAlchemy query and database engine URL.
 
@@ -19,13 +20,13 @@ class SQLAlchemyDask:
         :param engine_url: Database connection string for SQLAlchemy engine.
         :param chunk_size: Number of records per chunk for Dask partitions.
         :param logger: Logger instance for logging.
-        :param verbose: Whether to print detailed logs.
+        :param debug: Whether to print detailed logs.
         """
         self.query = None
         self.model = model
         self.filters = filters
         self.chunk_size = chunk_size
-        self.verbose = verbose
+        self.debug = debug
         self.engine = create_engine(engine_url)
         self.Session = sessionmaker(bind=self.engine)
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
@@ -118,14 +119,17 @@ class SQLAlchemyDask:
                     partitions.append(dd.from_pandas(df, npartitions=1))
 
                 # Concatenate all partitions
-                # print(partitions)
                 if partitions:
                     dask_df = dd.concat(partitions, axis=0, ignore_index=True)
                 else:
                     dask_df = dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)
 
-                if self.verbose:
+                if self.debug:
                     self.logger.info(f"Loaded {len(dask_df)} rows into Dask DataFrame.")
+
+                if isinstance(dask_df, dask_expr._collection.DataFrame):
+                    dask_df = dask_df.to_legacy_dataframe()
+
                 return dask_df
 
             except Exception as e:

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_load_from_db.py

@@ -1,22 +1,13 @@
-from typing import Dict
-
 import dask.dataframe as dd
 import pandas as pd
-from sqlalchemy.inspection import inspect
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import select
-#from sqlmodel import Session, select
 
-from sibi_dst.df_helper.core import ParamsConfig, QueryConfig, sqlalchemy_field_conversion_map_dask, \
-    normalize_sqlalchemy_type
+from sibi_dst.df_helper.core import ParamsConfig, QueryConfig
 from sibi_dst.utils import Logger
 from ._io_sqlalchemy_dask import SQLAlchemyDask
-from ._sqlachemy_filter_handler import SqlAlchemyFilterHandler
 from ._sqlalchemy_db_connection import SqlAlchemyConnectionConfig
 
-
 class SqlAlchemyLoadFromDb:
-    df: dd.DataFrame
+    df: dd.DataFrame = None
 
     def __init__(
             self,
@@ -43,17 +34,25 @@ class SqlAlchemyLoadFromDb:
         """
         Load data into a Dask DataFrame based on the query and parameters.
         """
-        self.df = self._build_and_load()
+        self._build_and_load()
         return self.df
 
     def _build_and_load(self) -> dd.DataFrame:
+
         try:
-            reader = SQLAlchemyDask(model=self.model, filters=self.params_config.filters,engine_url=self.engine.url, logger=self.logger, chunk_size=1000, verbose=self.debug)
-            df = reader.read_frame()
-            if df is None or len(df.index) == 0:
+            # reader = SQLAlchemyDask(model=self.model, filters=self.params_config.filters,engine_url=self.engine.url, logger=self.logger, chunk_size=1000, debug=self.debug)
+            self.df = SQLAlchemyDask(
+                model=self.model,
+                filters=self.params_config.filters,
+                engine_url=self.engine.url,
+                logger=self.logger,
+                chunk_size=1000,
+                debug=self.debug).read_frame()
+            if self.df is None or len(self.df.head().index) == 0:
                 self.logger.warning("Query returned no results.")
                 return dd.from_pandas(pd.DataFrame(), npartitions=1)
-            return df
+
+            return self.df
         except Exception as e:
             self.logger.error(f"Failed to load data into Dask DataFrame.{e}")
             return dd.from_pandas(pd.DataFrame(), npartitions=1)

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_model_builder.py

@@ -82,9 +82,12 @@ class SqlAlchemyModelBuilder:
             dict: Dictionary of column attributes.
         """
         columns = {}
+        reserved_names = ["metadata", "class_", "table"]
+
         for column in table.columns:
             column_name = self.normalize_column_name(column.name)
-            columns[column_name] = column
+            if column_name not in reserved_names:
+                columns[column_name] = column
         return columns
 
     def add_relationships(self, attrs, table: Table):

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/utils/_clickhouse_writer.py

@@ -31,9 +31,9 @@ class ClickHouseWriter:
         self.order_by=kwargs.setdefault('order_by','id')
 
     def save_to_clickhouse(self, df, **kwargs):
-        self.df = df
+        self.df = df.copy()
         self.order_by = kwargs.setdefault('order_by',self.order_by)
-        if len(self.df.index) == 0:
+        if len(self.df.head().index) == 0:
             self.logger.info("Dataframe is empty")
             return
         self._handle_missing_values()
@@ -122,7 +122,7 @@ class ClickHouseWriter:
         """
         Writes the Dask DataFrame to a ClickHouse table partition by partition.
         """
-        if len(self.df.index) == 0:
+        if len(self.df.head().index) == 0:
             self.logger.info("No data found. Nothing written.")
             return
 

{sibi_dst-0.3.12 → sibi_dst-0.3.15}/sibi_dst/utils/_data_utils.py

@@ -1,77 +1,32 @@
-import pandas as pd
 import dask.dataframe as dd
+import pandas as pd
+
 from sibi_dst.utils import Logger
 
+
 class DataUtils:
 
-    def __init__(self, logger=None):
+    def __init__(self, logger=None, **kwargs):
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.debug = kwargs.get('debug', False)
 
     def transform_numeric_cols(self, df, columns, fill_value=0, dtype=int):
         if not columns:
             self.logger.warning('No columns specified')
-
+        self.logger.debug(f'Dataframe type:{type(df)}')
         columns = [column for column in columns if column in df.columns]
         for col in columns:
-            if isinstance(df, dd.DataFrame):
-                # Replace NaN with 0, then convert to boolean
-                df[col] = df[col].map_partitions(
-                    lambda s: pd.to_numeric(s, errors='coerce')  # Convert to numeric, invalid to NaN
-                    .fillna(fill_value)  # Replace NaN with 0
-                    .astype(dtype),
-                    meta=(col, dtype)
-                )
-            else:
-                # For Pandas DataFrame, handle mixed types and invalid values
-                df[col] = pd.to_numeric(df[col], errors='coerce')  # Convert to numeric, invalid to NaN
-                df[col] = df[col].fillna(fill_value).astype(dtype)
+            # Replace NaN with 0, then convert to boolean
+            df[col] = df[col].map_partitions(
+                lambda s: pd.to_numeric(s, errors='coerce')  # Convert to numeric, invalid to NaN
+                .fillna(fill_value)  # Replace NaN with 0
+                .astype(dtype),
+                meta=(col, dtype)
+            )
 
         return df
 
-    @staticmethod
-    def transform_numeric_columns(df, columns=None, fill_value=0, transform_func=None):
-        """
-        Transform numeric columns in a DataFrame (Pandas or Dask), handling missing values and applying optional transformations.
-
-        Parameters:
-        - df (pandas.DataFrame or dask.dataframe.DataFrame): The DataFrame.
-        - columns (list of str, optional): Specific columns to transform. If None, all numeric columns are transformed.
-        - fill_value (int or float): The value to replace NA values with.
-        - transform_func (callable, optional): The transformation function to apply.
-          If None, no additional transformation is applied.
-
-        Returns:
-        - pandas.DataFrame or dask.dataframe.DataFrame: Updated DataFrame with transformed numeric columns.
-        """
-        if columns is None:
-            # Detect numeric columns
-            columns = df.select_dtypes(include=['number']).columns.tolist()
-
-        if not columns:
-            return df
-
-        columns = [column for column in columns if column in df.columns]
-        # Default transformation function (identity) if none is provided
-        if transform_func is None:
-            transform_func = lambda x: x
-
-        # Batch processing for Dask
-        if isinstance(df, dd.DataFrame):
-            def transform_partition(partition):
-                # Apply transformations for all numeric columns in a single pass
-                partition[columns] = partition[columns].fillna(fill_value).map(transform_func)
-                return partition
-
-            # Apply the transformation function to all specified columns
-            df = df.map_partitions(transform_partition, meta=df)
-        else:
-            # Pandas: Vectorized operations for all specified columns
-            df[columns] = df[columns].fillna(fill_value).map(transform_func)
-
-        return df
-
-    @staticmethod
-    def transform_boolean_columns(df, columns=None):
+    def transform_boolean_columns(self, df, columns=None):
         """
         Detect if the provided columns in a DataFrame (Pandas or Dask) contain only 0 and 1
         and convert them to boolean. Detection is performed using a sample.
@@ -84,23 +39,20 @@ class DataUtils:
         Returns:
         - pandas.DataFrame or dask.dataframe.DataFrame: Updated DataFrame with transformed boolean columns.
         """
+
         # Apply transformation to each specified column
         for col in columns:
             if col in df.columns:
-                if isinstance(df, dd.DataFrame):
-                    # Replace NaN with 0, then convert to boolean
-                    df[col] = df[col].map_partitions(
-                        lambda s: pd.to_numeric(s, errors='coerce')  # Convert to numeric, invalid to NaN
-                        .fillna(0)  # Replace NaN with 0
-                        .astype(int)  # Ensure integer type
-                        .astype(bool),  # Convert to boolean
-                        meta=(col, 'bool')
-                    )
-                else:
-                    # For Pandas DataFrame, handle mixed types and invalid values
-                    df[col] = pd.to_numeric(df[col], errors='coerce')  # Convert to numeric, invalid to NaN
-                    df[col] = df[col].fillna(0).astype(int).astype(bool)
-
+                # Replace NaN with 0, then convert to boolean
+                df[col] = df[col].map_partitions(
+                    lambda s: pd.to_numeric(s, errors='coerce')  # Convert to numeric, invalid to NaN
+                    .fillna(0)  # Replace NaN with 0
+                    .astype(int)  # Ensure integer type
+                    .astype(bool),  # Convert to boolean
+                    meta=(col, 'bool')
+                )
+        if self.debug:
+            self.logger.debug(f'Dataframe type:{type(df)}, boolean applied to columns: {columns}')
         return df
 
     def merge_lookup_data(self, classname, df, **kwargs):
@@ -141,12 +93,19 @@ class DataUtils:
 
         # Get unique IDs from source column
         ids = df[source_col].dropna().unique()
-        if isinstance(ids, dd.Series):
+        # Compute if it's a Dask Series
+        if isinstance(ids, dd.core.Series):
             ids = ids.compute()
+
+        # Check if any IDs are found
         if not len(ids):
             self.logger.info(f"No IDs found in the source column: {source_col}")
             return df
-        ids = sorted(ids.tolist())
+
+        # Convert to a list only if necessary and sort
+        if not isinstance(ids, list):
+            ids = ids.tolist()
+        ids = sorted(ids)
         # Prepare kwargs for loading lookup data
         load_kwargs = kwargs.copy()
         load_kwargs.update({
@@ -167,14 +126,13 @@ class DataUtils:
         df = df.merge(result, how='left', left_on=source_col, right_on=temp_join_col)
 
         if fillna_source_description_alias and source_description_alias in df.columns:
-            df[source_description_alias]=df[source_description_alias].fillna('')
+            df[source_description_alias] = df[source_description_alias].fillna('')
 
         # Drop temp_join_col if present
         df = df.drop(columns='temp_join_col', errors='ignore')
 
         return df
 
-
     def is_dataframe_empty(self, df):
         """
         Check if a DataFrame (Pandas or Dask) is empty.