sibi-dst 0.3.10__tar.gz → 0.3.12__tar.gz

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/PKG-INFO

@@ -1,7 +1,7 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.10
-Summary: 
+Version: 0.3.12
+Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
 Requires-Python: >=3.11,<4.0
@@ -9,6 +9,8 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
+Requires-Dist: chardet (>=5.2.0,<6.0.0)
+Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2024.11.1,<2025.0.0)
@@ -29,6 +31,7 @@ Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
 Requires-Dist: sqlmodel (>=0.0.22,<0.0.23)
 Requires-Dist: tornado (>=6.4.1,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
+Requires-Dist: uvicorn (>=0.32.1,<0.33.0)
 Description-Content-Type: text/markdown
 
 # sibi-dst

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.10"
-description = ""
+version = "0.3.12"
+description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
 packages = [{include = "sibi_dst"}]
@@ -29,6 +29,9 @@ pytest = "^8.3.3"
 clickhouse-connect = "^0.8.7"
 clickhouse-driver = "^0.2.9"
 paramiko = "^3.5.0"
+chardet = "^5.2.0"
+charset-normalizer = "^3.4.0"
+uvicorn = "^0.32.1"
 
 
 [build-system]

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/sibi_dst/df_helper/_df_helper.py

@@ -42,6 +42,7 @@ class DfHelper:
         self.dt_field=kwargs.setdefault("dt_field", None)
         self.as_pandas = kwargs.setdefault("as_pandas", False)
         kwargs.setdefault("live", True)
+        kwargs.setdefault("logger", self.logger)
         self.post_init(**kwargs)
 
 
@@ -101,6 +102,8 @@ class DfHelper:
 
     def _load_from_sqlalchemy(self, **options):
         try:
+            options.setdefault("debug", self.debug)
+            options.setdefault("verbose_debug", self.verbose_debug)
             db_loader = SqlAlchemyLoadFromDb(
                 self.plugin_sqlalchemy,
                 self.plugin_query,
@@ -120,6 +123,8 @@ class DfHelper:
 
     def _load_from_db(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         try:
+            options.setdefault("debug", self.debug)
+            options.setdefault("verbose_debug", self.verbose_debug)
             db_loader = DjangoLoadFromDb(
                 self.plugin_django_connection,
                 self.plugin_query,
@@ -207,6 +212,7 @@ class DfHelper:
     def save_to_parquet(self, parquet_filename: Optional[str] = None):
         ps = ParquetSaver(self.df, self.parquet_storage_path, self.logger)
         ps.save_to_parquet(parquet_filename)
+        self.logger.info(f"Parquet saved to {parquet_filename} in parquet storage: {self.parquet_storage_path}.")
 
     def save_to_clickhouse(self, database, table, order_by=None, **credentials):
         click_config ={
@@ -215,13 +221,14 @@ class DfHelper:
             'order_by': order_by or 'id',
         }
         credentials = {**credentials, **click_config}
-        cs=ClickHouseWriter(**credentials)
+        cs=ClickHouseWriter(logger=self.logger, **credentials)
         cs.save_to_clickhouse(self.df)
+        self.logger.info("Save to ClickHouse completed.")
 
     def _load_from_parquet(self, **options) -> Union[pd.DataFrame, dd.DataFrame]:
         self.df = self.plugin_parquet.load_files()
         if options:
-            self.df = ParquetFilterHandler().apply_filters_dask(self.df, options)
+            self.df = ParquetFilterHandler(logger=self.logger).apply_filters_dask(self.df, options)
         return self.df
 
     def load_period(self, **kwargs):

sibi_dst-0.3.12/sibi_dst/df_helper/core/__init__.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from ._params_config import ParamsConfig
+from ._query_config import QueryConfig
+from ._defaults import (
+    django_field_conversion_map_pandas,
+    django_field_conversion_map_dask,
+    sqlalchemy_field_conversion_map_dask,
+    normalize_sqlalchemy_type)
+
+__all__ = [
+    "ParamsConfig",
+    "QueryConfig",
+    "django_field_conversion_map_pandas",
+    "django_field_conversion_map_dask",
+    "sqlalchemy_field_conversion_map_dask",
+    "normalize_sqlalchemy_type"
+]

sibi_dst-0.3.12/sibi_dst/df_helper/core/_defaults.py

@@ -0,0 +1,133 @@
+#  Copyright (c) 2023. ISTMO Center S.A.  All Rights Reserved
+#
+import json
+from typing import Dict
+
+import pandas as pd
+from sqlalchemy import String, Text, Integer, BigInteger, SmallInteger, Float, Boolean, DateTime, Date, Time, JSON, \
+    Numeric, UUID
+from sqlalchemy.dialects.mysql import TINYINT, MEDIUMTEXT
+
+# This is the defaults configuration file for the df_helper module.
+
+# conversion_map is a dictionary that maps the field types to their corresponding data type conversion functions.
+# Each entry in the dictionary is a pair of a field type (as a string) and a callable function that performs the
+# conversion. This mapping is used to convert the values in a pandas DataFrame to the appropriate data types based on
+# the Django field type.
+
+django_field_conversion_map_pandas: Dict[str, callable] = {
+    "CharField": lambda x: x.astype(str),
+    "TextField": lambda x: x.astype(str),
+    "IntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "AutoField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "BigAutoField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "BigIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "SmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "PositiveIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "PositiveSmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "FloatField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "DecimalField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "BooleanField": lambda x: x.astype(bool),
+    "NullBooleanField": lambda x: x.astype(bool),
+    "DateTimeField": lambda x: pd.to_datetime(x, errors="coerce"),
+    "DateField": lambda x: pd.to_datetime(x, errors="coerce").dt.date,
+    "TimeField": lambda x: pd.to_datetime(x, errors="coerce").dt.time,
+    "DurationField": lambda x: pd.to_timedelta(x, errors="coerce"),
+    # for JSONField, assuming JSON objects are represented as string in df
+    "JSONField": lambda x: x.apply(json.loads),
+    "ArrayField": lambda x: x.apply(eval),
+    "UUIDField": lambda x: x.astype(str),
+}
+
+django_field_conversion_map_dask: Dict[str, callable] = {
+    "CharField": lambda x: x.astype(str),
+    "TextField": lambda x: x.astype(str),
+    "IntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "AutoField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "BigAutoField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "BigIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "SmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "PositiveIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "PositiveSmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "FloatField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "DecimalField": lambda x: pd.to_numeric(x, errors="coerce"),
+    "BooleanField": lambda x: x.astype(bool),
+    "NullBooleanField": lambda x: x.astype(bool),
+    "DateTimeField": lambda x: pd.to_datetime(x, errors="coerce"),
+    "DateField": lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda x: x.dt.date, meta=("date", "object")),
+    "TimeField": lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda x: x.dt.time, meta=("time", "object")),
+    "DurationField": lambda x: pd.to_timedelta(x, errors="coerce"),
+    "JSONField": lambda x: x.map_partitions(lambda s: s.apply(json.loads), meta=("json", "object")),
+    "ArrayField": lambda x: x.map_partitions(lambda s: s.apply(eval), meta=("array", "object")),
+    "UUIDField": lambda x: x.astype(str),
+}
+
+sqlalchemy_field_conversion_map_dask: Dict[str, callable] = {
+    String.__name__: lambda x: x.astype(str).fillna(""),
+    Text.__name__: lambda x: x.fillna('').astype(str),
+    Integer.__name__: lambda x: x.fillna(0).astype(int),
+    BigInteger.__name__: lambda x: pd.to_numeric(x, errors="coerce"),
+    SmallInteger.__name__: lambda x: pd.to_numeric(x, errors="coerce"),
+    Float.__name__: lambda x: pd.to_numeric(x, errors="coerce"),
+    Numeric.__name__: lambda x: pd.to_numeric(x, errors="coerce"),
+    Boolean.__name__: lambda x: x.astype(bool),
+    DateTime.__name__: lambda x: pd.to_datetime(x, errors="coerce"),
+    Date.__name__: lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda x: x.dt.date, meta=("date", "object")),
+    Time.__name__: lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda x: x.dt.time, meta=("time", "object")),
+    JSON.__name__: lambda x: x.map_partitions(lambda s: s.apply(json.loads), meta=("json", "object")),
+    UUID.__name__: lambda x: x.astype(str),
+}
+
+# Conversion map with normalized SQLAlchemy field types
+# sqlalchemy_field_conversion_map_dask: Dict[str, callable] = {
+#     "String": lambda x: x.map_partitions(lambda s: s.astype(str), meta=("string", "string")),
+#     "Text": lambda x: x.map_partitions(lambda s: s.astype(str), meta=("text", "string")),
+#     "Integer": lambda x: pd.to_numeric(x, errors="coerce"),
+#     "SmallInteger": lambda x: pd.to_numeric(x, errors="coerce"),
+#     "BigInteger": lambda x: pd.to_numeric(x, errors="coerce"),
+#     "Float": lambda x: pd.to_numeric(x, errors="coerce"),
+#     "Numeric": lambda x: pd.to_numeric(x, errors="coerce"),
+#     "Boolean": lambda x: x.map_partitions(lambda s: s.fillna(False).astype(bool), meta=("boolean", "bool")),
+#     "DateTime": lambda x: pd.to_datetime(x, errors="coerce"),
+#     "Date": lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda s: s.dt.date, meta=("date", "object")),
+#     "Time": lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda s: s.dt.time, meta=("time", "object")),
+#     "JSON": lambda x: x.map_partitions(lambda s: s.apply(json.loads), meta=("json", "object")),
+# }
+
+
+def normalize_sqlalchemy_type(field_type):
+    """
+    Normalize SQLAlchemy field types to generic type names.
+    Handles dialect-specific types (e.g., MySQL).
+    """
+    # Map of generic SQLAlchemy types
+    type_mapping = {
+        String: "String",
+        Text: "Text",
+        Integer: "Integer",
+        SmallInteger: "SmallInteger",
+        BigInteger: "BigInteger",
+        Float: "Float",
+        Numeric: "Numeric",
+        Boolean: "Boolean",
+        DateTime: "DateTime",
+        Date: "Date",
+        Time: "Time",
+        JSON: "JSON",
+    }
+
+    # Dialect-specific types
+    dialect_mapping = {
+        TINYINT: "SmallInteger",
+        MEDIUMTEXT: "Text",
+    }
+
+    # Check if the field matches a generic or dialect-specific type
+    for sql_type, name in {**type_mapping, **dialect_mapping}.items():
+        if isinstance(field_type, sql_type):
+            return name
+
+    # Fallback to raw class name
+    return field_type.__class__.__name__
+
+

sibi_dst-0.3.12/sibi_dst/df_helper/plugins/django/_django_load_from_db.py

@@ -0,0 +1,121 @@
+import dask.dataframe as dd
+import pandas as pd
+from django.db.models import Q
+
+from sibi_dst.df_helper.plugins.django import ReadFrameDask
+from sibi_dst.utils import Logger
+from sibi_dst.df_helper.core import django_field_conversion_map_dask
+
+class DjangoLoadFromDb:
+    df: dd.DataFrame
+
+    def __init__(self, db_connection, db_query, db_params, logger, **kwargs):
+        self.connection_config = db_connection
+        self.debug = kwargs.pop('debug', False)
+        self.verbose_debug = kwargs.pop('verbose_debug', False)
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        if self.connection_config.model is None:
+            if self.debug:
+                self.logger.critical('Model must be specified')
+                if self.verbose_debug:
+                    print('Model must be specified')
+            raise ValueError('Model must be specified')
+
+        self.query_config = db_query
+        self.params_config = db_params
+        self.params_config.parse_params(kwargs)
+
+    def build_and_load(self):
+        self.df = self._build_and_load()
+        #self.df = self._convert_columns(self.df)
+        return self.df
+
+
+    def _build_and_load(self) -> dd.DataFrame:
+        query = self.connection_config.model.objects.using(self.connection_config.connection_name)
+        if not self.params_config.filters:
+            # IMPORTANT: if no filters are provided show only the first n_records
+            # this is to prevent loading the entire table by mistake
+            n_records = self.query_config.n_records if self.query_config.n_records else 100
+            queryset=query.all()[:n_records]
+        else:
+            q_objects = self.__build_query_objects(self.params_config.filters, self.query_config.use_exclude)
+            queryset = query.filter(q_objects)
+        if queryset is not None:
+            try:
+                self.df = ReadFrameDask(queryset, **self.params_config.df_params).read_frame()
+            except Exception as e:
+                self.logger.critical(f'Error loading query: {str(queryset.query)}, error message: {e}')
+                self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
+        else:
+            self.df = dd.from_pandas(pd.DataFrame(), npartitions=1)
+
+        return self.df
+
+    @staticmethod
+    def __build_query_objects(filters: dict, use_exclude: bool):
+        q_objects = Q()
+        for key, value in filters.items():
+            if not use_exclude:
+                q_objects.add(Q(**{key: value}), Q.AND)
+            else:
+                q_objects.add(~Q(**{key: value}), Q.AND)
+        return q_objects
+
+    def _convert_columns(self, df: dd.DataFrame) -> dd.DataFrame:
+        """
+        Convert the data types of columns in a Dask DataFrame based on the field type in the Django model.
+
+        :param df: Dask DataFrame whose columns' data types are to be converted.
+        :return: Dask DataFrame with converted column data types.
+        """
+
+        def log_debug(message: str, is_verbose: bool = False):
+            """Helper to handle debug and verbose debug logging."""
+            if self.debug:
+                self.logger.debug(message)
+                if is_verbose and self.verbose_debug:
+                    print(message)
+
+        if self.debug:
+            self.logger.info(f'Converting columns: {list(df.columns)}')
+
+        # Get field information from the Django model
+        model_fields = self.connection_config.model._meta.get_fields()
+        field_type_map = {field.name: type(field).__name__ for field in model_fields}
+        # Simplified loop to apply conversions partition-wise
+        for field_name, field_type in field_type_map.items():
+            if field_name not in df.columns:
+
+                log_debug(f"Column '{field_name}' not found in DataFrame columns.")
+                continue
+
+            conversion_func = django_field_conversion_map_dask.get(field_type)
+            if not conversion_func:
+                message=f"Field type '{field_type}' not found in conversion_map."
+                log_debug(message, is_verbose=True)
+                continue
+
+            def apply_conversion(partition):
+                """
+                Apply the conversion function to a single partition for the given column.
+                """
+                try:
+                    if field_name in partition.columns:
+                        partition[field_name] = conversion_func(partition[field_name])
+                except Exception as e:
+                    self.logger.error(f"Error converting column '{field_name}' in partition: {str(e)}")
+                return partition
+
+            try:
+                # Apply conversion lazily to each partition
+                df = df.map_partitions(
+                    apply_conversion,
+                    meta=df,
+                )
+                log_debug(f"Successfully queued conversion for column '{field_name}' to type '{field_type}'.",
+                          is_verbose=True)
+            except Exception as e:
+                log_debug(f"Failed to queue conversion for column '{field_name}': {str(e)}", is_verbose=True)
+
+        return df

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/http/_http_config.py

@@ -13,11 +13,10 @@ class HttpConfig(BaseModel):
     api_key: Optional[SecretStr] = None
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
-    def __init__(self, **data):
+    def __init__(self, logger=None, **data):
         super().__init__(**data)
         # Initialize the logger if not provided
-        if not self.logger:
-            self.logger = Logger(log_dir='./logs/', logger_name="HttpDataSource", log_file='http_data_source.log')
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
 
     async def fetch_data(self, **options) -> dd.DataFrame:
         """Asynchronously fetch JSON data from HTTP endpoint, substituting options into the URL path."""

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/parquet/_parquet_filter_handler.py

@@ -1,7 +1,11 @@
 import pandas as pd
 import dask.dataframe as dd
+from sibi_dst.utils import Logger
 
 class ParquetFilterHandler(object):
+    def __init__(self, logger=None):
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+
     @staticmethod
     def apply_filters_dask(df, filters):
         dt_operators = ['date', 'time']

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/parquet/_parquet_options.py

@@ -25,6 +25,8 @@ class ParquetConfig(BaseModel):
     @model_validator(mode='after')
     def check_parquet_params(self):
         # Configure paths based on fsspec
+        if self.logger is None:
+            self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
         self.fs = fsspec.filesystem("file") if "://" not in str(self.parquet_storage_path) else fsspec.filesystem(str(self.parquet_storage_path).split("://")[0])
 
         # Validation for parquet path

sibi_dst-0.3.12/sibi_dst/df_helper/plugins/sql_alchemy/_io_sqlalchemy_dask.py

@@ -0,0 +1,133 @@
+import itertools
+
+import dask.dataframe as dd
+import pandas as pd
+from sqlalchemy import create_engine, inspect, select
+from sqlalchemy.orm import sessionmaker
+
+from sibi_dst.df_helper.plugins.sql_alchemy._sqlachemy_filter_handler import SqlAlchemyFilterHandler
+from sibi_dst.utils import Logger
+
+
+class SQLAlchemyDask:
+    def __init__(self, model, filters, engine_url, chunk_size=1000, logger=None, verbose=True):
+        """
+        Initialize with an SQLAlchemy query and database engine URL.
+
+        :param model: SQLAlchemy ORM model.
+        :param filters: Filters to apply on the query.
+        :param engine_url: Database connection string for SQLAlchemy engine.
+        :param chunk_size: Number of records per chunk for Dask partitions.
+        :param logger: Logger instance for logging.
+        :param verbose: Whether to print detailed logs.
+        """
+        self.query = None
+        self.model = model
+        self.filters = filters
+        self.chunk_size = chunk_size
+        self.verbose = verbose
+        self.engine = create_engine(engine_url)
+        self.Session = sessionmaker(bind=self.engine)
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+
+    @staticmethod
+    def infer_dtypes_from_model(model):
+        """
+        Infer data types for Dask DataFrame based on SQLAlchemy ORM model columns.
+        """
+        mapper = inspect(model)
+        sqlalchemy_to_dask_dtype = {
+            #'INTEGER': pd.to_numeric(x, errors="coerce"),
+            'INTEGER': 'Int64',
+            'SMALLINT': 'Int64',
+            'BIGINT': 'Int64',
+            'FLOAT': 'float64',
+            'NUMERIC': 'float64',
+            'BOOLEAN': 'bool',
+            'VARCHAR': 'object',
+            'TEXT': 'object',
+            'DATE': 'datetime64[ns]',
+            'DATETIME': 'datetime64[ns]',
+            'TIME': 'object',
+            'UUID': 'object',
+        }
+
+        dtypes = {}
+        for column in mapper.columns:
+            dtype = sqlalchemy_to_dask_dtype.get(str(column.type).upper(), 'object')
+            dtypes[column.name] = dtype
+
+        return dtypes
+
+    def read_frame(self, fillna_value=None):
+        """
+        Load data from an SQLAlchemy query into a Dask DataFrame.
+
+        :param fillna_value: Value to replace NaN or NULL values with, if any.
+        :return: Dask DataFrame.
+        """
+        with self.Session() as session:
+            try:
+                # Build query
+                self.query = select(self.model)
+                if self.filters:
+                    self.query = SqlAlchemyFilterHandler.apply_filters_sqlalchemy(self.query, self.model, self.filters)
+                else:
+                    n_records = 100
+                    self.query = self.query.limit(n_records)
+
+                # Infer dtypes
+                dtypes = self.infer_dtypes_from_model(self.model)
+                # Get the column order from the SQLAlchemy model
+                ordered_columns = [column.name for column in self.model.__table__.columns]
+
+                # Execute query and fetch results in chunks
+                result_proxy = session.execute(self.query)
+                results = result_proxy.scalars().all()  # Fetch all rows
+                iterator = iter(results)
+
+                partitions = []
+
+                while True:
+                    chunk = list(itertools.islice(iterator, self.chunk_size))
+                    if not chunk:
+                        break
+
+                    # Convert chunk to Pandas DataFrame
+                    df = pd.DataFrame.from_records(
+                        [row._asdict() if hasattr(row, '_asdict') else row.__dict__ for row in chunk]
+                    )
+                    # Drop internal SQLAlchemy state if it exists
+                    df = df.loc[:, ~df.columns.str.contains('_sa_instance_state')]
+
+                    # Reorder columns to match the model's order
+                    df = df[ordered_columns]
+
+                    # Fill NaN values
+                    if fillna_value is not None:
+                        df = df.fillna(fillna_value)
+
+                    # Convert timezone-aware columns to naive
+                    for col in df.columns:
+                        if isinstance(df[col].dtype, pd.DatetimeTZDtype):
+                            df[col] = df[col].dt.tz_localize(None)
+
+                    # Apply inferred dtypes
+                    df = df.astype(dtypes)
+                    # Create a Dask partition
+                    partitions.append(dd.from_pandas(df, npartitions=1))
+
+                # Concatenate all partitions
+                # print(partitions)
+                if partitions:
+                    dask_df = dd.concat(partitions, axis=0, ignore_index=True)
+                else:
+                    dask_df = dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)
+
+                if self.verbose:
+                    self.logger.info(f"Loaded {len(dask_df)} rows into Dask DataFrame.")
+                return dask_df
+
+            except Exception as e:
+                self.logger.error(f"Error executing query: {str(e)}")
+                return dd.from_pandas(pd.DataFrame(columns=ordered_columns), npartitions=1)

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/sibi_dst/df_helper/plugins/sql_alchemy/_sqlachemy_filter_handler.py

@@ -50,6 +50,7 @@ class SqlAlchemyFilterHandler:
                     return [datetime.date.fromisoformat(v) for v in value]
             return value
 
+
         def handle_date_operator(column, date_op):
             """
             Handle filtering on specific datetime parts (e.g., year, month).
@@ -90,6 +91,7 @@ class SqlAlchemyFilterHandler:
 
             # Get the column from the model
             column = getattr(model, field_name, None)
+            #column = model.__table__.columns.get(field_name)
             if not column:
                 raise AttributeError(f"Field '{field_name}' not found in model '{model.__name__}'")
 

sibi_dst-0.3.12/sibi_dst/df_helper/plugins/sql_alchemy/_sqlalchemy_load_from_db.py

@@ -0,0 +1,59 @@
+from typing import Dict
+
+import dask.dataframe as dd
+import pandas as pd
+from sqlalchemy.inspection import inspect
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy import select
+#from sqlmodel import Session, select
+
+from sibi_dst.df_helper.core import ParamsConfig, QueryConfig, sqlalchemy_field_conversion_map_dask, \
+    normalize_sqlalchemy_type
+from sibi_dst.utils import Logger
+from ._io_sqlalchemy_dask import SQLAlchemyDask
+from ._sqlachemy_filter_handler import SqlAlchemyFilterHandler
+from ._sqlalchemy_db_connection import SqlAlchemyConnectionConfig
+
+
+class SqlAlchemyLoadFromDb:
+    df: dd.DataFrame
+
+    def __init__(
+            self,
+            plugin_sqlalchemy: SqlAlchemyConnectionConfig,  # Expected to be an instance of SqlAlchemyConnection
+            plugin_query: QueryConfig = None,
+            plugin_params: ParamsConfig = None,
+            logger: Logger = None,
+            **kwargs,
+    ):
+        """
+        Initialize the loader with database connection, query, and parameters.
+        """
+        self.db_connection = plugin_sqlalchemy
+        self.table_name = self.db_connection.table
+        self.model = self.db_connection.model
+        self.engine = self.db_connection.engine
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.query_config = plugin_query
+        self.params_config = plugin_params
+        self.debug = kwargs.pop("debug", False)
+        self.verbose_debug = kwargs.pop("verbose_debug", False)
+
+    def build_and_load(self) -> dd.DataFrame:
+        """
+        Load data into a Dask DataFrame based on the query and parameters.
+        """
+        self.df = self._build_and_load()
+        return self.df
+
+    def _build_and_load(self) -> dd.DataFrame:
+        try:
+            reader = SQLAlchemyDask(model=self.model, filters=self.params_config.filters,engine_url=self.engine.url, logger=self.logger, chunk_size=1000, verbose=self.debug)
+            df = reader.read_frame()
+            if df is None or len(df.index) == 0:
+                self.logger.warning("Query returned no results.")
+                return dd.from_pandas(pd.DataFrame(), npartitions=1)
+            return df
+        except Exception as e:
+            self.logger.error(f"Failed to load data into Dask DataFrame.{e}")
+            return dd.from_pandas(pd.DataFrame(), npartitions=1)

{sibi_dst-0.3.10 → sibi_dst-0.3.12}/sibi_dst/utils/__init__.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 from ._credentials import ConfigManager, ConfigLoader
 from ._log_utils import Logger
-from ._date_utils import DateUtils
+from ._date_utils import DateUtils, BusinessDays
 from ._data_utils import DataUtils
 from ._file_utils import FileUtils
 from ._filepath_generator import FilePathGenerator
@@ -17,6 +17,7 @@ __all__=[
     "ConfigLoader",
     "Logger",
     "DateUtils",
+    "BusinessDays",
     "FileUtils",
     "DataWrapper",
     "DataUtils",