sibi-dst 0.3.64__tar.gz → 2025.1.2__tar.gz

sibi_dst-2025.1.2/PKG-INFO

@@ -0,0 +1,55 @@
+Metadata-Version: 2.1
+Name: sibi-dst
+Version: 2025.1.2
+Summary: Data Science Toolkit
+Author: Luis Valverde
+Author-email: lvalverdeb@gmail.com
+Requires-Python: >=3.12,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: clickhouse-connect (>=0.8.18,<0.9.0)
+Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
+Requires-Dist: dask[complete] (>=2025.5.1,<2026.0.0)
+Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
+Requires-Dist: pandas (>=2.3.1,<3.0.0)
+Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
+Requires-Dist: pydantic (>=2.11.7,<3.0.0)
+Requires-Dist: pymysql (>=1.1.1,<2.0.0)
+Requires-Dist: s3fs (>=2025.5.1,<2026.0.0)
+Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
+Requires-Dist: tqdm (>=4.67.1,<5.0.0)
+Requires-Dist: webdav4 (>=0.10.0,<0.11.0)
+Description-Content-Type: text/markdown
+
+### SIBI-DST
+
+Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, NetworkX, SQLAlchemy, GeoPandas, and Folium.
+
+## Example Use Cases
+
+1. **Build DataCubes, DataSets, and DataObjects** from diverse data sources, including **relational databases, Parquet files, Excel (`.xlsx`), delimited tables (`.csv`, `.tsv`), JSON, and RESTful APIs**.
+2. **Comprehensive DataFrame Management** utilities for efficient data handling, transformation, and optimization using **Pandas** and **Dask**.
+3. **Flexible Data Sharing** with client applications by writing to **Data Warehouses in Clickhouse, local filesystems, and cloud storage platforms** such as **S3**.
+4. **Microservices for Data Access** – Build scalable **API-driven services** using **RESTful APIs (`Django REST Framework`, `FastAPI`)** for high-performance data exchange.
+5. **Geospatial Analysis** – Utilize **OpenStreetMaps** and **GeoPandas** for advanced geospatial data processing and visualization.
+
+## Supported Technologies
+
+- **Data Processing**: Pandas, Dask
+- **Databases & Storage**: SQLAlchemy, Parquet, S3, Clickhouse
+- **Mapping & Geospatial Analysis**: OpenStreetMaps, OSMnx, Geopy
+- **API Development**: Django REST Framework, FastAPI
+
+## Installation
+
+```bash
+# with pip
+
+pip install sibi-dst  # Install only the main package
+pip install sibi-dst[geospatial] # Install with geospatial dependencies
+pip install sibi-dst[dev,test,geospatial]  # Install all optional dependencies
+
+
+```
+

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/README.md

@@ -22,15 +22,9 @@ Data Science Toolkit built with Python, Pandas, Dask, OpenStreetMaps, NetworkX,
 ```bash
 # with pip
 
-pip install sibi-dst[complete]  # Install all dependencies
-pip install sibi-dst[df_helper]  # Install only df_helper dependencies
-pip install sibi-dst[geospatial]  # Install only geospatial dependencies
-
-# with poetry
-
-poetry add "sibi-dst[complete]"  # Install all dependencies
-poetry add "sibi-dst[df_helper]"  # Install only df_helper dependencies
-poetry add "sibi-dst[geospatial]"  # Install only geospatial dependencies
+pip install sibi-dst  # Install only the main package
+pip install sibi-dst[geospatial] # Install with geospatial dependencies
+pip install sibi-dst[dev,test,geospatial]  # Install all optional dependencies
 
 
 ```

sibi_dst-2025.1.2/pyproject.toml

@@ -0,0 +1,49 @@
+[tool.poetry]
+name = "sibi-dst"
+version = "2025.1.2"
+description = "Data Science Toolkit"
+authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
+readme = "README.md"
+packages = [{ include = "sibi_dst" }]
+
+[tool.poetry.dependencies]
+python = ">=3.12,<4.0"
+pandas = "^2.3.1"
+dask = {extras = ["complete"], version = "^2025.5.1"}
+psycopg2 = "^2.9.10"
+mysqlclient = "^2.2.7"
+webdav4 = "^0.10.0"
+clickhouse-connect = "^0.8.18"
+clickhouse-driver = "^0.2.9"
+tqdm = "^4.67.1"
+s3fs = "^2025.5.1"
+pydantic = "^2.11.7"
+sqlalchemy = "^2.0.41"
+pymysql = "^1.1.1"
+
+[tool.poetry.group.dev]
+optional = true
+
+[tool.poetry.group.dev.dependencies]
+jupyter = "^1.1.1"
+python-dotenv = "^1.1.1"
+black = "^25.1.0"
+
+[tool.poetry.group.test]
+optional = true
+
+[tool.poetry.group.test.dependencies]
+pytest = "^8.4.1"
+pytest-cov = "^6.2.1"
+
+[tool.poetry.group.geospatial]
+optional = true
+
+[tool.poetry.group.geospatial.dependencies]
+osmnx = "^2.0.5"
+geopy = "^2.4.1"
+folium = "^0.20.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/sibi_dst/df_helper/_df_helper.py

@@ -26,6 +26,7 @@ class BaseBackend:
     def __init__(self, helper: DfHelper):
         self.helper = helper
         self.logger = helper.logger
+        self.debug = helper.debug
 
     def load(self, **options) -> dd.DataFrame | pd.DataFrame:
         """Synchronous data loading method. Must be implemented by sync backends."""
@@ -47,7 +48,8 @@ class SqlAlchemyBackend(BaseBackend):
                 plugin_sqlalchemy=self.helper.backend_db_connection,
                 plugin_query=self.helper._backend_query,
                 plugin_params=self.helper._backend_params,
-                logger=self.logger
+                logger=self.logger,
+                debug= self.debug
             )
             return db_loader.build_and_load()
         except Exception as e:
@@ -62,10 +64,10 @@ class ParquetBackend(BaseBackend):
         try:
             df = self.helper.backend_parquet.load_files()
             if options and df is not None:
-                df = FilterHandler('dask', self.logger).apply_filters(df, filters=options)
+                df = FilterHandler('dask', logger=self.logger, debug=False).apply_filters(df, filters=options)
             return df
         except Exception as e:
-            self.logger.error(f"Failed to load data from parquet: {e}", exc_info=self.debug)
+            self.logger.error(f"Failed to load data from parquet: {e}", exc_info=True)
             return dd.from_pandas(pd.DataFrame(), npartitions=1)
 
 

sibi_dst-2025.1.2/sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py

@@ -0,0 +1,329 @@
+
+from typing import Type
+
+import dask
+import dask.dataframe as dd
+import pandas as pd
+from sqlalchemy import (
+    inspect,
+    select
+)
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import declarative_base
+import time
+from sqlalchemy.exc import TimeoutError as SASQLTimeoutError, OperationalError
+import sqlalchemy as sa
+from sibi_dst.df_helper.core import FilterHandler
+from sibi_dst.utils import Logger
+
+
+class SQLAlchemyDask:
+    """
+    Loads data from a database into a Dask DataFrame using a memory-safe,
+    non-parallel, paginated approach.
+
+    This class avoids using a numeric `index_col for parallel loading.
+    """
+
+    _SQLALCHEMY_TO_DASK_DTYPE = {
+        "INTEGER": "Int64",
+        "SMALLINT": "Int64",
+        "BIGINT": "Int64",
+        "FLOAT": "float64",
+        "NUMERIC": "float64",
+        "BOOLEAN": "bool",
+        "VARCHAR": "object",
+        "TEXT": "object",
+        "DATE": "datetime64[ns]",
+        "DATETIME": "datetime64[ns]",
+        "TIME": "object",
+        "UUID": "object",
+    }
+
+    def __init__(
+            self,
+            model: Type[declarative_base()],
+            filters: dict,
+            engine: Engine,
+            chunk_size: int = 1000,
+            logger=None,
+            debug: bool = False,
+    ):
+        """
+        Initializes the data loader.
+
+        Args:
+            model: The SQLAlchemy ORM model for the table.
+            filters: A dictionary of filters to apply to the query.
+            engine: An SQLAlchemy Engine instance.
+            chunk_size: The number of records to fetch in each database query.
+            logger: A logger instance.
+            debug: Whether to enable detailed logging.
+        """
+        self.model = model
+        self.filters = filters
+        self.engine = engine
+        self.chunk_size = chunk_size
+        self.debug = debug
+        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
+        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
+        self.filter_handler_cls = FilterHandler
+
+    @classmethod
+    def infer_meta_from_model(cls, model: Type[declarative_base()]) -> dict:
+        """
+        Infers a metadata dictionary for Dask based on the SQLAlchemy model.
+        This helps Dask understand the DataFrame structure without reading data.
+        """
+        mapper = inspect(model)
+        dtypes = {}
+        for column in mapper.columns:
+            dtype_str = str(column.type).upper().split("(")[0]
+            dtype = cls._SQLALCHEMY_TO_DASK_DTYPE.get(dtype_str, "object")
+            dtypes[column.name] = dtype
+        return dtypes
+
+    def read_frame(self, fillna_value=None) -> dd.DataFrame:
+        """
+        Builds and executes a query to load data into a Dask DataFrame.
+
+        This method works by first running a COUNT query to get the total
+        size, then creating a series of delayed tasks that each fetch a
+        chunk of data using LIMIT/OFFSET.
+
+        Args:
+            fillna_value: Value to replace NaN or NULL values with, if any.
+
+        Returns:
+            A lazy Dask DataFrame.
+        """
+        # 1. Build the base query and apply filters
+        query = select(self.model)
+        if self.filters:
+            query = self.filter_handler_cls(
+                backend="sqlalchemy", logger=self.logger, debug=self.debug
+            ).apply_filters(query, model=self.model, filters=self.filters)
+        else:
+            query = query.limit(self.chunk_size)
+        self.logger.debug(f"Base query for pagination: {query}")
+
+        # 2. Get metadata for the Dask DataFrame structure
+        ordered_columns = [column.name for column in self.model.__table__.columns]
+        meta_dtypes = self.infer_meta_from_model(self.model)
+        meta_df = pd.DataFrame(columns=ordered_columns).astype(meta_dtypes)
+
+        # 3. Get the total record count to calculate the number of chunks
+
+        retry_attempts = 3
+        backoff_factor = 0.5  # start with a 0.5-second delay
+
+        for attempt in range(retry_attempts):
+            try:
+                with self.engine.connect() as connection:
+                    count_query = sa.select(sa.func.count()).select_from(query.alias())
+                    total_records = connection.execute(count_query).scalar_one()
+
+                # If successful, break the loop
+                break
+
+            except SASQLTimeoutError:
+                if attempt < retry_attempts - 1:
+                    self.logger.warning(
+                        f"Connection pool limit reached. Retrying in {backoff_factor} seconds..."
+                    )
+                    time.sleep(backoff_factor)
+                    backoff_factor *= 2  # Double the backoff time for the next attempt
+                else:
+                    self.logger.error(
+                        "Failed to get a connection from the pool after several retries.",
+                        exc_info=True
+                    )
+                    return dd.from_pandas(meta_df, npartitions=1)
+            except OperationalError as oe:
+                # sometimes the DB driver wraps timeouts in OperationalError
+                if "timeout" in str(oe).lower():
+                    self.logger.warning("OperationalTimeout, retrying…", exc_info=True)
+                    time.sleep(backoff_factor)
+                    backoff_factor *= 2
+                    continue
+                else:
+                    self.logger.error("OperationalError", exc_info=True)
+                    return dd.from_pandas(meta_df, npartitions=1)
+            except Exception as e:
+                self.logger.error(f"An unexpected error occurred: {e}", exc_info=True)
+                return dd.from_pandas(meta_df, npartitions=1)
+
+        if total_records == 0:
+            self.logger.warning("Query returned 0 records.")
+            return dd.from_pandas(meta_df, npartitions=1)
+
+        self.logger.debug(f"Total records to fetch: {total_records}. Chunk size: {self.chunk_size}.")
+
+        # 4. Create a list of Dask Delayed objects, one for each chunk
+        @dask.delayed
+        def get_chunk(sql_query, chunk_offset):
+            """A Dask-delayed function to fetch one chunk of data."""
+            # LIMIT/OFFSET must be applied in the delayed function
+            paginated_query = sql_query.limit(self.chunk_size).offset(chunk_offset)
+            df = pd.read_sql(paginated_query, self.engine)
+
+            if fillna_value is not None:
+                df = df.fillna(fillna_value)
+
+            # Ensure column order and types match the meta
+            return df[ordered_columns].astype(meta_dtypes)
+
+        offsets = range(0, total_records, self.chunk_size)
+        delayed_chunks = [get_chunk(query, offset) for offset in offsets]
+
+        # 5. Construct the final lazy Dask DataFrame from the delayed chunks
+        ddf = dd.from_delayed(delayed_chunks, meta=meta_df)
+        self.logger.debug(f"Successfully created a lazy Dask DataFrame with {ddf.npartitions} partitions.")
+
+        return ddf
+
+## Dask-Only Solution to test in better hardware
+
+# from typing import Type, Dict, Any
+# import math
+# import time
+# import pandas as pd
+# import dask
+# import dask.dataframe as dd
+#
+# import sqlalchemy as sa
+# from sqlalchemy import select, func
+# from sqlalchemy.engine import Engine
+# from sqlalchemy.exc import TimeoutError as SASQLTimeoutError, OperationalError
+# from sqlalchemy.orm import declarative_base
+#
+# from sibi_dst.df_helper.core import FilterHandler
+# from sibi_dst.utils import Logger
+#
+#
+# class SQLAlchemyDask:
+#     """
+#     Loads data into a Dask DataFrame.  If there’s exactly one integer PK,
+#     use dask.dataframe.read_sql_table; otherwise fall back to offset‐based
+#     pagination pushed into dask.delayed to keep memory use minimal.
+#     """
+#
+#     def __init__(
+#         self,
+#         model: Type[declarative_base()],
+#         filters: Dict[str, Any],
+#         engine: Engine,
+#         chunk_size: int = 1_000,
+#         logger=None,
+#         debug: bool = False,
+#     ):
+#         self.model      = model
+#         self.filters    = filters or {}
+#         self.engine     = engine
+#         self.chunk_size = chunk_size
+#         self.logger     = logger or Logger.default_logger(self.__class__.__name__)
+#         self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
+#         self.filter_handler_cls = FilterHandler
+#         self.debug = debug
+#
+#     def read_frame(self, fillna_value=None) -> dd.DataFrame:
+#         # 1) Build base query + filters
+#         base_q = select(self.model)
+#         if self.filters:
+#             base_q = self.filter_handler_cls(
+#                 backend="sqlalchemy",
+#                 logger=self.logger,
+#                 debug=self.debug,
+#             ).apply_filters(base_q, model=self.model, filters=self.filters)
+#
+#         # 2) Zero-row meta for dtype inference
+#         meta = pd.read_sql_query(base_q.limit(0), self.engine).iloc[:0]
+#         if meta.shape[1] == 0:
+#             self.logger.warning("No columns detected; returning empty DataFrame.")
+#             return dd.from_pandas(meta, npartitions=1)
+#
+#         # 3) Single‐PK parallel path?
+#         pk_cols = list(self.model.__table__.primary_key.columns)
+#         if (
+#             len(pk_cols) == 1
+#             and pd.api.types.is_integer_dtype(meta[pk_cols[0].name])
+#         ):
+#             try:
+#                 return self._ddf_via_read_sql_table(pk_cols[0], meta, fillna_value)
+#             except Exception:
+#                 self.logger.warning(
+#                     "read_sql_table path failed, falling back to offset pagination",
+#                     exc_info=True,
+#                 )
+#
+#         # 4) Composite PK or fallback → offset pagination in delayed tasks
+#         return self._offset_paginated_ddf(base_q, meta, fillna_value)
+#
+#     def _offset_paginated_ddf(self, base_q, meta, fillna):
+#         # 1) count total rows
+#         try:
+#             with self.engine.connect() as conn:
+#                 total = conn.execute(
+#                     select(func.count()).select_from(base_q.alias())
+#                 ).scalar_one()
+#         except Exception:
+#             self.logger.error("Failed to count records; returning empty DataFrame", exc_info=True)
+#             return dd.from_pandas(meta, npartitions=1)
+#
+#         if total == 0:
+#             self.logger.warning("Query returned 0 records.")
+#             return dd.from_pandas(meta, npartitions=1)
+#         self.logger.debug(f"Total records to fetch: {total}. Chunk size: {self.chunk_size}.")
+#         # 2) create delayed tasks per offset
+#         @dask.delayed
+#         def _fetch_chunk(offset: int) -> pd.DataFrame:
+#             q = base_q.limit(self.chunk_size).offset(offset)
+#             df = pd.read_sql_query(q, self.engine)
+#             if fillna is not None:
+#                 df = df.fillna(fillna)
+#             return df[meta.columns].astype(meta.dtypes.to_dict())
+#
+#         offsets = range(0, total, self.chunk_size)
+#         parts = [_fetch_chunk(off) for off in offsets]
+#
+#         ddf = dd.from_delayed(parts, meta=meta)
+#         self.logger.debug(f"Offset‐paginated read → {len(parts)} partitions")
+#         return ddf
+#
+#     def _ddf_via_read_sql_table(self, pk_col, meta, fillna) -> dd.DataFrame:
+#         # same as before: min/max + dd.read_sql_table
+#         backoff = 0.5
+#         for attempt in range(3):
+#             try:
+#                 with self.engine.connect() as conn:
+#                     min_id, max_id = conn.execute(
+#                         select(func.min(pk_col), func.max(pk_col))
+#                         .select_from(self.model.__table__)
+#                     ).one()
+#                 break
+#             except (SASQLTimeoutError, OperationalError) as e:
+#                 if "timeout" in str(e).lower() and attempt < 2:
+#                     self.logger.warning(f"Timeout fetching PK bounds; retrying in {backoff}s")
+#                     time.sleep(backoff)
+#                     backoff *= 2
+#                 else:
+#                     raise
+#
+#         if min_id is None or max_id is None:
+#             self.logger.warning("Table empty—no PK bounds.")
+#             return dd.from_pandas(meta, npartitions=1)
+#
+#         total = max_id - min_id + 1
+#         nparts = max(1, math.ceil(total / self.chunk_size))
+#         ddf = dd.read_sql_table(
+#             table=self.model.__table__.name,
+#             uri=str(self.engine.url),
+#             index_col=pk_col.name,
+#             limits=(min_id, max_id),
+#             npartitions=nparts,
+#             columns=list(meta.columns),
+#         )
+#         if fillna is not None:
+#             ddf = ddf.fillna(fillna)
+#         self.logger.debug(f"Parallel read via dask.read_sql_table → {nparts} partitions")
+#         return ddf

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/sibi_dst/df_helper/core/__init__.py

@@ -1,8 +1,6 @@
 from __future__ import annotations
 
 from ._defaults import (
-    django_field_conversion_map_pandas,
-    django_field_conversion_map_dask,
     sqlalchemy_field_conversion_map_dask,
     normalize_sqlalchemy_type)
 from ._filter_handler import FilterHandler
@@ -12,8 +10,6 @@ from ._query_config import QueryConfig
 __all__ = [
     "ParamsConfig",
     "QueryConfig",
-    "django_field_conversion_map_pandas",
-    "django_field_conversion_map_dask",
     "sqlalchemy_field_conversion_map_dask",
     "normalize_sqlalchemy_type",
     "FilterHandler",

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/sibi_dst/df_helper/core/_defaults.py

@@ -13,56 +13,7 @@ from sqlalchemy.dialects.mysql import TINYINT, MEDIUMTEXT
 # conversion_map is a dictionary that maps the field types to their corresponding data type conversion functions.
 # Each entry in the dictionary is a pair of a field type (as a string) and a callable function that performs the
 # conversion. This mapping is used to convert the values in a pandas DataFrame to the appropriate data types based on
-# the Django field type.
-
-django_field_conversion_map_pandas: Dict[str, callable] = {
-    "CharField": lambda x: x.astype(str),
-    "TextField": lambda x: x.astype(str),
-    "IntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "AutoField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "BigAutoField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "BigIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "SmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "PositiveIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "PositiveSmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "FloatField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "DecimalField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "BooleanField": lambda x: x.astype(bool),
-    "NullBooleanField": lambda x: x.astype(bool),
-    "DateTimeField": lambda x: pd.to_datetime(x, errors="coerce"),
-    "DateField": lambda x: pd.to_datetime(x, errors="coerce").dt.date,
-    "TimeField": lambda x: pd.to_datetime(x, errors="coerce").dt.time,
-    "DurationField": lambda x: pd.to_timedelta(x, errors="coerce"),
-    # for JSONField, assuming JSON objects are represented as string in df
-    "JSONField": lambda x: x.apply(json.loads),
-    "ArrayField": lambda x: x.apply(eval),
-    "UUIDField": lambda x: x.astype(str),
-}
-
-django_field_conversion_map_dask: Dict[str, callable] = {
-    "CharField": lambda x: x.astype(str),
-    "TextField": lambda x: x.astype(str),
-    "IntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "AutoField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "BigAutoField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "BigIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "SmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "PositiveIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "PositiveSmallIntegerField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "FloatField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "DecimalField": lambda x: pd.to_numeric(x, errors="coerce"),
-    "BooleanField": lambda x: x.astype(bool),
-    "NullBooleanField": lambda x: x.astype(bool),
-    "DateTimeField": lambda x: pd.to_datetime(x, errors="coerce"),
-    "DateField": lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda x: x.dt.date,
-                                                                             meta=("date", "object")),
-    "TimeField": lambda x: pd.to_datetime(x, errors="coerce").map_partitions(lambda x: x.dt.time,
-                                                                             meta=("time", "object")),
-    "DurationField": lambda x: pd.to_timedelta(x, errors="coerce"),
-    "JSONField": lambda x: x.map_partitions(lambda s: s.apply(json.loads), meta=("json", "object")),
-    "ArrayField": lambda x: x.map_partitions(lambda s: s.apply(eval), meta=("array", "object")),
-    "UUIDField": lambda x: x.astype(str),
-}
+# the db field type.
 
 sqlalchemy_field_conversion_map_dask: Dict[str, callable] = {
     String.__name__: lambda x: x.astype(str).fillna(""),

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/sibi_dst/utils/__init__.py

@@ -10,7 +10,6 @@ from .df_utils import DfUtils
 from .storage_manager import StorageManager
 from .parquet_saver import ParquetSaver
 from .clickhouse_writer import ClickHouseWriter
-from .airflow_manager import AirflowDAGManager
 from .credentials import *
 from .update_planner import UpdatePlanner
 from .data_wrapper import DataWrapper
@@ -35,7 +34,6 @@ __all__ = [
     "StorageManager",
     "DfUtils",
     "ClickHouseWriter",
-    "AirflowDAGManager",
     "StorageConfig",
     "FsRegistry",
     "DataFromHttpSource",

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/sibi_dst/utils/data_wrapper.py

@@ -38,7 +38,7 @@ class DataWrapper:
             logger: Logger = None,
             show_progress: bool = False,
             timeout: float = 30,
-            max_threads: int = 1,
+            max_threads: int = 3,
             **kwargs: Any,
     ):
         self.dataclass = dataclass
@@ -66,6 +66,7 @@ class DataWrapper:
         self.benchmarks: Dict[datetime.date, Dict[str, float]] = {}
         self.mmanifest = kwargs.get("mmanifest", None)
         self.update_planner=kwargs.get("update_planner", None)
+        self.datacls = self.dataclass(**self.class_params)
 
     def __enter__(self):
         """Context manager entry"""
@@ -164,28 +165,24 @@ class DataWrapper:
     def _process_single_date(self, date: datetime.date):
         """Core date processing logic with load/save timing and thread reporting"""
         path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/"
-        self.logger.info(f"Processing date {date.isoformat()} for {path}")
-        # self.logger.info(f"Path {path} in {self.skipped}: {path in self.skipped}")
+        self.logger.debug(f"Processing date {date.isoformat()} for {path}")
         if path in self.update_planner.skipped and self.update_planner.ignore_missing:
             self.logger.info(f"Skipping {date} as it exists in the skipped list")
             return
         full_path = f"{path}{self.parquet_filename}"
 
         thread_name = threading.current_thread().name
-        self.logger.info(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
+        self.logger.debug(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
 
         overall_start = time.perf_counter()
         try:
             load_start = time.perf_counter()
-            with self.dataclass(**self.class_params) as data:
-                df = data.load_period(
-                    dt_field=self.date_field,
-                    start=date,
-                    end=date,
-                    **self.load_params
-                )
+            date_filter = {f"{self.date_field}__date": {date.isoformat()}}
+            self.logger.debug(f"Loading data for {date} with filter: {date_filter}")
+            # Load data using the dataclass with the provided date filter
+            self.load_params.update(date_filter)
+            df = self.datacls.load(**self.load_params)
             load_time = time.perf_counter() - load_start
-
             if df.head(1, compute=True).empty:
                 if self.mmanifest:
                     schema = df._meta.dtypes.astype(str).to_dict()

{sibi_dst-0.3.64 → sibi_dst-2025.1.2}/sibi_dst/utils/update_planner.py

@@ -73,6 +73,8 @@ class UpdatePlanner:
         self.show_progress = show_progress
         self.logger = logger or Logger.default_logger(logger_name="update_planner")
         self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
+        self.debug = debug
+        self.verbose = verbose
 
         # Filesystem and age helper
         self.fs = fs or fsspec.filesystem(filesystem_type, **(filesystem_options or {}))