sibi-dst 2025.1.5__tar.gz → 2025.1.6__tar.gz

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.5
+Version: 2025.1.6
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -13,6 +13,8 @@ Requires-Dist: clickhouse-connect (>=0.8.18,<0.9.0)
 Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
 Requires-Dist: dask[complete] (>=2025.5.1,<2026.0.0)
 Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
+Requires-Dist: opentelemetry-exporter-otlp (>=1.35.0,<2.0.0)
+Requires-Dist: opentelemetry-sdk (>=1.35.0,<2.0.0)
 Requires-Dist: pandas (>=2.3.1,<3.0.0)
 Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
 Requires-Dist: pyarrow (>=20.0.0,<21.0.0)

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "2025.1.5"
+version = "2025.1.6"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -22,6 +22,8 @@ sqlalchemy = "^2.0.41"
 pymysql = "^1.1.1"
 pyarrow = "^20.0.0"
 rich = "^14.0.0"
+opentelemetry-exporter-otlp = "^1.35.0"
+opentelemetry-sdk = "^1.35.0"
 
 [tool.poetry.group.dev]
 optional = true

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/df_helper/_artifact_updater_multi_wrapper.py

@@ -2,7 +2,7 @@ import datetime
 import time
 import random
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Any, Callable, Dict, List, Optional, Type
+from typing import Any, Callable, Dict, List, Optional, Type, ClassVar
 
 from sibi_dst.utils import  ManagedResource
 
@@ -14,7 +14,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
     This version is refactored for a pure multi-threaded environment, aligning
     the orchestration model with the underlying threaded workers (DataWrapper).
     """
-
+    wrapped_classes: Dict[str, List[Type]]
     def __init__(
             self,
             wrapped_classes: Dict[str, List[Type]],
@@ -26,7 +26,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
             backoff_jitter: float = 0.1,
             priority_fn: Optional[Callable[[Type], int]] = None,
             artifact_class_kwargs: Optional[Dict[str, Any]] = None,
-            **kwargs: Any,
+            **kwargs: Dict[str, Any]
     ) -> None:
         super().__init__(**kwargs)
         self.wrapped_classes = wrapped_classes
@@ -75,14 +75,14 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
             except Exception as e:
                 self.logger.warning(f"priority_fn error for {name}: {e}")
 
-        # Fallback to size estimate if available
-        if hasattr(artifact_cls, 'get_size_estimate'):
-            try:
-                # This performs blocking I/O
-                return artifact_cls(**self.artifact_class_kwargs).get_size_estimate()
-
-            except Exception as e:
-                self.logger.warning(f"get_size_estimate failed for {name}: {e}")
+        # # Fallback to size estimate if available
+        # if hasattr(artifact_cls, 'get_size_estimate'):
+        #     try:
+        #         # This performs blocking I/O
+        #         return artifact_cls(**self.artifact_class_kwargs).get_size_estimate()
+        #
+        #     except Exception as e:
+        #         self.logger.warning(f"get_size_estimate failed for {name}: {e}")
 
         # Default priority
         return 999
@@ -115,7 +115,7 @@ class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
         # If all retries fail, raise an exception to be caught by the main loop
         raise RuntimeError(f"{name} failed after {self.retry_attempts} attempts.")
 
-    def update_data(self, data_type: str, **kwargs: Any) -> None:
+    async def update_data(self, data_type: str, **kwargs: Any) -> None:
         """
         Entry point to update all artifacts of a given type using a ThreadPoolExecutor.
         """

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/df_helper/_parquet_artifact.py

@@ -1,8 +1,7 @@
 from __future__ import annotations
 import datetime
-import logging
 import threading
-from typing import Optional, Any, Dict
+from typing import Optional, Any, Dict, ClassVar
 
 import dask.dataframe as dd
 import fsspec
@@ -55,7 +54,7 @@ class ParquetArtifact(DfHelper):
     :ivar fs: Filesystem object used for storage operations.
     :type fs: fsspec.AbstractFileSystem
     """
-    DEFAULT_CONFIG = {
+    DEFAULT_CONFIG: ClassVar[Dict[str, str]] = {
         'backend': 'parquet'
     }
 
@@ -91,8 +90,6 @@ class ParquetArtifact(DfHelper):
         }
         self.df: Optional[dd.DataFrame] = None
         super().__init__(**self.config)
-        #self._own_logger = False
-        #self._setup_logging()
         self.data_wrapper_class = data_wrapper_class
 
         self.date_field = self._validate_required('date_field')
@@ -101,16 +98,6 @@ class ParquetArtifact(DfHelper):
         self.parquet_start_date = self._validate_required('parquet_start_date')
         self.parquet_end_date = self._validate_required('parquet_end_date')
 
-        # Filesystem setup
-        #self.filesystem_type = self.config.setdefault('filesystem_type', 'file')
-        #self.filesystem_options = self.config.setdefault('filesystem_options', {})
-        #self.fs = self.config.setdefault('fs', None)
-        #self._own_fs = self.fs is None
-        #if self.fs is None:
-        #    self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
-        #    self._own_fs = True
-        #self.config.setdefault('fs', self.fs)
-        ## Populate to parameters to pass to data_wrapper_class
         self.class_params = self.config.pop('class_params', {
             'debug': self.debug,
             'logger': self.logger,
@@ -125,15 +112,6 @@ class ParquetArtifact(DfHelper):
         self.update_planner_params = {}
         self.datawrapper_params = {}
 
-    #def _setup_logging(self):
-    #    """Initialize logger and debug settings."""
-    #    self.debug = self.config.get('debug', False)
-    #    logger = self.config.get('logger', None)
-    #    self._own_logger = logger is None
-    #    self.logger = logger or Logger.default_logger(
-    #                                      logger_name=f'Parquet_Artifact_InstanceOf_{self.__class__.__name__}')
-    #    self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
-
     def _validate_required(self, key: str) -> Any:
         """Validate required configuration fields."""
         value = self.config.setdefault(key, None)
@@ -211,28 +189,28 @@ class ParquetArtifact(DfHelper):
         """
         Synchronously estimates artifact size for use in multi-threaded environments.
 
-        This method uses the filesystem's own .sync() method to safely execute
-        asynchronous I/O operations from a synchronous context, preventing
-        event loop conflicts.
+        This method safely executes asynchronous I/O operations from a synchronous
+        context, handling variations in fsspec filesystem implementations.
         """
 
         async def _get_total_bytes_async():
             """A helper async coroutine to perform the I/O."""
             import asyncio
 
-            # Use the async versions of fsspec methods (e.g., _glob, _size)
             files = await self.fs._glob(f"{self.parquet_storage_path}/*.parquet")
             if not files:
                 return 0
 
-            # Concurrently gather the size of all files for performance
             size_tasks = [self.fs._size(f) for f in files]
             sizes = await asyncio.gather(*size_tasks)
             return sum(s for s in sizes if s is not None)
 
-        # Use the filesystem's own built-in sync method. This is the most
-        # reliable way to bridge the sync/async gap for fsspec.
-        total_bytes = self.fs.sync(_get_total_bytes_async())
+        try:
+            # Attempt the standard fsspec method first
+            total_bytes = self.fs.sync(_get_total_bytes_async())
+        except AttributeError:
+            #  fallback for filesystems like s3fs that lack .sync()
+            total_bytes = self.fs.loop.run_until_complete(_get_total_bytes_async())
 
         # Convert to megabytes, ensuring a minimum of 1
         return max(1, int(total_bytes / (1024 ** 2)))

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/df_helper/_parquet_reader.py

@@ -1,11 +1,10 @@
 import logging
-from typing import Optional
+from typing import Optional, ClassVar, Dict
 
 import dask.dataframe as dd
 import fsspec
 
 from sibi_dst.df_helper import DfHelper
-from sibi_dst.utils import Logger
 
 class ParquetReader(DfHelper):
     """
@@ -44,7 +43,7 @@ class ParquetReader(DfHelper):
         Parquet storage.
     :type fs: fsspec.AbstractFileSystem
     """
-    DEFAULT_CONFIG = {
+    DEFAULT_CONFIG: ClassVar[Dict[str, int]] = {
         'backend': 'parquet'
     }
 

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -85,7 +85,8 @@ class ParquetConfig(BaseModel):
         if self.logger is None:
             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
         self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
-
+        if self.fs is None:
+            raise ValueError('Parquet Options: File system (fs) must be specified')
 
         if self.parquet_storage_path is None:
             raise ValueError('Parquet storage path must be specified')

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/utils/base.py

@@ -1,3 +1,4 @@
+import asyncio
 from .log_utils import Logger
 
 class ManagedResource:
@@ -61,7 +62,7 @@ class ManagedResource:
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         """Exit the runtime context and trigger cleanup for 'async with' statements."""
-        self.cleanup()
+        await self.acleanup()
         return False  # Propagate exceptions
 
     def __repr__(self) -> str:
@@ -80,7 +81,7 @@ class ManagedResource:
 
     def cleanup(self):
         """
-        Clean up resources managed by this instance.
+        Cleanup resources managed by this instance.
         """
         if self._own_fs and hasattr(self.fs, "clear_instance_cache"):
             if self.logger:
@@ -88,10 +89,29 @@ class ManagedResource:
             self.fs.clear_instance_cache()
 
         if self._own_logger and hasattr(self.logger, "shutdown"):
-            # Ensure logger exists before trying to use or shut it down
+            # Ensure the logger exists before trying to use or shut it down
             if self.logger:
                 self.logger.debug(f"'{self.__class__.__name__}' is shutting down its own logger.")
                 self.logger.shutdown()
             self.logger = None  # Set to None after shutdown
 
         self._entered = False
+
+    async def acleanup(self):
+        """
+        Async Cleanup resources managed by this instance.
+        """
+        if self._own_fs and hasattr(self.fs, "clear_instance_cache"):
+            if self.logger:
+                self.logger.debug(f"'{self.__class__.__name__}' is clearing its own filesystem cache.")
+            self.fs.clear_instance_cache()
+
+        if self._own_logger and hasattr(self.logger, "shutdown"):
+            # Ensure the logger exists before trying to use or shut it down
+            if self.logger:
+                self.logger.debug(f"'{self.__class__.__name__}' is shutting down its own logger.")
+                self.logger.shutdown()
+            self.logger = None  # Set to None after shutdown
+
+        self._entered = False
+

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/utils/clickhouse_writer.py

@@ -1,4 +1,5 @@
 from concurrent.futures import ThreadPoolExecutor
+from typing import ClassVar, Dict
 
 import clickhouse_connect
 import pandas as pd
@@ -36,7 +37,7 @@ class ClickHouseWriter(ManagedResource):
     :ivar order_by: Field or column name to use for table ordering.
     :type order_by: str
     """
-    dtype_to_clickhouse = {
+    dtype_to_clickhouse:  ClassVar[Dict[str, str]] = {
         'int64': 'Int64',
         'int32': 'Int32',
         'float64': 'Float64',

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/utils/data_wrapper.py

@@ -3,7 +3,7 @@ import logging
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Type, Any, Dict, Optional, Union, List
+from typing import Type, Any, Dict, Optional, Union, List, ClassVar
 
 import fsspec
 import pandas as pd
@@ -15,15 +15,15 @@ from .parquet_saver import ParquetSaver
 
 
 class DataWrapper(ManagedResource):
-    DEFAULT_PRIORITY_MAP = {
+    DEFAULT_PRIORITY_MAP: ClassVar[Dict[str, int]] = {
         "overwrite": 1,
         "missing_in_history": 2,
         "existing_but_stale": 3,
         "missing_outside_history": 4,
         "file_is_recent": 0
     }
-    DEFAULT_MAX_AGE_MINUTES = 1440
-    DEFAULT_HISTORY_DAYS_THRESHOLD = 30
+    DEFAULT_MAX_AGE_MINUTES: int = 1440
+    DEFAULT_HISTORY_DAYS_THRESHOLD: int = 30
 
     def __init__(
             self,
@@ -31,12 +31,8 @@ class DataWrapper(ManagedResource):
             date_field: str,
             data_path: str,
             parquet_filename: str,
-            #fs: Optional[fsspec.AbstractFileSystem] = None,
-            #debug: bool = False,
-            #verbose: bool = False,
             class_params: Optional[Dict] = None,
             load_params: Optional[Dict] = None,
-            #logger: Logger = None,
             show_progress: bool = False,
             timeout: float = 30,
             max_threads: int = 3,
@@ -47,14 +43,8 @@ class DataWrapper(ManagedResource):
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
         self.parquet_filename = parquet_filename
-        #self.fs = fs or None
         if self.fs is None:
             raise ValueError("Datawrapper requires a File system (fs) to be provided .")
-        #self.debug = debug
-        #self.verbose = verbose
-        #self._own_logger = logger is None
-        #self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
-        #self.logger.set_level(logging.DEBUG if debug else logging.INFO)
         self.show_progress = show_progress
         self.timeout = timeout
         self.max_threads = max_threads
@@ -71,7 +61,6 @@ class DataWrapper(ManagedResource):
         self.benchmarks: Dict[datetime.date, Dict[str, float]] = {}
         self.mmanifest = kwargs.get("mmanifest", None)
         self.update_planner=kwargs.get("update_planner", None)
-        # self.datacls = self.dataclass(**self.class_params)
 
 
     def __exit__(self, exc_type, exc_val, exc_tb):
@@ -174,7 +163,7 @@ class DataWrapper(ManagedResource):
             load_time = time.perf_counter() - load_start
 
             if hasattr(local_class_instance, "total_records"):
-                self.logger.debug(f"Total records loaded by {local_class_instance}: {local_class_instance.total_records}")
+                self.logger.debug(f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
                 if int(local_class_instance.total_records) == 0:  # If no records were loaded but not due to an error
                     if self.mmanifest:
                         self.mmanifest.record(
@@ -183,12 +172,13 @@ class DataWrapper(ManagedResource):
                     self.logger.info(f"No data found for {date}. Logged to missing manifest.")
                     return
             save_start = time.perf_counter()
-            ParquetSaver(
+            with ParquetSaver(
                 df_result=df,
                 parquet_storage_path=path,
                 fs=self.fs,
                 logger=self.logger
-            ).save_to_parquet(self.parquet_filename, overwrite=True)
+            ) as ps:
+                ps.save_to_parquet(self.parquet_filename, overwrite=True)
             save_time = time.perf_counter() - save_start
 
             total_time = time.perf_counter() - overall_start
@@ -218,4 +208,4 @@ class DataWrapper(ManagedResource):
             return
         df_bench = pd.DataFrame.from_records([{"date": d, **m} for d, m in self.benchmarks.items()])
         df_bench = df_bench.set_index("date").sort_index(ascending=not self.update_planner.reverse_order)
-        self.logger.info("Benchmark Summary:\n" + df_bench.to_string())
+        self.logger.info(f"Benchmark Summary:\n {self.dataclass.__name__}\n" + df_bench.to_string())

{sibi_dst-2025.1.5 → sibi_dst-2025.1.6}/sibi_dst/utils/update_planner.py

@@ -1,6 +1,6 @@
 import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import List, Optional, Dict, Union, Tuple, Set, Iterator
+from typing import List, Optional, Dict, Union, Tuple, Set, Iterator, ClassVar
 import pandas as pd
 from .date_utils import FileAgeChecker
 from pydantic import BaseModel, Field
@@ -55,7 +55,7 @@ class UpdatePlanner(ManagedResource):
         generate_plan() will overwrite self.plan and self.df_req, and returns a DataFrame of required updates.
     """
 
-    DEFAULT_PRIORITY_MAP = {
+    DEFAULT_PRIORITY_MAP: ClassVar[Dict[str, int]]={
         "file_is_recent": 0,
         "missing_ignored": 0,
         "overwrite_forced": 1,
@@ -64,8 +64,8 @@ class UpdatePlanner(ManagedResource):
         "stale_in_history": 4,
     }
 
-    DEFAULT_MAX_AGE_MINUTES = 1440
-    DEFAULT_HISTORY_DAYS_THRESHOLD = 30
+    DEFAULT_MAX_AGE_MINUTES: int = 1440
+    DEFAULT_HISTORY_DAYS_THRESHOLD: int = 30
 
     def __init__(
             self,
@@ -217,7 +217,7 @@ class UpdatePlanner(ManagedResource):
         for priority in sorted(required_updates["update_priority"].unique()):
             dates_df = required_updates[required_updates["update_priority"] == priority]
             # Sort dates within the priority group
-            sorted_dates = dates_df.sort_values(by="date", ascending=not self.reverse_order)
+            sorted_dates = dates_df.sort_values(by=["date"], ascending=not self.reverse_order)
             dates = sorted_dates["date"].tolist()
             if dates:
                 yield priority, dates