sibi-dst 2025.1.7__tar.gz → 2025.1.9__tar.gz

{sibi_dst-2025.1.7 → sibi_dst-2025.1.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.7
+Version: 2025.1.9
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.1.7 → sibi_dst-2025.1.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "2025.1.7"
+version = "2025.1.9"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-2025.1.7 → sibi_dst-2025.1.9}/sibi_dst/utils/data_wrapper.py

@@ -1,16 +1,13 @@
 import datetime
-import logging
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Type, Any, Dict, Optional, Union, List, ClassVar
 
-import fsspec
 import pandas as pd
 from tqdm import tqdm
 
 from . import ManagedResource
-from .log_utils import Logger
 from .parquet_saver import ParquetSaver
 
 
@@ -60,8 +57,7 @@ class DataWrapper(ManagedResource):
         self.processed_dates: List[datetime.date] = []
         self.benchmarks: Dict[datetime.date, Dict[str, float]] = {}
         self.mmanifest = kwargs.get("mmanifest", None)
-        self.update_planner=kwargs.get("update_planner", None)
-
+        self.update_planner = kwargs.get("update_planner", None)
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Context manager exit"""
@@ -104,7 +100,6 @@ class DataWrapper(ManagedResource):
             if self.update_planner.show_progress:
                 self.show_benchmark_summary()
 
-
     def _execute_task_batch(self, priority: int, dates: List[datetime.date], max_retries: int):
         """Executes a single batch of tasks (dates) using a thread pool."""
         desc = f"Processing {self.dataclass.__name__}, priority: {priority}"
@@ -135,7 +130,7 @@ class DataWrapper(ManagedResource):
                     time.sleep(2 ** attempt)  # Exponential backoff
                 else:
                     self.logger.error(f"Failed processing {date} after {max_retries} attempts.")
-                    #raise
+                    # raise
 
     def _process_single_date(self, date: datetime.date):
         """Core date processing logic with load/save timing and thread reporting"""
@@ -146,8 +141,8 @@ class DataWrapper(ManagedResource):
             return
         full_path = f"{path}{self.parquet_filename}"
 
-        #thread_name = threading.current_thread().name
-        #self.logger.debug(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
+        # thread_name = threading.current_thread().name
+        # self.logger.debug(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
 
         overall_start = time.perf_counter()
         try:
@@ -159,35 +154,43 @@ class DataWrapper(ManagedResource):
             local_load_params = self.load_params.copy()
             local_load_params.update(date_filter)
             local_class_instance = self.dataclass(**self.class_params)
-            df=local_class_instance.load(**local_load_params)
+            df = local_class_instance.load(**local_load_params)
             load_time = time.perf_counter() - load_start
 
             if hasattr(local_class_instance, "total_records"):
-                self.logger.debug(f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
+                self.logger.debug(
+                    f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
                 if int(local_class_instance.total_records) == 0:  # If no records were loaded but not due to an error
                     if self.mmanifest:
                         self.mmanifest.record(
                             full_path=path
                         )
-                    self.logger.info(f"No data found for {date}. Logged to missing manifest.")
-                    return
-            save_start = time.perf_counter()
-            with ParquetSaver(
-                df_result=df,
-                parquet_storage_path=path,
-                fs=self.fs,
-                logger=self.logger
-            ) as ps:
-                ps.save_to_parquet(self.parquet_filename, overwrite=True)
-            save_time = time.perf_counter() - save_start
-
-            total_time = time.perf_counter() - overall_start
-            self.benchmarks[date] = {
-                "load_duration": load_time,
-                "save_duration": save_time,
-                "total_duration": total_time
-            }
-            self._log_success(date, total_time, full_path)
+                    self.logger.info(f"No data found for {full_path}. Logged to missing manifest.")
+                elif int(local_class_instance.total_records) < 0:
+                    self.logger.warning(
+                        f"Negative record count ({local_class_instance.total_records}) for {full_path}. "
+                        "This may indicate an error in the data loading process."
+                    )
+                else:
+                    save_start = time.perf_counter()
+                    parquet_params ={
+                        "df_result": df,
+                        "parquet_storage_path": path,
+                        "fs": self.fs,
+                        "logger": self.logger,
+                        "debug": self.debug,
+                    }
+                    with ParquetSaver(**parquet_params) as ps:
+                        ps.save_to_parquet(self.parquet_filename, overwrite=True)
+                    save_time = time.perf_counter() - save_start
+
+                    total_time = time.perf_counter() - overall_start
+                    self.benchmarks[date] = {
+                        "load_duration": load_time,
+                        "save_duration": save_time,
+                        "total_duration": total_time
+                    }
+                    self._log_success(date, total_time, full_path)
         except Exception as e:
             self._log_failure(date, e)
             raise

{sibi_dst-2025.1.7 → sibi_dst-2025.1.9}/sibi_dst/utils/parquet_saver.py

@@ -26,10 +26,6 @@ class ParquetSaver(ManagedResource):
         super().__init__(**kwargs)
         self.df_result = df_result
         self.parquet_storage_path = parquet_storage_path.rstrip("/")
-        #self.debug = debug
-        #self.logger = logger or Logger.default_logger(self.__class__.__name__)
-        #self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
-        #self.fs = fs
         # Determine protocol for special handling (e.g., 's3')
         if not self.fs:
             raise ValueError("File system (fs) must be provided to ParquetSaver.")

{sibi_dst-2025.1.7 → sibi_dst-2025.1.9}/sibi_dst/utils/update_planner.py

@@ -72,9 +72,6 @@ class UpdatePlanner(ManagedResource):
             data_path: str,
             filename: str,
             description: str = "Update Planner",
-            #fs: Optional[fsspec.AbstractFileSystem] = None,
-            #filesystem_type: str = "file",
-            #filesystem_options: Optional[Dict] = None,
             reference_date: Union[str, datetime.date] = None,
             history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
             max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
@@ -199,11 +196,12 @@ class UpdatePlanner(ManagedResource):
         for _, row in self.plan.iterrows():
             table.add_row(*(str(item) for item in row))
 
-        console.print(table)
+        console = Console()
+        with console.capture() as capture:
+            console.print(table)
+        plan_string = capture.get()
 
-        plan_string = console.export_text()
-
-        self.logger.info(f"Full Update Plan:\n{plan_string}")
+        self.logger.info(f"Full Update Plan:\n{plan_string.strip()}")
 
     def get_tasks_by_priority(self) -> Iterator[Tuple[int, List[datetime.date]]]:
         """Yields batches of dates to be processed, grouped and sorted by priority."""