sibi-dst 2025.1.7__py3-none-any.whl → 2025.1.8__py3-none-any.whl

sibi_dst/utils/data_wrapper.py

@@ -1,16 +1,13 @@
 import datetime
-import logging
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Type, Any, Dict, Optional, Union, List, ClassVar
 
-import fsspec
 import pandas as pd
 from tqdm import tqdm
 
 from . import ManagedResource
-from .log_utils import Logger
 from .parquet_saver import ParquetSaver
 
 
@@ -60,8 +57,7 @@ class DataWrapper(ManagedResource):
         self.processed_dates: List[datetime.date] = []
         self.benchmarks: Dict[datetime.date, Dict[str, float]] = {}
         self.mmanifest = kwargs.get("mmanifest", None)
-        self.update_planner=kwargs.get("update_planner", None)
-
+        self.update_planner = kwargs.get("update_planner", None)
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """Context manager exit"""
@@ -104,7 +100,6 @@ class DataWrapper(ManagedResource):
             if self.update_planner.show_progress:
                 self.show_benchmark_summary()
 
-
     def _execute_task_batch(self, priority: int, dates: List[datetime.date], max_retries: int):
         """Executes a single batch of tasks (dates) using a thread pool."""
         desc = f"Processing {self.dataclass.__name__}, priority: {priority}"
@@ -135,7 +130,7 @@ class DataWrapper(ManagedResource):
                     time.sleep(2 ** attempt)  # Exponential backoff
                 else:
                     self.logger.error(f"Failed processing {date} after {max_retries} attempts.")
-                    #raise
+                    # raise
 
     def _process_single_date(self, date: datetime.date):
         """Core date processing logic with load/save timing and thread reporting"""
@@ -146,8 +141,8 @@ class DataWrapper(ManagedResource):
             return
         full_path = f"{path}{self.parquet_filename}"
 
-        #thread_name = threading.current_thread().name
-        #self.logger.debug(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
+        # thread_name = threading.current_thread().name
+        # self.logger.debug(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
 
         overall_start = time.perf_counter()
         try:
@@ -159,35 +154,43 @@ class DataWrapper(ManagedResource):
             local_load_params = self.load_params.copy()
             local_load_params.update(date_filter)
             local_class_instance = self.dataclass(**self.class_params)
-            df=local_class_instance.load(**local_load_params)
+            df = local_class_instance.load(**local_load_params)
             load_time = time.perf_counter() - load_start
 
             if hasattr(local_class_instance, "total_records"):
-                self.logger.debug(f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
+                self.logger.debug(
+                    f"Total records loaded by {local_class_instance.__class__.__name__}: {local_class_instance.total_records}")
                 if int(local_class_instance.total_records) == 0:  # If no records were loaded but not due to an error
                     if self.mmanifest:
                         self.mmanifest.record(
                             full_path=path
                         )
-                    self.logger.info(f"No data found for {date}. Logged to missing manifest.")
-                    return
-            save_start = time.perf_counter()
-            with ParquetSaver(
-                df_result=df,
-                parquet_storage_path=path,
-                fs=self.fs,
-                logger=self.logger
-            ) as ps:
-                ps.save_to_parquet(self.parquet_filename, overwrite=True)
-            save_time = time.perf_counter() - save_start
-
-            total_time = time.perf_counter() - overall_start
-            self.benchmarks[date] = {
-                "load_duration": load_time,
-                "save_duration": save_time,
-                "total_duration": total_time
-            }
-            self._log_success(date, total_time, full_path)
+                    self.logger.info(f"No data found for {full_path}. Logged to missing manifest.")
+                elif int(local_class_instance.total_records) < 0:
+                    self.logger.warning(
+                        f"Negative record count ({local_class_instance.total_records}) for {full_path}. "
+                        "This may indicate an error in the data loading process."
+                    )
+                else:
+                    save_start = time.perf_counter()
+                    parquet_params ={
+                        "df_result": df,
+                        "parquet_storage_path": path,
+                        "fs": self.fs,
+                        "logger": self.logger,
+                        "debug": self.debug,
+                    }
+                    with ParquetSaver(**parquet_params) as ps:
+                        ps.save_to_parquet(self.parquet_filename, overwrite=True)
+                    save_time = time.perf_counter() - save_start
+
+                    total_time = time.perf_counter() - overall_start
+                    self.benchmarks[date] = {
+                        "load_duration": load_time,
+                        "save_duration": save_time,
+                        "total_duration": total_time
+                    }
+                    self._log_success(date, total_time, full_path)
         except Exception as e:
             self._log_failure(date, e)
             raise

sibi_dst/utils/parquet_saver.py

@@ -26,10 +26,6 @@ class ParquetSaver(ManagedResource):
         super().__init__(**kwargs)
         self.df_result = df_result
         self.parquet_storage_path = parquet_storage_path.rstrip("/")
-        #self.debug = debug
-        #self.logger = logger or Logger.default_logger(self.__class__.__name__)
-        #self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
-        #self.fs = fs
         # Determine protocol for special handling (e.g., 's3')
         if not self.fs:
             raise ValueError("File system (fs) must be provided to ParquetSaver.")

{sibi_dst-2025.1.7.dist-info → sibi_dst-2025.1.8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.7
+Version: 2025.1.8
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.1.7.dist-info → sibi_dst-2025.1.8.dist-info}/RECORD

@@ -38,14 +38,14 @@ sibi_dst/utils/clickhouse_writer.py,sha256=mNUJoYOreIdRrEFv2mQ6pdtLi1Iz_2rALDyO6
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=8lbKe85XJB0VhIR9GRo7-R_9DBhRZvNv8MsqSbsupeA,8945
+sibi_dst/utils/data_wrapper.py,sha256=deUz2760T_v42Ni1twLUcGS4ucIQM63vJnC6p8sWsb4,9470
 sibi_dst/utils/date_utils.py,sha256=8fwPpOYqSdM3nHeNykh7Ftk-uPdFa44cEAy5S8iUNw4,18667
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
 sibi_dst/utils/log_utils.py,sha256=_YnpCnMcjT--ou3BU0EGJma1xMULrA4V5v5UU4IbjAo,14102
 sibi_dst/utils/manifest_manager.py,sha256=Rw7i2phoKJjGlPHYLg_1kr40syVKxd9LJEmfxvZPeDg,8544
-sibi_dst/utils/parquet_saver.py,sha256=zau_s0Mn2ccz9ivVtRbTkBmCghUgCofI1LsCdy1dfDw,4943
+sibi_dst/utils/parquet_saver.py,sha256=B1ztPZMJvsulbgXMBnJdSkPhLFvdv8sRnmyqjjBBRTI,4735
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
 sibi_dst/utils/storage_config.py,sha256=uaCBF8rgCeYkk-lxVSCjsic8O8HJKAu455MR-OBliCo,4325
 sibi_dst/utils/storage_manager.py,sha256=yyZqT8XjTf4MKFrfznCmxXxOYz_TiWgtQhzqPoXR9So,6569
@@ -71,6 +71,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.1.7.dist-info/METADATA,sha256=AaJunhF_PdvxT9KpA0mVzQhoY1sAZoe5HTA9ClabYPs,2610
-sibi_dst-2025.1.7.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.1.7.dist-info/RECORD,,
+sibi_dst-2025.1.8.dist-info/METADATA,sha256=DcMnd1S5S6CGThkl1-ebn4HWkmGC0ujPHZYHMPrJHd0,2610
+sibi_dst-2025.1.8.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.1.8.dist-info/RECORD,,