sibi-dst 0.3.58__py3-none-any.whl → 0.3.59__py3-none-any.whl

sibi_dst/utils/data_wrapper.py

@@ -1,17 +1,16 @@
 import datetime
 import logging
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Type, Any, Dict, Optional, Union, List, Tuple
 import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Type, Any, Dict, Optional, Union, List
+
 import fsspec
 import pandas as pd
-from IPython.display import display
 from tqdm import tqdm
 
 from .log_utils import Logger
-from .date_utils import FileAgeChecker
 from .parquet_saver import ParquetSaver
-from .update_planner import UpdatePlanner
 
 
 class DataWrapper:
@@ -25,77 +24,80 @@ class DataWrapper:
     DEFAULT_MAX_AGE_MINUTES = 1440
     DEFAULT_HISTORY_DAYS_THRESHOLD = 30
 
-    def __init__(self,
-                 dataclass: Type,
-                 date_field: str,
-                 data_path: str,
-                 parquet_filename: str,
-                 start_date: Any,
-                 end_date: Any,
-                 fs: Optional[fsspec.AbstractFileSystem] = None,
-                 filesystem_type: str = "file",
-                 filesystem_options: Optional[Dict] = None,
-                 debug: bool = False,
-                 verbose: bool = False,
-                 class_params: Optional[Dict] = None,
-                 load_params: Optional[Dict] = None,
-                 reverse_order: bool = False,
-                 overwrite: bool = False,
-                 ignore_missing: bool = False,
-                 logger: Logger = None,
-                 max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
-                 history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
-                 show_progress: bool = False,
-                 timeout: float = 60,
-                 reference_date: datetime.date = None,
-                 custom_priority_map: Dict[str, int] = None,
-                 max_threads: int = 3):
+    def __init__(
+            self,
+            dataclass: Type,
+            date_field: str,
+            data_path: str,
+            parquet_filename: str,
+            #start_date: Any,
+            #end_date: Any,
+            fs: Optional[fsspec.AbstractFileSystem] = None,
+            #filesystem_type: str = "file",
+            #filesystem_options: Optional[Dict] = None,
+            debug: bool = False,
+            verbose: bool = False,
+            class_params: Optional[Dict] = None,
+            load_params: Optional[Dict] = None,
+            #reverse_order: bool = False,
+            #overwrite: bool = False,
+            #ignore_missing: bool = False,
+            logger: Logger = None,
+            #max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
+            #history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
+            show_progress: bool = False,
+            timeout: float = 60,
+            #reference_date: datetime.date = None,
+            #custom_priority_map: Dict[str, int] = None,
+            max_threads: int = 3,
+            **kwargs: Any,
+    ):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
         self.parquet_filename = parquet_filename
-        self.filesystem_type = filesystem_type
-        self.filesystem_options = filesystem_options or {}
-        self.fs = fs or self._init_filesystem()
+        #self.filesystem_type = filesystem_type
+        #self.filesystem_options = filesystem_options or {}
+        self.fs = fs or None
         self.debug = debug
         self.verbose = verbose
-        self.class_params = class_params or {}
-        self.load_params = load_params or {}
-        self.reverse_order = reverse_order
-        self.overwrite = overwrite
-        self.ignore_missing = ignore_missing
+        # self.reverse_order = reverse_order
+        # self.overwrite = overwrite
+        # self.ignore_missing = ignore_missing
         self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
         self.logger.set_level(logging.DEBUG if debug else logging.INFO)
-        self.max_age_minutes = max_age_minutes
-        self.history_days_threshold = history_days_threshold
+        # self.max_age_minutes = max_age_minutes
+        # self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
         self.timeout = timeout
-        self.reference_date = reference_date or datetime.date.today()
-        self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
+        #self.reference_date = reference_date or datetime.date.today()
+        #self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
         self.max_threads = max_threads
-
-        self.start_date = self._convert_to_date(start_date)
-        self.end_date = self._convert_to_date(end_date)
-        self._lock = threading.Lock()
-        self.processed_dates = []
-        self.age_checker = FileAgeChecker(logger=self.logger)
-
-        self.update_planner_params = {
-            "data_path": self.data_path,
-            "filename": self.parquet_filename,
-            "fs": self.fs,
-            "debug": self.debug,
-            "logger": self.logger,
-            "reverse_order": self.reverse_order,
-            "overwrite": self.overwrite,
-            "ignore_missing": self.ignore_missing,
-            "history_days_threshold": history_days_threshold,
-            "max_age_minutes": max_age_minutes,
-            "show_progress": self.show_progress,
-            "description": f"{self.dataclass.__name__}"
+        self.class_params = class_params or {
+            'debug': self.debug,
+            'logger': self.logger,
+            'fs': self.fs,
+            'verbose': self.verbose,
         }
-        self.update_plan = UpdatePlanner(**self.update_planner_params).generate_plan(self.start_date, self.end_date)
+        self.load_params = load_params or {}
 
+        self._lock = threading.Lock()
+        self.processed_dates: List[datetime.date] = []
+        self.benchmarks: Dict[datetime.date, Dict[str, float]] = {}
+        self.mmanifest = kwargs.get("mmanifest", None)
+        self.update_planner=kwargs.get("update_planner", None)
+
+    def __enter__(self):
+        """Context manager entry"""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit"""
+        if self.mmanifest and self.mmanifest._new_records:
+            self.mmanifest.save()
+        if exc_type is not None:
+            self.logger.error(f"Exception occurred: {exc_val}")
+        return False
 
     def _init_filesystem(self) -> fsspec.AbstractFileSystem:
         with self._lock:
@@ -114,100 +116,86 @@ class DataWrapper:
     def _ensure_forward_slash(path: str) -> str:
         return path.rstrip('/') + '/'
 
-    def generate_date_range(self) -> List[datetime.date]:
-        """Generate ordered date range with future date handling"""
-        date_range = pd.date_range(
-            start=self.start_date,
-            end=self.end_date,
-            freq='D'
-        ).date.tolist()
-
-        if self.reverse_order:
-            date_range.reverse()
-
-        return [
-            d for d in date_range
-            if d <= self.reference_date or self.overwrite
-        ]
-
     def process(self, max_retries: int = 3):
-        """Process updates with priority-based execution and retries"""
-        #update_plan = self.generate_update_plan()
-        update_plan = self.update_plan
-        if update_plan.empty:
+        """Process updates with priority-based execution, retries, benchmarking and progress updates"""
+        overall_start = time.perf_counter()
+        plan = self.update_planner.plan
+        # Use len(plan.index) instead of plan.empty for Dask compatibility
+        plan_count = len(plan.index)
+        if plan_count == 0:
             self.logger.info("No updates required")
             return
-        # Filter for required updates first
-        #update_plan = update_plan[update_plan["update_required"] == True]
-
-        if self.show_progress:
-            #display(self._enhanced_display_table(update_plan))
-            display(update_plan)
-
-        for priority in sorted(update_plan["update_priority"].unique()):
-            self._process_priority_group(update_plan, priority, max_retries)
-
-    def _process_priority_group(self,
-                                update_plan: pd.DataFrame,
-                                priority: int,
-                                max_retries: int):
-        """Process a single priority group with parallel execution"""
-        dates = update_plan[update_plan["update_priority"] == priority]["date"].tolist()
+        self.logger.info(f"Update plan for {self.dataclass.__name__} includes {plan_count} items for update")
+
+        if self.verbose:
+            self.update_planner.show_update_plan()
+
+        for priority in sorted(plan["update_priority"].unique()):
+            self._process_priority_group(plan, priority, max_retries)
+
+        total_time = time.perf_counter() - overall_start
+        processed = len(self.processed_dates)
+        if processed:
+            self.logger.info(
+                f"Processed {processed} dates in {total_time:.1f}s "
+                f"(avg {total_time / processed:.1f}s per date)"
+            )
+            if self.show_progress or self.verbose:
+                self.show_benchmark_summary()
+
+    def _process_priority_group(
+            self,
+            plan: pd.DataFrame,
+            priority: int,
+            max_retries: int
+    ):
+        """Process a single priority group with parallel execution and timing"""
+        dates = plan[plan["update_priority"] == priority]["date"].tolist()
         if not dates:
             return
-
-        desc = f"Processing {self.dataclass.__name__}, task: {self._priority_label(priority)}"
+        desc = f"Processing {self.dataclass.__name__}, priority: {priority}"
         self.logger.debug(f"Starting {desc.lower()}")
-        max_threads = min(len(dates), self.max_threads)
-        self.logger.debug(f"DataWrapper Max threads set at: {max_threads}")
-        with ThreadPoolExecutor(max_workers=max_threads) as executor:
-            futures = {
-                executor.submit(self._process_date_with_retry, date, max_retries): date
-                for date in dates
-            }
-
-            for future in tqdm(as_completed(futures),
-                               total=len(futures),
-                               desc=desc,
-                               disable=not self.show_progress):
+        group_start = time.perf_counter()
+        max_thr = min(len(dates), self.max_threads)
+        self.logger.debug(f"Max threads for priority {priority}: {max_thr}")
+        with ThreadPoolExecutor(max_workers=max_thr) as executor:
+            futures = {executor.submit(self._process_date_with_retry, date, max_retries): date for date in dates}
+            for future in tqdm(as_completed(futures), total=len(futures), desc=desc, disable=not self.show_progress):
                 date = futures[future]
                 try:
                     future.result(timeout=self.timeout)
                 except Exception as e:
-                    self.logger.error(f"Permanent failure processing {date}: {str(e)}")
-
-    def _priority_label(self, priority: int) -> str:
-        """Get human-readable label for priority level"""
-        return next(
-            (k for k, v in self.priority_map.items() if v == priority),
-            f"Unknown Priority {priority}"
-        )
+                    self.logger.error(f"Permanent failure processing {date}: {e}")
+        group_time = time.perf_counter() - group_start
+        self.logger.info(f"Priority {priority} group processed {len(dates)} dates in {group_time:.1f}s")
 
     def _process_date_with_retry(self, date: datetime.date, max_retries: int):
-        """Process a date with retry logic"""
         for attempt in range(1, max_retries + 1):
             try:
                 self._process_single_date(date)
                 return
             except Exception as e:
                 if attempt < max_retries:
-                    self.logger.warning(f"Retry {attempt}/{max_retries} for {date}: {str(e)}")
+                    self.logger.warning(f"Retry {attempt}/{max_retries} for {date}: {e}")
                 else:
                     raise RuntimeError(f"Failed processing {date} after {max_retries} attempts") from e
 
     def _process_single_date(self, date: datetime.date):
-        """Core date processing logic"""
+        """Core date processing logic with load/save timing and thread reporting"""
         path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/"
+        self.logger.info(f"Processing date {date.isoformat()} for {path}")
+        # self.logger.info(f"Path {path} in {self.skipped}: {path in self.skipped}")
+        #if path in self.skipped:
+        #    self.logger.info(f"Skipping {date} as it exists in the skipped list")
+        #    return
         full_path = f"{path}{self.parquet_filename}"
 
-        self.logger.info(f"Processing {date} ({full_path})")
-        start_time = datetime.datetime.now()
+        thread_name = threading.current_thread().name
+        self.logger.info(f"[{thread_name}] Executing date: {date} -> saving to: {full_path}")
 
+        overall_start = time.perf_counter()
         try:
-            self.logger.debug(f"Class Params: {self.class_params}")
-            self.logger.debug(f"Load Params: {self.load_params}")
-
-            df = pd.DataFrame()
+            load_start = time.perf_counter()
             with self.dataclass(**self.class_params) as data:
                 df = data.load_period(
                     dt_field=self.date_field,
@@ -215,11 +203,22 @@ class DataWrapper:
                     end=date,
                     **self.load_params
                 )
-
-            if len(df.index)==0:
-                self.logger.warning(f"No data found for {date}")
+            load_time = time.perf_counter() - load_start
+
+            if df.head(1, compute=True).empty:
+                if self.mmanifest:
+                    schema = df._meta.dtypes.astype(str).to_dict()
+                    self.mmanifest.record(
+                        full_path=path
+                    )
+                self.logger.info(f"No data found for {date}. Logged to missing manifest.")
                 return
+            # Dask-compatible empty check
+            # if len(df.index) == 0:
+            #    self.logger.warning(f"No data found for {date}")
+            #    return
 
+            save_start = time.perf_counter()
             with self._lock:
                 ParquetSaver(
                     df_result=df,
@@ -227,23 +226,33 @@ class DataWrapper:
                     fs=self.fs,
                     logger=self.logger
                 ).save_to_parquet(self.parquet_filename)
+            save_time = time.perf_counter() - save_start
 
-            duration = (datetime.datetime.now() - start_time).total_seconds()
-            self._log_success(date, duration, full_path)
-
+            total_time = time.perf_counter() - overall_start
+            self.benchmarks[date] = {
+                "load_duration": load_time,
+                "save_duration": save_time,
+                "total_duration": total_time
+            }
+            self._log_success(date, total_time, full_path)
         except Exception as e:
             self._log_failure(date, e)
             raise
 
     def _log_success(self, date: datetime.date, duration: float, path: str):
-        """Handle successful processing logging"""
         msg = f"Completed {date} in {duration:.1f}s | Saved to {path}"
         self.logger.info(msg)
         self.processed_dates.append(date)
 
     def _log_failure(self, date: datetime.date, error: Exception):
-        """Handle error logging"""
-        msg = f"Failed processing {date}: {str(error)}"
+        msg = f"Failed processing {date}: {error}"
         self.logger.error(msg)
 
-
+    def show_benchmark_summary(self):
+        """Display a summary of load/save timings per date"""
+        if not self.benchmarks:
+            self.logger.info("No benchmarking data to show")
+            return
+        df_bench = pd.DataFrame.from_records([{"date": d, **m} for d, m in self.benchmarks.items()])
+        df_bench = df_bench.set_index("date").sort_index(ascending=not self.reverse_order)
+        self.logger.info("Benchmark Summary:\n" + df_bench.to_string())

sibi_dst/utils/date_utils.py

@@ -149,9 +149,9 @@ class DateUtils:
 
 
 class FileAgeChecker:
-    def __init__(self, logger=None):
+    def __init__(self, debug=False, logger=None):
         self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-
+        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
     def is_file_older_than(
             self,
             file_path: str,
@@ -171,15 +171,15 @@ class FileAgeChecker:
         :return: True if older than max_age_minutes, False otherwise.
         """
         fs = fs or fsspec.filesystem("file")
-        self.logger.info(f"Checking age for {file_path}...")
+        self.logger.debug(f"Checking age for {file_path}...")
 
         try:
             if not fs.exists(file_path):
-                self.logger.info(f"Path not found: {file_path}.")
+                self.logger.debug(f"Path not found: {file_path}.")
                 return not ignore_missing
 
             if fs.isdir(file_path):
-                self.logger.info(f"Found directory: {file_path}")
+                self.logger.debug(f"Found directory: {file_path}")
                 age = self._get_directory_age_minutes(file_path, fs, verbose)
             elif fs.isfile(file_path):
                 age = self._get_file_age_minutes(file_path, fs, verbose)
@@ -208,7 +208,7 @@ class FileAgeChecker:
         fs = fs or fsspec.filesystem("file")
         try:
             if not fs.exists(file_path):
-                self.logger.info(f"Path not found: {file_path}")
+                self.logger.debug(f"Path not found: {file_path}")
                 return float("inf")
 
             if fs.isdir(file_path):
@@ -237,7 +237,7 @@ class FileAgeChecker:
             return float("inf")
 
         if not all_files:
-            self.logger.info(f"Empty directory: {dir_path}")
+            self.logger.debug(f"Empty directory: {dir_path}")
             return float("inf")
 
         modification_times = []
@@ -255,7 +255,7 @@ class FileAgeChecker:
 
         oldest = min(modification_times)
         age = (datetime.datetime.now(datetime.timezone.utc) - oldest).total_seconds() / 60
-        self.logger.info(f"Oldest in {dir_path}: {age:.2f} minutes")
+        self.logger.debug(f"Oldest in {dir_path}: {age:.2f} minutes")
 
         return age
 

sibi_dst/utils/log_utils.py

@@ -77,7 +77,7 @@ class Logger:
         formatter.converter = time.localtime  # << Set local time explicitly
 
         # Create a file handler
-        file_handler = logging.FileHandler(log_file_path)
+        file_handler = logging.FileHandler(log_file_path, delay=True)
         file_handler.setFormatter(formatter)
         self.logger.addHandler(file_handler)
 

sibi_dst/utils/manifest_manager.py

@@ -0,0 +1,154 @@
+import pandas as pd
+import fsspec
+import threading
+import uuid
+from typing import List, Optional, Set, Dict, Any
+
+from sibi_dst.utils import Logger
+
+
+class MissingManifestManager:
+    """
+    Thread-safe manager for a “missing-partitions” manifest (Parquet file).
+    """
+
+    def __init__(
+        self,
+        fs: fsspec.AbstractFileSystem,
+        manifest_path: str,
+        clear_existing: bool = False,
+        **kwargs: Any,
+    ):
+        self.fs = fs
+        self.manifest_path = manifest_path.rstrip("/")
+        self.clear_existing = clear_existing
+
+        self.debug: bool = kwargs.get("debug", False)
+        self.logger = kwargs.get(
+            "logger",
+            Logger.default_logger(logger_name="missing_manifest_manager")
+        )
+        self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
+
+        # In-memory list for new paths
+        self._new_records: List[Dict[str, str]] = []
+        # Cached set of existing paths
+        self._loaded_paths: Optional[Set[str]] = None
+
+        # Use a reentrant lock so save() can call load_existing() safely
+        self._lock = threading.RLock()
+
+    def _safe_exists(self, path: str) -> bool:
+        try:
+            return self.fs.exists(path)
+        except PermissionError:
+            if self.debug:
+                self.logger.debug(f"Permission denied checking existence of '{path}'")
+            return False
+        except Exception as e:
+            self.logger.warning(f"Error checking existence of '{path}': {e}")
+            return False
+
+    def load_existing(self) -> Set[str]:
+        """
+        Load and cache existing manifest paths.
+        """
+        with self._lock:
+            if self._loaded_paths is not None:
+                return self._loaded_paths
+
+            if not self._safe_exists(self.manifest_path):
+                self._loaded_paths = set()
+                return self._loaded_paths
+
+            try:
+                df = pd.read_parquet(self.manifest_path, filesystem=self.fs)
+                paths = (
+                    df.get("path", pd.Series(dtype=str))
+                      .dropna().astype(str)
+                      .loc[lambda s: s.str.strip().astype(bool)]
+                )
+                self._loaded_paths = set(paths.tolist())
+            except Exception as e:
+                self.logger.warning(f"Failed to load manifest '{self.manifest_path}': {e}")
+                self._loaded_paths = set()
+
+            return self._loaded_paths
+
+    def record(self, full_path: str) -> None:
+        """
+        Register a missing file path.
+        """
+        if not full_path or not isinstance(full_path, str):
+            return
+        with self._lock:
+            self._new_records.append({"path": full_path})
+
+    def save(self) -> None:
+        """
+        Merge new records into the manifest and write it out atomically.
+        """
+        with self._lock:
+            # Build DataFrame of new entries
+            new_df = pd.DataFrame(self._new_records)
+            should_overwrite = self.clear_existing or not self._safe_exists(self.manifest_path)
+            if new_df.empty and not should_overwrite:
+                return
+
+            # Clean new_df
+            new_df = (
+                new_df.get("path", pd.Series(dtype=str))
+                      .dropna().astype(str)
+                      .loc[lambda s: s.str.strip().astype(bool)]
+                      .to_frame()
+            )
+
+            # Merge or overwrite
+            if should_overwrite:
+                out_df = new_df
+            else:
+                try:
+                    old_df = pd.read_parquet(self.manifest_path, filesystem=self.fs)
+                    old_paths = (
+                        old_df.get("path", pd.Series(dtype=str))
+                              .dropna().astype(str)
+                              .loc[lambda s: s.str.strip().astype(bool)]
+                              .to_frame()
+                    )
+                    out_df = pd.concat([old_paths, new_df], ignore_index=True)
+                except Exception as e:
+                    self.logger.warning(f"Could not merge manifest, overwriting: {e}")
+                    out_df = new_df
+
+            out_df = out_df.drop_duplicates(subset=["path"]).reset_index(drop=True)
+
+            # Ensure parent dir
+            parent = self.manifest_path.rsplit("/", 1)[0]
+            try:
+                self.fs.makedirs(parent, exist_ok=True)
+            except Exception as e:
+                self.logger.warning(f"Could not create manifest directory '{parent}': {e}")
+
+            # Write atomically: temp file + rename
+            temp_path = f"{self.manifest_path}.tmp-{uuid.uuid4().hex}"
+            try:
+                out_df.to_parquet(
+                    temp_path,
+                    filesystem=self.fs,
+                    index=False
+                )
+                # rename into place (atomic in most filesystems)
+                self.fs.mv(temp_path, self.manifest_path, recursive=False)
+            except Exception as e:
+                self.logger.error(f"Failed to write or rename manifest: {e}")
+                # Clean up temp if it exists
+                try:
+                    if self.fs.exists(temp_path):
+                        self.fs.rm(temp_path, recursive=True)
+                except Exception:
+                    pass
+                raise
+
+            # Reset memory & cache
+            self._new_records.clear()
+            self._loaded_paths = set(out_df["path"].tolist())