sibi-dst 0.3.21__tar.gz → 0.3.22__tar.gz

{sibi_dst-0.3.21 → sibi_dst-0.3.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.21
+Version: 0.3.22
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.21 → sibi_dst-0.3.22}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.21"
+version = "0.3.22"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.21 → sibi_dst-0.3.22}/sibi_dst/utils/_data_wrapper.py

@@ -1,4 +1,5 @@
 import datetime
+from concurrent.futures import ThreadPoolExecutor
 from typing import Type, Any, Dict, Optional
 
 import fsspec
@@ -32,7 +33,8 @@ class DataWrapper:
                  logger: Optional[Logger] = None,
                  max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
                  history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
-                 show_progress: bool = False):
+                 show_progress: bool = False,
+                 timeout: Optional[int] = 300):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self.ensure_forward_slash(data_path)
@@ -50,6 +52,7 @@ class DataWrapper:
         self.max_age_minutes = max_age_minutes
         self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
+        self.timeout = timeout
 
         self.start_date = self.convert_to_date(start_date)
         self.end_date = self.convert_to_date(end_date)
@@ -76,31 +79,79 @@ class DataWrapper:
             yield date.date()
 
     def process(self):
-        """Execute the update plan following the specified hierarchy."""
+        """Execute the update plan using 'update_priority' to determine processing order."""
         update_plan_table = self.generate_update_plan_with_conditions()
 
-        # Display the update plan table to the user if show_progress is True
+        # Display the update plan table to the user if requested
         if self.show_progress:
             display(update_plan_table)
 
-        # Process files according to the hierarchy, considering only `update_required` dates
-        for category, description in [
-            ("overwrite", "Processing files due to overwrite=True"),
-            ("history_days", "Processing files within history_days_threshold"),
-            ("missing_files", "Processing missing files")
-        ]:
-            # Filter dates in the category where `update_required` is True
+        # Filter out rows that do not require updates (priority 0 means skip)
+        update_plan_table = update_plan_table[
+            (update_plan_table["update_required"] == True) & (update_plan_table["update_priority"] != 0)
+            ]
+
+        # Group by priority
+        priorities = sorted(update_plan_table["update_priority"].unique())
+
+        # We will process each priority level in its own thread.
+        # Each thread will handle all dates associated with that priority.
+        def process_priority(priority):
+            # Extract dates for the current priority
             dates_to_process = update_plan_table[
-                (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+                update_plan_table["update_priority"] == priority
                 ]["date"].tolist()
 
+            # If show_progress is True, wrap in a progress bar
             date_iterator = dates_to_process
             if self.show_progress:
-                date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
+                date_iterator = tqdm(date_iterator, desc=f"Processing priority {priority}:{self.dataclass.__name__}",
+                                     unit="date")
 
+            # Process each date for this priority
             for current_date in date_iterator:
                 self.process_date(current_date)
 
+        # Launch a separate thread for each priority
+        with ThreadPoolExecutor(max_workers=len(priorities)) as executor:
+            futures = {executor.submit(process_priority, p): p for p in priorities}
+            for future in futures:
+                try:
+                    future.result(timeout=self.timeout)
+                except TimeoutError:
+                    self.logger.error(f"Thread for {self.dataclass.__name__} timed out. Thread cancelled.")
+                    future.cancel()
+                    priority = futures[future]
+                    new_future = executor.submit(process_priority, priority)
+                    futures[new_future] = priority
+                    self.logger.info(f"Resubmitted task for priority {priority} after timeout.")
+
+    # def process(self):
+    #     """Execute the update plan following the specified hierarchy."""
+    #     update_plan_table = self.generate_update_plan_with_conditions()
+    #
+    #     # Display the update plan table to the user if show_progress is True
+    #     if self.show_progress:
+    #         display(update_plan_table)
+    #
+    #     # Process files according to the hierarchy, considering only `update_required` dates
+    #     for category, description in [
+    #         ("overwrite", "Processing files due to overwrite=True"),
+    #         ("history_days", "Processing files within history_days_threshold"),
+    #         ("missing_files", "Processing missing files")
+    #     ]:
+    #         # Filter dates in the category where `update_required` is True
+    #         dates_to_process = update_plan_table[
+    #             (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+    #             ]["date"].tolist()
+    #
+    #         date_iterator = dates_to_process
+    #         if self.show_progress:
+    #             date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
+    #
+    #         for current_date in date_iterator:
+    #             self.process_date(current_date)
+
     def is_file_older_than(self, file_path: str) -> bool:
         """
         Check if a file is older than the specified max_age_minutes.
@@ -181,12 +232,14 @@ class DataWrapper:
                     category = "history_days"
                     update_required = True
                 else:
+                    category = "file age is recent"
                     update_required = False
             # Hierarchy 3: Missing files
             elif missing_file and current_date <= today:
                 category = "missing_files"
                 update_required = True
             else:
+                category = "No Update Required"
                 update_required = False
 
             # Collect condition descriptions for the update plan table
@@ -199,6 +252,16 @@ class DataWrapper:
                 "update_category": category,
                 "datawrapper class": self.dataclass.__name__
             })
+            priority_map = {
+                "overwrite": 1,
+                "history_days": 2,
+                "missing_files": 3
+            }
+
+            for row in rows:
+                category = row.get("update_category")
+                # Default to None if no category assigned (no update required)
+                row["update_priority"] = priority_map.get(category, 0)
 
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table

{sibi_dst-0.3.21 → sibi_dst-0.3.22}/sibi_dst/df_helper/backends/django/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
+from ._io_dask import ReadFrameDask
 from ._django_db_connection import DjangoConnectionConfig
 from ._django_load_from_db import DjangoLoadFromDb
-from ._io_dask import ReadFrameDask
 
 __all__ = [
     "DjangoConnectionConfig",

{sibi_dst-0.3.21 → sibi_dst-0.3.22}/sibi_dst/utils/__init__.py

@@ -1,22 +1,22 @@
 from __future__ import annotations
 
-from ._airflow_manager import AirflowDAGManager
-from ._clickhouse_writer import ClickHouseWriter
-from ._credentials import *
-from ._data_utils import DataUtils
-from ._data_wrapper import DataWrapper
+from ._log_utils import Logger
 from ._date_utils import *
-from ._df_utils import DfUtils
+from ._data_utils import DataUtils
 from ._file_utils import FileUtils
 from ._filepath_generator import FilePathGenerator
-from ._log_utils import Logger
-from ._parquet_saver import ParquetSaver
+from ._df_utils import DfUtils
 from ._storage_manager import StorageManager
+from ._parquet_saver import ParquetSaver
+from ._clickhouse_writer import ClickHouseWriter
+from ._airflow_manager import AirflowDAGManager
+from ._credentials import *
+from ._data_wrapper import DataWrapper
 
 __all__ = [
+    "Logger",
     "ConfigManager",
     "ConfigLoader",
-    "Logger",
     "DateUtils",
     "BusinessDays",
     "FileUtils",