sibi-dst 0.3.21__py3-none-any.whl → 0.3.22__py3-none-any.whl

sibi_dst/df_helper/backends/django/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
+from ._io_dask import ReadFrameDask
 from ._django_db_connection import DjangoConnectionConfig
 from ._django_load_from_db import DjangoLoadFromDb
-from ._io_dask import ReadFrameDask
 
 __all__ = [
     "DjangoConnectionConfig",

sibi_dst/utils/__init__.py

@@ -1,22 +1,22 @@
 from __future__ import annotations
 
-from ._airflow_manager import AirflowDAGManager
-from ._clickhouse_writer import ClickHouseWriter
-from ._credentials import *
-from ._data_utils import DataUtils
-from ._data_wrapper import DataWrapper
+from ._log_utils import Logger
 from ._date_utils import *
-from ._df_utils import DfUtils
+from ._data_utils import DataUtils
 from ._file_utils import FileUtils
 from ._filepath_generator import FilePathGenerator
-from ._log_utils import Logger
-from ._parquet_saver import ParquetSaver
+from ._df_utils import DfUtils
 from ._storage_manager import StorageManager
+from ._parquet_saver import ParquetSaver
+from ._clickhouse_writer import ClickHouseWriter
+from ._airflow_manager import AirflowDAGManager
+from ._credentials import *
+from ._data_wrapper import DataWrapper
 
 __all__ = [
+    "Logger",
     "ConfigManager",
     "ConfigLoader",
-    "Logger",
     "DateUtils",
     "BusinessDays",
     "FileUtils",

sibi_dst/utils/_data_wrapper.py

@@ -1,4 +1,5 @@
 import datetime
+from concurrent.futures import ThreadPoolExecutor
 from typing import Type, Any, Dict, Optional
 
 import fsspec
@@ -32,7 +33,8 @@ class DataWrapper:
                  logger: Optional[Logger] = None,
                  max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
                  history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
-                 show_progress: bool = False):
+                 show_progress: bool = False,
+                 timeout: Optional[int] = 300):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self.ensure_forward_slash(data_path)
@@ -50,6 +52,7 @@ class DataWrapper:
         self.max_age_minutes = max_age_minutes
         self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
+        self.timeout = timeout
 
         self.start_date = self.convert_to_date(start_date)
         self.end_date = self.convert_to_date(end_date)
@@ -76,31 +79,79 @@ class DataWrapper:
             yield date.date()
 
     def process(self):
-        """Execute the update plan following the specified hierarchy."""
+        """Execute the update plan using 'update_priority' to determine processing order."""
         update_plan_table = self.generate_update_plan_with_conditions()
 
-        # Display the update plan table to the user if show_progress is True
+        # Display the update plan table to the user if requested
         if self.show_progress:
             display(update_plan_table)
 
-        # Process files according to the hierarchy, considering only `update_required` dates
-        for category, description in [
-            ("overwrite", "Processing files due to overwrite=True"),
-            ("history_days", "Processing files within history_days_threshold"),
-            ("missing_files", "Processing missing files")
-        ]:
-            # Filter dates in the category where `update_required` is True
+        # Filter out rows that do not require updates (priority 0 means skip)
+        update_plan_table = update_plan_table[
+            (update_plan_table["update_required"] == True) & (update_plan_table["update_priority"] != 0)
+            ]
+
+        # Group by priority
+        priorities = sorted(update_plan_table["update_priority"].unique())
+
+        # We will process each priority level in its own thread.
+        # Each thread will handle all dates associated with that priority.
+        def process_priority(priority):
+            # Extract dates for the current priority
             dates_to_process = update_plan_table[
-                (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+                update_plan_table["update_priority"] == priority
                 ]["date"].tolist()
 
+            # If show_progress is True, wrap in a progress bar
             date_iterator = dates_to_process
             if self.show_progress:
-                date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
+                date_iterator = tqdm(date_iterator, desc=f"Processing priority {priority}:{self.dataclass.__name__}",
+                                     unit="date")
 
+            # Process each date for this priority
             for current_date in date_iterator:
                 self.process_date(current_date)
 
+        # Launch a separate thread for each priority
+        with ThreadPoolExecutor(max_workers=len(priorities)) as executor:
+            futures = {executor.submit(process_priority, p): p for p in priorities}
+            for future in futures:
+                try:
+                    future.result(timeout=self.timeout)
+                except TimeoutError:
+                    self.logger.error(f"Thread for {self.dataclass.__name__} timed out. Thread cancelled.")
+                    future.cancel()
+                    priority = futures[future]
+                    new_future = executor.submit(process_priority, priority)
+                    futures[new_future] = priority
+                    self.logger.info(f"Resubmitted task for priority {priority} after timeout.")
+
+    # def process(self):
+    #     """Execute the update plan following the specified hierarchy."""
+    #     update_plan_table = self.generate_update_plan_with_conditions()
+    #
+    #     # Display the update plan table to the user if show_progress is True
+    #     if self.show_progress:
+    #         display(update_plan_table)
+    #
+    #     # Process files according to the hierarchy, considering only `update_required` dates
+    #     for category, description in [
+    #         ("overwrite", "Processing files due to overwrite=True"),
+    #         ("history_days", "Processing files within history_days_threshold"),
+    #         ("missing_files", "Processing missing files")
+    #     ]:
+    #         # Filter dates in the category where `update_required` is True
+    #         dates_to_process = update_plan_table[
+    #             (update_plan_table["update_category"] == category) & (update_plan_table["update_required"])
+    #             ]["date"].tolist()
+    #
+    #         date_iterator = dates_to_process
+    #         if self.show_progress:
+    #             date_iterator = tqdm(date_iterator, desc=f"{description}:{self.dataclass.__name__}", unit="date")
+    #
+    #         for current_date in date_iterator:
+    #             self.process_date(current_date)
+
     def is_file_older_than(self, file_path: str) -> bool:
         """
         Check if a file is older than the specified max_age_minutes.
@@ -181,12 +232,14 @@ class DataWrapper:
                     category = "history_days"
                     update_required = True
                 else:
+                    category = "file age is recent"
                     update_required = False
             # Hierarchy 3: Missing files
             elif missing_file and current_date <= today:
                 category = "missing_files"
                 update_required = True
             else:
+                category = "No Update Required"
                 update_required = False
 
             # Collect condition descriptions for the update plan table
@@ -199,6 +252,16 @@ class DataWrapper:
                 "update_category": category,
                 "datawrapper class": self.dataclass.__name__
             })
+            priority_map = {
+                "overwrite": 1,
+                "history_days": 2,
+                "missing_files": 3
+            }
+
+            for row in rows:
+                category = row.get("update_category")
+                # Default to None if no category assigned (no update required)
+                row["update_priority"] = priority_map.get(category, 0)
 
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table

{sibi_dst-0.3.21.dist-info → sibi_dst-0.3.22.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.21
+Version: 0.3.22
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.21.dist-info → sibi_dst-0.3.22.dist-info}/RECORD

@@ -4,7 +4,7 @@ sibi_dst/df_helper/_df_helper.py,sha256=3fibDnRAX4R0v-xgfG87BKLR-ZCg8AZWrKDIO7qb
 sibi_dst/df_helper/_parquet_artifact.py,sha256=nx1wTEyrjARpCCPNwBxYiBROee3CSb6c-u7Cpme_tdk,4978
 sibi_dst/df_helper/_parquet_reader.py,sha256=sbe8DsScNT2h6huNsz8mUxVnUGpJeRzbaONZ3u2sQeQ,1685
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sibi_dst/df_helper/backends/django/__init__.py,sha256=Py4XGV8OnWv_6qkdlJ0hhc1_xT_NLMS712N00CJOg0w,270
+sibi_dst/df_helper/backends/django/__init__.py,sha256=6LNllh46ZQ10QpEAxYizEOybx0__nUmpyLVCp8uyHyc,270
 sibi_dst/df_helper/backends/django/_django_db_connection.py,sha256=GypF84Ej7ViTID4r3UEReRGmLiyfMtEknPI_NINSm3g,1641
 sibi_dst/df_helper/backends/django/_django_load_from_db.py,sha256=GLsAsuEQD1cXfEm7BuxofZfR32VwZNEfwR9c-AZn-x0,5555
 sibi_dst/df_helper/backends/django/_django_sql_model_builder.py,sha256=xyOq0JY0k9380aBeV66RXbeXL-mF22CczbQQoXLDhuo,14884
@@ -29,12 +29,12 @@ sibi_dst/df_helper/core/_defaults.py,sha256=eNpHD2sZxir-2xO0b3_V16ryw8YP_5FfpIKK
 sibi_dst/df_helper/core/_filter_handler.py,sha256=1-IdviSYi5Hc28KckO4dkYHDfQ8X9SUb6kwfobm16_E,8580
 sibi_dst/df_helper/core/_params_config.py,sha256=mM1CnF29zls5LXx7rpKY8uix_GyOG5smO4ry_OX31IU,3477
 sibi_dst/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
-sibi_dst/utils/__init__.py,sha256=5WeBMxhNGB8ZpHUrp1NOQf8Kn0bLOtjrerFjQdFTa7U,787
+sibi_dst/utils/__init__.py,sha256=-7gtDN58v5f09bd2OGKAqMyccEyDP8tANWdw3RwI408,787
 sibi_dst/utils/_airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
 sibi_dst/utils/_clickhouse_writer.py,sha256=dL5pixjn4cj0Rwpc3POfCcY2D-aQCMbPSECX0dKATyE,8587
 sibi_dst/utils/_credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/_data_utils.py,sha256=ch4j5FEs8ZnniUzpbeLO-b4Yco_6nwCu71xHaVqMGi4,7050
-sibi_dst/utils/_data_wrapper.py,sha256=BmKFO70xVX3AjpGRzqrc6HS1Uw4xerZDx3IpFrcrIIo,9020
+sibi_dst/utils/_data_wrapper.py,sha256=4U0sKVXK7qDTObhufO19jxTzJa6ohs2VOh3WAhhzLCU,11982
 sibi_dst/utils/_date_utils.py,sha256=CMAZBNwVj7cvERcNiTA8Pf7_5EjV9By9yxkYJpkqz1g,10656
 sibi_dst/utils/_df_utils.py,sha256=NHnEJG9KDeRuqfE60kwBOO21B6Hvjh7PzE5B8cQrIXc,10986
 sibi_dst/utils/_file_utils.py,sha256=JpsybYj3XvVJisSBeVU6YSaZnYRm4_6YWTI3TLnnY4Y,1257
@@ -42,6 +42,6 @@ sibi_dst/utils/_filepath_generator.py,sha256=hjI7gQwfwRToPeuzoUQDayHKQrr4Ivhi4Ch
 sibi_dst/utils/_log_utils.py,sha256=AvKu5Qmi9LXezA7gdkG7yV-MvzZeav8c3PK8s-DwTGE,2314
 sibi_dst/utils/_parquet_saver.py,sha256=hLrWr1G132y94eLopDPPGQGDsAiR1lQ8id4QQtGYPE4,4349
 sibi_dst/utils/_storage_manager.py,sha256=7nkfeBW_2xlF59pGj7V2aY5TLwpJnPQuPVclqjavJOA,3856
-sibi_dst-0.3.21.dist-info/METADATA,sha256=P0GRxm9kh8V1mSjJgfvBSQplvfTuDnPbZZ71uOzwPJQ,2134
-sibi_dst-0.3.21.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-sibi_dst-0.3.21.dist-info/RECORD,,
+sibi_dst-0.3.22.dist-info/METADATA,sha256=syXtIBRGv51uoahxxWFHbWvFkHIezDeAWzXCpkAOZWA,2134
+sibi_dst-0.3.22.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+sibi_dst-0.3.22.dist-info/RECORD,,