sibi-dst 0.3.36__tar.gz → 0.3.38__tar.gz

{sibi_dst-0.3.36 → sibi_dst-0.3.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.36
+Version: 0.3.38
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -10,7 +10,6 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: apache-airflow-client (>=2.10.0,<3.0.0)
-Requires-Dist: boto3 (>=1.36.3,<2.0.0)
 Requires-Dist: chardet (>=5.2.0,<6.0.0)
 Requires-Dist: charset-normalizer (>=3.4.0,<4.0.0)
 Requires-Dist: clickhouse-connect (>=0.8.7,<0.9.0)

{sibi_dst-0.3.36 → sibi_dst-0.3.38}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.36"
+version = "0.3.38"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"
@@ -43,7 +43,6 @@ geopandas = "^1.0.1"
 osmnx = "^2.0.1"
 gunicorn = "^23.0.0"
 uvicorn-worker = "^0.3.0"
-boto3 = "^1.36.3"
 
 
 [build-system]

{sibi_dst-0.3.36 → sibi_dst-0.3.38}/sibi_dst/utils/data_wrapper.py

@@ -1,13 +1,13 @@
 import datetime
 from concurrent.futures import ThreadPoolExecutor
-from typing import Type, Any, Dict, Optional
-
+from typing import Type, Any, Dict, Optional, Union
+from threading import Lock
 import fsspec
 import pandas as pd
 from IPython.display import display
 from tqdm import tqdm
 
-from sibi_dst.utils import Logger
+from sibi_dst.utils import Logger, DateUtils
 from sibi_dst.utils import ParquetSaver
 
 
@@ -87,11 +87,11 @@ class DataWrapper:
                  reverse_order: bool = False,
                  overwrite: bool = False,
                  ignore_missing: bool = False,
-                 logger: Optional[Logger] = None,
+                 logger: Logger = None,
                  max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
                  history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
                  show_progress: bool = False,
-                 timeout: Optional[int] = 300):
+                 timeout: float = 300):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self.ensure_forward_slash(data_path)
@@ -113,9 +113,12 @@ class DataWrapper:
 
         self.start_date = self.convert_to_date(start_date)
         self.end_date = self.convert_to_date(end_date)
+        self._lock = Lock()
+        self.processed_dates = []
+        self.date_utils = DateUtils(logger=self.logger)
 
     @staticmethod
-    def convert_to_date(date: Any) -> datetime.date:
+    def convert_to_date(date: Union[datetime.date, str]) -> datetime.date:
         if isinstance(date, datetime.date):
             return date
         try:
@@ -135,7 +138,7 @@ class DataWrapper:
         for date in date_range:
             yield date.date()
 
-    def process(self):
+    def process(self, max_retries: int = 3):
         """
         Processes update tasks by generating an update plan, filtering required updates, and distributing
         the workload across threads based on priority levels.
@@ -145,8 +148,8 @@ class DataWrapper:
         Each thread handles the updates for a specific priority level, ensuring a streamlined approach
         to handling the updates efficiently.
 
+        :param max_retries: Maximum number of retries for a task after a timeout. Defaults to 3.
         :raises TimeoutError: If a thread processing a priority level exceeds the allowed timeout duration.
-
         :return: None
         """
         update_plan_table = self.generate_update_plan_with_conditions()
@@ -156,91 +159,63 @@ class DataWrapper:
             display(update_plan_table)
 
         # Filter out rows that do not require updates (priority 0 means skip)
-        update_plan_table = update_plan_table[
-            (update_plan_table["update_required"] == True) & (update_plan_table["update_priority"] != 0)
-            ]
+        with self._lock:
+            update_plan_table = update_plan_table[
+                (update_plan_table["update_required"] == True) & (update_plan_table["update_priority"] != 0)
+                ]
 
         # Group by priority
-        priorities = sorted(update_plan_table["update_priority"].unique())
+        with self._lock:
+            priorities = sorted(update_plan_table["update_priority"].unique())
 
         # We will process each priority level in its own thread.
         # Each thread will handle all dates associated with that priority.
         def process_priority(priority):
             # Extract dates for the current priority
-            dates_to_process = update_plan_table[
-                update_plan_table["update_priority"] == priority
-                ]["date"].tolist()
-
-            # If show_progress is True, wrap in a progress bar
-            date_iterator = dates_to_process
-            if self.show_progress:
-                date_iterator = tqdm(date_iterator, desc=f"Processing priority {priority}:{self.dataclass.__name__}",
-                                     unit="date")
-
-            # Process each date for this priority
-            for current_date in date_iterator:
-                self.process_date(current_date)
+            with self._lock:
+                dates_to_process = update_plan_table[
+                    update_plan_table["update_priority"] == priority
+                    ]["date"].tolist()
+
+                # If show_progress is True, wrap in a progress bar
+                date_iterator = dates_to_process
+                if self.show_progress:
+                    date_iterator = tqdm(date_iterator,
+                                         desc=f"Processing priority {priority}:{self.dataclass.__name__}",
+                                         unit="date")
+
+                # Process each date for this priority
+                for current_date in date_iterator:
+                    self.process_date(current_date)
 
         # Launch a separate thread for each priority
         with ThreadPoolExecutor(max_workers=len(priorities)) as executor:
             futures = {executor.submit(process_priority, p): p for p in priorities}
-            for future in futures:
-                try:
-                    future.result(timeout=self.timeout)
-                except TimeoutError:
-                    self.logger.error(f"Thread for {self.dataclass.__name__} timed out. Thread cancelled.")
-                    future.cancel()
-                    priority = futures[future]
-                    new_future = executor.submit(process_priority, priority)
-                    futures[new_future] = priority
-                    self.logger.info(f"Resubmitted task for priority {priority} after timeout.")
-
-    def is_file_older_than(self, file_path: str) -> bool:
-        """
-        Check if a file is older than the specified max_age_minutes.
-
-        :param file_path: Path to the file.
-        :return: True if the file is older than max_age_minutes, False otherwise.
-        """
-        try:
-            # Get file info
-            info = self.fs.info(file_path)
-            self.logger.info(f"File info for {file_path}: {info}")
-
-            # Determine the modification time from available keys
-            file_modification_time = None
-            if "mtime" in info:  # Local filesystem
-                file_modification_time = info["mtime"]
-                file_modification_datetime = datetime.datetime.fromtimestamp(
-                    file_modification_time, tz=datetime.timezone.utc
-                )
-            elif "LastModified" in info:  # S3-compatible filesystem
-                file_modification_datetime = (
-                    info["LastModified"] if isinstance(info["LastModified"], datetime.datetime)
-                    else datetime.datetime.strptime(info["LastModified"], "%Y-%m-%dT%H:%M:%S.%fZ")
-                )
-                self.logger.info(f"S3 File modification time: {file_modification_datetime}")
-            else:
-                self.logger.warning(f"Modification time not available for {file_path}.")
-                return True  # Assume file is too old if we cannot determine its age
-
-            # Compare file age
-            current_time = datetime.datetime.now(datetime.timezone.utc)
-            file_age_minutes = (current_time - file_modification_datetime).total_seconds() / 60
-            self.logger.info(
-                f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
-                f"(threshold: {self.max_age_minutes} minutes)"
-            )
-            return file_age_minutes > self.max_age_minutes
-
-        except FileNotFoundError:
-            self.logger.warning(f"File {file_path} not found.")
-            if self.ignore_missing:
-                return False
-            return True  # File is considered old if it doesn't exist
-        except Exception as e:
-            self.logger.error(f"Error checking file age for {file_path}: {str(e)}")
-            return True
+            retries = {p: 0 for p in priorities}  # Track retry counts for each priority
+
+            while futures:
+                for future in list(futures.keys()):
+                    try:
+                        future.result(timeout=self.timeout)
+                        del futures[future]  # Remove completed future
+                    except TimeoutError:
+                        priority = futures[future]
+                        retries[priority] += 1
+
+                        if retries[priority] <= max_retries:
+                            self.logger.warning(
+                                f"Thread for priority {priority} timed out. Retrying ({retries[priority]}/{max_retries})..."
+                            )
+                            new_future = executor.submit(process_priority, priority)
+                            futures[new_future] = priority
+                        else:
+                            self.logger.error(
+                                f"Thread for priority {priority} timed out. Max retries ({max_retries}) exceeded. Skipping."
+                            )
+                        del futures[future]  # Remove the timed-out future
+                    except Exception as e:
+                        self.logger.error(f"Error processing priority {futures[future]}: {e}")
+                        del futures[future]  # Remove the failed future
 
     def process_date(self, date: datetime.date):
         """
@@ -257,27 +232,31 @@ class DataWrapper:
         :type date: datetime.date
         :return: None
         """
-        folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
-        full_parquet_filename = f"{folder}{self.parquet_filename}"
+        with self._lock:
+            folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
+            full_parquet_filename = f"{folder}{self.parquet_filename}"
 
-        start_time = datetime.datetime.now()
-        self.logger.info(f"Processing {full_parquet_filename}...")
+            start_time = datetime.datetime.now()
+            self.logger.info(f"Processing date: {date}")
+            self.logger.info(f"Processing {full_parquet_filename}...")
 
-        data_object = self.dataclass(**self.class_params)
-        df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
+            data_object = self.dataclass(**self.class_params)
+            df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
 
-        if len(df.index) == 0:
-            self.logger.error("No data found for the specified date.")
-            return
+            if len(df.index) == 0:
+                self.logger.error("No data found for the specified date.")
+                return
 
-        parquet_saver = ParquetSaver(df, parquet_storage_path=folder, logger=self.logger, fs=self.fs)
-        parquet_saver.save_to_parquet(self.parquet_filename, clear_existing=True)
+            parquet_saver = ParquetSaver(df, parquet_storage_path=folder, logger=self.logger, fs=self.fs)
+            parquet_saver.save_to_parquet(self.parquet_filename, clear_existing=True)
 
-        end_time = datetime.datetime.now()
-        duration_seconds = (end_time - start_time).total_seconds()
-        self.logger.info(
-            f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
-        )
+            end_time = datetime.datetime.now()
+            duration_seconds = (end_time - start_time).total_seconds()
+            self.logger.info(
+                f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
+            )
+            self.processed_dates.append(date)
+            self.logger.info(f"Finished processing date: {date}")
 
     def generate_update_plan_with_conditions(self):
         """
@@ -297,7 +276,12 @@ class DataWrapper:
 
         today = datetime.date.today()
         history_start_date = today - datetime.timedelta(days=self.history_days_threshold)
-
+        priority_map = {
+            "file is recent":0,
+            "overwrite": 1,
+            "history_days": 2,
+            "missing_files": 3
+        }
         date_range = self.generate_date_range()
         if self.show_progress:
             date_range = tqdm(date_range, desc=f"Evaluating update plan:{self.dataclass.__name__}", unit="date")
@@ -317,11 +301,17 @@ class DataWrapper:
                 update_required = True
             # Hierarchy 2: History threshold evaluation
             elif within_history:
-                if self.is_file_older_than(full_parquet_filename):
+                if self.date_utils.is_file_older_than(
+                        full_parquet_filename,
+                        max_age_minutes=self.max_age_minutes,
+                        fs=self.fs,
+                        ignore_missing=self.ignore_missing,
+                        verbose=self.verbose
+                ):
                     category = "history_days"
                     update_required = True
                 else:
-                    category = "file age is recent"
+                    category = "file is recent"
                     update_required = False
             # Hierarchy 3: Missing files
             elif missing_file and current_date <= today:
@@ -332,20 +322,18 @@ class DataWrapper:
                 update_required = False
 
             # Collect condition descriptions for the update plan table
-            rows.append({
+            row = {
                 "date": current_date,
                 "file_exists": file_exists,
                 "within_history": within_history,
                 "missing_file": missing_file,
                 "update_required": update_required,
                 "update_category": category,
-                "datawrapper class": self.dataclass.__name__
-            })
-            priority_map = {
-                "overwrite": 1,
-                "history_days": 2,
-                "missing_files": 3
+                "datawrapper class": self.dataclass.__name__,
+                "update_priority": priority_map.get(category, 0)
             }
+            rows.append(row)
+
 
             for row in rows:
                 category = row.get("update_category")

{sibi_dst-0.3.36 → sibi_dst-0.3.38}/sibi_dst/utils/date_utils.py

@@ -1,6 +1,7 @@
 import datetime
-from typing import Union, Tuple, Callable, Dict
+from typing import Union, Tuple, Callable, Dict, Optional
 
+import fsspec
 import numpy as np
 import pandas as pd
 
@@ -143,6 +144,158 @@ class DateUtils:
             'ytd': lambda: (datetime.date(today().year, 1, 1), today()),
         }
 
+    def is_file_older_than(self, file_path: str, max_age_minutes: int, fs: Optional[fsspec.AbstractFileSystem] = None,
+                           ignore_missing: bool = False, verbose: bool = False) -> bool:
+        """
+        Check if a file or a partitioned Parquet dataset is older than the specified max_age_minutes.
+
+        :param file_path: Path to the file or dataset.
+        :param max_age_minutes: Maximum allowed age in minutes.
+        :param fs: Filesystem object (e.g., S3, local). If not provided, defaults to the local filesystem.
+        :param ignore_missing: If True, treat missing files as not old. Defaults to False.
+        :param verbose: If True, log detailed information. Defaults to False.
+        :return: True if the file or dataset is older than max_age_minutes, False otherwise.
+        """
+        fs = fs or fsspec.filesystem("file")
+        self.logger.info(f"Checking age for {file_path}...")
+
+        try:
+            if not fs.exists(file_path):
+                self.logger.info(f"Path not found: {file_path}.")
+                return not ignore_missing
+
+            if fs.isdir(file_path):
+                self.logger.info(f"Found that {file_path} is a directory...")
+                return self._is_directory_older_than(file_path, max_age_minutes, fs, verbose)
+
+            elif fs.isfile(file_path):
+                return self._is_file_older_than(file_path, max_age_minutes, fs, verbose)
+
+            else:
+                self.logger.warning(f"Path {file_path} is neither a file nor a directory.")
+                return True
+
+        except Exception as e:
+            self.logger.warning(f"Error checking age for {file_path}: {str(e)}")
+            return True
+
+    def _is_directory_older_than(self, dir_path: str, max_age_minutes: int, fs: fsspec.AbstractFileSystem,
+                                 verbose: bool) -> bool:
+        """
+        Check if the oldest file in a directory is older than the specified max_age_minutes.
+
+        :param dir_path: Path to the directory.
+        :param max_age_minutes: Maximum allowed age in minutes.
+        :param fs: Filesystem object.
+        :param verbose: If True, log detailed information.
+        :return: True if the oldest file is older than max_age_minutes, False otherwise.
+        """
+        all_files = fs.ls(dir_path)
+        if not all_files:
+            self.logger.info(f"No files found in dataset: {dir_path}.")
+            return True
+
+        modification_times = [
+            self._get_modification_time(fs.info(file), file)
+            for file in all_files
+            if self._is_valid_file(file, fs)
+        ]
+
+        if not modification_times:
+            self.logger.warning(f"No valid modification times found for dataset: {dir_path}. Assuming dataset is old.")
+            return True
+
+        oldest_modification_time = min(modification_times)
+        dataset_age_minutes = (datetime.datetime.now(
+            datetime.timezone.utc) - oldest_modification_time).total_seconds() / 60
+
+        if verbose:
+            self.logger.info(
+                f"Oldest file in dataset {dir_path} is {round(dataset_age_minutes, 2)} minutes old "
+                f"(threshold: {max_age_minutes} minutes)"
+            )
+
+        return dataset_age_minutes > max_age_minutes
+
+    def _is_file_older_than(self, file_path: str, max_age_minutes: int, fs: fsspec.AbstractFileSystem,
+                            verbose: bool) -> bool:
+        """
+        Check if a single file is older than the specified max_age_minutes.
+
+        :param file_path: Path to the file.
+        :param max_age_minutes: Maximum allowed age in minutes.
+        :param fs: Filesystem object.
+        :param verbose: If True, log detailed information.
+        :return: True if the file is older than max_age_minutes, False otherwise.
+        """
+        info = fs.info(file_path)
+        if verbose:
+            self.logger.debug(f"File info for {file_path}: {info}")
+
+        file_modification_datetime = self._get_modification_time(info, file_path)
+        file_age_minutes = (datetime.datetime.now(
+            datetime.timezone.utc) - file_modification_datetime).total_seconds() / 60
+
+        if verbose:
+            self.logger.debug(
+                f"File {file_path} is {round(file_age_minutes, 2)} minutes old "
+                f"(threshold: {max_age_minutes} minutes)"
+            )
+
+        return file_age_minutes > max_age_minutes
+
+    def _is_valid_file(self, file_path: str, fs: fsspec.AbstractFileSystem) -> bool:
+        """
+        Check if a file is valid (exists and has a valid modification time).
+
+        :param file_path: Path to the file.
+        :param fs: Filesystem object.
+        :return: True if the file is valid, False otherwise.
+        """
+        try:
+            fs.info(file_path)
+            return True
+        except Exception as e:
+            self.logger.warning(f"Error checking file age for {file_path}: {str(e)}")
+            return False
+
+    def _get_modification_time(self, info: Dict, file_path: str) -> datetime.datetime:
+        """
+        Extract the modification time from file info.
+
+        :param info: File info dictionary.
+        :param file_path: Path to the file (for logging purposes).
+        :return: Modification time as a timezone-aware datetime object.
+        """
+        if "LastModified" in info:  # S3-compatible filesystem
+            last_modified = info["LastModified"]
+            if isinstance(last_modified, datetime.datetime):
+                return last_modified
+            else:
+                return datetime.datetime.strptime(last_modified, "%Y-%m-%dT%H:%M:%S.%fZ").replace(
+                    tzinfo=datetime.timezone.utc)
+
+        elif "mtime" in info:  # Local filesystem
+            return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc)
+
+        elif "modified" in info:  # FTP or SSH filesystem
+            modified_str = info["modified"]
+            try:
+                return datetime.datetime.strptime(modified_str, "%Y-%m-%d %H:%M:%S").replace(
+                    tzinfo=datetime.timezone.utc)
+            except ValueError:
+                try:
+                    return datetime.datetime.strptime(modified_str, "%b %d %H:%M").replace(
+                        year=datetime.datetime.now().year, tzinfo=datetime.timezone.utc
+                    )
+                except ValueError:
+                    self.logger.warning(f"Unsupported modification time format for {file_path}: {modified_str}")
+                    raise ValueError("Unsupported modification time format")
+
+        else:  # Fallback for unsupported filesystems
+            self.logger.warning(f"Modification time not available for {file_path}.")
+            raise ValueError("Modification time not available")
+
 
 class BusinessDays:
     """