sibi-dst 0.3.38__tar.gz → 0.3.39__tar.gz

{sibi_dst-0.3.38 → sibi_dst-0.3.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.38
+Version: 0.3.39
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.38 → sibi_dst-0.3.39}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.38"
+version = "0.3.39"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.38 → sibi_dst-0.3.39}/sibi_dst/utils/data_wrapper.py

@@ -98,7 +98,7 @@ class DataWrapper:
         self.parquet_filename = parquet_filename
         self.filesystem_type = filesystem_type
         self.filesystem_options = filesystem_options or {}
-        self.fs = fs or fsspec.filesystem(filesystem_type, **self.filesystem_options)
+        self.fs = fs
         self.verbose = verbose
         self.class_params = class_params or {}
         self.load_params = load_params or {}
@@ -116,6 +116,10 @@ class DataWrapper:
         self._lock = Lock()
         self.processed_dates = []
         self.date_utils = DateUtils(logger=self.logger)
+        if self.fs is None:
+            with self._lock:
+                if self.fs is None:
+                    self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
 
     @staticmethod
     def convert_to_date(date: Union[datetime.date, str]) -> datetime.date:
@@ -172,21 +176,20 @@ class DataWrapper:
         # Each thread will handle all dates associated with that priority.
         def process_priority(priority):
             # Extract dates for the current priority
-            with self._lock:
-                dates_to_process = update_plan_table[
-                    update_plan_table["update_priority"] == priority
-                    ]["date"].tolist()
+            dates_to_process = update_plan_table[
+                update_plan_table["update_priority"] == priority
+                ]["date"].tolist()
 
-                # If show_progress is True, wrap in a progress bar
-                date_iterator = dates_to_process
-                if self.show_progress:
-                    date_iterator = tqdm(date_iterator,
-                                         desc=f"Processing priority {priority}:{self.dataclass.__name__}",
-                                         unit="date")
+            # If show_progress is True, wrap in a progress bar
+            date_iterator = dates_to_process
+            if self.show_progress:
+                date_iterator = tqdm(date_iterator,
+                                     desc=f"Processing priority {priority}:{self.dataclass.__name__}",
+                                     unit="date")
 
-                # Process each date for this priority
-                for current_date in date_iterator:
-                    self.process_date(current_date)
+            # Process each date for this priority
+            for current_date in date_iterator:
+                self.process_date(current_date)
 
         # Launch a separate thread for each priority
         with ThreadPoolExecutor(max_workers=len(priorities)) as executor:
@@ -232,21 +235,21 @@ class DataWrapper:
         :type date: datetime.date
         :return: None
         """
-        with self._lock:
-            folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
-            full_parquet_filename = f"{folder}{self.parquet_filename}"
+        folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
+        full_parquet_filename = f"{folder}{self.parquet_filename}"
 
-            start_time = datetime.datetime.now()
-            self.logger.info(f"Processing date: {date}")
-            self.logger.info(f"Processing {full_parquet_filename}...")
+        start_time = datetime.datetime.now()
+        self.logger.info(f"Processing date: {date}")
+        self.logger.info(f"Processing {full_parquet_filename}...")
 
-            data_object = self.dataclass(**self.class_params)
-            df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
+        data_object = self.dataclass(**self.class_params)
+        df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
 
-            if len(df.index) == 0:
-                self.logger.error("No data found for the specified date.")
-                return
+        if len(df.index) == 0:
+            self.logger.error("No data found for the specified date.")
+            return
 
+        with self._lock:
             parquet_saver = ParquetSaver(df, parquet_storage_path=folder, logger=self.logger, fs=self.fs)
             parquet_saver.save_to_parquet(self.parquet_filename, clear_existing=True)
 
@@ -255,8 +258,9 @@ class DataWrapper:
             self.logger.info(
                 f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
             )
+
             self.processed_dates.append(date)
-            self.logger.info(f"Finished processing date: {date}")
+        self.logger.info(f"Finished processing date: {date}")
 
     def generate_update_plan_with_conditions(self):
         """
@@ -301,18 +305,23 @@ class DataWrapper:
                 update_required = True
             # Hierarchy 2: History threshold evaluation
             elif within_history:
-                if self.date_utils.is_file_older_than(
+                if file_exists:
+                    if self.date_utils.is_file_older_than(
                         full_parquet_filename,
                         max_age_minutes=self.max_age_minutes,
                         fs=self.fs,
                         ignore_missing=self.ignore_missing,
                         verbose=self.verbose
-                ):
-                    category = "history_days"
-                    update_required = True
+                    ):
+                        category = "history_days"
+                        update_required = True
+                    else:
+                        category = "file is recent"
+                        update_required = False
                 else:
-                    category = "file is recent"
-                    update_required = False
+                    category = "missing_files"
+                    update_required = True
+
             # Hierarchy 3: Missing files
             elif missing_file and current_date <= today:
                 category = "missing_files"
@@ -334,12 +343,6 @@ class DataWrapper:
             }
             rows.append(row)
 
-
-            for row in rows:
-                category = row.get("update_category")
-                # Default to None if no category assigned (no update required)
-                row["update_priority"] = priority_map.get(category, 0)
-
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table