sibi-dst 0.3.38__py3-none-any.whl → 0.3.39__py3-none-any.whl

sibi_dst/utils/data_wrapper.py

@@ -98,7 +98,7 @@ class DataWrapper:
         self.parquet_filename = parquet_filename
         self.filesystem_type = filesystem_type
         self.filesystem_options = filesystem_options or {}
-        self.fs = fs or fsspec.filesystem(filesystem_type, **self.filesystem_options)
+        self.fs = fs
         self.verbose = verbose
         self.class_params = class_params or {}
         self.load_params = load_params or {}
@@ -116,6 +116,10 @@ class DataWrapper:
         self._lock = Lock()
         self.processed_dates = []
         self.date_utils = DateUtils(logger=self.logger)
+        if self.fs is None:
+            with self._lock:
+                if self.fs is None:
+                    self.fs = fsspec.filesystem(self.filesystem_type, **self.filesystem_options)
 
     @staticmethod
     def convert_to_date(date: Union[datetime.date, str]) -> datetime.date:
@@ -172,21 +176,20 @@ class DataWrapper:
         # Each thread will handle all dates associated with that priority.
         def process_priority(priority):
             # Extract dates for the current priority
-            with self._lock:
-                dates_to_process = update_plan_table[
-                    update_plan_table["update_priority"] == priority
-                    ]["date"].tolist()
+            dates_to_process = update_plan_table[
+                update_plan_table["update_priority"] == priority
+                ]["date"].tolist()
 
-                # If show_progress is True, wrap in a progress bar
-                date_iterator = dates_to_process
-                if self.show_progress:
-                    date_iterator = tqdm(date_iterator,
-                                         desc=f"Processing priority {priority}:{self.dataclass.__name__}",
-                                         unit="date")
+            # If show_progress is True, wrap in a progress bar
+            date_iterator = dates_to_process
+            if self.show_progress:
+                date_iterator = tqdm(date_iterator,
+                                     desc=f"Processing priority {priority}:{self.dataclass.__name__}",
+                                     unit="date")
 
-                # Process each date for this priority
-                for current_date in date_iterator:
-                    self.process_date(current_date)
+            # Process each date for this priority
+            for current_date in date_iterator:
+                self.process_date(current_date)
 
         # Launch a separate thread for each priority
         with ThreadPoolExecutor(max_workers=len(priorities)) as executor:
@@ -232,21 +235,21 @@ class DataWrapper:
         :type date: datetime.date
         :return: None
         """
-        with self._lock:
-            folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
-            full_parquet_filename = f"{folder}{self.parquet_filename}"
+        folder = f'{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/'
+        full_parquet_filename = f"{folder}{self.parquet_filename}"
 
-            start_time = datetime.datetime.now()
-            self.logger.info(f"Processing date: {date}")
-            self.logger.info(f"Processing {full_parquet_filename}...")
+        start_time = datetime.datetime.now()
+        self.logger.info(f"Processing date: {date}")
+        self.logger.info(f"Processing {full_parquet_filename}...")
 
-            data_object = self.dataclass(**self.class_params)
-            df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
+        data_object = self.dataclass(**self.class_params)
+        df = data_object.load_period(dt_field=self.date_field, start=date, end=date)
 
-            if len(df.index) == 0:
-                self.logger.error("No data found for the specified date.")
-                return
+        if len(df.index) == 0:
+            self.logger.error("No data found for the specified date.")
+            return
 
+        with self._lock:
             parquet_saver = ParquetSaver(df, parquet_storage_path=folder, logger=self.logger, fs=self.fs)
             parquet_saver.save_to_parquet(self.parquet_filename, clear_existing=True)
 
@@ -255,8 +258,9 @@ class DataWrapper:
             self.logger.info(
                 f"Data saved to {full_parquet_filename}. Processing time: {duration_seconds:.2f} seconds"
             )
+
             self.processed_dates.append(date)
-            self.logger.info(f"Finished processing date: {date}")
+        self.logger.info(f"Finished processing date: {date}")
 
     def generate_update_plan_with_conditions(self):
         """
@@ -301,18 +305,23 @@ class DataWrapper:
                 update_required = True
             # Hierarchy 2: History threshold evaluation
             elif within_history:
-                if self.date_utils.is_file_older_than(
+                if file_exists:
+                    if self.date_utils.is_file_older_than(
                         full_parquet_filename,
                         max_age_minutes=self.max_age_minutes,
                         fs=self.fs,
                         ignore_missing=self.ignore_missing,
                         verbose=self.verbose
-                ):
-                    category = "history_days"
-                    update_required = True
+                    ):
+                        category = "history_days"
+                        update_required = True
+                    else:
+                        category = "file is recent"
+                        update_required = False
                 else:
-                    category = "file is recent"
-                    update_required = False
+                    category = "missing_files"
+                    update_required = True
+
             # Hierarchy 3: Missing files
             elif missing_file and current_date <= today:
                 category = "missing_files"
@@ -334,12 +343,6 @@ class DataWrapper:
             }
             rows.append(row)
 
-
-            for row in rows:
-                category = row.get("update_category")
-                # Default to None if no category assigned (no update required)
-                row["update_priority"] = priority_map.get(category, 0)
-
         update_plan_table = pd.DataFrame(rows)
         return update_plan_table
 

{sibi_dst-0.3.38.dist-info → sibi_dst-0.3.39.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.38
+Version: 0.3.39
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.38.dist-info → sibi_dst-0.3.39.dist-info}/RECORD

@@ -42,7 +42,7 @@ sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWd
 sibi_dst/utils/clickhouse_writer.py,sha256=syXGN9NG1FS8soHuMj6QNRqTRWi-thuYUF-_BWDc_KI,9883
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_utils.py,sha256=j-lEKt6EJL2fm0z7adcjtVG7yFYLRpQL8xSgh2CVmJg,8769
-sibi_dst/utils/data_wrapper.py,sha256=k9fTKbF2njAmRTeLAtW3HU-vwF0y43KnROvwWLqR7Bk,16340
+sibi_dst/utils/data_wrapper.py,sha256=SMmr4hwnUvCrN6nouI8N7aRiczuatjSMdYzNOBcDnr8,16283
 sibi_dst/utils/date_utils.py,sha256=UppOs1vfm41Si9JITAM7Qn9qqOi9yb6ukYz1E2mnA1I,19214
 sibi_dst/utils/df_utils.py,sha256=OFEtcwVKIilvf9qVf-IfIOHp4jcFAHX5l2IDGudhPZg,10989
 sibi_dst/utils/file_utils.py,sha256=JpsybYj3XvVJisSBeVU6YSaZnYRm4_6YWTI3TLnnY4Y,1257
@@ -50,6 +50,6 @@ sibi_dst/utils/filepath_generator.py,sha256=volVm0SSlBrtZp1RpTHxyui5rj5asNcVsWEB
 sibi_dst/utils/log_utils.py,sha256=XUbeXa1JsOlcEJyW8jnBlWo295rLUnuYi-HMzyhHwJg,3145
 sibi_dst/utils/parquet_saver.py,sha256=_QkXL3IiC2b4m7sxHpCSeqPwBWxXeiP5sH_WheSMEm4,8042
 sibi_dst/utils/storage_manager.py,sha256=-zlMrRo_6o6mCd_OHknKqNQl7m0I9VW89grDAUO1V5c,4229
-sibi_dst-0.3.38.dist-info/METADATA,sha256=ydwsS0rynG0HRVwwyRRxu_Kj4nA-0mGPb8JohGnUWEQ,2564
-sibi_dst-0.3.38.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.38.dist-info/RECORD,,
+sibi_dst-0.3.39.dist-info/METADATA,sha256=SZJ04u1MedYe00Z3h_YexNMEzB2Nfd9BFyTdDXVX6ZM,2564
+sibi_dst-0.3.39.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.39.dist-info/RECORD,,