sibi-dst 0.3.61__py3-none-any.whl → 0.3.62__py3-none-any.whl

sibi_dst/utils/data_wrapper.py

@@ -30,25 +30,14 @@ class DataWrapper:
             date_field: str,
             data_path: str,
             parquet_filename: str,
-            #start_date: Any,
-            #end_date: Any,
             fs: Optional[fsspec.AbstractFileSystem] = None,
-            #filesystem_type: str = "file",
-            #filesystem_options: Optional[Dict] = None,
             debug: bool = False,
             verbose: bool = False,
             class_params: Optional[Dict] = None,
             load_params: Optional[Dict] = None,
-            #reverse_order: bool = False,
-            #overwrite: bool = False,
-            #ignore_missing: bool = False,
             logger: Logger = None,
-            #max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
-            #history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
             show_progress: bool = False,
             timeout: float = 60,
-            #reference_date: datetime.date = None,
-            #custom_priority_map: Dict[str, int] = None,
             max_threads: int = 3,
             **kwargs: Any,
     ):
@@ -56,22 +45,13 @@ class DataWrapper:
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
         self.parquet_filename = parquet_filename
-        #self.filesystem_type = filesystem_type
-        #self.filesystem_options = filesystem_options or {}
         self.fs = fs or None
         self.debug = debug
         self.verbose = verbose
-        # self.reverse_order = reverse_order
-        # self.overwrite = overwrite
-        # self.ignore_missing = ignore_missing
         self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
         self.logger.set_level(logging.DEBUG if debug else logging.INFO)
-        # self.max_age_minutes = max_age_minutes
-        # self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
         self.timeout = timeout
-        #self.reference_date = reference_date or datetime.date.today()
-        #self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
         self.max_threads = max_threads
         self.class_params = class_params or {
             'debug': self.debug,
@@ -95,6 +75,7 @@ class DataWrapper:
         """Context manager exit"""
         if self.mmanifest and self.mmanifest._new_records:
             self.mmanifest.save()
+            self.mmanifest.cleanup_temp_manifests()
         if exc_type is not None:
             self.logger.error(f"Exception occurred: {exc_val}")
         return False
@@ -185,9 +166,9 @@ class DataWrapper:
         path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/"
         self.logger.info(f"Processing date {date.isoformat()} for {path}")
         # self.logger.info(f"Path {path} in {self.skipped}: {path in self.skipped}")
-        #if path in self.skipped:
-        #    self.logger.info(f"Skipping {date} as it exists in the skipped list")
-        #    return
+        if path in self.update_planner.skipped and self.update_planner.ignore_missing:
+            self.logger.info(f"Skipping {date} as it exists in the skipped list")
+            return
         full_path = f"{path}{self.parquet_filename}"
 
         thread_name = threading.current_thread().name
@@ -254,5 +235,5 @@ class DataWrapper:
             self.logger.info("No benchmarking data to show")
             return
         df_bench = pd.DataFrame.from_records([{"date": d, **m} for d, m in self.benchmarks.items()])
-        df_bench = df_bench.set_index("date").sort_index(ascending=not self.reverse_order)
+        df_bench = df_bench.set_index("date").sort_index(ascending=not self.update_planner.reverse_order)
         self.logger.info("Benchmark Summary:\n" + df_bench.to_string())

{sibi_dst-0.3.61.dist-info → sibi_dst-0.3.62.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.61
+Version: 0.3.62
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.61.dist-info → sibi_dst-0.3.62.dist-info}/RECORD

@@ -44,7 +44,7 @@ sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcW
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=DFkqi84DIGxcrf36FfbgmeF9Hu7PZjMO9otNerV8ZYk,10546
+sibi_dst/utils/data_wrapper.py,sha256=o7zAdiUl2ohXhISYi4cv7osxppdongNh-A_OXTs-CXU,9641
 sibi_dst/utils/date_utils.py,sha256=T3ij-WOQu3cIfmNAweSVMWWr-hVtuBcTGjEY-cMJIvU,18627
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
@@ -77,6 +77,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.61.dist-info/METADATA,sha256=GZ-Yz9oiehgGgI2iJoCejdExgtclAlaz-N-sI5hGIi0,4292
-sibi_dst-0.3.61.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.61.dist-info/RECORD,,
+sibi_dst-0.3.62.dist-info/METADATA,sha256=wqtWlXfA57ZXflNzp6WYvnHWyZ0JkDu8P1gxmEqF3rs,4292
+sibi_dst-0.3.62.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.62.dist-info/RECORD,,