sibi-dst 0.3.61__py3-none-any.whl → 0.3.63__py3-none-any.whl

sibi_dst/df_helper/_parquet_artifact.py

@@ -158,6 +158,7 @@ class ParquetArtifact(DfHelper):
         if not manifest_exists:
             self.logger.info(f"Creating new manifest at {self.missing_manifest_path}")
             self.mmanifest.save()
+            self.mmanifest.cleanup_temp_manifests()
         else:
             self.logger.info(f"Manifest already exists at {self.missing_manifest_path}")
 
@@ -206,6 +207,7 @@ class ParquetArtifact(DfHelper):
         try:
             if self.mmanifest and self.mmanifest._new_records:
                 self.mmanifest.save()
+                self.mmanifest.cleanup_temp_manifests()
             if getattr(self, "_entered", False) and self.fs and self._own_fs:
                 self.fs.close()
         except Exception as e:

sibi_dst/utils/data_wrapper.py

@@ -30,48 +30,28 @@ class DataWrapper:
             date_field: str,
             data_path: str,
             parquet_filename: str,
-            #start_date: Any,
-            #end_date: Any,
             fs: Optional[fsspec.AbstractFileSystem] = None,
-            #filesystem_type: str = "file",
-            #filesystem_options: Optional[Dict] = None,
             debug: bool = False,
             verbose: bool = False,
             class_params: Optional[Dict] = None,
             load_params: Optional[Dict] = None,
-            #reverse_order: bool = False,
-            #overwrite: bool = False,
-            #ignore_missing: bool = False,
             logger: Logger = None,
-            #max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
-            #history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
             show_progress: bool = False,
-            timeout: float = 60,
-            #reference_date: datetime.date = None,
-            #custom_priority_map: Dict[str, int] = None,
-            max_threads: int = 3,
+            timeout: float = 30,
+            max_threads: int = 1,
             **kwargs: Any,
     ):
         self.dataclass = dataclass
         self.date_field = date_field
         self.data_path = self._ensure_forward_slash(data_path)
         self.parquet_filename = parquet_filename
-        #self.filesystem_type = filesystem_type
-        #self.filesystem_options = filesystem_options or {}
         self.fs = fs or None
         self.debug = debug
         self.verbose = verbose
-        # self.reverse_order = reverse_order
-        # self.overwrite = overwrite
-        # self.ignore_missing = ignore_missing
         self.logger = logger or Logger.default_logger(logger_name=self.dataclass.__name__)
         self.logger.set_level(logging.DEBUG if debug else logging.INFO)
-        # self.max_age_minutes = max_age_minutes
-        # self.history_days_threshold = history_days_threshold
         self.show_progress = show_progress
         self.timeout = timeout
-        #self.reference_date = reference_date or datetime.date.today()
-        #self.priority_map = custom_priority_map or self.DEFAULT_PRIORITY_MAP
         self.max_threads = max_threads
         self.class_params = class_params or {
             'debug': self.debug,
@@ -95,6 +75,7 @@ class DataWrapper:
         """Context manager exit"""
         if self.mmanifest and self.mmanifest._new_records:
             self.mmanifest.save()
+            self.mmanifest.cleanup_temp_manifests()
         if exc_type is not None:
             self.logger.error(f"Exception occurred: {exc_val}")
         return False
@@ -185,9 +166,9 @@ class DataWrapper:
         path = f"{self.data_path}{date.year}/{date.month:02d}/{date.day:02d}/"
         self.logger.info(f"Processing date {date.isoformat()} for {path}")
         # self.logger.info(f"Path {path} in {self.skipped}: {path in self.skipped}")
-        #if path in self.skipped:
-        #    self.logger.info(f"Skipping {date} as it exists in the skipped list")
-        #    return
+        if path in self.update_planner.skipped and self.update_planner.ignore_missing:
+            self.logger.info(f"Skipping {date} as it exists in the skipped list")
+            return
         full_path = f"{path}{self.parquet_filename}"
 
         thread_name = threading.current_thread().name
@@ -254,5 +235,5 @@ class DataWrapper:
             self.logger.info("No benchmarking data to show")
             return
         df_bench = pd.DataFrame.from_records([{"date": d, **m} for d, m in self.benchmarks.items()])
-        df_bench = df_bench.set_index("date").sort_index(ascending=not self.reverse_order)
+        df_bench = df_bench.set_index("date").sort_index(ascending=not self.update_planner.reverse_order)
         self.logger.info("Benchmark Summary:\n" + df_bench.to_string())

{sibi_dst-0.3.61.dist-info → sibi_dst-0.3.63.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.61
+Version: 0.3.63
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.61.dist-info → sibi_dst-0.3.63.dist-info}/RECORD

@@ -2,7 +2,7 @@ sibi_dst/__init__.py,sha256=3pbriM7Ym5f9gew7n9cO4G_p9n-0bnxdmQ0hwBdJjr4,253
 sibi_dst/df_helper/__init__.py,sha256=McYrw2N0MsMgtawLrONXTGdyHfQWVOBUvIDbklfjb54,342
 sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=-Y4i5KAxKY2BNkmoVeMEZxjTFD7zaM9oQ0aRsvUbQrs,9340
 sibi_dst/df_helper/_df_helper.py,sha256=uKP6i-7dasZQ5zViD8-VJU0lNHumrdZG6IXvDFijZ18,31214
-sibi_dst/df_helper/_parquet_artifact.py,sha256=6y8nJ-HDAdmy3XNSvnEdA2zBXDhUIVoUeKgXLmVMGCo,14879
+sibi_dst/df_helper/_parquet_artifact.py,sha256=Nio5GSD6rTYl52nf_TSpQhYIF0hKqRrB3H3A4zYnaG8,14987
 sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/django/__init__.py,sha256=uWHi-DtQX5re7b2HcqoXUH3_FZWOw1VTmDf552FAkNs,256
@@ -44,7 +44,7 @@ sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcW
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=DFkqi84DIGxcrf36FfbgmeF9Hu7PZjMO9otNerV8ZYk,10546
+sibi_dst/utils/data_wrapper.py,sha256=Tb9bHIHI6qVsdH791BOFN1VrPb-7GS4fHhhHV8hktec,9641
 sibi_dst/utils/date_utils.py,sha256=T3ij-WOQu3cIfmNAweSVMWWr-hVtuBcTGjEY-cMJIvU,18627
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
@@ -77,6 +77,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.61.dist-info/METADATA,sha256=GZ-Yz9oiehgGgI2iJoCejdExgtclAlaz-N-sI5hGIi0,4292
-sibi_dst-0.3.61.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.61.dist-info/RECORD,,
+sibi_dst-0.3.63.dist-info/METADATA,sha256=ZsVn8AeFIUeVrb0Ybxmjk393FdUyn2j2fOnGQ8MXM1k,4292
+sibi_dst-0.3.63.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.63.dist-info/RECORD,,