sibi-dst 2025.1.4__py3-none-any.whl → 2025.1.5__py3-none-any.whl

sibi_dst/utils/parquet_saver.py

@@ -1,196 +1,121 @@
-import base64
-import hashlib
 import logging
-import threading
 import warnings
 from typing import Optional
 
+import dask.dataframe as dd
 import pyarrow as pa
+from fsspec import AbstractFileSystem
 
-# Suppress the specific UserWarning message
-warnings.filterwarnings("ignore")
-from .log_utils import Logger
+from . import ManagedResource
 
+warnings.filterwarnings("ignore", message="Passing 'overwrite=True' to to_parquet is deprecated")
 
-class ParquetSaver:
-    def __init__(self, df_result, parquet_storage_path, logger=None, fs=None, debug=False):
-        """
-        Initialize ParquetSaver.
-        :param df_result: Dask DataFrame to save.
-        :param parquet_storage_path: Base storage path (e.g., "s3://bucket-name/path/").
-        :param logger: Logger instance for logging messages.
-        :param fs: Pre-initialized fsspec filesystem instance. Defaults to 'file' if None.
-        """
+
+class ParquetSaver(ManagedResource):
+    """
+    Saves Dask DataFrames to Parquet, with a workaround for S3-compatible
+    storage that fails on batch delete operations.
+    """
+
+    def __init__(
+            self,
+            df_result: dd.DataFrame,
+            parquet_storage_path: str,
+            **kwargs,
+    ):
+        super().__init__(**kwargs)
         self.df_result = df_result
         self.parquet_storage_path = parquet_storage_path.rstrip("/")
-        self.debug = debug
-        self.logger = logger or Logger.default_logger(logger_name=self.__class__.__name__)
-        self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
-        self.fs = fs
-        self.protocol = self.parquet_storage_path.split(":")[0]
-        self._lock = threading.Lock()
-
-    def save_to_parquet(self, parquet_filename: Optional[str] = None, clear_existing=True):
+        #self.debug = debug
+        #self.logger = logger or Logger.default_logger(self.__class__.__name__)
+        #self.logger.set_level(logging.DEBUG if self.debug else logging.INFO)
+        #self.fs = fs
+        # Determine protocol for special handling (e.g., 's3')
+        if not self.fs:
+            raise ValueError("File system (fs) must be provided to ParquetSaver.")
+
+        self.protocol = "file"
+        if "://" in self.parquet_storage_path:
+            self.protocol = self.parquet_storage_path.split(":", 1)[0]
+
+    def save_to_parquet(self, output_directory_name: str = "default_output", overwrite: bool = True):
         """
-        Save the DataFrame to Parquet format.
-        :param parquet_filename: Filename for the Parquet file.
-        :param clear_existing: Whether to clear existing files in the target directory.
+        Saves the DataFrame to a Parquet dataset.
+
+        If overwrite is True, it manually clears the destination directory before
+        writing to avoid issues with certain S3-compatible storage providers.
         """
-        with self._lock:
-            full_path = self._construct_full_path(parquet_filename)
-            self.logger.info(f"Save method for :{full_path}")
-            # Ensure directory exists and clear if necessary
-            self._ensure_directory_exists(full_path, clear_existing=clear_existing)
-
-            # Define schema and save DataFrame to Parquet
-            schema = self._define_schema()
-            self._convert_dtypes(schema)
-            self._save_dataframe_to_parquet(full_path, schema)
-            # Close the filesystem if the close method exists
-            if hasattr(self.fs, 'close') and callable(getattr(self.fs, 'close', None)):
-                self.fs.close()
+        full_path = f"{self.parquet_storage_path}/{output_directory_name}"
 
-    def _define_schema(self) -> pa.Schema:
-        """Define a PyArrow schema dynamically based on df_result column types."""
-        pandas_dtype_to_pa = {
-            "object": pa.string(),
-            "string": pa.string(),
-            "Int64": pa.int64(),
-            "int64": pa.int64(),
-            "float64": pa.float64(),
-            "float32": pa.float32(),
-            "bool": pa.bool_(),
-            "boolean": pa.bool_(),  # pandas nullable boolean
-            "datetime64[ns]": pa.timestamp("ns"),
-            "timedelta[ns]": pa.duration("ns"),
-        }
+        if overwrite and self.fs and self.fs.exists(full_path):
+            self.logger.info(f"Overwrite is True, clearing destination path: {full_path}")
+            self._clear_directory_safely(full_path)
 
-        dtypes = self.df_result.dtypes
+        # Ensure the base directory exists after clearing
+        self.fs.mkdirs(full_path, exist_ok=True)
 
-        fields = [
-            pa.field(col, pandas_dtype_to_pa.get(str(dtype), pa.string()))
-            for col, dtype in dtypes.items()
-        ]
-        return pa.schema(fields)
+        schema = self._define_schema()
+        self.logger.info(f"Saving DataFrame to Parquet dataset at: {full_path}")
+        self.df_result = self.df_result.persist()
+        try:
+            # We call to_parquet with overwrite=False because we have already
+            # handled the directory clearing manually.
+            self.df_result.to_parquet(
+                path=full_path,
+                engine="pyarrow",
+                schema=schema,
+                overwrite=False,
+                filesystem=self.fs,
+                write_index=False,
+                compute=True,  # Use compute=True over persisted ddf for immediate execution.
+            )
+            self.logger.info(f"Successfully saved Parquet dataset to: {full_path}")
+        except Exception as e:
+            self.logger.error(f"Failed to save Parquet dataset to {full_path}: {e}")
+            raise
 
-    def _convert_dtypes(self, schema: pa.Schema):
-        """Convert DataFrame columns to match the specified schema."""
-        dtype_mapping = {}
-        for field in schema:
-            col_name = field.name
-            if col_name in self.df_result.columns:
-                if pa.types.is_string(field.type):
-                    dtype_mapping[col_name] = "string"
-                elif pa.types.is_int64(field.type):
-                    dtype_mapping[col_name] = "Int64"
-                elif pa.types.is_float64(field.type):
-                    dtype_mapping[col_name] = "float64"
-                elif pa.types.is_float32(field.type):
-                    dtype_mapping[col_name] = "float32"
-                elif pa.types.is_boolean(field.type):
-                    dtype_mapping[col_name] = "boolean"
-                elif pa.types.is_timestamp(field.type):
-                    dtype_mapping[col_name] = "datetime64[ns]"
-                else:
-                    dtype_mapping[col_name] = "object"
-        self.df_result = self.df_result.astype(dtype_mapping)
-
-    def _construct_full_path(self, parquet_filename: Optional[str]) -> str:
-        """Construct and return the full path for the Parquet file."""
-        parquet_filename = parquet_filename or "default.parquet"
-        return f"{self.parquet_storage_path}/{parquet_filename}"
-
-    def _ensure_directory_exists(self, full_path: str, clear_existing=False):
+    def _clear_directory_safely(self, directory: str):
         """
-        Ensure that the directory for the path exists, clearing it if specified.
-        :param full_path: Full path for the target file.
-        :param clear_existing: Whether to clear existing files/directories.
+        Clears the contents of a directory robustly.
+        - For S3, it deletes files one-by-one to bypass the 'MissingContentMD5' error.
+        - For other filesystems, it uses the standard recursive remove.
         """
-        directory = "/".join(full_path.split("/")[:-1])
-        if self.fs.exists(directory):
-            self.logger.info(f"Directory already exists: {directory}")
-            if clear_existing:
-                self._clear_directory(directory)
-
-        if not self.fs.exists(directory):
-            self.logger.info(f"Creating directory: {directory}")
-            self.fs.mkdirs(directory, exist_ok=True)
-
-    def _save_dataframe_to_parquet(self, full_path: str, schema: pa.Schema):
-        """Save the DataFrame to Parquet using the specified schema."""
-        #self._clear_directory(full_path)
-        self.logger.info(f"Saving/Overwrite Parquet file to: {full_path}")
-        self.df_result.to_parquet(
-             path=full_path,
-             engine="pyarrow",
-             schema=schema,
-             #overwrite=True,
-             filesystem=self.fs,
-             write_index=False,
-        )
-
-    def _clear_directory(self, directory: str):
-        """
-        Clears the specified directory by removing all the files within it. If the
-        directory is not associated with the "s3" protocol, it will be removed using
-        the local filesystem's functionality. For directories using the "s3" protocol,
-        the bucket name and prefix are parsed, and files are deleted using the S3
-        client's `delete_objects` method.
-
-        :param directory: The directory path to clear. If the protocol is not "s3", it
-                          represents a local filesystem path. Otherwise, it is assumed
-                          to be an S3 path (e.g., "s3://bucket_name/prefix").
-        :type directory: str
-        """
-        if self.protocol != "s3":
-            if self.fs.exists(directory):
-                self.logger.info(f"Clearing directory: {directory}")
-                self.fs.rm(directory, recursive=True)
-            return
-        # Parse bucket name and prefix
-        bucket_name, prefix = self._parse_s3_path(directory)
-
-        # List files in the directory
-        files = self.fs.ls(directory, detail=True)
-        if not files:
-            self.logger.info(f"No files to delete in directory: {directory}")
-            return
-
-        # Generate the delete payload
-        objects_to_delete = [{"Key": file["name"].replace(f"{bucket_name}/", "", 1)} for file in files]
-        delete_payload = {
-            "Objects": objects_to_delete,
-            "Quiet": True,
-        }
-
-        # Calculate Content-MD5
-        payload_bytes = str(delete_payload).encode("utf-8")
-        md5_hash = hashlib.md5(payload_bytes).digest()
-        content_md5 = base64.b64encode(md5_hash).decode("utf-8")
-
-        # Use the underlying s3 client to delete objects
-        try:
-            self.fs.s3.delete_objects(
-                Bucket=bucket_name,
-                Delete=delete_payload,
-                ContentMD5=content_md5,
+        if self.protocol == "s3":
+            self.logger.warning(
+                "Using single-file S3 deletion for compatibility. "
+                "This may be slow for directories with many files."
             )
-            self.logger.info(f"Successfully deleted files in {directory}.")
-        except Exception as e:
-            self.logger.error(f"Failed to delete objects in {directory}: {e}")
-            pass
+            # Glob all contents (files and subdirs) and delete them individually.
+            # Calling fs.rm() on a single file path should trigger a single
+            # DeleteObject call, avoiding the faulty batch operation.
+            # We sort by length descending to delete contents of subdirectories first.
+            all_paths = self.fs.glob(f"{directory}/**")
+            paths_to_delete = sorted([p for p in all_paths if p != directory], key=len, reverse=True)
+
+            for path in paths_to_delete:
+                self.logger.debug(f"Deleting: {path}")
+                self.fs.rm_file(path)
+        else:
+            # Standard, fast deletion for other filesystems (local, etc.)
+            self.fs.rm(directory, recursive=True)
 
-    @staticmethod
-    def _parse_s3_path(s3_path: str):
+    def _define_schema(self) -> pa.Schema:
         """
-        Parse an S3 path into bucket name and prefix.
-        :param s3_path: Full S3 path (e.g., s3://bucket-name/path/).
-        :return: Tuple of bucket name and prefix.
+        Defines a PyArrow schema dynamically based on DataFrame's column types.
         """
-        if not s3_path.startswith("s3://"):
-            raise ValueError("Invalid S3 path. Must start with 's3://'.")
-        path_parts = s3_path[5:].split("/", 1)
-        bucket_name = path_parts[0]
-        prefix = path_parts[1] if len(path_parts) > 1 else ""
-        return bucket_name, prefix
+        pandas_dtype_to_pa = {
+            "object": pa.string(), "string": pa.string(),
+            "int64": pa.int64(), "Int64": pa.int64(),
+            "int32": pa.int32(), "Int32": pa.int32(),
+            "float64": pa.float64(), "float32": pa.float32(),
+            "bool": pa.bool_(), "boolean": pa.bool_(),
+            "datetime64[ns]": pa.timestamp("ns"),
+            "datetime64[ns, UTC]": pa.timestamp("ns", tz="UTC"),
+            "category": pa.string(),
+        }
+        fields = [
+            pa.field(c, pandas_dtype_to_pa.get(str(d), pa.string()))
+            for c, d in self.df_result.dtypes.items()
+        ]
+        return pa.schema(fields)
 

sibi_dst/utils/storage_config.py

@@ -31,6 +31,12 @@ class StorageConfig:
                 "use_listings_cache": False,
                 "client_kwargs": {
                     "endpoint_url": self.conf.get('fs_endpoint')
+                },
+                "config_kwargs" :{
+                    "signature_version": "s3v4",
+                    's3': {
+                      'addressing_style': 'path'
+                    }
                 }
             }
         elif self.filesystem_type == "webdav":

sibi_dst/utils/storage_manager.py

@@ -20,6 +20,7 @@ class StorageManager:
             self._initialize_webdav()
         else:
             self.fs = fsspec.filesystem(fs_type, **self.fs_options)
+
         self.depot_paths = {}
         self.depot_name = None
 
@@ -161,4 +162,4 @@ class StorageManager:
             self.webdav_client.download_file(remote_path, local_path)
         else:
             # Use fsspec's get method for other filesystems
-            self.fs.get(remote_path, local_path)
+            self.fs.get(remote_path, local_path)

sibi_dst/utils/update_planner.py

@@ -1,13 +1,38 @@
 import datetime
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import List, Optional, Dict, Union, Tuple, Set
+from typing import List, Optional, Dict, Union, Tuple, Set, Iterator
 import pandas as pd
-import fsspec
-from sibi_dst.utils import Logger
 from .date_utils import FileAgeChecker
+from pydantic import BaseModel, Field
+from rich.console import Console
+from rich.table import Table
 
+from sibi_dst.utils import ManagedResource
 
-class UpdatePlanner:
+
+class UpdateConfig(BaseModel):
+    """
+    A unified Pydantic model for the data update process configuration.
+    Acts as a single source of truth for all settings.
+    """
+    overwrite: bool = False
+    reverse_order: bool = True
+    ignore_missing: bool = False
+    history_days_threshold: int = 30
+    max_age_minutes: int = 1440  # 24 hours
+    show_progress: bool = False
+    verbose: bool = False
+    debug: bool = False
+    start_date: datetime.date
+    end_date: datetime.date
+    custom_priority_map: Optional[Dict[str, int]] = None
+    max_threads: int = 3
+    timeout: float = 30.0
+
+    class Config:
+        arbitrary_types_allowed = True
+
+class UpdatePlanner(ManagedResource):
     """
     A utility class to scan a date-partitioned filesystem and
     generate an update plan indicating which dates need processing.
@@ -47,9 +72,9 @@ class UpdatePlanner:
             data_path: str,
             filename: str,
             description: str = "Update Planner",
-            fs: Optional[fsspec.AbstractFileSystem] = None,
-            filesystem_type: str = "file",
-            filesystem_options: Optional[Dict] = None,
+            #fs: Optional[fsspec.AbstractFileSystem] = None,
+            #filesystem_type: str = "file",
+            #filesystem_options: Optional[Dict] = None,
             reference_date: Union[str, datetime.date] = None,
             history_days_threshold: int = DEFAULT_HISTORY_DAYS_THRESHOLD,
             max_age_minutes: int = DEFAULT_MAX_AGE_MINUTES,
@@ -58,12 +83,11 @@ class UpdatePlanner:
             custom_priority_map: Optional[Dict[str, int]] = None,
             reverse_order: bool = False,
             show_progress: bool = False,
-            verbose: bool = False,
-            debug: bool = False,
-            logger: Optional[Logger] = None,
             skipped: Optional[List[str]] = None,
+            **kwargs
     ):
         # Initialize state
+        super().__init__(**kwargs)
         self.plan: pd.DataFrame = pd.DataFrame()
         self.df_req: pd.DataFrame = pd.DataFrame()
         self.description = description
@@ -71,14 +95,7 @@ class UpdatePlanner:
         self.filename = filename
         self.reverse_order = reverse_order
         self.show_progress = show_progress
-        self.logger = logger or Logger.default_logger(logger_name="update_planner")
-        self.logger.set_level(Logger.DEBUG if debug else Logger.INFO)
-        self.debug = debug
-        self.verbose = verbose
-
-        # Filesystem and age helper
-        self.fs = fs or fsspec.filesystem(filesystem_type, **(filesystem_options or {}))
-        self.age_checker = FileAgeChecker(debug=debug, logger=self.logger)
+        self.age_checker = FileAgeChecker(debug=self.debug, logger=self.logger)
 
         # Normalize reference date
         if reference_date is None:
@@ -127,8 +144,12 @@ class UpdatePlanner:
                 )
             for future in iterator:
                 d = futures[future]
-                exists, age = future.result()
-                rows.append(self._make_row(d, history_start, exists, age))
+                try:
+                    exists, age = future.result()
+                    rows.append(self._make_row(d, history_start, exists, age))
+                except Exception as exc:
+                    self.logger.error(f"Error processing date {d}: {exc}")
+                    rows.append(self._make_row(d, history_start, False, None))
 
         df = pd.DataFrame(rows)
         df = df.sort_values(
@@ -169,8 +190,37 @@ class UpdatePlanner:
         if self.plan.empty:
             self.logger.warning("No update plan available. Call generate_plan() first.")
             return
-        from IPython.display import display
-        display(self.plan)
+
+        console = Console(record=True)
+
+        table = Table(title=f"Update Plan for {self.data_path}", show_header=True, header_style="bold magenta")
+        for column in self.plan.columns:
+            table.add_column(column, justify="left")
+        for _, row in self.plan.iterrows():
+            table.add_row(*(str(item) for item in row))
+
+        console.print(table)
+
+        plan_string = console.export_text()
+
+        self.logger.info(f"Full Update Plan:\n{plan_string}")
+
+    def get_tasks_by_priority(self) -> Iterator[Tuple[int, List[datetime.date]]]:
+        """Yields batches of dates to be processed, grouped and sorted by priority."""
+        if self.plan.empty:
+            return
+
+        required_updates = self.plan[self.plan['update_required']].copy()
+        if required_updates.empty:
+            return
+
+        for priority in sorted(required_updates["update_priority"].unique()):
+            dates_df = required_updates[required_updates["update_priority"] == priority]
+            # Sort dates within the priority group
+            sorted_dates = dates_df.sort_values(by="date", ascending=not self.reverse_order)
+            dates = sorted_dates["date"].tolist()
+            if dates:
+                yield priority, dates
 
     def _get_file_status(
             self,

{sibi_dst-2025.1.4.dist-info → sibi_dst-2025.1.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.1.4
+Version: 2025.1.5
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com
@@ -18,6 +18,7 @@ Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
 Requires-Dist: pyarrow (>=20.0.0,<21.0.0)
 Requires-Dist: pydantic (>=2.11.7,<3.0.0)
 Requires-Dist: pymysql (>=1.1.1,<2.0.0)
+Requires-Dist: rich (>=14.0.0,<15.0.0)
 Requires-Dist: s3fs (>=2025.5.1,<2026.0.0)
 Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
 Requires-Dist: tqdm (>=4.67.1,<5.0.0)

{sibi_dst-2025.1.4.dist-info → sibi_dst-2025.1.5.dist-info}/RECORD

@@ -1,9 +1,9 @@
-sibi_dst/__init__.py,sha256=3pbriM7Ym5f9gew7n9cO4G_p9n-0bnxdmQ0hwBdJjr4,253
-sibi_dst/df_helper/__init__.py,sha256=McYrw2N0MsMgtawLrONXTGdyHfQWVOBUvIDbklfjb54,342
-sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=-Y4i5KAxKY2BNkmoVeMEZxjTFD7zaM9oQ0aRsvUbQrs,9340
-sibi_dst/df_helper/_df_helper.py,sha256=DJRQWTihnEtgBm3X0ar9nH-xcE1PCkWmh1JgID3WDsY,10939
-sibi_dst/df_helper/_parquet_artifact.py,sha256=Nio5GSD6rTYl52nf_TSpQhYIF0hKqRrB3H3A4zYnaG8,14987
-sibi_dst/df_helper/_parquet_reader.py,sha256=L6mr2FeKtTeIn37G9EGpvOx8PwMqXb6qnEECqBaiwxo,3954
+sibi_dst/__init__.py,sha256=j8lZpGCJlxlLgEgeIMxZnWdqJ0g3MCs7-gsnbvPn_KY,285
+sibi_dst/df_helper/__init__.py,sha256=VJE1qvKO-7QsFADZxSY5s4LVoWnPKfz0rP3nYO2ljhA,358
+sibi_dst/df_helper/_artifact_updater_multi_wrapper.py,sha256=LvFCNr4VKFV-b-NS_TeRkaoKsM4tdsPtAgSIwMvKgGA,18043
+sibi_dst/df_helper/_df_helper.py,sha256=iBoWz2iVgLzQ3hA1EwllL62dkraKamRx2sXseu30FVI,11914
+sibi_dst/df_helper/_parquet_artifact.py,sha256=vDZOtSugBuWuZ3W6l2Y7IBO6RohIrA_sVNuPHp8e8h0,15438
+sibi_dst/df_helper/_parquet_reader.py,sha256=o5ijxHtD1EMzUUD9e6PIoGMeuSLHDItvZIouGfVZhgA,3817
 sibi_dst/df_helper/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/df_helper/backends/http/__init__.py,sha256=d1pfgYxbiYg7E0Iw8RbJ7xfqIfJShqqTBQQGU_S6OOo,105
 sibi_dst/df_helper/backends/http/_http_config.py,sha256=eGPFdqZ5M3Tscqx2P93B6XoBEEzlmdt7yNg7PXUQnNQ,4726
@@ -11,10 +11,10 @@ sibi_dst/df_helper/backends/parquet/__init__.py,sha256=esWJ9aSuYC26d-T01z9dPrJ1u
 sibi_dst/df_helper/backends/parquet/_filter_handler.py,sha256=TvDf0RXta7mwJv11GNQttYJsXgFf2XDj4oLIjt4xTzA,5219
 sibi_dst/df_helper/backends/parquet/_parquet_options.py,sha256=TaU5_wG1Y3lQC8DVCItVvMnc6ZJmECLu3avssVEMbaM,10591
 sibi_dst/df_helper/backends/sqlalchemy/__init__.py,sha256=LjWm9B7CweTvlvFOgB90XjSe0lVLILAIYMWKPkFXFm8,265
-sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=gppZrXLGK8U8xfkzRQPZCIFoWY-miP04nDNHpV8lXtU,10600
-sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=ph4w8Sd9eVr_jUIZuDhGyEwtDn0KQkb0lUkERrIXKGM,12852
-sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=NXVhtYF2mYsrW2fXBkL29VQ5gxAlOYPJkYa8HZKYUyM,2846
-sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=Q93O_xqK0SdrS3IrijVcqky_Zf6xKjtPHdI3qnf1g8E,7457
+sibi_dst/df_helper/backends/sqlalchemy/_db_connection.py,sha256=Rsvh1nfVtqzfMhv968vNTYYIqVxYsEs4PB-O5CTSYdk,10935
+sibi_dst/df_helper/backends/sqlalchemy/_io_dask.py,sha256=NqBSHqeYv_1vHt6J0tez0GdMwKrP_sIRcXYXu869ZkY,13313
+sibi_dst/df_helper/backends/sqlalchemy/_load_from_db.py,sha256=ibxeVqpIEsSVusP2bgcd1MNV_wJIoNgXwacltUbwTas,3194
+sibi_dst/df_helper/backends/sqlalchemy/_sql_model_builder.py,sha256=d_-ip-dQnWOlM8btCjoywAXpaiSuN6AaavkTGJsVQfY,3576
 sibi_dst/df_helper/core/__init__.py,sha256=LfmTqFh6GUZup-g95bcXgAxX7J5Hkve7ftLE_CJg_AE,409
 sibi_dst/df_helper/core/_defaults.py,sha256=9UMEMu2wXznO5UzEhnQ82f_ZazZ20JRyRXIi3HP3gDw,4043
 sibi_dst/df_helper/core/_filter_handler.py,sha256=Pmbzygry2mpkNPVS7DBMulHpAb1yYZNFqUU0bJTWJF0,11214
@@ -32,23 +32,24 @@ sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqT
 sibi_dst/osmnx_helper/utils.py,sha256=BzuY8CtYnBAAO8UAr_M7EOk6CP1zcifNLs8pkdFZEFg,20577
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
-sibi_dst/utils/__init__.py,sha256=H0Yr_Xo4dBTf03_Si_cggmPNSv6cf8_BBetoHJ86Tiw,1162
-sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcWxIQJ60,9877
+sibi_dst/utils/__init__.py,sha256=PQsG188_lnqgSFljkCc15Nyv933HnvmQ7XYs02m77Vc,1217
+sibi_dst/utils/base.py,sha256=MBshlQKg-WNeTeuE_aMLQjcBRaa4O_TFED7bVKDhJ-o,3783
+sibi_dst/utils/clickhouse_writer.py,sha256=5XDRjXU9d0Vhb9GFdCiRoNXrucJvTm6h8auCAQbEwW0,9917
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=MqbwXk33BuANWeKKmsabHouhb8GZswSmbM-VetWWE-M,10357
-sibi_dst/utils/data_wrapper.py,sha256=69aPQFP178-QTJ_joJYqymP--wNxa1qzri_KkvvUTIw,9688
-sibi_dst/utils/date_utils.py,sha256=T3ij-WOQu3cIfmNAweSVMWWr-hVtuBcTGjEY-cMJIvU,18627
+sibi_dst/utils/data_wrapper.py,sha256=Vx3At8SlAoMCTaXmVsTiTGynfjV2isB9W6yL0cWZ7g4,9346
+sibi_dst/utils/date_utils.py,sha256=8fwPpOYqSdM3nHeNykh7Ftk-uPdFa44cEAy5S8iUNw4,18667
 sibi_dst/utils/df_utils.py,sha256=TzIAUCLbgOn3bvCFvzkc1S9YU-OlZTImdCj-88dtg8g,11401
 sibi_dst/utils/file_utils.py,sha256=Z99CZ_4nPDIaZqbCfzzUDfAYJjSudWDj-mwEO8grhbc,1253
 sibi_dst/utils/filepath_generator.py,sha256=-HHO0U-PR8fysDDFwnWdHRlgqksh_RkmgBZLWv9hM7s,6669
-sibi_dst/utils/log_utils.py,sha256=C2wkbxGC2n6hZIEU-z8rHrunDcq95MHkf3B1zGynHnE,4904
-sibi_dst/utils/manifest_manager.py,sha256=FPlKMB1L6hXJSccllIec3ogjCoqAJvraUiaNU-zhLmc,18184
-sibi_dst/utils/parquet_saver.py,sha256=O62xwPfphOpKgEiHqnts20CPSU96pxs49Cg7PVetLK0,8193
+sibi_dst/utils/log_utils.py,sha256=_YnpCnMcjT--ou3BU0EGJma1xMULrA4V5v5UU4IbjAo,14102
+sibi_dst/utils/manifest_manager.py,sha256=Rw7i2phoKJjGlPHYLg_1kr40syVKxd9LJEmfxvZPeDg,8544
+sibi_dst/utils/parquet_saver.py,sha256=zau_s0Mn2ccz9ivVtRbTkBmCghUgCofI1LsCdy1dfDw,4943
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
-sibi_dst/utils/storage_config.py,sha256=TE15H-7d0mqwYPSUgrdidK9U7N7p87Z8JfUQH4-jdPs,4123
-sibi_dst/utils/storage_manager.py,sha256=btecX7ggNb7rfu5EK9Xuu2q_FZA7r_rB_tfhQ8V96qc,6567
-sibi_dst/utils/update_planner.py,sha256=t9A5DLE9cDiYNO8ctQIWVyVWnkMSV0PfbBJ43A0bQv4,9742
+sibi_dst/utils/storage_config.py,sha256=uaCBF8rgCeYkk-lxVSCjsic8O8HJKAu455MR-OBliCo,4325
+sibi_dst/utils/storage_manager.py,sha256=yyZqT8XjTf4MKFrfznCmxXxOYz_TiWgtQhzqPoXR9So,6569
+sibi_dst/utils/update_planner.py,sha256=2ZVsuZlghKDRv7IhqaraS-7GRIY6nGRpFnpBdXYo7Io,11538
 sibi_dst/utils/webdav_client.py,sha256=pYF1UsGOuxYeGLq7aBfwZFvkvD4meOcbbaiZ4d6GW9I,7107
 sibi_dst/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/v2/df_helper/__init__.py,sha256=XuH6jKYAPg2DdRbsxxBSxp9X3x-ARyaT0xe27uILrVo,99
@@ -70,10 +71,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst/v3/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sibi_dst/v3/backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sibi_dst/v3/df_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-sibi_dst/v3/df_helper/_df_helper.py,sha256=NKIQ4Y-Tn-e841sbZxzLh3Q071_Zo9Vu4y3OAXcsO98,3900
-sibi_dst-2025.1.4.dist-info/METADATA,sha256=1YbANTdPXYdETuWFKKldnB237pYywn7rw53Oww3NiC0,2459
-sibi_dst-2025.1.4.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.1.4.dist-info/RECORD,,
+sibi_dst-2025.1.5.dist-info/METADATA,sha256=T0zFKtNSQ7if1S590EwTZ_CN96oiDe8t559-xFQ-XWM,2498
+sibi_dst-2025.1.5.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-2025.1.5.dist-info/RECORD,,