sibi-dst 2025.9.11__py3-none-any.whl → 2025.9.13__py3-none-any.whl

sibi_dst/__init__.py

@@ -10,12 +10,17 @@ try:
 except version_reader.PackageNotFoundError:
     __version__ = "unknown"
 
-__all__ = [
-    "__version__",
-]
-
-import sibi_dst.df_helper as df_helper
+from sibi_dst.df_helper import *
 from sibi_dst.osmnx_helper import *
 from sibi_dst.geopy_helper import *
-from sibi_dst.utils import *
+from sibi_dst import utils as sibiutils
 
+
+__all__ = [
+    "__version__",
+    "DfHelper",
+    "ParquetArtifact",
+    "ParquetReader",
+    "ArtifactUpdaterMultiWrapperAsync",
+    "sibiutils"
+]

sibi_dst/df_helper/__init__.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
 from ._parquet_reader import ParquetReader
-#from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded, ArtifactUpdaterMultiWrapperAsync
 from ._artifact_updater_async import ArtifactUpdaterMultiWrapperAsync
 from ._artifact_updater_threaded import ArtifactUpdaterMultiWrapperThreaded
 

sibi_dst/df_helper/_artifact_updater_async.py

@@ -4,28 +4,24 @@ import asyncio
 import datetime
 import random
 import time
-from contextlib import ExitStack
+import pickle
+from contextlib import ExitStack, suppress
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Sequence, Type
 
-from sibi_dst.utils import ManagedResource
-
-try:
-    from dask.distributed import Client, LocalCluster
-except ImportError:
-    Client = None
-    LocalCluster = None
+from sibi_dst.utils import ManagedResource, Logger
+from sibi_dst.utils.dask_utils import DaskClientMixin
 
 
 @dataclass(slots=True)
 class _RetryCfg:
+    """Retry and backoff configuration."""
     attempts: int = 3
     backoff_base: float = 2.0
     backoff_max: float = 60.0
     jitter: float = 0.15
 
 
-# ---------------- Worker (safe for Dask pickling) ----------------
 def run_artifact_update(
     cls: Type,
     artifact_class_kwargs: Dict[str, Any],
@@ -33,21 +29,43 @@ def run_artifact_update(
     period: str,
     artifact_kwargs: Dict[str, Any],
 ) -> Dict[str, Any]:
-    """Standalone worker — safe for Dask distributed execution."""
+    """
+    Executed inside Dask worker.
+    Instantiates artifact and runs update_parquet() with retry logic.
+    Reconstructs logger and filesystem if not provided (worker isolation safe).
+    """
     import logging
+    import fsspec
+    from sibi_dst.utils import Logger
 
-    logger = logging.getLogger(cls.__name__)
+    # ---- Reinitialize a lightweight logger for the worker
+    worker_logger = Logger.default_logger(logger_name=cls.__name__) if hasattr(Logger, "default_logger") else logging.getLogger(cls.__name__)
+    worker_logger.set_level(logging.INFO)
+
+    # ---- Ensure fs is recreated if missing
+    fs = artifact_class_kwargs.get("fs")
+    if fs is None or isinstance(fs, str):
+        try:
+            fs_protocol = fs if isinstance(fs, str) else "file"
+            fs = fsspec.filesystem(fs_protocol)
+        except Exception:
+            fs = fsspec.filesystem("file")
+
+    # ---- Merge reconstructed environment into kwargs
+    artifact_kwargs_final = {
+        **artifact_class_kwargs,
+        "logger": worker_logger,
+        "fs": fs,
+    }
 
-    start_wall = datetime.datetime.now()
-    attempt_count = 0
-    success = False
-    error_msg = None
+    start_time = datetime.datetime.now()
+    success, error_msg, attempts = False, None, 0
 
     for attempt in range(1, retry.attempts + 1):
-        attempt_count = attempt
+        attempts = attempt
         try:
             with ExitStack() as stack:
-                inst = cls(**artifact_class_kwargs)
+                inst = cls(**artifact_kwargs_final)
                 inst = stack.enter_context(inst)
                 inst.update_parquet(period=period, **artifact_kwargs)
             success = True
@@ -59,31 +77,40 @@ def run_artifact_update(
                 delay *= 1 + random.uniform(0, retry.jitter)
                 time.sleep(delay)
 
-    end_wall = datetime.datetime.now()
-    duration = (end_wall - start_wall).total_seconds()
+    duration = (datetime.datetime.now() - start_time).total_seconds()
+    status = "😀" if success else "😩"
+    worker_logger.info(
+        f"{status} {cls.__name__} [{period}] finished in {duration:.2f}s ({attempts} attempt(s))"
+    )
 
     return {
         "artifact": cls.__name__,
         "period": period,
-        "start": start_wall.isoformat(),
-        "end": end_wall.isoformat(),
-        "processing_time": duration,
-        "retries": attempt_count - 1 if success else attempt_count,
         "success": success,
         "error": error_msg,
+        "attempts": attempts,
+        "duration_seconds": duration,
+        "started_at": start_time.isoformat(),
+        "ended_at": datetime.datetime.now().isoformat(),
     }
 
 
-class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
+# ---------------- Async Orchestrator ----------------
+class ArtifactUpdaterMultiWrapperAsync(DaskClientMixin, ManagedResource):
     """
-    Async/Threaded orchestrator.
-    Dask-enabled if a Client is passed (or created automatically).
+    Async orchestrator for concurrent artifact updates.
+
+    • Uses Dask client (via DaskClientMixin) or local threads.
+    • Automatically sanitizes non-picklable arguments (e.g., loggers, fs).
+    • Provides structured retries, async orchestration, and safe cleanup.
     """
 
     def __init__(
         self,
         wrapped_classes: Dict[str, Sequence[Type]],
         *,
+        logger: Logger,
+        fs,
         max_workers: int = 3,
         retry_attempts: int = 3,
         update_timeout_seconds: int = 600,
@@ -92,88 +119,127 @@ class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
         backoff_jitter: float = 0.15,
         priority_fn: Optional[Callable[[Type], int]] = None,
         artifact_class_kwargs: Optional[Dict[str, Any]] = None,
-        dask_client: Optional[Client] = None,
         use_dask: bool = True,
+        dask_client: Optional[Any] = None,
+        debug: bool = False,
+        verbose: bool = False,
         **kwargs: Any,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__(logger=logger, fs=fs, debug=debug, verbose=verbose)
+
+        # ---- Client lifecycle management
+
+        self.own_dask_client = dask_client is None
+        self._init_dask_client(dask_client, logger=logger)
+        self.use_dask = use_dask
 
+        # ---- Core configuration
         self.wrapped_classes = wrapped_classes
-        self.max_workers = int(max_workers)
-        self.update_timeout_seconds = int(update_timeout_seconds)
+        self.max_workers = max_workers
         self.priority_fn = priority_fn
-        self.use_dask = use_dask
-        self.client: Optional[Client] = dask_client
-        self._owns_client = False
+        self.update_timeout_seconds = update_timeout_seconds
 
+        # ---- Retry configuration
         self._retry = _RetryCfg(
-            attempts=int(retry_attempts),
-            backoff_base=float(backoff_base),
-            backoff_max=float(backoff_max),
-            jitter=float(backoff_jitter),
+            attempts=retry_attempts,
+            backoff_base=backoff_base,
+            backoff_max=backoff_max,
+            jitter=backoff_jitter,
         )
 
-        # Safe kwargs for artifacts
-        if self.use_dask:
-            self.artifact_class_kwargs = {
-                "debug": self.debug,
-                "verbose": self.verbose,
-                **(artifact_class_kwargs or {}),
-            }
-        else:
-            self.artifact_class_kwargs = {
-                "logger": self.logger,
-                "fs": self.fs,
-                "debug": self.debug,
-                "verbose": self.verbose,
-                **(artifact_class_kwargs or {}),
-            }
+        # ---- Artifact instantiation arguments
+        self.artifact_class_kwargs = {
+            "logger": logger,
+            "fs": fs,
+            "debug": debug,
+            "verbose": verbose,
+            **(artifact_class_kwargs or {}),
+        }
 
+        # ---- Runtime tracking
         self.completion_secs: Dict[str, float] = {}
         self.failed: List[str] = []
-        self._stop = asyncio.Event()
-
-        if self.use_dask and Client is None:
-            raise RuntimeError("Dask is not installed, cannot use Dask mode")
-
-        # auto-start local client if requested
-        if self.use_dask and not self.client:
-            self.client = Client(
-                LocalCluster(
-                    n_workers=max_workers,
-                    threads_per_worker=1,
-                    dashboard_address=None,
-                )
-            )
-            self._owns_client = True
+        self._stop_event = asyncio.Event()
 
-    # ---- Internals ------------------------------------------------------------
+        self.logger_extra = {"sibi_dst_component": self.__class__.__name__}
+
+        if self.use_dask:
+            self.logger.debug(f"Initialized with Dask client: {self.dask_client}")
+        else:
+            self.logger.debug(f"Running in local thread-based mode.")
+
+    async def update_data(self, period: str, **kwargs: Any) -> List[Dict[str, Any]]:
+        """Runs updates for all artifacts in a given period."""
+        self.completion_secs.clear()
+        self.failed.clear()
+        classes = self._classes_for(period)
+
+        self.logger.info(
+            f"Starting artifact updates for period '{period}' ({len(classes)} artifacts).",
+            extra=self.logger_extra,
+        )
 
-    def _classes_for(self, period: str) -> List[Type]:
         try:
-            classes = list(self.wrapped_classes[period])
-        except KeyError:
-            raise ValueError(f"Unsupported period '{period}'.")
-        if not classes:
-            raise ValueError(f"No artifact classes configured for '{period}'.")
-        if self.priority_fn:
+            if self.use_dask:
+                futures = [self._submit_one_dask(cls, period, kwargs) for cls in classes]
+                results = await asyncio.to_thread(lambda: self.dask_client.gather(futures))
+            else:
+                sem = asyncio.Semaphore(self.max_workers)
+                tasks = [self._run_one_async(cls, period, sem, kwargs) for cls in classes]
+                results = await asyncio.gather(*tasks)
+
+            self.logger.info(
+                f"Completed {len(results)} artifact updates for period '{period}'.",
+                extra=self.logger_extra,
+            )
+            return results
+
+        finally:
+            # Always cleanup if we own the client
+            if getattr(self, "own_dask_client", False):
+                self._close_dask_client()
+
+
+    def _sanitize_kwargs_for_dask(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Removes non-picklable runtime objects (e.g., loggers, fs) before sending to Dask.
+        """
+        clean: Dict[str, Any] = {}
+        for k, v in kwargs.items():
             try:
-                classes.sort(key=self.priority_fn)
-            except Exception as e:
-                self.logger.warning(f"priority_fn failed; using listed order: {e}")
-        return classes
+                pickle.dumps(v)
+                clean[k] = v
+            except Exception:
+                self.logger.debug(f"Skipping non-picklable key '{k}' for Dask worker.")
+        return clean
 
     def _submit_one_dask(self, cls: Type, period: str, artifact_kwargs: Dict[str, Any]):
-        return self.client.submit(
+        """Submit one artifact job to Dask."""
+        safe_kwargs = self._sanitize_kwargs_for_dask(self.artifact_class_kwargs)
+        return self.dask_client.submit(
             run_artifact_update,
             cls,
-            dict(self.artifact_class_kwargs),
+            safe_kwargs,
             self._retry,
             period,
             artifact_kwargs,
             pure=False,
         )
 
+    def _classes_for(self, period: str) -> List[Type]:
+        """Selects artifact classes for the given period."""
+        try:
+            classes = list(self.wrapped_classes[period])
+        except KeyError:
+            raise ValueError(f"No artifacts configured for period '{period}'.")
+        if not classes:
+            raise ValueError(f"No artifact classes found for '{period}'.")
+
+        if self.priority_fn:
+            with suppress(Exception):
+                classes.sort(key=self.priority_fn)
+        return classes
+
     async def _run_one_async(
         self,
         cls: Type,
@@ -181,112 +247,70 @@ class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
         sem: asyncio.Semaphore,
         artifact_kwargs: Dict[str, Any],
     ) -> Dict[str, Any]:
-        """Async/threaded fallback execution."""
+        """Fallback local async execution (no Dask)."""
         name = cls.__name__
-        self.logger.info(f"▶️ Starting {name} for period '{period}'")
-        start_wall = datetime.datetime.now()
-
-        attempt_count = 0
-        success = False
-        error_msg = None
-
-        try:
-            async with sem:
-                for attempt in range(1, self._retry.attempts + 1):
-                    attempt_count = attempt
-                    try:
-                        def _sync_block():
-                            with ExitStack() as stack:
-                                inst = cls(**self.artifact_class_kwargs)
-                                inst = stack.enter_context(inst)
-                                inst.update_parquet(period=period, **artifact_kwargs)
-
-                        await asyncio.wait_for(
-                            asyncio.to_thread(_sync_block),
-                            timeout=self.update_timeout_seconds,
-                        )
-                        success = True
-                        break
-                    except Exception as e:
-                        error_msg = str(e)
-                        if attempt < self._retry.attempts and not self._stop.is_set():
-                            delay = min(
-                                self._retry.backoff_base ** (attempt - 1),
-                                self._retry.backoff_max,
-                            )
-                            delay *= 1 + random.uniform(0, self._retry.jitter)
-                            await asyncio.sleep(delay)
-        finally:
-            end_wall = datetime.datetime.now()
-            duration = (end_wall - start_wall).total_seconds()
-
-            result = {
-                "artifact": name,
-                "period": period,
-                "start": start_wall.isoformat(),
-                "end": end_wall.isoformat(),
-                "processing_time": duration,
-                "retries": attempt_count - 1 if success else attempt_count,
-                "success": success,
-                "error": error_msg,
-            }
-
-            if success:
-                self.logger.info(f"✅ Artifact {name} succeeded", extra=result)
-                self.completion_secs[name] = duration
-            else:
-                self.logger.error(f"❌ Artifact {name} failed", extra=result)
-                self.failed.append(name)
-
-            return result
+        start_time = datetime.datetime.now()
+
+        async with sem:
+            for attempt in range(1, self._retry.attempts + 1):
+                try:
+                    def _sync_block():
+                        with ExitStack() as stack:
+                            inst = cls(**self.artifact_class_kwargs)
+                            inst = stack.enter_context(inst)
+                            inst.update_parquet(period=period, **artifact_kwargs)
+
+                    await asyncio.wait_for(
+                        asyncio.to_thread(_sync_block),
+                        timeout=self.update_timeout_seconds,
+                    )
+                    duration = (datetime.datetime.now() - start_time).total_seconds()
+                    self.completion_secs[name] = duration
+                    self.logger.info(f"✅ {name} completed in {duration:.2f}s")
+                    return {
+                        "artifact": name,
+                        "period": period,
+                        "success": True,
+                        "attempts": attempt,
+                        "duration_seconds": duration,
+                    }
+
+                except Exception as e:
+                    if attempt < self._retry.attempts:
+                        delay = min(self._retry.backoff_base ** attempt, self._retry.backoff_max)
+                        delay *= 1 + random.uniform(0, self._retry.jitter)
+                        self.logger.warning(f"Retry {attempt}/{self._retry.attempts} for {name}: {e}")
+                        await asyncio.sleep(delay)
+                    else:
+                        duration = (datetime.datetime.now() - start_time).total_seconds()
+                        self.failed.append(name)
+                        self.logger.error(f"❌ {name} failed after {attempt} attempts: {e}")
+                        return {
+                            "artifact": name,
+                            "period": period,
+                            "success": False,
+                            "attempts": attempt,
+                            "error": str(e),
+                            "duration_seconds": duration,
+                        }
 
-    # ---- Public API -----------------------------------------------------------
-
-    async def update_data(self, period: str, **kwargs: Any) -> List[Dict[str, Any]]:
-        self.completion_secs.clear()
-        self.failed.clear()
-        classes = self._classes_for(period)
-
-        try:
-            if self.use_dask:
-                futures = [self._submit_one_dask(cls, period, kwargs) for cls in classes]
-                results = await asyncio.to_thread(lambda: self.client.gather(futures))
-            else:
-                sem = asyncio.Semaphore(self.max_workers)
-                tasks = [
-                    asyncio.create_task(self._run_one_async(cls, period, sem, kwargs))
-                    for cls in classes
-                ]
-                results = await asyncio.gather(*tasks)
-            return results
-        finally:
-            # only shut down if we own the client
-            if self._owns_client:
-                self.close()
 
     def get_update_status(self) -> Dict[str, Any]:
+        """Returns summary of completed, failed, and pending artifacts."""
         done = set(self.completion_secs)
         fail = set(self.failed)
-        all_names = {c.__name__ for v in self.wrapped_classes.values() for c in v}
+        all_names = {cls.__name__ for v in self.wrapped_classes.values() for cls in v}
         return {
             "total": len(all_names),
             "completed": sorted(done),
             "failed": sorted(fail),
             "pending": sorted(all_names - done - fail),
-            "completion_times": dict(self.completion_secs),
+            "completion_times": self.completion_secs,
         }
 
-    # ---- Lifecycle ------------------------------------------------------------
-
     def _cleanup(self) -> None:
-        """Release any resources created by this wrapper."""
-        if self._owns_client and self.client is not None:
-            try:
-                cluster = getattr(self.client, "cluster", None)
-                self.client.close()
-                if cluster is not None:
-                    cluster.close()
-            finally:
-                self.client = None
-                self._owns_client = False
+        """Ensures safe resource closure."""
+        with suppress(Exception):
+            if getattr(self, "own_dask_client", False):
+                self._close_dask_client()
 

sibi_dst/osmnx_helper/__init__.py

@@ -1,7 +1,9 @@
 from .base_osm_map import BaseOsmMap
 from .utils import PBFHandler
+from .route_path_builder import RoutePathBuilder, RoutePathBuilderConfig
 __all__ = [
     "BaseOsmMap",
-    #"MapConfig",
+    "RoutePathBuilder",
+    "RoutePathBuilderConfig",
     "PBFHandler",
 ]

sibi_dst/utils/__init__.py

@@ -24,6 +24,7 @@ from .manifest_manager import MissingManifestManager
 __all__ = [
     "Logger",
     "ManagedResource",
+
     "ConfigManager",
     "ConfigLoader",
     "DateUtils",
@@ -42,5 +43,5 @@ __all__ = [
     "FsRegistry",
     "DataFromHttpSource",
     "WebDAVClient",
-    "MissingManifestManager"
+    "MissingManifestManager",
 ]

sibi_dst/utils/boilerplate/base_pipeline.py

@@ -93,7 +93,7 @@ class BasePipeline(ManagedResource):
         df[self.date_field] = dd.to_datetime(df[self.date_field], errors="coerce")
         df["partition_date"] = df[self.date_field].dt.date.astype(str)
 
-        out_path = self.storage_path.rstrip("/")+"/"+self._get_output_filename(fmt="parquet")
+        out_path = self.storage_path.rstrip("/")
         self.logger.info("Saving dataset to %s", out_path)
         ps = ParquetSaver(
             df_result=df,
@@ -111,7 +111,6 @@ class BasePipeline(ManagedResource):
             parquet_start_date=self.start_date,
             parquet_end_date=self.end_date,
             parquet_storage_path=self.storage_path,
-            parquet_filename=self._get_output_filename(),
             fs=self.fs,
             debug=self.debug,
             logger=self.logger,

sibi_dst/utils/dask_utils.py

@@ -1,7 +1,11 @@
+from __future__ import annotations
+
 import asyncio
+import logging
 from typing import List, Any, Dict
 
 import dask
+# dask.config.set({"distributed.worker.daemon": False})
 import dask.dataframe as dd
 
 def _to_int_safe(x) -> int:
@@ -58,4 +62,139 @@ class UniqueValuesExtractor:
             return col, await self.compute_to_list(ser)
 
         pairs = await asyncio.gather(*(one(c) for c in columns))
-        return dict(pairs)
+        return dict(pairs)
+
+from contextlib import suppress, asynccontextmanager
+from dask.distributed import Client, LocalCluster, get_client
+import os
+
+class DaskClientMixin:
+    """
+    Provides shared Dask client lifecycle management.
+    Ensures reuse of an existing client if available,
+    or creates a local in-process Dask cluster for fallback.
+    """
+
+    def _init_dask_client(
+        self,
+        dask_client=None,
+        logger=None,
+        *,
+        n_workers: int = 1,
+        threads_per_worker: int = 1,
+        processes: bool = False,
+        asynchronous: bool = False,
+        memory_limit: str = "auto",
+        #dashboard_address: str | None = None,
+        local_directory: str | None = None,
+        silence_logs: str = "info",
+        resources: dict | None = None,
+        timeout: int = 30,
+    ):
+        self.dask_client = dask_client
+        self.own_dask_client = False
+        self.logger = logger
+        # Apply log filters globally
+        logging.getLogger("distributed.shuffle._scheduler_plugin").setLevel(
+            logging.ERROR
+        )
+        logging.getLogger("distributed.scheduler").setLevel(logging.WARNING)
+        logging.getLogger("distributed.worker").setLevel(logging.WARNING)
+
+        if self.dask_client is None:
+            with suppress(ValueError, RuntimeError):
+                # Try to attach to an existing client (common in shared Dask setups)
+                self.dask_client = get_client()
+
+        if self.dask_client is None:
+            # Default to half of logical cores if not specified
+            n_workers = n_workers or max(2, os.cpu_count() // 2)
+
+            cluster = LocalCluster(
+                n_workers=n_workers,
+                threads_per_worker=threads_per_worker,
+                processes=processes,
+                asynchronous=asynchronous,
+                memory_limit=memory_limit,
+                local_directory=local_directory,
+                silence_logs=silence_logs,
+                resources=resources,
+                timeout=timeout,
+            )
+
+            self.dask_client = Client(cluster)
+            self.own_dask_client = True
+
+            if self.logger:
+                self.logger.info(
+                    f"Started local Dask cluster with {n_workers} workers × {threads_per_worker} threads "
+                    f"({memory_limit} memory per worker). Dashboard: {self.dask_client.dashboard_link}"
+                )
+        else:
+            if self.logger:
+                self.logger.debug(
+                    f"Using existing Dask client: {self.dask_client.dashboard_link}"
+                )
+
+    def _close_dask_client(self):
+        """Close the Dask client if this instance created it."""
+        if getattr(self, "own_dask_client", False) and self.dask_client is not None:
+            try:
+                cluster = getattr(self.dask_client, "cluster", None)
+                self.dask_client.close()
+                if cluster is not None:
+                    cluster.close()
+                if self.logger:
+                    self.logger.info("Closed local Dask client and cluster.")
+            except Exception as e:
+                if self.logger:
+                    self.logger.warning(f"Error while closing Dask client: {e}")
+
+@asynccontextmanager
+async def shared_dask_session(**kwargs):
+    mixin = DaskClientMixin()
+    mixin._init_dask_client(**kwargs)
+    try:
+        yield mixin.dask_client
+    finally:
+        mixin._close_dask_client()
+
+# from contextlib import suppress
+# from dask.distributed import Client, get_client
+#
+# class DaskClientMixin:
+#     """
+#     Provides shared Dask client lifecycle management.
+#     Ensures reuse of existing client when available, otherwise creates a lightweight local one.
+#     """
+#
+#     def _init_dask_client(self, dask_client=None, logger=None):
+#         self.dask_client = dask_client
+#         self.own_dask_client = False
+#         self.logger = logger
+#
+#         if self.dask_client is None:
+#             with suppress(ValueError, RuntimeError):
+#                 # Try to attach to an existing active client if running inside a Dask context
+#                 self.dask_client = get_client()
+#
+#         if self.dask_client is None:
+#             # Start a local in-process scheduler for fallback
+#             self.dask_client = Client(processes=False)
+#             self.own_dask_client = True
+#             if self.logger:
+#                 self.logger.info(f"Started local Dask client: {self.dask_client.dashboard_link}")
+#         else:
+#             if self.logger:
+#                 self.logger.debug(f"Using existing Dask client: {self.dask_client.dashboard_link}")
+#
+#     def _close_dask_client(self):
+#         """Close client only if this instance created it."""
+#         if getattr(self, "own_dask_client", False) and self.dask_client is not None:
+#             try:
+#                 self.dask_client.close()
+#                 if self.logger:
+#                     self.logger.info("Closed local Dask client.")
+#             except Exception as e:
+#                 if self.logger:
+#                     self.logger.warning(f"Error while closing Dask client: {e}")

sibi_dst/utils/data_wrapper.py

@@ -87,17 +87,6 @@ class DataWrapper(ManagedResource):
             "dataclass": self.dataclass.__name__
         })
 
-    # --------------------- Context Management ---------------------
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        """Ensure manifest is saved and resources are cleaned up on context exit."""
-        if self.mmanifest:
-            try:
-                self.mmanifest.save()
-            except Exception as e:
-                self.logger.error(f"Failed to save manifest in __exit__: {e}", extra=self.logger_extra)
-        # Call parent's __exit__ which triggers _cleanup
-        return super().__exit__(exc_type, exc_val, exc_tb)
-
     # --------------------- Cleanup ---------------------
     def _cleanup(self) -> None:
         """Signal shutdown during class-specific cleanup."""

{sibi_dst-2025.9.11.dist-info → sibi_dst-2025.9.13.dist-info}/METADATA

@@ -1,34 +1,31 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: sibi-dst
-Version: 2025.9.11
-Summary: Data Science Toolkit
-Author: Luis Valverde
-Author-email: lvalverdeb@gmail.com
-Requires-Python: >=3.11,<4.0
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: clickhouse-connect (>=0.8.18,<0.9.0)
-Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
-Requires-Dist: dask[complete] (>=2025.9.0,<2026.0.0)
-Requires-Dist: distributed (>=2025.9.1,<2026.0.0)
-Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
-Requires-Dist: opentelemetry-exporter-otlp (>=1.35.0,<2.0.0)
-Requires-Dist: opentelemetry-sdk (>=1.35.0,<2.0.0)
-Requires-Dist: pandas (>=2.3.1,<3.0.0)
-Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
-Requires-Dist: pyarrow (>=20.0.0,<21.0.0)
-Requires-Dist: pydantic (>=2.11.7,<3.0.0)
-Requires-Dist: pyiceberg[hive,s3fs] (>=0.9.1,<0.10.0)
-Requires-Dist: pymysql (>=1.1.1,<2.0.0)
-Requires-Dist: pyrosm (>=0.6.2,<0.7.0)
-Requires-Dist: s3fs (>=2025.5.1,<2026.0.0)
-Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
-Requires-Dist: sse-starlette (>=3.0.2,<4.0.0)
-Requires-Dist: tqdm (>=4.67.1,<5.0.0)
-Requires-Dist: webdav4 (>=0.10.0,<0.11.0)
+Version: 2025.9.13
+Summary: A data science toolkit for scalable data processing and analysis.
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
+Requires-Dist: clickhouse-connect>=0.9.2
+Requires-Dist: clickhouse-driver>=0.2.9
+Requires-Dist: dask>=2025.9.1
+Requires-Dist: distributed>=2025.9.1
+Requires-Dist: fastapi>=0.118.0
+Requires-Dist: folium>=0.20.0
+Requires-Dist: mysqlclient>=2.2.7
+Requires-Dist: opentelemetry-api>=1.37.0
+Requires-Dist: opentelemetry-exporter-otlp>=1.37.0
+Requires-Dist: opentelemetry-sdk>=1.37.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: psycopg2>=2.9.10
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pydantic>=2.11.10
+Requires-Dist: pymysql>=1.1.2
+Requires-Dist: redis>=6.4.0
+Requires-Dist: s3fs>=2025.9.0
+Requires-Dist: sqlalchemy>=2.0.43
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: uvicorn>=0.37.0
+Requires-Dist: webdav4>=0.10.0
+Requires-Dist: wheel>=0.45.1
 
 ### SIBI-DST
 
@@ -60,4 +57,3 @@ pip install sibi-dst[dev,test,geospatial]  # Install all optional dependencies
 
 
 ```
-

{sibi_dst-2025.9.11.dist-info → sibi_dst-2025.9.13.dist-info}/RECORD

@@ -1,6 +1,6 @@
-sibi_dst/__init__.py,sha256=D4TMsAMGRl54J5PNYMDf_z0NcCnS_lZK1YHze2eJvpc,464
-sibi_dst/df_helper/__init__.py,sha256=aOQFmWpLeSq-7O8IYEKaxR1nvuIe9F7SJsOsgwj1ECg,579
-sibi_dst/df_helper/_artifact_updater_async.py,sha256=0VBwGMIQXKagr4MpshvfuhplwePD2VQbN2MOUTgQyJs,9992
+sibi_dst/__init__.py,sha256=QQVT3Xlj8iZN17sSMfRQFSb_DHr8A7giJP8hn02K2Oo,585
+sibi_dst/df_helper/__init__.py,sha256=7rUdMybgCNZhQL_J7IFTTHz_xtFin81xavi5-PUExkA,463
+sibi_dst/df_helper/_artifact_updater_async.py,sha256=AZp0vM3vji0tjiaScr8a9SUMH15NjPIKYPdRQ7SJe3Y,11372
 sibi_dst/df_helper/_artifact_updater_threaded.py,sha256=M5GNZismOqMmBrcyfolP1DPv87VILQf_P18is_epn50,7238
 sibi_dst/df_helper/_df_helper.py,sha256=rgVP4ggiCW6tTHmUz2UqUvLznwOtY5IyoVS3WSlg73U,17005
 sibi_dst/df_helper/_parquet_artifact.py,sha256=UXkhDSAVRNKp9DykVhJd3agnryCZT0Sj2qhdhUZomuM,19421
@@ -22,39 +22,30 @@ sibi_dst/df_helper/core/_defaults.py,sha256=9UMEMu2wXznO5UzEhnQ82f_ZazZ20JRyRXIi
 sibi_dst/df_helper/core/_filter_handler.py,sha256=9C30zrT8wSGy1X8ryiTWc0XfnbpeoHndHgoOcHKOPOo,19309
 sibi_dst/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxRrQKE5FQRxcEWsac,6736
 sibi_dst/df_helper/core/_query_config.py,sha256=1ApqmuSGXTC3CdF-xMsSbCa3V2Z5hOP3Wq5huhzZwqY,439
-sibi_dst/df_helper/data_cleaner.py,sha256=lkxQoXLvGzXCicFUimnA5nen5qkrO1oxgl_p2Be2o8w,5183
 sibi_dst/geopy_helper/__init__.py,sha256=Q1RJiUZIOlV0QNNLjxZ_2IZS5LqIe5jRbeQkfD1Vm60,112
 sibi_dst/geopy_helper/geo_location_service.py,sha256=1ArI980QF_gRw096ZsABHwJt-m55jrfOlB8tPwL1BvY,2959
 sibi_dst/geopy_helper/utils.py,sha256=Sb7qfSqIyWh-AZ4GBdB9-z5FrQPWtrdtQLLcNjph0yw,3351
-sibi_dst/osmnx_helper/__init__.py,sha256=On2_pD13HmzZjP-YrXV9BA9uFK-z26QkQE-MliGdv5w,134
+sibi_dst/osmnx_helper/__init__.py,sha256=hTv1uCnN35MSUEYNWk30OHHSflDb1qidwfLDXhUU8cE,243
 sibi_dst/osmnx_helper/base_osm_map.py,sha256=L7g3VBiayHX41BcCBTOCS0iJOKzp2ZZYcrp8N-mnU90,19392
+sibi_dst/osmnx_helper/route_path_builder.py,sha256=XJJyu4YXegAkCRjE-knyQncwXaxDVXZhalYacLcb7e0,3557
+sibi_dst/osmnx_helper/utils.py,sha256=7-lFVhGn4rHjGz6FvpXtC2jY8UzGIVyKR3MVyEfB7nw,14407
 sibi_dst/osmnx_helper/basemaps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/osmnx_helper/basemaps/calendar_html.py,sha256=UArt6FDgoCgoRte45Xo3IHqd-RNzW0YgitgZYfOFasY,4031
 sibi_dst/osmnx_helper/basemaps/route_map_plotter.py,sha256=rsJidieojcqIoe0kBanZbrxcelrS6nWoAyWoQXWdPiQ,11849
 sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqTadrxMx-YK4djYhqPqfQ,10941
-sibi_dst/osmnx_helper/route_path_builder.py,sha256=XJJyu4YXegAkCRjE-knyQncwXaxDVXZhalYacLcb7e0,3557
-sibi_dst/osmnx_helper/utils.py,sha256=7-lFVhGn4rHjGz6FvpXtC2jY8UzGIVyKR3MVyEfB7nw,14407
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_baseclass.py,sha256=5huAwjWo_SOEZR2_0y5w9qUmw5G7pVdm8X1OTG87JK0,11562
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
-sibi_dst/utils/__init__.py,sha256=vShNCOMPw8KKwlb4tq5XGrpjqakJ_OE8YDc_xDAWAxI,1302
+sibi_dst/utils/__init__.py,sha256=eoW7iROrCUVYjNT1owMgGvW6U7lolbNy3FrBb_wInPs,1304
 sibi_dst/utils/async_utils.py,sha256=53aywfgq1Q6-0OVr9qR1Sf6g7Qv3I9qunAAR4fjFXBE,351
 sibi_dst/utils/base.py,sha256=sFngliI7Ku8bZMz0YdVhppuaPNZ0dvqRwCsPe9XdF1A,16256
-sibi_dst/utils/boilerplate/__init__.py,sha256=89oepkDCnjegnhzd6Kgga71AH17KSvNMlgD83CAAmg8,582
-sibi_dst/utils/boilerplate/base_attacher.py,sha256=iZftWNUx8y370OJP_kGCs5v3t2RgPuARIK_jQeFfbAU,2089
-sibi_dst/utils/boilerplate/base_data_cube.py,sha256=ErKTM2kT8LsSXADcyYvT436O_Mp0J2hm8xs1IUircb4,2760
-sibi_dst/utils/boilerplate/base_parquet_artifact.py,sha256=oqPbjHFfChA9j1WL-eDAh7XLA3zmf-Rq7s_kzITVniA,3753
-sibi_dst/utils/boilerplate/base_parquet_reader.py,sha256=3kN9_bbxyX-WuJLMBWejeApW2V_BDArSljhSUOAOhVU,692
-sibi_dst/utils/boilerplate/base_pipeline.py,sha256=LQZBACksqHO3tQ8OhWShfqjiGyda7UhrmllRq3eWQfU,5690
-sibi_dst/utils/boilerplate/base_pipeline_template.py,sha256=D5HFA4odsR2wlTY6iLg1tm57Tsh91QkoYjjX8eUgrjU,1574
-sibi_dst/utils/boilerplate/hybrid_data_loader.py,sha256=Tazn7QL3FmWKVMXxzkvxPrG_2ucsPHvSotIW9dBLoNc,6018
 sibi_dst/utils/business_days.py,sha256=DPZExTXTt7n3IbAaEuVacm-vZgbR_Ug2bJTPBUaoP3g,6694
 sibi_dst/utils/clickhouse_writer.py,sha256=8W_dTEOKQp4pXANznVSxRqFA2H5oD8UJifiBAONpXWY,17001
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
-sibi_dst/utils/dask_utils.py,sha256=QhFcmpH4fXAy6b3DugIX5JvH4h-P3M3hXKnBYTLRkq0,1991
+sibi_dst/utils/dask_utils.py,sha256=FT85GPvHxaR7BDhxgeQT4mgQ_lFmNT5CVmiyhRv9Fns,7251
 sibi_dst/utils/data_from_http_source.py,sha256=AcpKNsqTgN2ClNwuhgUpuNCx62r5_DdsAiKY8vcHEBA,1867
 sibi_dst/utils/data_utils.py,sha256=7bLidEjppieNoozDFb6OuRY0W995cxg4tiGAlkGfePI,7768
-sibi_dst/utils/data_wrapper.py,sha256=-a5LkkjBVXUPwDYngn5K98oSNYsQf87zNaih9IJxs-I,18543
+sibi_dst/utils/data_wrapper.py,sha256=9HTuDXgvfhmFAOyNG_GEOaHuojxE3639yyzOoBt7Unc,18000
 sibi_dst/utils/date_utils.py,sha256=hBVWu9_cqiZ-XsLR7QY9Iek09DQKLwrY1ZlYxWlXj7g,31101
 sibi_dst/utils/df_utils.py,sha256=bQGromLOEdRTvbVVcuHq0vQ0fIgqhwOoD_eIp5v7VEY,10899
 sibi_dst/utils/file_age_checker.py,sha256=44B3lwH_PLwzMfiKkgvJKjKx-qSgITIXxKfNbdf_VeA,11552
@@ -66,15 +57,23 @@ sibi_dst/utils/manifest_manager.py,sha256=9y4cV-Ig8O-ekhApp_UObTY-cTsl-bGnvKIThI
 sibi_dst/utils/parquet_saver.py,sha256=Itctsf8UnBCnD6NrP00FK0y9KzZgKYfjUk1CC-0x-F0,20486
 sibi_dst/utils/periods.py,sha256=8eTGi-bToa6_a8Vwyg4fkBPryyzft9Nzy-3ToxjqC8c,1434
 sibi_dst/utils/phone_formatter.py,sha256=oeM22nLjhObENrpItCNeVpkYS4pXRm5hSxdk0M4nvwU,4580
-sibi_dst/utils/progress/__init__.py,sha256=VELVxzo2cePN_-LL0veel8-F3po6tokY5MOOpu6pz1A,92
-sibi_dst/utils/progress/jobs.py,sha256=nE58ng9GPCPZhnaCDltr1tQgu3AJVqBJ1dWbGcCH4xo,3089
-sibi_dst/utils/progress/sse_runner.py,sha256=NttASZH_ayXo1Zi6I4tSwYnWySLxexOYQGlqzOZiXlI,4965
 sibi_dst/utils/storage_config.py,sha256=DLtP5jKVM0mdFdgRw6LQfRqyavMjJcCVU7GhsUCRH78,4427
 sibi_dst/utils/storage_hive.py,sha256=eZ3nq2YWLUUG-06iJubSC15cwSHEbKKdKIwoVhD_I_E,8568
 sibi_dst/utils/storage_manager.py,sha256=La1NY79bhRAmHWXp7QcXJZtbHoRboJMgoXOSXbIl1SA,6643
 sibi_dst/utils/update_planner.py,sha256=1UOh4MjZSfaA_ZO-nKailOGal5EY-xVR8KSCJzo7p_g,16834
 sibi_dst/utils/webdav_client.py,sha256=D9J5d1f1qQwHGm5FE5AMVpOPwcU5oD7K8JZoKGP8NpM,5811
 sibi_dst/utils/write_gatekeeper.py,sha256=V8sY9YMO-JuN8Ps7prqwVSjP4f1HGH9KiVV-aTPCC_k,569
+sibi_dst/utils/boilerplate/__init__.py,sha256=89oepkDCnjegnhzd6Kgga71AH17KSvNMlgD83CAAmg8,582
+sibi_dst/utils/boilerplate/base_attacher.py,sha256=iZftWNUx8y370OJP_kGCs5v3t2RgPuARIK_jQeFfbAU,2089
+sibi_dst/utils/boilerplate/base_data_cube.py,sha256=ErKTM2kT8LsSXADcyYvT436O_Mp0J2hm8xs1IUircb4,2760
+sibi_dst/utils/boilerplate/base_parquet_artifact.py,sha256=oqPbjHFfChA9j1WL-eDAh7XLA3zmf-Rq7s_kzITVniA,3753
+sibi_dst/utils/boilerplate/base_parquet_reader.py,sha256=3kN9_bbxyX-WuJLMBWejeApW2V_BDArSljhSUOAOhVU,692
+sibi_dst/utils/boilerplate/base_pipeline.py,sha256=cPVvjGxd0tDDvP_c27MNDHNuNnM6k3yx5bivN5ZFgrQ,5587
+sibi_dst/utils/boilerplate/base_pipeline_template.py,sha256=D5HFA4odsR2wlTY6iLg1tm57Tsh91QkoYjjX8eUgrjU,1574
+sibi_dst/utils/boilerplate/hybrid_data_loader.py,sha256=Tazn7QL3FmWKVMXxzkvxPrG_2ucsPHvSotIW9dBLoNc,6018
+sibi_dst/utils/progress/__init__.py,sha256=VELVxzo2cePN_-LL0veel8-F3po6tokY5MOOpu6pz1A,92
+sibi_dst/utils/progress/jobs.py,sha256=nE58ng9GPCPZhnaCDltr1tQgu3AJVqBJ1dWbGcCH4xo,3089
+sibi_dst/utils/progress/sse_runner.py,sha256=NttASZH_ayXo1Zi6I4tSwYnWySLxexOYQGlqzOZiXlI,4965
 sibi_dst/v2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/v2/df_helper/__init__.py,sha256=XuH6jKYAPg2DdRbsxxBSxp9X3x-ARyaT0xe27uILrVo,99
 sibi_dst/v2/df_helper/_df_helper.py,sha256=9pED3bjQ2Z81zqzJrZ9e7SguoO4-hBmNTJK4WOKrr4M,9297
@@ -95,6 +94,7 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-2025.9.11.dist-info/METADATA,sha256=LPlDJ-9iwPB3hKALqJuAX3vvZX2EHv83BPD4hwVs6Hw,2761
-sibi_dst-2025.9.11.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-2025.9.11.dist-info/RECORD,,
+sibi_dst-2025.9.13.dist-info/METADATA,sha256=v8QE7j-yP0-1s6i4UcOhHIjsDpvSJGtBvzSkLy3f-v8,2413
+sibi_dst-2025.9.13.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sibi_dst-2025.9.13.dist-info/top_level.txt,sha256=g3Cj4R-rciuNyJgcxuxNgw5nhN0n4TCB0ujcTEjZNiU,9
+sibi_dst-2025.9.13.dist-info/RECORD,,

{sibi_dst-2025.9.11.dist-info → sibi_dst-2025.9.13.dist-info}/WHEEL

@@ -1,4 +1,5 @@
 Wheel-Version: 1.0
-Generator: poetry-core 1.9.1
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
+

sibi_dst-2025.9.13.dist-info/top_level.txt

@@ -0,0 +1 @@
+sibi_dst

sibi_dst/df_helper/data_cleaner.py

@@ -1,132 +0,0 @@
-import re
-from nltk.corpus import stopwords
-from nltk.stem import SnowballStemmer
-import dask.dataframe as dd
-from dask_ml.preprocessing import OneHotEncoder, LabelEncoder
-import nltk
-
-class DataCleaner:
-    def __init__(self, dataframe):
-        self.original_df = dataframe
-        self.df = dataframe.copy()
-        self.duplicates_df = None
-
-    def handle_missing_values(self, strategy='mean'):
-        if strategy == 'mean':
-            self.df = self.df.fillna(self.df.mean())
-        elif strategy == 'median':
-            self.df = self.df.fillna(self.df.median())
-        elif strategy == 'mode':
-            self.df = self.df.fillna(self.df.mode().iloc[0])
-        elif strategy == 'drop':
-            self.df = self.df.dropna()
-        return self
-
-    def identify_duplicates(self, subset=None):
-        self.duplicates_df = self.df.map_partitions(lambda df: df[df.duplicated(subset=subset, keep=False)])
-        return self.duplicates_df
-
-    def remove_duplicates(self):
-        if self.duplicates_df is not None:
-            self.df = self.df[~self.df.index.isin(self.duplicates_df.index)]
-        return self
-
-    def validate_date_fields(self, date_columns=None):
-        if date_columns is None:
-            date_columns = self.df.select_dtypes(include=['datetime', 'datetime64[ns]', 'datetime64[ns, UTC]']).columns
-        for col in date_columns:
-            print('Validating date field: ', col)
-            self.df[col] = dd.to_datetime(self.df[col], errors='coerce')
-        return self
-
-    def clean_text(self, text_columns=None, language='english'):
-        nltk.download('stopwords')
-        stop_words = set(stopwords.words(language))
-        stemmer = SnowballStemmer(language)
-
-        def clean_text(text):
-            if isinstance(text, str):
-                text = text.strip().lower()  # Remove leading/trailing whitespace and convert to lowercase
-                text = re.sub(r'[^\w\s]', '', text)  # Remove special characters and punctuation
-                words = text.split()
-                words = [word for word in words if word not in stop_words]  # Remove stop words
-                words = [stemmer.stem(word) for word in words]  # Apply stemming
-                return ' '.join(words)
-            return text
-
-        if text_columns is None:
-            text_columns = self.df.select_dtypes(include=['object', 'string']).columns
-            text_columns = [col for col in text_columns if self.df[col].dtype != 'bool']
-
-        for col in text_columns:
-            print('Cleaning text field: ', col)
-            self.df[col] = self.df[col].map(clean_text, meta=('cleaned_text', 'object'))
-        return self
-
-    def validate_numeric_fields(self, int_columns=None, float_columns=None):
-        if int_columns is None:
-            int_columns = self.df.select_dtypes(include=['int64', 'int32']).columns
-        if float_columns is None:
-            float_columns = self.df.select_dtypes(include=['float64', 'float32']).columns
-
-        for col in int_columns:
-            print('Validating integer field: ', col)
-            self.df[col] = dd.to_numeric(self.df[col], errors='coerce', downcast='integer')
-
-        for col in float_columns:
-            print('Validating float field: ', col)
-            self.df[col] = dd.to_numeric(self.df[col], errors='coerce', downcast='float')
-
-        return self
-
-    def detect_categorical_columns(self, threshold=0.05):
-        """
-        Detect columns that can be converted to 'category' dtype.
-
-        Parameters:
-        threshold (float): The maximum ratio of unique values to total values for a column to be considered categorical.
-
-        Returns:
-        List of column names that can be converted to 'category' dtype.
-        """
-        categorical_columns = []
-
-        def unique_ratio(partition, col):
-            return partition[col].nunique() / len(partition)
-
-        for col in self.df.columns:
-            print("Detecting categorical columns: ", col)
-            unique_ratios = self.df.map_partitions(unique_ratio, col=col).compute()
-            overall_unique_ratio = unique_ratios.sum() / len(self.df)
-            if overall_unique_ratio < threshold:
-                print(f'Column {col} is categorical')
-                categorical_columns.append(col)
-
-        return categorical_columns
-
-    def handle_categorical_variables(self, columns=None, method='onehot', threshold=0.05):
-        if columns is None:
-            columns = self.detect_categorical_columns(threshold)
-
-        if method == 'onehot':
-            for col in columns:
-                self.df[col] = self.df[col].astype('category')
-            encoder = OneHotEncoder(sparse_output=False)
-            self.df = encoder.fit_transform(self.df)
-        elif method == 'label':
-            encoder = LabelEncoder()
-            for col in columns:
-                self.df[col] = encoder.fit_transform(self.df[col])
-        return self
-
-    def analyze_dtypes(self):
-        return self.df.dtypes
-
-    def get_cleaned_dataframe(self):
-        return self.df
-
-    def get_original_dataframe(self):
-        return self.original_df
-
-    def get_duplicates_dataframe(self):
-        return self.duplicates_df