sibi-dst 2025.9.10__py3-none-any.whl → 2025.9.12__py3-none-any.whl

sibi_dst/__init__.py

@@ -10,12 +10,17 @@ try:
 except version_reader.PackageNotFoundError:
     __version__ = "unknown"
 
-__all__ = [
-    "__version__",
-]
-
-import sibi_dst.df_helper as df_helper
+from sibi_dst.df_helper import *
 from sibi_dst.osmnx_helper import *
 from sibi_dst.geopy_helper import *
-from sibi_dst.utils import *
+from sibi_dst import utils as sibiutils
 
+
+__all__ = [
+    "__version__",
+    "DfHelper",
+    "ParquetArtifact",
+    "ParquetReader",
+    "ArtifactUpdaterMultiWrapperAsync",
+    "sibiutils"
+]

sibi_dst/df_helper/__init__.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
 from ._parquet_reader import ParquetReader
-#from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded, ArtifactUpdaterMultiWrapperAsync
 from ._artifact_updater_async import ArtifactUpdaterMultiWrapperAsync
 from ._artifact_updater_threaded import ArtifactUpdaterMultiWrapperThreaded
 

sibi_dst/df_helper/_artifact_updater_async.py

@@ -4,28 +4,24 @@ import asyncio
 import datetime
 import random
 import time
-from contextlib import ExitStack
+import pickle
+from contextlib import ExitStack, suppress
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Optional, Sequence, Type
 
-from sibi_dst.utils import ManagedResource
-
-try:
-    from dask.distributed import Client, LocalCluster
-except ImportError:
-    Client = None
-    LocalCluster = None
+from sibi_dst.utils import ManagedResource, Logger
+from sibi_dst.utils.dask_utils import DaskClientMixin
 
 
 @dataclass(slots=True)
 class _RetryCfg:
+    """Retry and backoff configuration."""
     attempts: int = 3
     backoff_base: float = 2.0
     backoff_max: float = 60.0
     jitter: float = 0.15
 
 
-# ---------------- Worker (safe for Dask pickling) ----------------
 def run_artifact_update(
     cls: Type,
     artifact_class_kwargs: Dict[str, Any],
@@ -33,21 +29,43 @@ def run_artifact_update(
     period: str,
     artifact_kwargs: Dict[str, Any],
 ) -> Dict[str, Any]:
-    """Standalone worker — safe for Dask distributed execution."""
+    """
+    Executed inside Dask worker.
+    Instantiates artifact and runs update_parquet() with retry logic.
+    Reconstructs logger and filesystem if not provided (worker isolation safe).
+    """
     import logging
+    import fsspec
+    from sibi_dst.utils import Logger
 
-    logger = logging.getLogger(cls.__name__)
+    # ---- Reinitialize a lightweight logger for the worker
+    worker_logger = Logger.default_logger(logger_name=cls.__name__) if hasattr(Logger, "default_logger") else logging.getLogger(cls.__name__)
+    worker_logger.set_level(logging.INFO)
+
+    # ---- Ensure fs is recreated if missing
+    fs = artifact_class_kwargs.get("fs")
+    if fs is None or isinstance(fs, str):
+        try:
+            fs_protocol = fs if isinstance(fs, str) else "file"
+            fs = fsspec.filesystem(fs_protocol)
+        except Exception:
+            fs = fsspec.filesystem("file")
+
+    # ---- Merge reconstructed environment into kwargs
+    artifact_kwargs_final = {
+        **artifact_class_kwargs,
+        "logger": worker_logger,
+        "fs": fs,
+    }
 
-    start_wall = datetime.datetime.now()
-    attempt_count = 0
-    success = False
-    error_msg = None
+    start_time = datetime.datetime.now()
+    success, error_msg, attempts = False, None, 0
 
     for attempt in range(1, retry.attempts + 1):
-        attempt_count = attempt
+        attempts = attempt
         try:
             with ExitStack() as stack:
-                inst = cls(**artifact_class_kwargs)
+                inst = cls(**artifact_kwargs_final)
                 inst = stack.enter_context(inst)
                 inst.update_parquet(period=period, **artifact_kwargs)
             success = True
@@ -59,31 +77,40 @@ def run_artifact_update(
                 delay *= 1 + random.uniform(0, retry.jitter)
                 time.sleep(delay)
 
-    end_wall = datetime.datetime.now()
-    duration = (end_wall - start_wall).total_seconds()
+    duration = (datetime.datetime.now() - start_time).total_seconds()
+    status = "😀" if success else "😩"
+    worker_logger.info(
+        f"{status} {cls.__name__} [{period}] finished in {duration:.2f}s ({attempts} attempt(s))"
+    )
 
     return {
         "artifact": cls.__name__,
         "period": period,
-        "start": start_wall.isoformat(),
-        "end": end_wall.isoformat(),
-        "processing_time": duration,
-        "retries": attempt_count - 1 if success else attempt_count,
         "success": success,
         "error": error_msg,
+        "attempts": attempts,
+        "duration_seconds": duration,
+        "started_at": start_time.isoformat(),
+        "ended_at": datetime.datetime.now().isoformat(),
     }
 
 
-class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
+# ---------------- Async Orchestrator ----------------
+class ArtifactUpdaterMultiWrapperAsync(DaskClientMixin, ManagedResource):
     """
-    Async/Threaded orchestrator.
-    Dask-enabled if a Client is passed (or created automatically).
+    Async orchestrator for concurrent artifact updates.
+
+    • Uses Dask client (via DaskClientMixin) or local threads.
+    • Automatically sanitizes non-picklable arguments (e.g., loggers, fs).
+    • Provides structured retries, async orchestration, and safe cleanup.
     """
 
     def __init__(
         self,
         wrapped_classes: Dict[str, Sequence[Type]],
         *,
+        logger: Logger,
+        fs,
         max_workers: int = 3,
         retry_attempts: int = 3,
         update_timeout_seconds: int = 600,
@@ -92,88 +119,127 @@ class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
         backoff_jitter: float = 0.15,
         priority_fn: Optional[Callable[[Type], int]] = None,
         artifact_class_kwargs: Optional[Dict[str, Any]] = None,
-        dask_client: Optional[Client] = None,
         use_dask: bool = True,
+        dask_client: Optional[Any] = None,
+        debug: bool = False,
+        verbose: bool = False,
         **kwargs: Any,
     ) -> None:
-        super().__init__(**kwargs)
+        super().__init__(logger=logger, fs=fs, debug=debug, verbose=verbose)
+
+        # ---- Client lifecycle management
+
+        self.own_dask_client = dask_client is None
+        self._init_dask_client(dask_client, logger=logger)
+        self.use_dask = use_dask
 
+        # ---- Core configuration
         self.wrapped_classes = wrapped_classes
-        self.max_workers = int(max_workers)
-        self.update_timeout_seconds = int(update_timeout_seconds)
+        self.max_workers = max_workers
         self.priority_fn = priority_fn
-        self.use_dask = use_dask
-        self.client: Optional[Client] = dask_client
-        self._owns_client = False
+        self.update_timeout_seconds = update_timeout_seconds
 
+        # ---- Retry configuration
         self._retry = _RetryCfg(
-            attempts=int(retry_attempts),
-            backoff_base=float(backoff_base),
-            backoff_max=float(backoff_max),
-            jitter=float(backoff_jitter),
+            attempts=retry_attempts,
+            backoff_base=backoff_base,
+            backoff_max=backoff_max,
+            jitter=backoff_jitter,
         )
 
-        # Safe kwargs for artifacts
-        if self.use_dask:
-            self.artifact_class_kwargs = {
-                "debug": self.debug,
-                "verbose": self.verbose,
-                **(artifact_class_kwargs or {}),
-            }
-        else:
-            self.artifact_class_kwargs = {
-                "logger": self.logger,
-                "fs": self.fs,
-                "debug": self.debug,
-                "verbose": self.verbose,
-                **(artifact_class_kwargs or {}),
-            }
+        # ---- Artifact instantiation arguments
+        self.artifact_class_kwargs = {
+            "logger": logger,
+            "fs": fs,
+            "debug": debug,
+            "verbose": verbose,
+            **(artifact_class_kwargs or {}),
+        }
 
+        # ---- Runtime tracking
         self.completion_secs: Dict[str, float] = {}
         self.failed: List[str] = []
-        self._stop = asyncio.Event()
-
-        if self.use_dask and Client is None:
-            raise RuntimeError("Dask is not installed, cannot use Dask mode")
-
-        # auto-start local client if requested
-        if self.use_dask and not self.client:
-            self.client = Client(
-                LocalCluster(
-                    n_workers=max_workers,
-                    threads_per_worker=1,
-                    dashboard_address=None,
-                )
-            )
-            self._owns_client = True
+        self._stop_event = asyncio.Event()
 
-    # ---- Internals ------------------------------------------------------------
+        self.logger_extra = {"sibi_dst_component": self.__class__.__name__}
+
+        if self.use_dask:
+            self.logger.debug(f"Initialized with Dask client: {self.dask_client}")
+        else:
+            self.logger.debug(f"Running in local thread-based mode.")
+
+    async def update_data(self, period: str, **kwargs: Any) -> List[Dict[str, Any]]:
+        """Runs updates for all artifacts in a given period."""
+        self.completion_secs.clear()
+        self.failed.clear()
+        classes = self._classes_for(period)
+
+        self.logger.info(
+            f"Starting artifact updates for period '{period}' ({len(classes)} artifacts).",
+            extra=self.logger_extra,
+        )
 
-    def _classes_for(self, period: str) -> List[Type]:
         try:
-            classes = list(self.wrapped_classes[period])
-        except KeyError:
-            raise ValueError(f"Unsupported period '{period}'.")
-        if not classes:
-            raise ValueError(f"No artifact classes configured for '{period}'.")
-        if self.priority_fn:
+            if self.use_dask:
+                futures = [self._submit_one_dask(cls, period, kwargs) for cls in classes]
+                results = await asyncio.to_thread(lambda: self.dask_client.gather(futures))
+            else:
+                sem = asyncio.Semaphore(self.max_workers)
+                tasks = [self._run_one_async(cls, period, sem, kwargs) for cls in classes]
+                results = await asyncio.gather(*tasks)
+
+            self.logger.info(
+                f"Completed {len(results)} artifact updates for period '{period}'.",
+                extra=self.logger_extra,
+            )
+            return results
+
+        finally:
+            # Always cleanup if we own the client
+            if getattr(self, "own_dask_client", False):
+                self._close_dask_client()
+
+
+    def _sanitize_kwargs_for_dask(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Removes non-picklable runtime objects (e.g., loggers, fs) before sending to Dask.
+        """
+        clean: Dict[str, Any] = {}
+        for k, v in kwargs.items():
             try:
-                classes.sort(key=self.priority_fn)
-            except Exception as e:
-                self.logger.warning(f"priority_fn failed; using listed order: {e}")
-        return classes
+                pickle.dumps(v)
+                clean[k] = v
+            except Exception:
+                self.logger.debug(f"Skipping non-picklable key '{k}' for Dask worker.")
+        return clean
 
     def _submit_one_dask(self, cls: Type, period: str, artifact_kwargs: Dict[str, Any]):
-        return self.client.submit(
+        """Submit one artifact job to Dask."""
+        safe_kwargs = self._sanitize_kwargs_for_dask(self.artifact_class_kwargs)
+        return self.dask_client.submit(
             run_artifact_update,
             cls,
-            dict(self.artifact_class_kwargs),
+            safe_kwargs,
             self._retry,
             period,
             artifact_kwargs,
             pure=False,
         )
 
+    def _classes_for(self, period: str) -> List[Type]:
+        """Selects artifact classes for the given period."""
+        try:
+            classes = list(self.wrapped_classes[period])
+        except KeyError:
+            raise ValueError(f"No artifacts configured for period '{period}'.")
+        if not classes:
+            raise ValueError(f"No artifact classes found for '{period}'.")
+
+        if self.priority_fn:
+            with suppress(Exception):
+                classes.sort(key=self.priority_fn)
+        return classes
+
     async def _run_one_async(
         self,
         cls: Type,
@@ -181,112 +247,70 @@ class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
         sem: asyncio.Semaphore,
         artifact_kwargs: Dict[str, Any],
     ) -> Dict[str, Any]:
-        """Async/threaded fallback execution."""
+        """Fallback local async execution (no Dask)."""
         name = cls.__name__
-        self.logger.info(f"▶️ Starting {name} for period '{period}'")
-        start_wall = datetime.datetime.now()
-
-        attempt_count = 0
-        success = False
-        error_msg = None
-
-        try:
-            async with sem:
-                for attempt in range(1, self._retry.attempts + 1):
-                    attempt_count = attempt
-                    try:
-                        def _sync_block():
-                            with ExitStack() as stack:
-                                inst = cls(**self.artifact_class_kwargs)
-                                inst = stack.enter_context(inst)
-                                inst.update_parquet(period=period, **artifact_kwargs)
-
-                        await asyncio.wait_for(
-                            asyncio.to_thread(_sync_block),
-                            timeout=self.update_timeout_seconds,
-                        )
-                        success = True
-                        break
-                    except Exception as e:
-                        error_msg = str(e)
-                        if attempt < self._retry.attempts and not self._stop.is_set():
-                            delay = min(
-                                self._retry.backoff_base ** (attempt - 1),
-                                self._retry.backoff_max,
-                            )
-                            delay *= 1 + random.uniform(0, self._retry.jitter)
-                            await asyncio.sleep(delay)
-        finally:
-            end_wall = datetime.datetime.now()
-            duration = (end_wall - start_wall).total_seconds()
-
-            result = {
-                "artifact": name,
-                "period": period,
-                "start": start_wall.isoformat(),
-                "end": end_wall.isoformat(),
-                "processing_time": duration,
-                "retries": attempt_count - 1 if success else attempt_count,
-                "success": success,
-                "error": error_msg,
-            }
-
-            if success:
-                self.logger.info(f"✅ Artifact {name} succeeded", extra=result)
-                self.completion_secs[name] = duration
-            else:
-                self.logger.error(f"❌ Artifact {name} failed", extra=result)
-                self.failed.append(name)
-
-            return result
+        start_time = datetime.datetime.now()
+
+        async with sem:
+            for attempt in range(1, self._retry.attempts + 1):
+                try:
+                    def _sync_block():
+                        with ExitStack() as stack:
+                            inst = cls(**self.artifact_class_kwargs)
+                            inst = stack.enter_context(inst)
+                            inst.update_parquet(period=period, **artifact_kwargs)
+
+                    await asyncio.wait_for(
+                        asyncio.to_thread(_sync_block),
+                        timeout=self.update_timeout_seconds,
+                    )
+                    duration = (datetime.datetime.now() - start_time).total_seconds()
+                    self.completion_secs[name] = duration
+                    self.logger.info(f"✅ {name} completed in {duration:.2f}s")
+                    return {
+                        "artifact": name,
+                        "period": period,
+                        "success": True,
+                        "attempts": attempt,
+                        "duration_seconds": duration,
+                    }
+
+                except Exception as e:
+                    if attempt < self._retry.attempts:
+                        delay = min(self._retry.backoff_base ** attempt, self._retry.backoff_max)
+                        delay *= 1 + random.uniform(0, self._retry.jitter)
+                        self.logger.warning(f"Retry {attempt}/{self._retry.attempts} for {name}: {e}")
+                        await asyncio.sleep(delay)
+                    else:
+                        duration = (datetime.datetime.now() - start_time).total_seconds()
+                        self.failed.append(name)
+                        self.logger.error(f"❌ {name} failed after {attempt} attempts: {e}")
+                        return {
+                            "artifact": name,
+                            "period": period,
+                            "success": False,
+                            "attempts": attempt,
+                            "error": str(e),
+                            "duration_seconds": duration,
+                        }
 
-    # ---- Public API -----------------------------------------------------------
-
-    async def update_data(self, period: str, **kwargs: Any) -> List[Dict[str, Any]]:
-        self.completion_secs.clear()
-        self.failed.clear()
-        classes = self._classes_for(period)
-
-        try:
-            if self.use_dask:
-                futures = [self._submit_one_dask(cls, period, kwargs) for cls in classes]
-                results = await asyncio.to_thread(lambda: self.client.gather(futures))
-            else:
-                sem = asyncio.Semaphore(self.max_workers)
-                tasks = [
-                    asyncio.create_task(self._run_one_async(cls, period, sem, kwargs))
-                    for cls in classes
-                ]
-                results = await asyncio.gather(*tasks)
-            return results
-        finally:
-            # only shut down if we own the client
-            if self._owns_client:
-                self.close()
 
     def get_update_status(self) -> Dict[str, Any]:
+        """Returns summary of completed, failed, and pending artifacts."""
         done = set(self.completion_secs)
         fail = set(self.failed)
-        all_names = {c.__name__ for v in self.wrapped_classes.values() for c in v}
+        all_names = {cls.__name__ for v in self.wrapped_classes.values() for cls in v}
         return {
             "total": len(all_names),
             "completed": sorted(done),
             "failed": sorted(fail),
             "pending": sorted(all_names - done - fail),
-            "completion_times": dict(self.completion_secs),
+            "completion_times": self.completion_secs,
         }
 
-    # ---- Lifecycle ------------------------------------------------------------
-
     def _cleanup(self) -> None:
-        """Release any resources created by this wrapper."""
-        if self._owns_client and self.client is not None:
-            try:
-                cluster = getattr(self.client, "cluster", None)
-                self.client.close()
-                if cluster is not None:
-                    cluster.close()
-            finally:
-                self.client = None
-                self._owns_client = False
+        """Ensures safe resource closure."""
+        with suppress(Exception):
+            if getattr(self, "own_dask_client", False):
+                self._close_dask_client()
 

sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -231,7 +231,7 @@ class ParquetConfig(BaseModel):
         Builds a list of path patterns for dask.read_parquet.
         Respects partition_on + start/end date if given.
         """
-        print(f"_resolve_paths_for_read: {self.partition_on}")
+        self.logger.debug(f"_resolve_paths_for_read: {self.partition_on}")
         # Partitioned dataset by column
         if self.partition_on and self.parquet_start_date and self.parquet_end_date:
             if not isinstance(self.partition_on, (list, tuple)):
@@ -244,12 +244,10 @@ class ParquetConfig(BaseModel):
             days = pd.date_range(start=start, end=end, freq="D").date
 
             base = self.parquet_storage_path.rstrip("/")
-            print("base:",base)
             result= [
                 f"{base}/{parts[0]}={d.isoformat()}/*.parquet"
                 for d in days
             ]
-            print("result:",result)
             return result
 
         # Date-ranged folders (non-partitioned, using FilePathGenerator)

sibi_dst/osmnx_helper/__init__.py

@@ -1,7 +1,9 @@
 from .base_osm_map import BaseOsmMap
 from .utils import PBFHandler
+from .route_path_builder import RoutePathBuilder, RoutePathBuilderConfig
 __all__ = [
     "BaseOsmMap",
-    #"MapConfig",
+    "RoutePathBuilder",
+    "RoutePathBuilderConfig",
     "PBFHandler",
 ]

sibi_dst/utils/__init__.py

@@ -24,6 +24,7 @@ from .manifest_manager import MissingManifestManager
 __all__ = [
     "Logger",
     "ManagedResource",
+
     "ConfigManager",
     "ConfigLoader",
     "DateUtils",
@@ -42,5 +43,5 @@ __all__ = [
     "FsRegistry",
     "DataFromHttpSource",
     "WebDAVClient",
-    "MissingManifestManager"
+    "MissingManifestManager",
 ]

sibi_dst/utils/boilerplate/base_pipeline.py

@@ -93,7 +93,7 @@ class BasePipeline(ManagedResource):
         df[self.date_field] = dd.to_datetime(df[self.date_field], errors="coerce")
         df["partition_date"] = df[self.date_field].dt.date.astype(str)
 
-        out_path = self.storage_path.rstrip("/")+"/"+self._get_output_filename(fmt="parquet")
+        out_path = self.storage_path.rstrip("/")
         self.logger.info("Saving dataset to %s", out_path)
         ps = ParquetSaver(
             df_result=df,
@@ -111,7 +111,6 @@ class BasePipeline(ManagedResource):
             parquet_start_date=self.start_date,
             parquet_end_date=self.end_date,
             parquet_storage_path=self.storage_path,
-            parquet_filename=self._get_output_filename(),
             fs=self.fs,
             debug=self.debug,
             logger=self.logger,