sibi-dst 2025.9.10__tar.gz → 2025.9.12__tar.gz

{sibi_dst-2025.9.10 → sibi_dst-2025.9.12}/PKG-INFO

@@ -1,34 +1,31 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: sibi-dst
-Version: 2025.9.10
-Summary: Data Science Toolkit
-Author: Luis Valverde
-Author-email: lvalverdeb@gmail.com
-Requires-Python: >=3.11,<4.0
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: clickhouse-connect (>=0.8.18,<0.9.0)
-Requires-Dist: clickhouse-driver (>=0.2.9,<0.3.0)
-Requires-Dist: dask[complete] (>=2025.9.0,<2026.0.0)
-Requires-Dist: distributed (>=2025.9.1,<2026.0.0)
-Requires-Dist: mysqlclient (>=2.2.7,<3.0.0)
-Requires-Dist: opentelemetry-exporter-otlp (>=1.35.0,<2.0.0)
-Requires-Dist: opentelemetry-sdk (>=1.35.0,<2.0.0)
-Requires-Dist: pandas (>=2.3.1,<3.0.0)
-Requires-Dist: psycopg2 (>=2.9.10,<3.0.0)
-Requires-Dist: pyarrow (>=20.0.0,<21.0.0)
-Requires-Dist: pydantic (>=2.11.7,<3.0.0)
-Requires-Dist: pyiceberg[hive,s3fs] (>=0.9.1,<0.10.0)
-Requires-Dist: pymysql (>=1.1.1,<2.0.0)
-Requires-Dist: pyrosm (>=0.6.2,<0.7.0)
-Requires-Dist: s3fs (>=2025.5.1,<2026.0.0)
-Requires-Dist: sqlalchemy (>=2.0.41,<3.0.0)
-Requires-Dist: sse-starlette (>=3.0.2,<4.0.0)
-Requires-Dist: tqdm (>=4.67.1,<5.0.0)
-Requires-Dist: webdav4 (>=0.10.0,<0.11.0)
+Version: 2025.9.12
+Summary: A data science toolkit for scalable data processing and analysis.
+Requires-Python: >=3.11
 Description-Content-Type: text/markdown
+Requires-Dist: clickhouse-connect>=0.9.2
+Requires-Dist: clickhouse-driver>=0.2.9
+Requires-Dist: dask>=2025.9.1
+Requires-Dist: distributed>=2025.9.1
+Requires-Dist: fastapi>=0.118.0
+Requires-Dist: folium>=0.20.0
+Requires-Dist: mysqlclient>=2.2.7
+Requires-Dist: opentelemetry-api>=1.37.0
+Requires-Dist: opentelemetry-exporter-otlp>=1.37.0
+Requires-Dist: opentelemetry-sdk>=1.37.0
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: psycopg2>=2.9.10
+Requires-Dist: pyarrow>=21.0.0
+Requires-Dist: pydantic>=2.11.10
+Requires-Dist: pymysql>=1.1.2
+Requires-Dist: redis>=6.4.0
+Requires-Dist: s3fs>=2025.9.0
+Requires-Dist: sqlalchemy>=2.0.43
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: uvicorn>=0.37.0
+Requires-Dist: webdav4>=0.10.0
+Requires-Dist: wheel>=0.45.1
 
 ### SIBI-DST
 
@@ -60,4 +57,3 @@ pip install sibi-dst[dev,test,geospatial]  # Install all optional dependencies
 
 
 ```
-

sibi_dst-2025.9.12/pyproject.toml

@@ -0,0 +1,58 @@
+[project]
+name = "sibi-dst"
+version = "2025.9.12"
+description = "A data science toolkit for scalable data processing and analysis."
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "clickhouse-connect>=0.9.2",
+    "clickhouse-driver>=0.2.9",
+    "dask>=2025.9.1",
+    "distributed>=2025.9.1",
+    "fastapi>=0.118.0",
+    "folium>=0.20.0",
+    "mysqlclient>=2.2.7",
+    "opentelemetry-api>=1.37.0",
+    "opentelemetry-exporter-otlp>=1.37.0",
+    "opentelemetry-sdk>=1.37.0",
+    "pandas>=2.3.3",
+    "psycopg2>=2.9.10",
+    "pyarrow>=21.0.0",
+    "pydantic>=2.11.10",
+    "pymysql>=1.1.2",
+    "redis>=6.4.0",
+    "s3fs>=2025.9.0",
+    "sqlalchemy>=2.0.43",
+    "tqdm>=4.67.1",
+    "uvicorn>=0.37.0",
+    "webdav4>=0.10.0",
+    "wheel>=0.45.1",
+]
+
+
+[dependency-groups]
+dev = [
+    "black>=25.9.0",
+    "bokeh>=3.8.0",
+    "graphviz>=0.21",
+    "jupyter>=1.1.1",
+    "pytest>=8.4.2",
+    "python-dotenv>=1.1.1",
+    "wheel>=0.45.1",
+]
+geospatial = [
+    "folium>=0.20.0",
+    "geopandas>=1.1.1",
+    "geopy>=2.4.1",
+    "networkx>=3.5",
+    "osmnx>=2.0.6",
+    "scikit-learn>=1.7.2",
+]
+
+[build-system]
+requires = ["setuptools>=65", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["sibi_dst*"]

sibi_dst-2025.9.12/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

{sibi_dst-2025.9.10 → sibi_dst-2025.9.12}/sibi_dst/__init__.py

@@ -10,12 +10,17 @@ try:
 except version_reader.PackageNotFoundError:
     __version__ = "unknown"
 
-__all__ = [
-    "__version__",
-]
-
-import sibi_dst.df_helper as df_helper
+from sibi_dst.df_helper import *
 from sibi_dst.osmnx_helper import *
 from sibi_dst.geopy_helper import *
-from sibi_dst.utils import *
+from sibi_dst import utils as sibiutils
 
+
+__all__ = [
+    "__version__",
+    "DfHelper",
+    "ParquetArtifact",
+    "ParquetReader",
+    "ArtifactUpdaterMultiWrapperAsync",
+    "sibiutils"
+]

{sibi_dst-2025.9.10 → sibi_dst-2025.9.12}/sibi_dst/df_helper/__init__.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
 from ._parquet_reader import ParquetReader
-#from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded, ArtifactUpdaterMultiWrapperAsync
 from ._artifact_updater_async import ArtifactUpdaterMultiWrapperAsync
 from ._artifact_updater_threaded import ArtifactUpdaterMultiWrapperThreaded
 

sibi_dst-2025.9.12/sibi_dst/df_helper/_artifact_updater_async.py

@@ -0,0 +1,316 @@
+from __future__ import annotations
+
+import asyncio
+import datetime
+import random
+import time
+import pickle
+from contextlib import ExitStack, suppress
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type
+
+from sibi_dst.utils import ManagedResource, Logger
+from sibi_dst.utils.dask_utils import DaskClientMixin
+
+
+@dataclass(slots=True)
+class _RetryCfg:
+    """Retry and backoff configuration."""
+    attempts: int = 3
+    backoff_base: float = 2.0
+    backoff_max: float = 60.0
+    jitter: float = 0.15
+
+
+def run_artifact_update(
+    cls: Type,
+    artifact_class_kwargs: Dict[str, Any],
+    retry: _RetryCfg,
+    period: str,
+    artifact_kwargs: Dict[str, Any],
+) -> Dict[str, Any]:
+    """
+    Executed inside Dask worker.
+    Instantiates artifact and runs update_parquet() with retry logic.
+    Reconstructs logger and filesystem if not provided (worker isolation safe).
+    """
+    import logging
+    import fsspec
+    from sibi_dst.utils import Logger
+
+    # ---- Reinitialize a lightweight logger for the worker
+    worker_logger = Logger.default_logger(logger_name=cls.__name__) if hasattr(Logger, "default_logger") else logging.getLogger(cls.__name__)
+    worker_logger.set_level(logging.INFO)
+
+    # ---- Ensure fs is recreated if missing
+    fs = artifact_class_kwargs.get("fs")
+    if fs is None or isinstance(fs, str):
+        try:
+            fs_protocol = fs if isinstance(fs, str) else "file"
+            fs = fsspec.filesystem(fs_protocol)
+        except Exception:
+            fs = fsspec.filesystem("file")
+
+    # ---- Merge reconstructed environment into kwargs
+    artifact_kwargs_final = {
+        **artifact_class_kwargs,
+        "logger": worker_logger,
+        "fs": fs,
+    }
+
+    start_time = datetime.datetime.now()
+    success, error_msg, attempts = False, None, 0
+
+    for attempt in range(1, retry.attempts + 1):
+        attempts = attempt
+        try:
+            with ExitStack() as stack:
+                inst = cls(**artifact_kwargs_final)
+                inst = stack.enter_context(inst)
+                inst.update_parquet(period=period, **artifact_kwargs)
+            success = True
+            break
+        except Exception as e:
+            error_msg = str(e)
+            if attempt < retry.attempts:
+                delay = min(retry.backoff_base ** (attempt - 1), retry.backoff_max)
+                delay *= 1 + random.uniform(0, retry.jitter)
+                time.sleep(delay)
+
+    duration = (datetime.datetime.now() - start_time).total_seconds()
+    status = "😀" if success else "😩"
+    worker_logger.info(
+        f"{status} {cls.__name__} [{period}] finished in {duration:.2f}s ({attempts} attempt(s))"
+    )
+
+    return {
+        "artifact": cls.__name__,
+        "period": period,
+        "success": success,
+        "error": error_msg,
+        "attempts": attempts,
+        "duration_seconds": duration,
+        "started_at": start_time.isoformat(),
+        "ended_at": datetime.datetime.now().isoformat(),
+    }
+
+
+# ---------------- Async Orchestrator ----------------
+class ArtifactUpdaterMultiWrapperAsync(DaskClientMixin, ManagedResource):
+    """
+    Async orchestrator for concurrent artifact updates.
+
+    • Uses Dask client (via DaskClientMixin) or local threads.
+    • Automatically sanitizes non-picklable arguments (e.g., loggers, fs).
+    • Provides structured retries, async orchestration, and safe cleanup.
+    """
+
+    def __init__(
+        self,
+        wrapped_classes: Dict[str, Sequence[Type]],
+        *,
+        logger: Logger,
+        fs,
+        max_workers: int = 3,
+        retry_attempts: int = 3,
+        update_timeout_seconds: int = 600,
+        backoff_base: float = 2.0,
+        backoff_max: float = 60.0,
+        backoff_jitter: float = 0.15,
+        priority_fn: Optional[Callable[[Type], int]] = None,
+        artifact_class_kwargs: Optional[Dict[str, Any]] = None,
+        use_dask: bool = True,
+        dask_client: Optional[Any] = None,
+        debug: bool = False,
+        verbose: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(logger=logger, fs=fs, debug=debug, verbose=verbose)
+
+        # ---- Client lifecycle management
+
+        self.own_dask_client = dask_client is None
+        self._init_dask_client(dask_client, logger=logger)
+        self.use_dask = use_dask
+
+        # ---- Core configuration
+        self.wrapped_classes = wrapped_classes
+        self.max_workers = max_workers
+        self.priority_fn = priority_fn
+        self.update_timeout_seconds = update_timeout_seconds
+
+        # ---- Retry configuration
+        self._retry = _RetryCfg(
+            attempts=retry_attempts,
+            backoff_base=backoff_base,
+            backoff_max=backoff_max,
+            jitter=backoff_jitter,
+        )
+
+        # ---- Artifact instantiation arguments
+        self.artifact_class_kwargs = {
+            "logger": logger,
+            "fs": fs,
+            "debug": debug,
+            "verbose": verbose,
+            **(artifact_class_kwargs or {}),
+        }
+
+        # ---- Runtime tracking
+        self.completion_secs: Dict[str, float] = {}
+        self.failed: List[str] = []
+        self._stop_event = asyncio.Event()
+
+        self.logger_extra = {"sibi_dst_component": self.__class__.__name__}
+
+        if self.use_dask:
+            self.logger.debug(f"Initialized with Dask client: {self.dask_client}")
+        else:
+            self.logger.debug(f"Running in local thread-based mode.")
+
+    async def update_data(self, period: str, **kwargs: Any) -> List[Dict[str, Any]]:
+        """Runs updates for all artifacts in a given period."""
+        self.completion_secs.clear()
+        self.failed.clear()
+        classes = self._classes_for(period)
+
+        self.logger.info(
+            f"Starting artifact updates for period '{period}' ({len(classes)} artifacts).",
+            extra=self.logger_extra,
+        )
+
+        try:
+            if self.use_dask:
+                futures = [self._submit_one_dask(cls, period, kwargs) for cls in classes]
+                results = await asyncio.to_thread(lambda: self.dask_client.gather(futures))
+            else:
+                sem = asyncio.Semaphore(self.max_workers)
+                tasks = [self._run_one_async(cls, period, sem, kwargs) for cls in classes]
+                results = await asyncio.gather(*tasks)
+
+            self.logger.info(
+                f"Completed {len(results)} artifact updates for period '{period}'.",
+                extra=self.logger_extra,
+            )
+            return results
+
+        finally:
+            # Always cleanup if we own the client
+            if getattr(self, "own_dask_client", False):
+                self._close_dask_client()
+
+
+    def _sanitize_kwargs_for_dask(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Removes non-picklable runtime objects (e.g., loggers, fs) before sending to Dask.
+        """
+        clean: Dict[str, Any] = {}
+        for k, v in kwargs.items():
+            try:
+                pickle.dumps(v)
+                clean[k] = v
+            except Exception:
+                self.logger.debug(f"Skipping non-picklable key '{k}' for Dask worker.")
+        return clean
+
+    def _submit_one_dask(self, cls: Type, period: str, artifact_kwargs: Dict[str, Any]):
+        """Submit one artifact job to Dask."""
+        safe_kwargs = self._sanitize_kwargs_for_dask(self.artifact_class_kwargs)
+        return self.dask_client.submit(
+            run_artifact_update,
+            cls,
+            safe_kwargs,
+            self._retry,
+            period,
+            artifact_kwargs,
+            pure=False,
+        )
+
+    def _classes_for(self, period: str) -> List[Type]:
+        """Selects artifact classes for the given period."""
+        try:
+            classes = list(self.wrapped_classes[period])
+        except KeyError:
+            raise ValueError(f"No artifacts configured for period '{period}'.")
+        if not classes:
+            raise ValueError(f"No artifact classes found for '{period}'.")
+
+        if self.priority_fn:
+            with suppress(Exception):
+                classes.sort(key=self.priority_fn)
+        return classes
+
+    async def _run_one_async(
+        self,
+        cls: Type,
+        period: str,
+        sem: asyncio.Semaphore,
+        artifact_kwargs: Dict[str, Any],
+    ) -> Dict[str, Any]:
+        """Fallback local async execution (no Dask)."""
+        name = cls.__name__
+        start_time = datetime.datetime.now()
+
+        async with sem:
+            for attempt in range(1, self._retry.attempts + 1):
+                try:
+                    def _sync_block():
+                        with ExitStack() as stack:
+                            inst = cls(**self.artifact_class_kwargs)
+                            inst = stack.enter_context(inst)
+                            inst.update_parquet(period=period, **artifact_kwargs)
+
+                    await asyncio.wait_for(
+                        asyncio.to_thread(_sync_block),
+                        timeout=self.update_timeout_seconds,
+                    )
+                    duration = (datetime.datetime.now() - start_time).total_seconds()
+                    self.completion_secs[name] = duration
+                    self.logger.info(f"✅ {name} completed in {duration:.2f}s")
+                    return {
+                        "artifact": name,
+                        "period": period,
+                        "success": True,
+                        "attempts": attempt,
+                        "duration_seconds": duration,
+                    }
+
+                except Exception as e:
+                    if attempt < self._retry.attempts:
+                        delay = min(self._retry.backoff_base ** attempt, self._retry.backoff_max)
+                        delay *= 1 + random.uniform(0, self._retry.jitter)
+                        self.logger.warning(f"Retry {attempt}/{self._retry.attempts} for {name}: {e}")
+                        await asyncio.sleep(delay)
+                    else:
+                        duration = (datetime.datetime.now() - start_time).total_seconds()
+                        self.failed.append(name)
+                        self.logger.error(f"❌ {name} failed after {attempt} attempts: {e}")
+                        return {
+                            "artifact": name,
+                            "period": period,
+                            "success": False,
+                            "attempts": attempt,
+                            "error": str(e),
+                            "duration_seconds": duration,
+                        }
+
+
+    def get_update_status(self) -> Dict[str, Any]:
+        """Returns summary of completed, failed, and pending artifacts."""
+        done = set(self.completion_secs)
+        fail = set(self.failed)
+        all_names = {cls.__name__ for v in self.wrapped_classes.values() for cls in v}
+        return {
+            "total": len(all_names),
+            "completed": sorted(done),
+            "failed": sorted(fail),
+            "pending": sorted(all_names - done - fail),
+            "completion_times": self.completion_secs,
+        }
+
+    def _cleanup(self) -> None:
+        """Ensures safe resource closure."""
+        with suppress(Exception):
+            if getattr(self, "own_dask_client", False):
+                self._close_dask_client()
+

{sibi_dst-2025.9.10 → sibi_dst-2025.9.12}/sibi_dst/df_helper/backends/parquet/_parquet_options.py

@@ -231,7 +231,7 @@ class ParquetConfig(BaseModel):
         Builds a list of path patterns for dask.read_parquet.
         Respects partition_on + start/end date if given.
         """
-        print(f"_resolve_paths_for_read: {self.partition_on}")
+        self.logger.debug(f"_resolve_paths_for_read: {self.partition_on}")
         # Partitioned dataset by column
         if self.partition_on and self.parquet_start_date and self.parquet_end_date:
             if not isinstance(self.partition_on, (list, tuple)):
@@ -244,12 +244,10 @@ class ParquetConfig(BaseModel):
             days = pd.date_range(start=start, end=end, freq="D").date
 
             base = self.parquet_storage_path.rstrip("/")
-            print("base:",base)
             result= [
                 f"{base}/{parts[0]}={d.isoformat()}/*.parquet"
                 for d in days
             ]
-            print("result:",result)
             return result
 
         # Date-ranged folders (non-partitioned, using FilePathGenerator)

sibi_dst-2025.9.12/sibi_dst/osmnx_helper/__init__.py

@@ -0,0 +1,9 @@
+from .base_osm_map import BaseOsmMap
+from .utils import PBFHandler
+from .route_path_builder import RoutePathBuilder, RoutePathBuilderConfig
+__all__ = [
+    "BaseOsmMap",
+    "RoutePathBuilder",
+    "RoutePathBuilderConfig",
+    "PBFHandler",
+]

{sibi_dst-2025.9.10 → sibi_dst-2025.9.12}/sibi_dst/utils/__init__.py

@@ -24,6 +24,7 @@ from .manifest_manager import MissingManifestManager
 __all__ = [
     "Logger",
     "ManagedResource",
+
     "ConfigManager",
     "ConfigLoader",
     "DateUtils",
@@ -42,5 +43,5 @@ __all__ = [
     "FsRegistry",
     "DataFromHttpSource",
     "WebDAVClient",
-    "MissingManifestManager"
+    "MissingManifestManager",
 ]

{sibi_dst-2025.9.10 → sibi_dst-2025.9.12}/sibi_dst/utils/boilerplate/base_pipeline.py

@@ -93,7 +93,7 @@ class BasePipeline(ManagedResource):
         df[self.date_field] = dd.to_datetime(df[self.date_field], errors="coerce")
         df["partition_date"] = df[self.date_field].dt.date.astype(str)
 
-        out_path = self.storage_path.rstrip("/")+"/"+self._get_output_filename(fmt="parquet")
+        out_path = self.storage_path.rstrip("/")
         self.logger.info("Saving dataset to %s", out_path)
         ps = ParquetSaver(
             df_result=df,
@@ -111,7 +111,6 @@ class BasePipeline(ManagedResource):
             parquet_start_date=self.start_date,
             parquet_end_date=self.end_date,
             parquet_storage_path=self.storage_path,
-            parquet_filename=self._get_output_filename(),
             fs=self.fs,
             debug=self.debug,
             logger=self.logger,