sibi-dst 2025.8.1__tar.gz → 2025.8.3__tar.gz

{sibi_dst-2025.8.1 → sibi_dst-2025.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.8.1
+Version: 2025.8.3
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.8.1 → sibi_dst-2025.8.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "2025.8.1"
+version = "2025.8.3"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-2025.8.1 → sibi_dst-2025.8.3}/sibi_dst/df_helper/__init__.py

@@ -3,8 +3,9 @@ from __future__ import annotations
 from ._df_helper import DfHelper
 from ._parquet_artifact import ParquetArtifact
 from ._parquet_reader import ParquetReader
-from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded, ArtifactUpdaterMultiWrapperAsync
-
+#from ._artifact_updater_multi_wrapper import ArtifactUpdaterMultiWrapperThreaded, ArtifactUpdaterMultiWrapperAsync
+from ._artifact_updater_async import ArtifactUpdaterMultiWrapperAsync
+from ._artifact_updater_threaded import ArtifactUpdaterMultiWrapperThreaded
 __all__ = [
     'DfHelper',
     'ParquetArtifact',

sibi_dst-2025.8.3/sibi_dst/df_helper/_artifact_updater_async.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import asyncio
+import datetime
+import random
+import time
+from contextlib import ExitStack
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, Type
+
+from sibi_dst.utils import ManagedResource
+
+
+@dataclass(slots=True)
+class _RetryCfg:
+    attempts: int = 3
+    backoff_base: float = 2.0
+    backoff_max: float = 60.0
+    jitter: float = 0.15
+
+
+_ORCHESTRATOR_KEYS = {
+    "retry_attempts",
+    "backoff_base",
+    "backoff_max",
+    "backoff_jitter",
+    "update_timeout_seconds",
+    "max_workers",
+    "priority_fn",
+    "artifact_class_kwargs",
+}
+
+
+def _default_artifact_kwargs(resource: ManagedResource) -> Dict[str, Any]:
+    return {
+        "logger": resource.logger,
+        "debug": resource.debug,
+        "fs": resource.fs,
+        "verbose": resource.verbose,
+    }
+
+
+class ArtifactUpdaterMultiWrapperAsync(ManagedResource):
+    """
+    Backward-compatible async orchestrator with shutdown-aware scheduling.
+    """
+
+    def __init__(
+        self,
+        wrapped_classes: Dict[str, Sequence[Type]],
+        *,
+        max_workers: int = 3,
+        retry_attempts: int = 3,
+        update_timeout_seconds: int = 600,
+        backoff_base: float = 2.0,
+        backoff_max: float = 60.0,
+        backoff_jitter: float = 0.15,
+        priority_fn: Optional[Callable[[Type], int]] = None,
+        artifact_class_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.wrapped_classes = wrapped_classes
+        self.max_workers = int(max_workers)
+        self.update_timeout_seconds = int(update_timeout_seconds)
+        self.priority_fn = priority_fn
+
+        self._retry = _RetryCfg(
+            attempts=int(retry_attempts),
+            backoff_base=float(backoff_base),
+            backoff_max=float(backoff_max),
+            jitter=float(backoff_jitter),
+        )
+
+        self.artifact_class_kwargs = {
+            **_default_artifact_kwargs(self),
+            **(artifact_class_kwargs or {}),
+        }
+
+        self.completion_secs: Dict[str, float] = {}
+        self.failed: List[str] = []
+
+        # NEW: async stop gate — tripped on cleanup/cancel
+        self._stop = asyncio.Event()
+
+    # Trip stop gate on close paths
+    def _cleanup(self) -> None:
+        try:
+            loop = asyncio.get_running_loop()
+            loop.call_soon_threadsafe(self._stop.set)
+        except RuntimeError:
+            self._stop.set()
+
+    async def _acleanup(self) -> None:
+        self._stop.set()
+
+    # ---- internals -----------------------------------------------------------
+
+    def _classes_for(self, period: str) -> List[Type]:
+        try:
+            classes = list(self.wrapped_classes[period])
+        except KeyError:
+            raise ValueError(f"Unsupported period '{period}'.")
+        if not classes:
+            raise ValueError(f"No artifact classes configured for period '{period}'.")
+        if self.priority_fn:
+            try:
+                classes.sort(key=self.priority_fn)
+            except Exception as e:
+                self.logger.warning(f"priority_fn failed; using listed order: {e}")
+        return classes
+
+    @staticmethod
+    def _split_kwargs(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+        orch: Dict[str, Any] = {}
+        art: Dict[str, Any] = {}
+        for k, v in raw.items():
+            if k in _ORCHESTRATOR_KEYS:
+                orch[k] = v
+            else:
+                art[k] = v
+        return orch, art
+
+    async def _run_one(self, cls: Type, period: str, sem: asyncio.Semaphore, artifact_kwargs: Dict[str, Any]) -> None:
+        name = cls.__name__
+        if self._stop.is_set() or self.closed:
+            raise asyncio.CancelledError()
+
+        self.logger.info(f"Running {name} with period '{period}'", extra={"artifact": name, "period": period})
+        async with sem:
+            loop = asyncio.get_running_loop()
+            start = loop.time()
+            for attempt in range(1, self._retry.attempts + 1):
+                if self._stop.is_set() or self.closed:
+                    raise asyncio.CancelledError()
+                try:
+                    def _sync_block() -> None:
+                        with ExitStack() as stack:
+                            inst = cls(**self.artifact_class_kwargs)
+                            inst = stack.enter_context(inst)
+                            inst.update_parquet(period=period, **artifact_kwargs)
+
+                    await asyncio.wait_for(asyncio.to_thread(_sync_block), timeout=self.update_timeout_seconds)
+                    dt_secs = loop.time() - start
+                    self.completion_secs[name] = dt_secs
+                    self.logger.info(f"✅ {name} ({period}) in {dt_secs:.2f}s")
+                    return
+
+                except asyncio.TimeoutError:
+                    self.logger.warning(f"Timeout in {name} attempt {attempt}/{self._retry.attempts}")
+                except asyncio.CancelledError:
+                    raise
+                except Exception as e:
+                    self.logger.error(
+                        f"{name} attempt {attempt}/{self._retry.attempts} failed: {e}",
+                        exc_info=self.debug,
+                    )
+
+                if attempt < self._retry.attempts and not self._stop.is_set():
+                    delay = min(self._retry.backoff_base ** (attempt - 1), self._retry.backoff_max)
+                    delay *= 1 + random.uniform(0, self._retry.jitter)
+                    try:
+                        await asyncio.sleep(delay)
+                    except asyncio.CancelledError:
+                        raise
+
+            self.failed.append(name)
+            self.logger.error(f"✖️  {name} permanently failed")
+
+    # ---- public API ----------------------------------------------------------
+
+    async def update_data(self, period: str, **kwargs: Any) -> None:
+        """
+        Backward-compatible:
+          - Accepts orchestrator knobs in kwargs (we consume them).
+          - Forwards only artifact-friendly kwargs to update_parquet.
+        """
+        _, artifact_kwargs = self._split_kwargs(kwargs)
+
+        self.completion_secs.clear()
+        self.failed.clear()
+
+        classes = self._classes_for(period)
+        self.logger.info(
+            f"Starting update of {len(classes)} artifacts for period '{period}'",
+            extra={
+                "action_module_name": self.__class__.__name__,
+                "date_of_update": time.strftime('%Y-%m-%d'),
+                "start_time": time.strftime('%H:%M:%S'),
+                "period": period,
+            },
+        )
+
+        sem = asyncio.Semaphore(self.max_workers)
+        tasks = [asyncio.create_task(self._run_one(cls, period, sem, dict(artifact_kwargs))) for cls in classes]
+
+        try:
+            for t in asyncio.as_completed(tasks):
+                if self._stop.is_set():
+                    break
+                await t
+        except (asyncio.CancelledError, KeyboardInterrupt):
+            self._stop.set()
+            for t in tasks:
+                t.cancel()
+            raise
+        finally:
+            # Drain/cancel everything deterministically
+            for t in tasks:
+                if not t.done():
+                    t.cancel()
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+            self.logger.info(
+                f"Update completed for period: {period}",
+                extra={
+                    "action_module_name": self.__class__.__name__,
+                    "date_of_update": datetime.date.today().strftime('%Y-%m-%d'),
+                    "end_time": datetime.datetime.now().strftime('%H:%M:%S'),
+                    "period": period,
+                },
+            )
+            self.logger.info(
+                f"Artifacts processed: total={len(classes)}, "
+                f"completed={len(self.completion_secs)}, failed={len(self.failed)}"
+            )
+
+    def get_update_status(self) -> Dict[str, Any]:
+        done = set(self.completion_secs)
+        fail = set(self.failed)
+        all_names = {c.__name__ for v in self.wrapped_classes.values() for c in v}
+        return {
+            "total": len(all_names),
+            "completed": sorted(done),
+            "failed": sorted(fail),
+            "pending": sorted(all_names - done - fail),
+            "completion_times": dict(self.completion_secs),
+        }

sibi_dst-2025.8.3/sibi_dst/df_helper/_artifact_updater_threaded.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import datetime
+import time
+import random
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from contextlib import ExitStack
+from dataclasses import dataclass
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Tuple
+
+from sibi_dst.utils import ManagedResource
+
+
+@dataclass(slots=True)
+class _RetryCfg:
+    attempts: int = 3
+    backoff_base: float = 2.0
+    backoff_max: float = 60.0
+    jitter: float = 0.15
+
+
+_ORCHESTRATOR_KEYS = {
+    "retry_attempts",
+    "backoff_base",
+    "backoff_max",
+    "backoff_jitter",
+    "update_timeout_seconds",  # accepted but unused in pure-threads version
+    "max_workers",
+    "priority_fn",
+    "artifact_class_kwargs",
+}
+
+
+def _default_artifact_kwargs(resource: ManagedResource) -> Dict[str, Any]:
+    return {
+        "logger": resource.logger,
+        "debug": resource.debug,
+        "fs": resource.fs,
+        "verbose": resource.verbose,
+    }
+
+
+class ArtifactUpdaterMultiWrapperThreaded(ManagedResource):
+    """
+    Backward-compatible threaded orchestrator with shutdown-aware scheduling.
+    """
+
+    def __init__(
+        self,
+        wrapped_classes: Dict[str, Sequence[Type]],
+        *,
+        max_workers: int = 4,
+        retry_attempts: int = 3,
+        backoff_base: float = 2.0,
+        backoff_max: float = 60.0,
+        backoff_jitter: float = 0.15,
+        priority_fn: Optional[Callable[[Type], int]] = None,
+        artifact_class_kwargs: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.wrapped_classes = wrapped_classes
+        self.max_workers = int(max_workers)
+        self.priority_fn = priority_fn
+        self._retry = _RetryCfg(
+            attempts=int(retry_attempts),
+            backoff_base=float(backoff_base),
+            backoff_max=float(backoff_max),
+            jitter=float(backoff_jitter),
+        )
+        self.artifact_class_kwargs = {
+            **_default_artifact_kwargs(self),
+            **(artifact_class_kwargs or {}),
+        }
+        self.completion_secs: Dict[str, float] = {}
+        self.failed: List[str] = []
+
+        # NEW: stop gate — tripped on cleanup / Ctrl-C to stop scheduling & retries
+        self._stop_event = threading.Event()
+
+    # Trip the stop gate when this wrapper is closed
+    def _cleanup(self) -> None:
+        self._stop_event.set()
+
+    def _classes_for(self, period: str) -> List[Type]:
+        try:
+            classes = list(self.wrapped_classes[period])
+        except KeyError:
+            raise ValueError(f"Unsupported period '{period}'.")
+        if not classes:
+            raise ValueError(f"No artifact classes configured for period '{period}'.")
+        if self.priority_fn:
+            try:
+                classes.sort(key=self.priority_fn)
+            except Exception as e:
+                self.logger.warning(f"priority_fn failed; using listed order: {e}")
+        return classes
+
+    @staticmethod
+    def _split_kwargs(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+        orch: Dict[str, Any] = {}
+        art: Dict[str, Any] = {}
+        for k, v in raw.items():
+            if k in _ORCHESTRATOR_KEYS:
+                orch[k] = v
+            else:
+                art[k] = v
+        return orch, art
+
+    def _run_one(self, cls: Type, period: str, artifact_kwargs: Dict[str, Any]) -> str:
+        name = cls.__name__
+        start = time.monotonic()
+        for attempt in range(1, self._retry.attempts + 1):
+            if self._stop_event.is_set() or self.closed:
+                raise RuntimeError("shutting_down")
+            try:
+                with ExitStack() as stack:
+                    inst = cls(**self.artifact_class_kwargs)
+                    inst = stack.enter_context(inst)
+                    inst.update_parquet(period=period, **artifact_kwargs)
+                self.completion_secs[name] = time.monotonic() - start
+                return name
+            except Exception as e:
+                if attempt < self._retry.attempts and not self._stop_event.is_set():
+                    # interruptible backoff sleep
+                    delay = min(self._retry.backoff_base ** (attempt - 1), self._retry.backoff_max)
+                    delay *= 1 + random.uniform(0, self._retry.jitter)
+                    end = time.monotonic() + delay
+                    while not self._stop_event.is_set() and time.monotonic() < end:
+                        time.sleep(min(0.1, end - time.monotonic()))
+                    continue
+                raise RuntimeError(f"{name} failed after {self._retry.attempts} attempts: {e}") from e
+
+    def update_data(self, period: str, **kwargs: Any) -> None:
+        # Split kwargs to preserve backward compatibility
+        _, artifact_kwargs = self._split_kwargs(kwargs)
+
+        self.completion_secs.clear()
+        self.failed.clear()
+
+        classes = self._classes_for(period)
+
+        executor = ThreadPoolExecutor(max_workers=self.max_workers, thread_name_prefix="artifact-updater")
+        try:
+            fut2name: Dict[Any, str] = {}
+            for cls in classes:
+                if self._stop_event.is_set() or self.closed:
+                    break
+                try:
+                    fut = executor.submit(self._run_one, cls, period, dict(artifact_kwargs))
+                except RuntimeError as e:
+                    if "cannot schedule new futures after shutdown" in str(e).lower():
+                        self.logger.warning("Executor shutting down; halting new submissions.")
+                        break
+                    raise
+                fut2name[fut] = cls.__name__
+
+            for fut in as_completed(fut2name):
+                name = fut2name[fut]
+                try:
+                    fut.result()
+                    self.logger.info(f"✅ {name} ({period}) in {self.completion_secs[name]:.2f}s")
+                except Exception as e:
+                    self.failed.append(name)
+                    self.logger.error(f"✖️  {name} permanently failed: {e}")
+        except KeyboardInterrupt:
+            self.logger.warning("KeyboardInterrupt — stopping scheduling and shutting down.")
+            self._stop_event.set()
+            raise
+        finally:
+            # Ensure queued-but-not-started tasks are canceled
+            executor.shutdown(wait=True, cancel_futures=True)
+
+        self.logger.info(
+            f"Artifacts processed: total={len(classes)}, "
+            f"completed={len(self.completion_secs)}, failed={len(self.failed)}",
+            extra={
+                "date_of_update": time.strftime('%Y-%m-%d'),
+                "start_time": time.strftime('%H:%M:%S'),
+                "period": period,
+            },
+        )
+
+    def get_update_status(self) -> Dict[str, Any]:
+        done = set(self.completion_secs)
+        fail = set(self.failed)
+        all_names = {c.__name__ for v in self.wrapped_classes.values() for c in v}
+        return {
+            "total": len(all_names),
+            "completed": sorted(done),
+            "failed": sorted(fail),
+            "pending": sorted(all_names - done - fail),
+            "completion_times": dict(self.completion_secs),
+        }

{sibi_dst-2025.8.1 → sibi_dst-2025.8.3}/sibi_dst/df_helper/_df_helper.py

@@ -55,23 +55,42 @@ class ParquetBackend(BaseBackend):
     def load(self, **options):
         try:
             df = self.helper.backend_parquet.load_files()
-            if len(df.head(1)) == 0:
-                return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
+            if self._is_empty(df):
+                return -1, self._empty_like(df)
 
             if options and df is not None:
                 df = FilterHandler("dask", logger=self.logger, debug=False).apply_filters(df, filters=options)
-                if len(df.head(1)) == 0:
+                nrows = self._row_count(df)
+                if nrows == 0:
                     self.logger.debug("No records after filters; returning empty DataFrame.")
-                    return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
+                    return 0, self._empty_like(df)
 
             df = df.persist()
-            self.total_records = len(df) or -1
+            self.total_records = self._row_count(df) or -1
             return self.total_records, df
+
         except Exception as e:
-            self.total_records = -1
+            self.total_records = -1  # Reset total_records on failure
             self.logger.error(f"Failed to load data from parquet: {e}", exc_info=self.debug)
             return -1, dd.from_pandas(pd.DataFrame(), npartitions=1)
 
+    def _is_empty(self, ddf) -> bool:
+        """True if no rows across all partitions."""
+        try:
+            # head with npartitions=-1 walks partitions until it gets n rows
+            return ddf.head(1, npartitions=-1).shape[0] == 0
+        except Exception:
+            return True
+
+    def _row_count(self, ddf) -> int:
+        """Reliable row count for Dask DataFrame."""
+        return int(ddf.map_partitions(len).sum().compute())
+
+    def _empty_like(self, ddf):
+        """Return an empty Dask DF with the SAME columns/dtypes."""
+        empty_pdf = ddf._meta.iloc[0:0]
+        return dd.from_pandas(empty_pdf, npartitions=1)
+
 
 class HttpBackend(BaseBackend):
     def load(self, **options):
@@ -250,7 +269,7 @@ class DfHelper(ManagedResource):
         if hasattr(df, "npartitions") and df.npartitions == 1 and not len(df.head(1)):
             self.logger.warning("Cannot write to ClickHouse; DataFrame is empty.")
             return
-        with ClickHouseWriter(debug=self.debug, logger=self.logger, verbose=self.verbose, **credentials) as writer:
+        with ClickHouseWriter(debug=self.debug, logger=self.logger, fs=self.fs, verbose=self.verbose, **credentials) as writer:
             writer.save_to_clickhouse(df)
             self.logger.debug("Save to ClickHouse completed.")
 

{sibi_dst-2025.8.1 → sibi_dst-2025.8.3}/sibi_dst/df_helper/_parquet_artifact.py

@@ -242,13 +242,33 @@ class ParquetArtifact(ManagedResource):
             }
 
         def custom_config():
-            if "start_on" not in final_kwargs or "end_on" not in final_kwargs:
+            """
+                Prepare parameters for 'custom' period execution, ensuring `start_on` and `end_on`
+                are provided (with backward compatibility for `start_date`/`end_date` aliases).
+                """
+            # Backward compatibility: normalize aliases
+            alias_map = {
+                "start_on": ("start_date", "start"),
+                "end_on": ("end_date", "end"),
+            }
+            normalized_kwargs = dict(kwargs)  # shallow copy so we don't mutate original
+            for target, aliases in alias_map.items():
+                if target not in normalized_kwargs:
+                    for alias in aliases:
+                        if alias in normalized_kwargs:
+                            normalized_kwargs[target] = normalized_kwargs[alias]
+                            break
+
+            # Validation
+            missing = [k for k in ("start_on", "end_on") if k not in normalized_kwargs]
+            if missing:
                 raise ValueError(
-                    "For period 'custom', provide 'start_on' and 'end_on'."
+                    f"For period 'custom', the following required parameters are missing: {', '.join(missing)}"
                 )
+
             return {
-                "parquet_start_date": final_kwargs["start_on"],
-                "parquet_end_date": final_kwargs["end_on"],
+                "parquet_start_date": normalized_kwargs["start_on"],
+                "parquet_end_date": normalized_kwargs["end_on"],
             }
 
         if period == "itd":

{sibi_dst-2025.8.1 → sibi_dst-2025.8.3}/sibi_dst/df_helper/_parquet_reader.py

@@ -1,7 +1,8 @@
-from typing import Optional, ClassVar, Dict
+from typing import Optional, ClassVar, Dict, Any, Union
 
 import dask.dataframe as dd
 import fsspec
+import pandas as pd
 
 from sibi_dst.df_helper import DfHelper
 
@@ -23,7 +24,7 @@ class ParquetReader(DfHelper):
     :type config: dict
     :ivar df: Stores the loaded Dask DataFrame after the `load()` method is
         invoked. Initially set to None.
-    :type df: Optional[dd.DataFrame]
+    :type df: Optional[dd.DataFrame | pd.DataFrame]
     :ivar parquet_storage_path: The path to the Parquet storage directory.
     :type parquet_storage_path: str
     :ivar parquet_start_date: Start date for Parquet data selection. Must
@@ -32,19 +33,15 @@ class ParquetReader(DfHelper):
     :ivar parquet_end_date: End date for Parquet data selection. Must be
         set in the configuration.
     :type parquet_end_date: str
-    :ivar filesystem_type: The type of filesystem the Parquet files are
-        stored on (e.g., "file", "s3").
-    :type filesystem_type: str
-    :ivar filesystem_options: Any additional options required for the
-        specified filesystem type.
-    :type filesystem_options: dict
+
     :ivar fs: Instance of `fsspec` filesystem used to interact with the
         Parquet storage.
     :type fs: fsspec.AbstractFileSystem
     """
-    DEFAULT_CONFIG: ClassVar[Dict[str, int]] = {
+    DEFAULT_CONFIG: ClassVar[Dict[str, Any]] = {
         'backend': 'parquet'
     }
+    df: Optional[Union[dd.DataFrame, pd.DataFrame]] = None
 
     def __init__(self,  **kwargs):
         self.config = {
@@ -63,11 +60,13 @@ class ParquetReader(DfHelper):
         self.parquet_end_date = self.config.setdefault('parquet_end_date', None)
         if self.parquet_end_date is None:
             raise ValueError('parquet_end_date must be set')
+        if self.fs is None:
+            raise ValueError('Parquet Reader mush be supplied a fs instance')
 
         if not self.directory_exists():
             raise ValueError(f"{self.parquet_storage_path} does not exist")
 
-    def load(self, **kwargs):
+    def load(self, **kwargs) -> Union[pd.DataFrame, dd.DataFrame]:
         self.df = super().load(**kwargs)
         return self.df