sibi-dst 2025.9.12__tar.gz → 2025.9.14__tar.gz

{sibi_dst-2025.9.12 → sibi_dst-2025.9.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sibi-dst
-Version: 2025.9.12
+Version: 2025.9.14
 Summary: A data science toolkit for scalable data processing and analysis.
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{sibi_dst-2025.9.12 → sibi_dst-2025.9.14}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sibi-dst"
-version = "2025.9.12"
+version = "2025.9.14"
 description = "A data science toolkit for scalable data processing and analysis."
 readme = "README.md"
 requires-python = ">=3.11"

{sibi_dst-2025.9.12 → sibi_dst-2025.9.14}/sibi_dst/df_helper/_df_helper.py

@@ -137,7 +137,7 @@ class DfHelper(ManagedResource):
     def __init__(self, backend="sqlalchemy", **kwargs):
         self.default_config = self.default_config or {}
         kwargs = {**self.default_config.copy(), **kwargs}
-        kwargs.setdefault("auto_sse", True)
+        kwargs.setdefault("auto_sse", False)
         super().__init__(**kwargs)
         self.backend = backend
 

{sibi_dst-2025.9.12 → sibi_dst-2025.9.14}/sibi_dst/utils/base.py

@@ -14,7 +14,18 @@ from sibi_dst.utils import Logger
 
 # --------- Minimal built-in SSE sink (used when auto_sse=True) ----------
 class _QueueSSE:
-    """Async queue–backed SSE sink: async send/put, async iterator, graceful close."""
+    """
+    Handles asynchronous streaming of events with structured data.
+
+    This class provides the ability to manage an asynchronous queue for handling
+    streamed Server-Sent Events (SSE). It supports operations like sending events
+    with associated data, manually enqueuing items, and iterating over items in an
+    asynchronous loop. The class also includes mechanisms for clean closure of the
+    stream.
+
+    :ivar q: An asynchronous queue used to store events and data.
+    :type q: asyncio.Queue
+    """
     __slots__ = ("q", "_closed")
 
     def __init__(self) -> None:
@@ -46,11 +57,31 @@ class _QueueSSE:
 # ------------------------------ Base class ------------------------------
 class ManagedResource(abc.ABC):
     """
-    Owns a logger and optional fsspec filesystem. Can emit SSE events via:
-    - an async emitter callable:  await emitter(event, data)
-    - a sink exposing async send(event, data) or async put(item)
-    If neither is provided and auto_sse=True, a queue-backed sink is created eagerly.
-    Thread-safe lifecycle, CM support, GC finalizer.
+    Management of shared resources with configurable verbosity, logging,
+    and support for external file systems and server-sent events (SSE).
+
+    This class is designed to assist in managing resources such as logging,
+    file systems, and SSE within an asynchronous or synchronous environment.
+    It provides facilities for handling resource lifecycle, introspection,
+    and cleanup while ensuring resources are appropriately managed. The class
+    also supports lazy initialization of external dependencies via factories.
+
+    :ivar verbose: Controls verbosity of logging or operations. If set to True,
+        more detailed logging/output will be generated.
+    :type verbose: bool
+    :ivar debug: Enables debug-level logging and internal diagnostics when True.
+        Typically used for troubleshooting purposes.
+    :type debug: bool
+    :ivar logger: The logger instance used for this resource. If left unset,
+        a default logger will be created.
+    :type logger: Optional[Logger]
+    :ivar fs: The file system interface being used. Typically an instance of
+        `fsspec.AbstractFileSystem`. If not provided, it may be created lazily
+        using a supplied factory function.
+    :type fs: Optional[fsspec.AbstractFileSystem]
+    :ivar emitter: A callable, potentially asynchronous, function for emitting
+        events. Events are sent as a combination of event names and payload data.
+    :type emitter: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]]
     """
 
     __slots__ = (
@@ -74,28 +105,23 @@ class ManagedResource(abc.ABC):
         debug: bool = False,
         log_cleanup_errors: bool = True,
         logger: Optional[Logger] = None,
-        # filesystem
         fs: Optional[fsspec.AbstractFileSystem] = None,
         fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None,
-        # SSE
         emitter: Optional[Callable[[str, Dict[str, Any]], Awaitable[None]]] = None,
         emitter_factory: Optional[Callable[[], Callable[[str, Dict[str, Any]], Awaitable[None]]]] = None,
         sse: Optional[object] = None,
         sse_factory: Optional[Callable[[], object]] = None,
-        auto_sse: bool = False,           # eager auto-create if no emitter/sink is supplied
+        auto_sse: bool = False,
         **_: object,
     ) -> None:
-        # flags
         self.verbose = verbose
         self.debug = debug
         self._log_cleanup_errors = log_cleanup_errors
 
-        # lifecycle
         self._is_closed = False
         self._closing = False
         self._close_lock = threading.RLock()
 
-        # logger
         if logger is None:
             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
             self._owns_logger = True
@@ -105,7 +131,6 @@ class ManagedResource(abc.ABC):
             self.logger = logger
             self._owns_logger = False
 
-        # fs
         self.fs: Optional[fsspec.AbstractFileSystem] = None
         self._fs_factory = None
         self._owns_fs = False
@@ -119,7 +144,6 @@ class ManagedResource(abc.ABC):
             self._fs_factory = fs_factory
             self._owns_fs = True
 
-        # sse / emitter
         self._sse: Optional[object] = None
         self._sse_factory: Optional[Callable[[], object]] = None
         self._owns_sse = False
@@ -140,16 +164,15 @@ class ManagedResource(abc.ABC):
             self._sse_factory = sse_factory
             self._owns_sse = True
 
-        # EAGER auto-SSE: create sink+emitter now if none supplied
         if self._auto_sse and self._sse is None and self._emitter is None and self._sse_factory is None:
             self._create_auto_sse()
 
-        # GC finalizer
+        # Garbage Collector finaliser
         self._finalizer = weakref.finalize(self, self._finalize_static, weakref.ref(self))
 
         if self.debug:
             with contextlib.suppress(Exception):
-                self.logger.debug("Initialized %s %s", self.__class__.__name__, repr(self))
+                self.logger.debug("Initialised %s %s", self.__class__.__name__, repr(self))
 
     # ---------- Introspection ----------
     @property
@@ -222,7 +245,6 @@ class ManagedResource(abc.ABC):
 
     # ---------- SSE ----------
     def _create_auto_sse(self) -> None:
-        # internal helper: create queue sink + emitter, mark as owned
         sink = _QueueSSE()
         self._sse = sink
         self._owns_sse = True

{sibi_dst-2025.9.12 → sibi_dst-2025.9.14}/sibi_dst/utils/dask_utils.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 from typing import List, Any, Dict
 
 import dask
-#dask.config.set({"distributed.worker.daemon": False})
 import dask.dataframe as dd
 
 def _to_int_safe(x) -> int:
@@ -63,7 +63,7 @@ class UniqueValuesExtractor:
         pairs = await asyncio.gather(*(one(c) for c in columns))
         return dict(pairs)
 
-from contextlib import suppress
+from contextlib import suppress, asynccontextmanager
 from dask.distributed import Client, LocalCluster, get_client
 import os
 
@@ -93,6 +93,12 @@ class DaskClientMixin:
         self.dask_client = dask_client
         self.own_dask_client = False
         self.logger = logger
+        # Apply log filters globally
+        logging.getLogger("distributed.shuffle._scheduler_plugin").setLevel(
+            logging.ERROR
+        )
+        logging.getLogger("distributed.scheduler").setLevel(logging.WARNING)
+        logging.getLogger("distributed.worker").setLevel(logging.WARNING)
 
         if self.dask_client is None:
             with suppress(ValueError, RuntimeError):
@@ -143,42 +149,11 @@ class DaskClientMixin:
                 if self.logger:
                     self.logger.warning(f"Error while closing Dask client: {e}")
 
-# from contextlib import suppress
-# from dask.distributed import Client, get_client
-#
-# class DaskClientMixin:
-#     """
-#     Provides shared Dask client lifecycle management.
-#     Ensures reuse of existing client when available, otherwise creates a lightweight local one.
-#     """
-#
-#     def _init_dask_client(self, dask_client=None, logger=None):
-#         self.dask_client = dask_client
-#         self.own_dask_client = False
-#         self.logger = logger
-#
-#         if self.dask_client is None:
-#             with suppress(ValueError, RuntimeError):
-#                 # Try to attach to an existing active client if running inside a Dask context
-#                 self.dask_client = get_client()
-#
-#         if self.dask_client is None:
-#             # Start a local in-process scheduler for fallback
-#             self.dask_client = Client(processes=False)
-#             self.own_dask_client = True
-#             if self.logger:
-#                 self.logger.info(f"Started local Dask client: {self.dask_client.dashboard_link}")
-#         else:
-#             if self.logger:
-#                 self.logger.debug(f"Using existing Dask client: {self.dask_client.dashboard_link}")
-#
-#     def _close_dask_client(self):
-#         """Close client only if this instance created it."""
-#         if getattr(self, "own_dask_client", False) and self.dask_client is not None:
-#             try:
-#                 self.dask_client.close()
-#                 if self.logger:
-#                     self.logger.info("Closed local Dask client.")
-#             except Exception as e:
-#                 if self.logger:
-#                     self.logger.warning(f"Error while closing Dask client: {e}")
+@asynccontextmanager
+async def shared_dask_session(**kwargs):
+    mixin = DaskClientMixin()
+    mixin._init_dask_client(**kwargs)
+    try:
+        yield mixin.dask_client
+    finally:
+        mixin._close_dask_client()

{sibi_dst-2025.9.12 → sibi_dst-2025.9.14}/sibi_dst.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sibi-dst
-Version: 2025.9.12
+Version: 2025.9.14
 Summary: A data science toolkit for scalable data processing and analysis.
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown