sibi-dst 2025.9.4__tar.gz → 2025.9.6__tar.gz

{sibi_dst-2025.9.4 → sibi_dst-2025.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 2025.9.4
+Version: 2025.9.6
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-2025.9.4 → sibi_dst-2025.9.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "2025.9.4"
+version = "2025.9.6"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

sibi_dst-2025.9.6/sibi_dst/tests/test_baseclass.py

@@ -0,0 +1,403 @@
+import asyncio
+import json
+import threading
+from typing import Any, Dict
+from unittest.mock import MagicMock
+
+import fsspec
+
+from sibi_dst.utils import Logger
+from sibi_dst.utils import ManagedResource
+from sibi_dst.utils.base import _QueueSSE  # Replace 'your_module' with actual module name
+
+
+# ------------------------------ Test Fixtures ------------------------------
+
+class TestResource(ManagedResource):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.cleanup_called = False
+        self.acleanup_called = False
+
+    def _cleanup(self) -> None:
+        self.cleanup_called = True
+        super()._cleanup()
+
+    async def _acleanup(self) -> None:
+        self.acleanup_called = True
+        await super()._acleanup()
+
+
+class MockSSESink:
+    def __init__(self):
+        self.events = []
+        self.closed = False
+
+    async def send(self, event: str, data: Dict[str, Any]) -> None:
+        self.events.append({"event": event, "data": data})
+
+    async def aclose(self) -> None:
+        self.closed = True
+
+
+class MockSyncSSESink:
+    def __init__(self):
+        self.events = []
+        self.closed = False
+
+    def send(self, event: str, data: Dict[str, Any]) -> None:
+        self.events.append({"event": event, "data": data})
+
+    def close(self) -> None:
+        self.closed = True
+
+
+# ------------------------------ Mock fsspec filesystem ------------------------------
+
+class MockFileSystem(fsspec.AbstractFileSystem):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.closed = False
+
+    def close(self):
+        self.closed = True
+
+
+# ------------------------------ Utility for Event Loop ------------------------------
+
+def run_async_test(coro):
+    """Run async test safely in different environments."""
+    try:
+        # Try to get existing event loop (for Jupyter/IPython)
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            # In Jupyter, create a new task
+            task = loop.create_task(coro)
+            return task
+        else:
+            return loop.run_until_complete(coro)
+    except RuntimeError:
+        # No event loop running, use asyncio.run()
+        return asyncio.run(coro)
+
+
+# ------------------------------ Lifecycle Tests ------------------------------
+
+def test_double_close_no_error():
+    """Test that calling close() multiple times doesn't raise errors."""
+    resource = TestResource()
+    resource.close()
+    resource.close()  # Should not raise
+    assert resource.closed
+
+
+def test_double_aclose_no_error():
+    """Test that calling aclose() multiple times doesn't raise errors."""
+    async def test():
+        resource = TestResource()
+        await resource.aclose()
+        await resource.aclose()  # Should not raise
+        assert resource.closed
+
+    run_async_test(test())
+
+
+def test_context_manager_sync():
+    """Test sync context manager behavior."""
+    with TestResource() as resource:
+        assert not resource.closed
+    assert resource.closed
+    assert resource.cleanup_called
+
+
+def test_context_manager_async():
+    """Test async context manager behavior."""
+    async def test():
+        async with TestResource() as resource:
+            assert not resource.closed
+        assert resource.closed
+        assert resource.acleanup_called
+
+    run_async_test(test())
+
+
+# ------------------------------ SSE Emission Tests ------------------------------
+
+def test_auto_sse_creation():
+    """Test automatic SSE creation when auto_sse=True."""
+    resource = TestResource(auto_sse=True)
+    sse = resource.get_sse()
+    assert sse is not None
+    assert isinstance(sse, _QueueSSE)
+    assert resource._owns_sse
+
+
+def test_sse_emission_with_async_sink():
+    """Test SSE emission with async send method."""
+    async def test():
+        sink = MockSSESink()
+        resource = TestResource(sse=sink)
+
+        await resource.emit("test_event", key="value")
+
+        assert len(sink.events) == 1
+        assert sink.events[0]["event"] == "test_event"
+        assert sink.events[0]["data"] == {"key": "value"}
+
+    run_async_test(test())
+
+
+def test_sse_emission_with_sync_sink():
+    """Test SSE emission with sync send method wrapped in async."""
+    sink = MockSyncSSESink()
+    resource = TestResource(sse=sink)
+
+    async def test():
+        await resource.emit("test_event", key="value")
+
+        assert len(sink.events) == 1
+        assert sink.events[0]["event"] == "test_event"
+        assert sink.events[0]["data"] == {"key": "value"}
+
+    run_async_test(test())
+
+
+def test_sse_put_method_support():
+    """Test SSE emission with put method."""
+    class PutSink:
+        def __init__(self):
+            self.items = []
+
+        async def put(self, item: Dict[str, Any]) -> None:
+            self.items.append(item)
+
+    async def test():
+        sink = PutSink()
+        resource = TestResource(sse=sink)
+
+        await resource.emit("test_event", key="value")
+
+        assert len(sink.items) == 1
+        item = sink.items[0]
+        assert item["event"] == "test_event"
+        assert json.loads(item["data"]) == {"key": "value"}
+
+    run_async_test(test())
+
+
+def test_sse_no_emitter_no_error():
+    """Test that emit on resource without emitter doesn't raise."""
+    resource = TestResource()
+    # Should not raise error
+    async def test():
+        await resource.emit("test_event", key="value")
+
+    run_async_test(test())
+
+
+def test_sse_emission_after_close():
+    """Test that emit after close is no-op."""
+    async def test():
+        sink = MockSSESink()
+        resource = TestResource(sse=sink)
+
+        await resource.aclose()
+        await resource.emit("test_event", key="value")  # Should not raise
+
+        assert len(sink.events) == 0
+
+    run_async_test(test())
+
+
+# ------------------------------ Cleanup Interplay Tests ------------------------------
+
+def test_sync_cleanup_called_on_sync_close():
+    """Test that sync cleanup is called during sync close."""
+    resource = TestResource()
+    resource.close()
+    assert resource.cleanup_called
+    assert not resource.acleanup_called
+
+
+def test_async_cleanup_called_on_async_close():
+    """Test that async cleanup is called during async close."""
+    async def test():
+        resource = TestResource()
+        await resource.aclose()
+        assert resource.acleanup_called
+        assert not resource.cleanup_called
+
+    run_async_test(test())
+
+
+# ------------------------------ Logger Tests ------------------------------
+
+def test_logger_ownership():
+    """Test that logger is owned when not provided externally."""
+    resource = TestResource()
+    assert resource._owns_logger
+    assert resource.logger is not None
+
+
+def test_external_logger_not_owned():
+    """Test that external logger is not owned."""
+    external_logger = Logger.default_logger("test")
+    resource = TestResource(logger=external_logger)
+    assert not resource._owns_logger
+    assert resource.logger is external_logger
+
+
+def test_logger_level_configuration():
+    """Test logger level configuration based on verbose/debug flags."""
+    # Default (warning level)
+    resource = TestResource()
+    assert hasattr(resource.logger, 'level')
+
+    # Verbose (info level)
+    resource = TestResource(verbose=True)
+    assert hasattr(resource.logger, 'level')
+
+    # Debug (debug level)
+    resource = TestResource(debug=True)
+    assert hasattr(resource.logger, 'level')
+
+
+# ------------------------------ Lazy Instantiation Tests ------------------------------
+
+def test_lazy_fs_instantiation():
+    """Test lazy filesystem instantiation via factory."""
+    fs_instance = MockFileSystem()
+    factory_called = [False]
+
+    def fs_factory():
+        factory_called[0] = True
+        return fs_instance
+
+    resource = TestResource(fs_factory=fs_factory)
+    assert not factory_called[0]  # Not called yet
+
+    fs = resource._ensure_fs()
+    assert factory_called[0]
+    assert fs is fs_instance
+    assert resource.fs is fs_instance
+
+
+def test_lazy_sse_instantiation():
+    """Test lazy SSE instantiation via factory."""
+    sink_instance = MockSSESink()
+    factory_called = [False]
+
+    def sse_factory():
+        factory_called[0] = True
+        return sink_instance
+
+    resource = TestResource(sse_factory=sse_factory)
+    assert not factory_called[0]  # Not called yet
+
+    sse = resource._ensure_sse()
+    assert factory_called[0]
+    assert sse is sink_instance
+    assert resource._sse is sink_instance
+
+
+def test_lazy_fs_not_called_if_fs_provided():
+    """Test that factory is not called if fs is provided directly."""
+    fs_instance = MockFileSystem()
+    factory = MagicMock()
+
+    resource = TestResource(fs=fs_instance, fs_factory=factory)
+    fs = resource._ensure_fs()
+
+    assert fs is fs_instance
+    factory.assert_not_called()
+
+
+def test_lazy_sse_not_called_if_sse_provided():
+    """Test that factory is not called if sse is provided directly."""
+    sink_instance = MockSSESink()
+    factory = MagicMock()
+
+    resource = TestResource(sse=sink_instance, sse_factory=factory)
+    sse = resource._ensure_sse()
+
+    assert sse is sink_instance
+    factory.assert_not_called()
+
+
+# ------------------------------ Thread Safety Tests ------------------------------
+
+def test_thread_safe_close():
+    """Test that close operations are thread-safe."""
+    resource = TestResource()
+
+    results = []
+    errors = []
+
+    def close_resource():
+        try:
+            resource.close()
+            results.append("success")
+        except Exception as e:
+            errors.append(str(e))
+            results.append(f"error: {e}")
+
+    # Start multiple threads trying to close simultaneously
+    threads = [threading.Thread(target=close_resource) for _ in range(5)]
+    for t in threads:
+        t.start()
+    for t in threads:
+        t.join()
+
+    # Debug information
+    print(f"Results: {results}")
+    print(f"Errors: {errors}")
+    print(f"Resource closed: {resource.closed}")
+
+    # Should have at least one success (the first one) and no exceptions
+    success_count = results.count("success")
+    error_count = len([r for r in results if r.startswith("error")])
+
+    # At least one should succeed
+    assert success_count >= 1, f"Expected at least 1 success, got {success_count}"
+    # No errors should occur
+    assert error_count == 0, f"Expected 0 errors, got {error_count}"
+    # Resource should be closed
+    assert resource.closed, "Resource should be closed"
+
+
+# ------------------------------ Individual Test Functions ------------------------------
+
+# You can now run individual tests like this:
+if __name__ == "__main__":
+    # Run individual tests
+    test_double_close_no_error()
+    print("✓ test_double_close_no_error passed")
+
+    test_sync_cleanup_called_on_sync_close()
+    print("✓ test_sync_cleanup_called_on_sync_close passed")
+
+    test_logger_ownership()
+    print("✓ test_logger_ownership passed")
+
+    test_external_logger_not_owned()
+    print("✓ test_external_logger_not_owned passed")
+
+    test_lazy_fs_instantiation()
+    print("✓ test_lazy_fs_instantiation passed")
+
+    test_lazy_sse_instantiation()
+    print("✓ test_lazy_sse_instantiation passed")
+
+    test_lazy_fs_not_called_if_fs_provided()
+    print("✓ test_lazy_fs_not_called_if_fs_provided passed")
+
+    test_lazy_sse_not_called_if_sse_provided()
+    print("✓ test_lazy_sse_not_called_if_sse_provided passed")
+
+    test_thread_safe_close()
+    print("✓ test_thread_safe_close passed")
+
+    test_auto_sse_creation()
+    print("✓ test_auto_sse_creation passed")
+
+    print("All tests completed!")

{sibi_dst-2025.9.4 → sibi_dst-2025.9.6}/sibi_dst/utils/base.py

@@ -441,257 +441,3 @@ class ManagedResource(abc.ABC):
         except Exception:
             pass
 
-## Before SSE handling
-
-# import abc
-# import threading
-# import weakref
-# from typing import Self, Optional, Callable
-#
-# import fsspec
-#
-# from sibi_dst.utils import Logger
-#
-#
-# class ManagedResource(abc.ABC):
-#     """
-#     Boilerplate ABC for components that manage a logger and an optional fsspec filesystem,
-#     with sync/async lifecycle helpers, lazy FS creation via an optional factory, and
-#     configurable cleanup-error logging.
-#     """
-#
-#     def __init__(
-#         self,
-#         *,
-#         verbose: bool = False,
-#         debug: bool = False,
-#         log_cleanup_errors: bool = True,
-#         logger: Optional[Logger] = None,
-#         fs: Optional[fsspec.AbstractFileSystem] = None,
-#         fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None,
-#         **_: object,
-#     ) -> None:
-#         # ---- Declared upfront for type checkers
-#         self.logger: Logger
-#         self.fs: Optional[fsspec.AbstractFileSystem] = None
-#         self._fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None
-#         self._owns_logger: bool = False
-#         self._owns_fs: bool = False
-#         self._is_closed: bool = False
-#         self._closing: bool = False
-#         self._close_lock = threading.RLock()
-#
-#         self.verbose = verbose
-#         self.debug = debug
-#         self._log_cleanup_errors = log_cleanup_errors
-#
-#         # ---- Logger ownership
-#         if logger is None:
-#             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
-#             self._owns_logger = True
-#             level = Logger.DEBUG if self.debug else (Logger.INFO if self.verbose else Logger.WARNING)
-#             self.logger.set_level(level)
-#         else:
-#             self.logger = logger
-#             self._owns_logger = False  # do not mutate external logger
-#
-#         # ---- FS ownership & lazy creation
-#         if fs is not None:
-#             self.fs = fs
-#             self._owns_fs = False
-#             self._fs_factory = None
-#         elif fs_factory is not None:
-#             # Lazy: don't create until first use
-#             self._fs_factory = fs_factory
-#             self._owns_fs = True  # we will own it *if* created
-#             self.fs = None
-#         else:
-#             self.fs = None
-#             self._owns_fs = False
-#             self._fs_factory = None
-#
-#         # Register a GC-time finalizer that does not capture self
-#         self_ref = weakref.ref(self)
-#         self._finalizer = weakref.finalize(self, self._finalize_static, self_ref)
-#
-#         if self.debug:
-#             try:
-#                 self.logger.debug("Component %s initialized. %s", self.__class__.__name__, repr(self))
-#             except Exception:
-#                 pass
-#
-#     # ---------- Introspection ----------
-#     @property
-#     def is_closed(self) -> bool:
-#         return self._is_closed
-#
-#     @property
-#     def closed(self) -> bool:  # alias
-#         return self._is_closed
-#
-#     def __repr__(self) -> str:
-#         class_name = self.__class__.__name__
-#         logger_status = "own" if self._owns_logger else "external"
-#         if self.fs is None and self._fs_factory is not None:
-#             fs_status = "own(lazy)"
-#         elif self.fs is None:
-#             fs_status = "none"
-#         else:
-#             fs_status = "own" if self._owns_fs else "external"
-#         return (f"<{class_name} debug={self.debug} verbose={self.verbose} "
-#                 f"log_cleanup_errors={self._log_cleanup_errors} "
-#                 f"logger={logger_status} fs={fs_status}>")
-#
-#     # ---------- Subclass hooks ----------
-#     def _cleanup(self) -> None:
-#         """Sync cleanup for resources created BY THE SUBCLASS."""
-#         return
-#
-#     async def _acleanup(self) -> None:
-#         """Async cleanup for resources created BY THE SUBCLASS."""
-#         return
-#
-#     # ---------- FS helpers ----------
-#     def _ensure_fs(self) -> Optional[fsspec.AbstractFileSystem]:
-#         """Create the FS lazily if a factory was provided. Return fs (or None)."""
-#         if self.fs is None and self._fs_factory is not None:
-#             created = self._fs_factory()
-#             if not isinstance(created, fsspec.AbstractFileSystem):
-#                 raise TypeError(f"fs_factory() must return fsspec.AbstractFileSystem, got {type(created)!r}")
-#             self.fs = created
-#             # _owns_fs already True when factory is present
-#         return self.fs
-#
-#     def require_fs(self) -> fsspec.AbstractFileSystem:
-#         """Return a filesystem or raise if not configured/creatable."""
-#         fs = self._ensure_fs()
-#         if fs is None:
-#             raise RuntimeError(
-#                 f"{self.__class__.__name__}: filesystem is required but not configured"
-#             )
-#         return fs
-#
-#     # ---------- Shared shutdown helpers (no logging; safe for late shutdown) ----------
-#     def _release_owned_fs(self) -> None:
-#         if self._owns_fs:
-#             # ensure creation state is respected even if never used
-#             _ = self.fs or None  # no-op; if never created, nothing to close
-#             if self.fs is not None:
-#                 close = getattr(self.fs, "close", None)
-#                 try:
-#                     if callable(close):
-#                         close()
-#                 finally:
-#                     self.fs = None
-#
-#     def _shutdown_logger(self) -> None:
-#         if self._owns_logger:
-#             try:
-#                 self.logger.shutdown()
-#             except Exception:
-#                 pass
-#
-#     def _shutdown_owned_resources(self) -> None:
-#         self._release_owned_fs()
-#         self._shutdown_logger()
-#
-#     # ---------- Public lifecycle (sync) ----------
-#     def close(self) -> None:
-#         with self._close_lock:
-#             if self._is_closed or self._closing:
-#                 return
-#             self._closing = True
-#
-#         try:
-#             self._cleanup()
-#         except Exception:
-#             # Only include traceback when debug=True
-#             if self._log_cleanup_errors:
-#                 try:
-#                     self.logger.error(
-#                         "Error during %s._cleanup()", self.__class__.__name__,
-#                         exc_info=self.debug
-#                     )
-#                 except Exception:
-#                     pass
-#             raise
-#         finally:
-#             with self._close_lock:
-#                 self._is_closed = True
-#                 self._closing = False
-#             self._shutdown_owned_resources()
-#             if self.debug:
-#                 try:
-#                     self.logger.debug("Component %s closed.", self.__class__.__name__)
-#                 except Exception:
-#                     pass
-#
-#     # ---------- Public lifecycle (async) ----------
-#     async def aclose(self) -> None:
-#         with self._close_lock:
-#             if self._is_closed or self._closing:
-#                 return
-#             self._closing = True
-#
-#         try:
-#             await self._acleanup()
-#         except Exception:
-#             # Only include traceback when debug=True
-#             if self._log_cleanup_errors:
-#                 try:
-#                     self.logger.error(
-#                         "Error during %s._acleanup()", self.__class__.__name__,
-#                         exc_info=self.debug
-#                     )
-#                 except Exception:
-#                     pass
-#             raise
-#         finally:
-#             with self._close_lock:
-#                 self._is_closed = True
-#                 self._closing = False
-#             self._shutdown_owned_resources()
-#             if self.debug:
-#                 try:
-#                     self.logger.debug("Async component %s closed.", self.__class__.__name__)
-#                 except Exception:
-#                     pass
-#
-#     # ---------- Context managers ----------
-#     def __enter__(self) -> Self:
-#         return self
-#
-#     def __exit__(self, exc_type, exc, tb) -> bool:
-#         self.close()
-#         return False  # propagate exceptions
-#
-#     async def __aenter__(self) -> Self:
-#         return self
-#
-#     async def __aexit__(self, exc_type, exc, tb) -> bool:
-#         await self.aclose()
-#         return False
-#
-#     # ---------- Finalizer ( at Garbage Collection-time absolutely silent) ----------
-#     @staticmethod
-#     def _finalize_static(ref: "weakref.ReferenceType[ManagedResource]") -> None:
-#         obj = ref()
-#         if obj is None:
-#             return
-#         # No logging here; interpreter may be tearing down.
-#         # Best-effort silent cleanup; avoid locks and context managers.
-#         try:
-#             if not obj._is_closed:
-#                 try:
-#                     obj._cleanup()
-#                 except Exception:
-#                     pass
-#                 obj._is_closed = True
-#                 try:
-#                     obj._shutdown_owned_resources()
-#                 except Exception:
-#                     pass
-#         except Exception:
-#             # do not show anything at garbage collection time
-#             pass
-#

{sibi_dst-2025.9.4 → sibi_dst-2025.9.6}/sibi_dst/utils/boilerplate/__init__.py

@@ -2,10 +2,13 @@ from .base_parquet_artifact import BaseParquetArtifact
 from .base_data_cube import BaseDataCube
 from .base_attacher import make_attacher
 from .base_parquet_reader import BaseParquetReader
+from .hybrid_data_loader import HybridDataLoader
+
 __all__ = [
     "BaseDataCube",
     "BaseParquetArtifact",
     "make_attacher",
-    "BaseParquetReader"
+    "BaseParquetReader",
+    "HybridDataLoader",
 ]