sibi-dst 2025.8.1__py3-none-any.whl → 2025.8.2__py3-none-any.whl

sibi_dst/utils/base.py

@@ -10,8 +10,9 @@ from sibi_dst.utils import Logger
 
 class ManagedResource(abc.ABC):
     """
-    Boilerplate ABC for components that manage a logger and an fsspec filesystem
-    with sync/async lifecycle helpers.
+    Boilerplate ABC for components that manage a logger and an optional fsspec filesystem,
+    with sync/async lifecycle helpers, lazy FS creation via an optional factory, and
+    configurable cleanup-error logging.
     """
 
     def __init__(
@@ -19,46 +20,60 @@ class ManagedResource(abc.ABC):
         *,
         verbose: bool = False,
         debug: bool = False,
+        log_cleanup_errors: bool = True,
         logger: Optional[Logger] = None,
         fs: Optional[fsspec.AbstractFileSystem] = None,
         fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None,
         **_: object,
     ) -> None:
+        # ---- Declared upfront for type checkers
+        self.logger: Logger
+        self.fs: Optional[fsspec.AbstractFileSystem] = None
+        self._fs_factory: Optional[Callable[[], fsspec.AbstractFileSystem]] = None
+        self._owns_logger: bool = False
+        self._owns_fs: bool = False
+        self._is_closed: bool = False
+        self._closing: bool = False
+        self._close_lock = threading.RLock()
+
         self.verbose = verbose
         self.debug = debug
+        self._log_cleanup_errors = log_cleanup_errors
 
-        # --- Logger ownership ---
+        # ---- Logger ownership
         if logger is None:
             self.logger = Logger.default_logger(logger_name=self.__class__.__name__)
             self._owns_logger = True
-            self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
+            level = Logger.DEBUG if self.debug else (Logger.INFO if self.verbose else Logger.WARNING)
+            self.logger.set_level(level)
         else:
             self.logger = logger
-            self._owns_logger = False
-            # Do NOT mutate external logger level
+            self._owns_logger = False  # do not mutate external logger
 
-        # --- FS ownership ---
-        self._owns_fs = fs is None
+        # ---- FS ownership & lazy creation
         if fs is not None:
-            self.fs: Optional[fsspec.AbstractFileSystem] = fs
+            self.fs = fs
+            self._owns_fs = False
+            self._fs_factory = None
         elif fs_factory is not None:
-            created = fs_factory()
-            if not isinstance(created, fsspec.AbstractFileSystem):
-                raise TypeError(
-                    f"fs_factory() must return fsspec.AbstractFileSystem, got {type(created)!r}"
-                )
-            self.fs = created
+            # Lazy: don't create until first use
+            self._fs_factory = fs_factory
+            self._owns_fs = True  # we will own it *if* created
+            self.fs = None
         else:
-            self.fs = None  # optional; subclasses may not need fs
+            self.fs = None
+            self._owns_fs = False
+            self._fs_factory = None
 
-        self._is_closed = False
-        self._close_lock = threading.RLock()
+        # Register a GC-time finalizer that does not capture self
+        self_ref = weakref.ref(self)
+        self._finalizer = weakref.finalize(self, self._finalize_static, self_ref)
 
-        # register a best-effort finalizer
-        self._finalizer = weakref.finalize(self, self._finalize_silent)
-
-        # Early debug
-        self.logger.debug("Component %s initialized.", self.__class__.__name__)
+        if self.debug:
+            try:
+                self.logger.debug("Component %s initialized. %s", self.__class__.__name__, repr(self))
+            except Exception:
+                pass
 
     # ---------- Introspection ----------
     @property
@@ -72,10 +87,17 @@ class ManagedResource(abc.ABC):
     def __repr__(self) -> str:
         class_name = self.__class__.__name__
         logger_status = "own" if self._owns_logger else "external"
-        fs_status = "none" if self.fs is None else ("own" if self._owns_fs else "external")
-        return f"<{class_name} debug={self.debug} logger={logger_status} fs={fs_status}>"
+        if self.fs is None and self._fs_factory is not None:
+            fs_status = "own(lazy)"
+        elif self.fs is None:
+            fs_status = "none"
+        else:
+            fs_status = "own" if self._owns_fs else "external"
+        return (f"<{class_name} debug={self.debug} verbose={self.verbose} "
+                f"log_cleanup_errors={self._log_cleanup_errors} "
+                f"logger={logger_status} fs={fs_status}>")
 
-    # ---------- Hooks for subclasses ----------
+    # ---------- Subclass hooks ----------
     def _cleanup(self) -> None:
         """Sync cleanup for resources created BY THE SUBCLASS."""
         return
@@ -84,82 +106,111 @@ class ManagedResource(abc.ABC):
         """Async cleanup for resources created BY THE SUBCLASS."""
         return
 
-    # ---------- Owned resource shutdown ----------
-    def _shutdown_logger(self) -> None:
-        if not self._owns_logger:
-            self.logger.debug("%s: skipping logger shutdown (not owned).", self.__class__.__name__)
-            return
-        self.logger.debug("%s: shutting down owned logger.", self.__class__.__name__)
-        try:
-            self.logger.shutdown()
-        except Exception:  # keep shutdown robust
-            pass
-
-    def _shutdown_owned_resources(self) -> None:
-        # fsspec FS usually has no close; if it does, call it.
-        if self._owns_fs and self.fs is not None:
-            self.logger.debug("%s: releasing owned fsspec filesystem.", self.__class__.__name__)
-            close = getattr(self.fs, "close", None)
-            try:
-                if callable(close):
-                    close()
-            finally:
-                self.fs = None
-        else:
-            self.logger.debug(
-                "%s: skipping fs shutdown (not owned or none).", self.__class__.__name__
+    # ---------- FS helpers ----------
+    def _ensure_fs(self) -> Optional[fsspec.AbstractFileSystem]:
+        """Create the FS lazily if a factory was provided. Return fs (or None)."""
+        if self.fs is None and self._fs_factory is not None:
+            created = self._fs_factory()
+            if not isinstance(created, fsspec.AbstractFileSystem):
+                raise TypeError(f"fs_factory() must return fsspec.AbstractFileSystem, got {type(created)!r}")
+            self.fs = created
+            # _owns_fs already True when factory is present
+        return self.fs
+
+    def require_fs(self) -> fsspec.AbstractFileSystem:
+        """Return a filesystem or raise if not configured/creatable."""
+        fs = self._ensure_fs()
+        if fs is None:
+            raise RuntimeError(
+                f"{self.__class__.__name__}: filesystem is required but not configured"
             )
-        self._shutdown_logger()
+        return fs
+
+    # ---------- Shared shutdown helpers (no logging; safe for late shutdown) ----------
+    def _release_owned_fs(self) -> None:
+        if self._owns_fs:
+            # ensure creation state is respected even if never used
+            _ = self.fs or None  # no-op; if never created, nothing to close
+            if self.fs is not None:
+                close = getattr(self.fs, "close", None)
+                try:
+                    if callable(close):
+                        close()
+                finally:
+                    self.fs = None
 
-    async def _ashutdown_owned_resources(self) -> None:
-        # No async close in fsspec by default, keep parity with sync
-        if self._owns_fs and self.fs is not None:
-            self.logger.debug("%s: releasing owned fsspec filesystem (async).", self.__class__.__name__)
-            close = getattr(self.fs, "close", None)
+    def _shutdown_logger(self) -> None:
+        if self._owns_logger:
             try:
-                if callable(close):
-                    close()
-            finally:
-                self.fs = None
+                self.logger.shutdown()
+            except Exception:
+                pass
+
+    def _shutdown_owned_resources(self) -> None:
+        self._release_owned_fs()
         self._shutdown_logger()
 
-    # ---------- Public lifecycle ----------
+    # ---------- Public lifecycle (sync) ----------
     def close(self) -> None:
         with self._close_lock:
-            if self._is_closed:
+            if self._is_closed or self._closing:
                 return
-            self.logger.debug("Closing component %s...", self.__class__.__name__)
-            try:
-                self._cleanup()
-            except Exception:
-                # log and propagate — callers need to know
-                self.logger.error(
-                    "Error during %s._cleanup()", self.__class__.__name__, exc_info=True
-                )
-                raise
-            finally:
-                self._is_closed = True
-                self._shutdown_owned_resources()
-                self.logger.debug("Component %s closed.", self.__class__.__name__)
+            self._closing = True
 
+        try:
+            self._cleanup()
+        except Exception:
+            # Only include traceback when debug=True
+            if self._log_cleanup_errors:
+                try:
+                    self.logger.error(
+                        "Error during %s._cleanup()", self.__class__.__name__,
+                        exc_info=self.debug
+                    )
+                except Exception:
+                    pass
+            raise
+        finally:
+            with self._close_lock:
+                self._is_closed = True
+                self._closing = False
+            self._shutdown_owned_resources()
+            if self.debug:
+                try:
+                    self.logger.debug("Component %s closed.", self.__class__.__name__)
+                except Exception:
+                    pass
+
+    # ---------- Public lifecycle (async) ----------
     async def aclose(self) -> None:
         with self._close_lock:
-            if self._is_closed:
+            if self._is_closed or self._closing:
                 return
-            self.logger.debug("Asynchronously closing component %s...", self.__class__.__name__)
-        # run subclass async cleanup outside of lock
+            self._closing = True
+
         try:
             await self._acleanup()
         except Exception:
-            self.logger.error(
-                "Error during %s._acleanup()", self.__class__.__name__, exc_info=True
-            )
+            # Only include traceback when debug=True
+            if self._log_cleanup_errors:
+                try:
+                    self.logger.error(
+                        "Error during %s._acleanup()", self.__class__.__name__,
+                        exc_info=self.debug
+                    )
+                except Exception:
+                    pass
             raise
         finally:
             with self._close_lock:
                 self._is_closed = True
-            await self._ashutdown_owned_resources()
-            self.logger.debug("Async component %s closed.", self.__class__.__name__)
+                self._closing = False
+            self._shutdown_owned_resources()
+            if self.debug:
+                try:
+                    self.logger.debug("Async component %s closed.", self.__class__.__name__)
+                except Exception:
+                    pass
 
     # ---------- Context managers ----------
     def __enter__(self) -> Self:
@@ -176,148 +227,26 @@ class ManagedResource(abc.ABC):
         await self.aclose()
         return False
 
-    # ---------- Finalizer ----------
-    def _finalize_silent(self) -> None:
-        # Best-effort, no logging (avoid noisy GC-time logs).
+    # ---------- Finalizer ( at Garbage Collection-time absolutely silent) ----------
+    @staticmethod
+    def _finalize_static(ref: "weakref.ReferenceType[ManagedResource]") -> None:
+        obj = ref()
+        if obj is None:
+            return
+        # No logging here; interpreter may be tearing down.
+        # Best-effort silent cleanup; avoid locks and context managers.
         try:
-            if not self._is_closed:
-                self.close()
+            if not obj._is_closed:
+                try:
+                    obj._cleanup()
+                except Exception:
+                    pass
+                obj._is_closed = True
+                try:
+                    obj._shutdown_owned_resources()
+                except Exception:
+                    pass
         except Exception:
-            # absolutely swallow — GC context
+            # do not show anything at garbage collection time
             pass
 
-# import abc
-# from typing import Self, Optional, Callable, Any
-#
-# import fsspec
-#
-# from sibi_dst.utils import Logger
-#
-#
-# class ManagedResource(abc.ABC):
-#     """
-#     A unified boilerplate ABC for creating manageable components.
-#
-#     It provides integrated ownership and lifecycle management for a custom
-#     logger and a fsspec filesystem client, with full async support.
-#     """
-#
-#     def __init__(
-#             self,
-#             *,
-#             verbose: bool = False,
-#             debug: bool = False,
-#             logger: Optional[Logger] = None,
-#             fs: Optional[fsspec.AbstractFileSystem] = None,
-#             fs_factory: Optional[Callable[[], Any]] = None,
-#             **kwargs: Any,
-#     ) -> None:
-#         self.debug = debug
-#         self.verbose = verbose
-#
-#         self._is_closed = False
-#         self._owns_logger: bool
-#         self.fs, self._owns_fs = (fs, False) if fs else (None, True)
-#         if self._owns_fs and fs_factory:
-#             self.fs = fs_factory
-#         self.logger, self._owns_logger = (logger, False) if logger else (
-#             Logger.default_logger(logger_name=f"{self.__class__.__name__}"), True)
-#         self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
-#         self.logger.debug(f"Component: {self.__class__.__name__} initialized.")
-#
-#     @property
-#     def is_closed(self) -> bool:
-#         return self._is_closed
-#
-#     # Private methods for cleanup in the subclass
-#     def _cleanup(self) -> None:
-#         """Cleanup for resources created BY THE SUBCLASS."""
-#         pass
-#
-#     async def _acleanup(self) -> None:
-#         """Async cleanup for resources created BY THE SUBCLASS."""
-#         pass
-#
-#     # --- Private Shutdown Helpers ---
-#     def _shutdown_logger(self) -> None:
-#         # Your provided logger shutdown logic
-#         if not self._owns_logger:
-#             self.logger.debug(f"{self.__class__.__name__} is skipping logger shutdown (not owned).")
-#             return
-#         self.logger.debug(f"{self.__class__.__name__} is shutting down self-managed logger.")
-#         self.logger.shutdown()
-#
-#     def _shutdown_owned_resources(self) -> None:
-#         if self._owns_fs and isinstance(self.fs, fsspec.AbstractFileSystem):
-#             self.logger.debug(f"{self.__class__.__name__} is shutting down self-managed fsspec client synchronously.")
-#             del self.fs
-#         else:
-#             self.logger.debug(
-#                 f"{self.__class__.__name__} is skipping fsspec client shutdown (not owned or not an fsspec client).")
-#         self._shutdown_logger()
-#
-#     async def _ashutdown_owned_resources(self) -> None:
-#         """Internal method to shut down all owned resources ASYNCHRONOUSLY."""
-#
-#         if self._owns_fs and isinstance(self.fs, fsspec.AbstractFileSystem):
-#             self.logger.debug(f"{self.__class__.__name__} is shutting down self-managed fsspec client asynchronously.")
-#             del self.fs
-#
-#         self._shutdown_logger()
-#
-#     # Methods for Cleanup ---
-#     def close(self) -> None:
-#         if self._is_closed: return
-#         self.logger.debug(f"Closing component...{self.__class__.__name__}")
-#         try:
-#             self._cleanup()
-#         except Exception as e:
-#             self.logger.error(f"Error during subclass {self.__class__.__name__} cleanup: {e}", exc_info=True)
-#             raise
-#         finally:
-#             self._is_closed = True
-#             self._shutdown_owned_resources()
-#             self.logger.debug(f"Component {self.__class__.__name__} closed successfully.")
-#
-#     async def aclose(self) -> None:
-#         if self._is_closed: return
-#         self.logger.debug(f"Asynchronously closing component...{self.__class__.__name__}")
-#         try:
-#             await self._acleanup()
-#         except Exception as e:
-#             self.logger.error(f"Error during async subclass cleanup: {e}", exc_info=True)
-#             raise
-#         finally:
-#             self._is_closed = True
-#             await self._ashutdown_owned_resources()
-#             self.logger.debug(f"Async Component {self.__class__.__name__} closed successfully.")
-#
-#     def __repr__(self) -> str:
-#         """Return a string representation of the ManagedResource."""
-#         # Dynamically get the name of the class or subclass
-#         class_name = self.__class__.__name__
-#
-#         # Determine the status of the logger and filesystem
-#         logger_status = "own" if self._owns_logger else "external"
-#         fs_status = "own" if self._owns_fs else "external"
-#         return (
-#             f"<{class_name} debug={self.debug}, "
-#             f"logger='{logger_status}', fs='{fs_status}'>"
-#         )
-#
-#     # --- Context Management and Destruction ---
-#     def __enter__(self) -> Self:
-#         return self
-#
-#     def __exit__(self, *args) -> None:
-#         self.close()
-#
-#     async def __aenter__(self) -> Self:
-#         return self
-#
-#     async def __aexit__(self, *args) -> None:
-#         await self.aclose()
-#
-#     def __del__(self) -> None:
-#         if not self._is_closed:
-#             self.logger.critical(f"CRITICAL: Component {self!r} was not closed properly.")