sibi-dst 0.3.59__tar.gz → 0.3.60__tar.gz

{sibi_dst-0.3.59 → sibi_dst-0.3.60}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.59
+Version: 0.3.60
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.59 → sibi_dst-0.3.60}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "sibi-dst"
-version = "0.3.59"
+version = "0.3.60"
 description = "Data Science Toolkit"
 authors = ["Luis Valverde <lvalverdeb@gmail.com>"]
 readme = "README.md"

{sibi_dst-0.3.59 → sibi_dst-0.3.60}/sibi_dst/utils/__init__.py

@@ -14,7 +14,7 @@ from .airflow_manager import AirflowDAGManager
 from .credentials import *
 from .update_planner import UpdatePlanner
 from .data_wrapper import DataWrapper
-from .storage_config import StorageConfig
+from .storage_config import StorageConfig, FsRegistry
 from .data_from_http_source import DataFromHttpSource
 from .webdav_client import WebDAVClient
 from .manifest_manager import MissingManifestManager
@@ -37,7 +37,9 @@ __all__ = [
     "ClickHouseWriter",
     "AirflowDAGManager",
     "StorageConfig",
+    "FsRegistry",
     "DataFromHttpSource",
     "WebDAVClient",
     "MissingManifestManager",
 ]
+

{sibi_dst-0.3.59 → sibi_dst-0.3.60}/sibi_dst/utils/storage_config.py

@@ -1,3 +1,7 @@
+from threading import RLock
+from typing import Dict, Callable, Any
+
+from sibi_dst.utils import Logger
 from .storage_manager import StorageManager
 from .credentials import ConfigManager
 
@@ -46,4 +50,58 @@ class StorageConfig:
             # defaulting to local filesystem
             self.filesystem_type = 'file'
             self.filesystem_options = {}
-        self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}
+        self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}
+
+class FsRegistry:
+    def __init__(self, debug: bool = False, logger: Logger = None):
+        self._storage_registry: Dict[str, Callable[[], Any]]={}
+        self._fs_instance_cache: Dict[str, object] = {}
+        self._lock = RLock()
+        self.debug = debug
+
+        if logger:
+            self.logger = logger
+        else:
+            self.logger = Logger.default_logger(logger_name="FsRegistry")
+            self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
+
+    def register(self, name:str, manager: Any):
+        """
+        Registers a filesystem manager instance with a name.
+        :param name: Name of the filesystem instance.
+        :param manager: Filesystem manager instance to register.
+        """
+        if not hasattr(manager, 'get_fs_instance'):
+            raise TypeError("Manager must have a 'get_fs_instance' method.")
+        self._storage_registry[name] = lambda: manager
+
+
+    def get_fs_instance(self, name: str='source') -> object:
+        """
+        Retrieve a filesystem instance from a registered storage manager.
+        Caches instances per name.
+        """
+        if name in self._fs_instance_cache:
+            return self._fs_instance_cache[name]
+
+        if name not in self._storage_registry:
+            raise ValueError(f"Storage '{name}' has not been registered.")
+
+        manager = self._storage_registry[name]()
+        fs = manager.get_fs_instance()
+        self._fs_instance_cache[name] = fs
+        return fs
+
+    def unregister_fs(self, name: str):
+        """
+        Unregister a storage and clear its cached fs instance.
+        """
+        self._storage_registry.pop(name, None)
+        self._fs_instance_cache.pop(name, None)
+
+
+    def clear_fs_cache(self):
+        """
+        Clear all cached fs instances.
+        """
+        self._fs_instance_cache.clear()