sibi-dst 0.3.59__py3-none-any.whl → 0.3.60__py3-none-any.whl

sibi_dst/utils/__init__.py

@@ -14,7 +14,7 @@ from .airflow_manager import AirflowDAGManager
 from .credentials import *
 from .update_planner import UpdatePlanner
 from .data_wrapper import DataWrapper
-from .storage_config import StorageConfig
+from .storage_config import StorageConfig, FsRegistry
 from .data_from_http_source import DataFromHttpSource
 from .webdav_client import WebDAVClient
 from .manifest_manager import MissingManifestManager
@@ -37,7 +37,9 @@ __all__ = [
     "ClickHouseWriter",
     "AirflowDAGManager",
     "StorageConfig",
+    "FsRegistry",
     "DataFromHttpSource",
     "WebDAVClient",
     "MissingManifestManager",
 ]
+

sibi_dst/utils/storage_config.py

@@ -1,3 +1,7 @@
+from threading import RLock
+from typing import Dict, Callable, Any
+
+from sibi_dst.utils import Logger
 from .storage_manager import StorageManager
 from .credentials import ConfigManager
 
@@ -46,4 +50,58 @@ class StorageConfig:
             # defaulting to local filesystem
             self.filesystem_type = 'file'
             self.filesystem_options = {}
-        self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}
+        self.filesystem_options = {k: v for k, v in self.filesystem_options.items() if v}
+
+class FsRegistry:
+    def __init__(self, debug: bool = False, logger: Logger = None):
+        self._storage_registry: Dict[str, Callable[[], Any]]={}
+        self._fs_instance_cache: Dict[str, object] = {}
+        self._lock = RLock()
+        self.debug = debug
+
+        if logger:
+            self.logger = logger
+        else:
+            self.logger = Logger.default_logger(logger_name="FsRegistry")
+            self.logger.set_level(Logger.DEBUG if self.debug else Logger.INFO)
+
+    def register(self, name:str, manager: Any):
+        """
+        Registers a filesystem manager instance with a name.
+        :param name: Name of the filesystem instance.
+        :param manager: Filesystem manager instance to register.
+        """
+        if not hasattr(manager, 'get_fs_instance'):
+            raise TypeError("Manager must have a 'get_fs_instance' method.")
+        self._storage_registry[name] = lambda: manager
+
+
+    def get_fs_instance(self, name: str='source') -> object:
+        """
+        Retrieve a filesystem instance from a registered storage manager.
+        Caches instances per name.
+        """
+        if name in self._fs_instance_cache:
+            return self._fs_instance_cache[name]
+
+        if name not in self._storage_registry:
+            raise ValueError(f"Storage '{name}' has not been registered.")
+
+        manager = self._storage_registry[name]()
+        fs = manager.get_fs_instance()
+        self._fs_instance_cache[name] = fs
+        return fs
+
+    def unregister_fs(self, name: str):
+        """
+        Unregister a storage and clear its cached fs instance.
+        """
+        self._storage_registry.pop(name, None)
+        self._fs_instance_cache.pop(name, None)
+
+
+    def clear_fs_cache(self):
+        """
+        Clear all cached fs instances.
+        """
+        self._fs_instance_cache.clear()

{sibi_dst-0.3.59.dist-info → sibi_dst-0.3.60.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sibi-dst
-Version: 0.3.59
+Version: 0.3.60
 Summary: Data Science Toolkit
 Author: Luis Valverde
 Author-email: lvalverdeb@gmail.com

{sibi_dst-0.3.59.dist-info → sibi_dst-0.3.60.dist-info}/RECORD

@@ -38,7 +38,7 @@ sibi_dst/osmnx_helper/basemaps/router_plotter.py,sha256=UAiijn-J-jjX4YnL0_P9SFqT
 sibi_dst/osmnx_helper/utils.py,sha256=BzuY8CtYnBAAO8UAr_M7EOk6CP1zcifNLs8pkdFZEFg,20577
 sibi_dst/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_dst/tests/test_data_wrapper_class.py,sha256=6uFmZR2DxnxQz49L5jT2ehlKvlLnpUHMLFB_PqqUq7k,3336
-sibi_dst/utils/__init__.py,sha256=I-LeSCEnnOKunblmCrv_KTBzbsr7Rx1MEWM9TypDqWM,1203
+sibi_dst/utils/__init__.py,sha256=w0_q4rl3yD7x1Q5yWxH-GN_3Ju1XlebIzm3nJdrUeGE,1234
 sibi_dst/utils/airflow_manager.py,sha256=-d44EKUZNYJyp4wuNwRvilRQktunArPOB5fZuWdQv10,7526
 sibi_dst/utils/clickhouse_writer.py,sha256=iAUe4_Kn2WR1xZjpLW2FOWCWfOTw6fCGMTUcWxIQJ60,9877
 sibi_dst/utils/credentials.py,sha256=cHJPPsmVyijqbUQIq7WWPe-lIallA-mI5RAy3YUuRME,1724
@@ -53,7 +53,7 @@ sibi_dst/utils/log_utils.py,sha256=77xACRagKU83H9vn7aVeBzkQjxWlbe4dg4KuxPRCgvw,4
 sibi_dst/utils/manifest_manager.py,sha256=abm97TuWgJqNViPXMbpl5W7ttrg1BeiJkf2SMGc4hd8,5512
 sibi_dst/utils/parquet_saver.py,sha256=O62xwPfphOpKgEiHqnts20CPSU96pxs49Cg7PVetLK0,8193
 sibi_dst/utils/phone_formatter.py,sha256=tsVTDamuthFYgy4-5UwmQkPQ-FGTGH7MjZyH8utAkIY,4945
-sibi_dst/utils/storage_config.py,sha256=Cg8EOGLZ_5v9sunaQHZLYHdp5FDkgPrCVVNHF-ys5sQ,2181
+sibi_dst/utils/storage_config.py,sha256=TE15H-7d0mqwYPSUgrdidK9U7N7p87Z8JfUQH4-jdPs,4123
 sibi_dst/utils/storage_manager.py,sha256=btecX7ggNb7rfu5EK9Xuu2q_FZA7r_rB_tfhQ8V96qc,6567
 sibi_dst/utils/update_planner.py,sha256=dJXLC-KdbWrCs-MFe7Xa8F-ZhlNJq8P1szjLAzMJZk0,9684
 sibi_dst/utils/webdav_client.py,sha256=pYF1UsGOuxYeGLq7aBfwZFvkvD4meOcbbaiZ4d6GW9I,7107
@@ -77,6 +77,6 @@ sibi_dst/v2/df_helper/core/_params_config.py,sha256=DYx2drDz3uF-lSPzizPkchhy-kxR
 sibi_dst/v2/df_helper/core/_query_config.py,sha256=Y8LVSyaKuVkrPluRDkQoOwuXHQxner1pFWG3HPfnDHM,441
 sibi_dst/v2/utils/__init__.py,sha256=6H4cvhqTiFufnFPETBF0f8beVVMpfJfvUs6Ne0TQZNY,58
 sibi_dst/v2/utils/log_utils.py,sha256=rfk5VsLAt-FKpv6aPTC1FToIPiyrnHAFFBAkHme24po,4123
-sibi_dst-0.3.59.dist-info/METADATA,sha256=zYb_0a1ImTPUxB4raHM8cvip0_oZSot52N8okr3r_ZY,4292
-sibi_dst-0.3.59.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-sibi_dst-0.3.59.dist-info/RECORD,,
+sibi_dst-0.3.60.dist-info/METADATA,sha256=WuBvzuHMuZBUpHF2-dAO65MI1e5EhuZ1-Hvil3oQY6o,4292
+sibi_dst-0.3.60.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+sibi_dst-0.3.60.dist-info/RECORD,,