tracdap-runtime 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/_impl/{repos.py → core/repos.py}

@@ -13,10 +13,12 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
+import typing as _tp
+
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
 import tracdap.rt.exceptions as ex
-import tracdap.rt._impl.util as util
+import tracdap.rt._impl.core.logging as _logging
 
 # Import repo interfaces
 from tracdap.rt.ext.repos import *
@@ -26,8 +28,8 @@ class RepositoryManager:
 
     def __init__(self, sys_config: cfg.RuntimeConfig):
 
-        self._log = util.logger_for_object(self)
-        self._repos: tp.Dict[str, IModelRepository] = dict()
+        self._log = _logging.logger_for_object(self)
+        self._repos: _tp.Dict[str, IModelRepository] = dict()
 
         # Initialize all repos in the system config
         # Any errors for missing repo types (plugins) will be raised during startup
@@ -36,6 +38,13 @@ class RepositoryManager:
 
             try:
 
+                # Add global properties related to the repo protocol
+                related_props = {
+                    k: v for (k, v) in sys_config.properties.items()
+                    if k.startswith(f"{repo_config.protocol}.")}
+
+                repo_config.properties.update(related_props)
+
                 self._repos[repo_name] = plugins.PluginManager.load_plugin(IModelRepository, repo_config)
 
             except ex.EPluginNotAvailable as e:

tracdap/rt/_impl/{schemas.py → core/schemas.py}

@@ -21,10 +21,10 @@ import pyarrow as pa
 
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.storage as _storage
-import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _log
+import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.shim as _shim
 
 
 class SchemaLoader:
@@ -169,4 +169,4 @@ class SchemaLoader:
         raise _ex.EDataConformance(err)
 
 
-SchemaLoader._log = _util.logger_for_class(SchemaLoader)
+SchemaLoader._log = _log.logger_for_class(SchemaLoader)

tracdap/rt/_impl/{shim.py → core/shim.py}

@@ -30,8 +30,9 @@ import importlib.machinery as _ilm
 import importlib.resources as _ilr
 
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.guard_rails as _guard
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.guard_rails as _guard
+import tracdap.rt._impl.core.logging as _log
+import tracdap.rt._impl.core.util as _util
 
 
 class _Shim:
@@ -138,7 +139,7 @@ class _NamespaceShimFinder(_ila.MetaPathFinder):
     def __init__(self, shim_map: tp.Dict[str, _Shim], active_shim: _ActiveShim):
         self.__shim_map = shim_map
         self.__active_shim = active_shim
-        self._log = _util.logger_for_class(ShimLoader)
+        self._log = _log.logger_for_class(ShimLoader)
 
     def find_spec(
             self, fullname: str,
@@ -561,5 +562,5 @@ class ShimLoader:
                 raise _ex.ERuntimeValidation(err)
 
 
-ShimLoader._log = _util.logger_for_class(ShimLoader)
+ShimLoader._log = _log.logger_for_class(ShimLoader)
 ShimLoader._init()  # noqa

tracdap/rt/_impl/{storage.py → core/storage.py}

@@ -29,9 +29,10 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.util as _util
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.core.validation as _val
 
 # Import storage interfaces (using the internal version, it has extra bits that are not public)
 from tracdap.rt._impl.ext.storage import *
@@ -76,11 +77,11 @@ class StorageManager:
 
     def __init__(self, sys_config: _cfg.RuntimeConfig):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
-        self.__settings = sys_config.storage
+        self.__sys_config = sys_config
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
@@ -92,13 +93,20 @@ class StorageManager:
             self.create_storage(storage_key, storage_config)
 
     def default_storage_key(self):
-        return self.__settings.defaultBucket
+        return self.__sys_config.storage.defaultBucket
 
     def default_storage_format(self):
-        return self.__settings.defaultFormat
+        return self.__sys_config.storage.defaultFormat
 
     def create_storage(self, storage_key: str, storage_config: _cfg.PluginConfig):
 
+        # Add global properties related to the storage protocol
+        related_props = {
+            k: v for (k, v) in self.__sys_config.properties.items()
+            if k.startswith(f"{storage_config.protocol}.")}
+
+        storage_config.properties.update(related_props)
+
         if plugins.PluginManager.is_plugin_available(IStorageProvider, storage_config.protocol):
             self._create_storage_from_provider(storage_key, storage_config)
         else:
@@ -224,7 +232,7 @@ class CommonFileStorage(IFileStorage):
 
     def __init__(self, storage_key: str, storage_config: _cfg.PluginConfig, fs: pa_fs.SubTreeFileSystem):
 
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._key = storage_key
         self._config = storage_config
         self._fs = fs
@@ -495,7 +503,10 @@ class CommonFileStorage(IFileStorage):
 
         # For successful write streams, log the total size written
         if is_write and not error:
-            file_size = _util.format_file_size(stream.tell())
+            if not stream.closed:
+                file_size = _util.format_file_size(stream.tell())
+            else:
+                file_size = self._fs.get_file_info(storage_path).size
             self._log.info(f"File size [{self._key}]: {file_size} [{storage_path}]")
 
         # Close the stream - this may take time for write streams that are not flushed
@@ -731,7 +742,7 @@ class CommonDataStorage(IDataStorage):
             self, config: _cfg.PluginConfig, file_storage: IFileStorage,
             pushdown_pandas: bool = False, pushdown_spark: bool = False):
 
-        self.__log = _util.logger_for_object(self)
+        self.__log = _logging.logger_for_object(self)
 
         self.__config = config
         self.__file_storage = file_storage