tracdap-runtime 0.9.0b1__tar.gz → 0.9.0b2__tar.gz

{tracdap_runtime-0.9.0b1/tracdap_runtime.egg-info → tracdap_runtime-0.9.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracdap-runtime
-Version: 0.9.0b1
+Version: 0.9.0b2
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -16,11 +16,11 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <3.14,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: protobuf==5.29.3
+Requires-Dist: protobuf==5.29.5
 Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: dulwich==0.22.7
-Requires-Dist: requests==2.32.3
+Requires-Dist: requests==2.32.4
 Requires-Dist: typing_extensions<4.13
 Requires-Dist: urllib3<2.4.0
 Provides-Extra: grpc

{tracdap_runtime-0.9.0b1 → tracdap_runtime-0.9.0b2}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tracdap-runtime
-version = 0.9.0b1
+version = 0.9.0b2
 description = Runtime package for building models on the TRAC Data & Analytics Platform
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -40,11 +40,11 @@ package_dir =
 	tracdap.rt = src/tracdap/rt
 python_requires = >= 3.9, < 3.14
 install_requires = 
-	protobuf == 5.29.3
+	protobuf == 5.29.5
 	pyarrow == 18.1.0
 	pyyaml == 6.0.2
 	dulwich == 0.22.7
-	requests == 2.32.3
+	requests == 2.32.4
 	typing_extensions < 4.13
 	urllib3 < 2.4.0
 

{tracdap_runtime-0.9.0b1 → tracdap_runtime-0.9.0b2}/src/tracdap/rt/_impl/core/config_parser.py

@@ -45,6 +45,26 @@ except ModuleNotFoundError:
 _T = tp.TypeVar('_T')
 
 
+class ConfigKeys:
+
+    STORAGE_DEFAULT_LOCATION = "storage.default.location"
+    STORAGE_DEFAULT_FORMAT = "storage.default.format"
+    STORAGE_DEFAULT_LAYOUT = "storage.default.layout"
+
+    RESULT_ENABLED = "result.enabled"
+    RESULT_LOGS_ENABLED = "result.logs.enabled"
+    RESULT_STORAGE_LOCATION = "result.storage.location"
+    RESULT_STORAGE_PATH = "result.storage.path"
+    RESULT_FORMAT = "result.format"
+
+
+
+class ConfigKDefaults:
+
+    STORAGE_DEFAULT_FORMAT = "CSV"
+    STORAGE_DEFAULT_LAYOUT = "OBJECT_ID_LAYOUT"
+
+
 class ConfigManager:
 
     @classmethod
@@ -479,10 +499,16 @@ class ConfigParser(tp.Generic[_T]):
         # Now go back over the members and look for any that weren't declared in __init__
         # Members with non-generic default values can still be read from the input stream
 
-        for member_name in obj.__dict__:
+        def member_names():
+            if hasattr(obj, "__slots__"):
+                return filter(lambda s: hasattr(obj, s), obj.__slots__)
+            else:
+                return dir(obj)
+
+        for member_name in member_names():
 
             member_location = self._child_location(location, member_name)
-            default_value = obj.__dict__[member_name]
+            default_value = getattr(obj, member_name)
 
             if member_name in init_signature.parameters or member_name.startswith("_"):
                 continue
@@ -510,7 +536,7 @@ class ConfigParser(tp.Generic[_T]):
 
         for raw_name in raw_dict.keys():
 
-            if raw_name not in obj.__dict__:
+            if not hasattr(obj, raw_name):
                 self._error(location, f"Unrecognised config parameter '{raw_name}'")
 
             if raw_name == "self" or raw_name.startswith("_"):

{tracdap_runtime-0.9.0b1 → tracdap_runtime-0.9.0b2}/src/tracdap/rt/_impl/core/data.py

@@ -40,6 +40,7 @@ import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
+import tracdap.rt._impl.core.config_parser as _cfg_p
 import tracdap.rt._impl.core.logging as _log
 import tracdap.rt._impl.core.util as _util
 
@@ -70,6 +71,8 @@ class DataSpec:
     schema_id: tp.Optional[_meta.TagHeader] = None
     context_key: tp.Optional[str] = None
 
+    metadata: tp.Optional[_api.RuntimeMetadata] = None
+
     @staticmethod
     def create_data_spec(
             data_item: str,
@@ -108,37 +111,44 @@ class DataSpec:
             schema_id: tp.Optional[_meta.TagHeader] = None,
             context_key: tp.Optional[str] = None):
 
-        return DataSpec(
-            self.object_type, self.schema_type, self.data_item,
-            self.definition, self.storage, self.schema,
-            primary_id, storage_id, schema_id, context_key)
+        return dc.replace(self,
+            primary_id=primary_id,
+            storage_id=storage_id,
+            schema_id=schema_id,
+            context_key=context_key)
+            
+    def with_metadata(self, metadata: tp.Optional[_api.RuntimeMetadata]):
+        return dc.replace(self, metadata=metadata)
 
     def is_empty(self):
         return self.data_item is None or len(self.data_item) == 0
 
 
-class StorageLayout:
+class StorageLayout(metaclass=abc.ABCMeta):
 
-    __LAYOUTS: "tp.Dict[_meta.StorageLayout, StorageLayout]" = dict()
+    __LAYOUTS: "tp.Dict[str, StorageLayout]" = dict()
 
     @classmethod
-    def select(cls, layout_key: _meta.StorageLayout) -> "StorageLayout":
+    def select(cls, layout_key: tp.Union[str, _meta.StorageLayout]) -> "StorageLayout":
 
         # Legacy compatibility - layout key not set in storage definition
-        if not layout_key or layout_key.value == 0:
-            layout_key = _meta.StorageLayout.OBJECT_ID_LAYOUT
+        if not layout_key or layout_key == "":
+            layout_key = _meta.StorageLayout.OBJECT_ID_LAYOUT.name
+
+        if isinstance(layout_key, _meta.StorageLayout):
+            layout_key = layout_key.name
 
         layout = cls.__LAYOUTS.get(layout_key)
 
         if layout is not None:
             return layout
 
-        if layout_key == _meta.StorageLayout.OBJECT_ID_LAYOUT:
+        if layout_key == _meta.StorageLayout.OBJECT_ID_LAYOUT.name:
             layout = ObjectIdLayout()
-        elif layout_key == _meta.StorageLayout.DEVELOPER_LAYOUT:
+        elif layout_key == _meta.StorageLayout.DEVELOPER_LAYOUT.name:
             layout = DevelopmentLayout()
         else:
-            raise _ex.ETracInternal(f"Unknown storage layout [{layout_key.name}]")
+            raise _ex.ETracInternal(f"Unknown storage layout [{layout_key}]")
 
         cls.__LAYOUTS[layout_key] = layout
 
@@ -152,7 +162,7 @@ class StorageLayout:
     def new_data_spec(
             self, data_id: _meta.TagHeader, storage_id: _meta.TagHeader,
             context_key: tp.Optional[str], trac_schema: _meta.SchemaDefinition,
-            storage_config: _cfg.StorageConfig) -> DataSpec:
+            sys_config: _cfg.RuntimeConfig) -> DataSpec:
         pass
 
     @abc.abstractmethod
@@ -166,7 +176,7 @@ class StorageLayout:
     def new_file_spec(
             self, file_id: _meta.TagHeader, storage_id: _meta.TagHeader,
             context_key: str, file_type: _meta.FileType,
-            storage_config: _cfg.StorageConfig) -> DataSpec:
+            sys_config: _cfg.RuntimeConfig) -> DataSpec:
         pass
 
     @abc.abstractmethod
@@ -177,7 +187,7 @@ class StorageLayout:
         pass
 
 
-class BaseLayout(StorageLayout):
+class BaseLayout(StorageLayout, metaclass=abc.ABCMeta):
 
     __DATA_ITEM_TEMPLATE = "data/{}/{}/{}/snap-{:d}/delta-{:d}"
     __FILE_ITEM_TEMPLATE = "file/{}/version-{}"
@@ -198,7 +208,7 @@ class BaseLayout(StorageLayout):
     def new_data_spec(
             self, data_id: _meta.TagHeader, storage_id: _meta.TagHeader,
             context_key: tp.Optional[str], trac_schema: _meta.SchemaDefinition,
-            storage_config: _cfg.StorageConfig) -> DataSpec:
+            sys_config: _cfg.RuntimeConfig) -> DataSpec:
 
         part_key = _meta.PartKey("part-root", _meta.PartType.PART_ROOT)
         snap_index = 0
@@ -215,8 +225,11 @@ class BaseLayout(StorageLayout):
         data_def = self._add_new_snap(new_data_def, data_item, part_key, snap_index)
 
         # Take default location from the storage config
-        storage_key = storage_config.defaultBucket
-        storage_format = "JSON" if trac_schema.schemaType == _meta.SchemaType.STRUCT else storage_config.defaultFormat
+        storage_key = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+        if trac_schema.schemaType == _meta.SchemaType.STRUCT_SCHEMA:
+            storage_format = "text/json"
+        else:
+            storage_format = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_FORMAT, "text/csv")
         storage_path = self._data_storage_path(data_id, context_key, trac_schema, part_key, snap_index, 0, storage_format, prior_copy=None)
 
         storage_copy = _meta.StorageCopy(
@@ -276,7 +289,7 @@ class BaseLayout(StorageLayout):
     def new_file_spec(
             self, file_id: _meta.TagHeader, storage_id: _meta.TagHeader,
             context_key: str, file_type: _meta.FileType,
-            storage_config: _cfg.StorageConfig) -> DataSpec:
+            sys_config: _cfg.RuntimeConfig) -> DataSpec:
 
         data_item = self.__FILE_ITEM_TEMPLATE.format(file_id.objectId, file_id.objectVersion)
 
@@ -288,7 +301,7 @@ class BaseLayout(StorageLayout):
             storageId=_util.selector_for_latest(storage_id),
             size=0)
 
-        storage_key = storage_config.defaultBucket
+        storage_key = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
         storage_format = file_def.mimeType
         storage_path = self._file_storage_path(file_id, file_def, prior_copy=None)
 
@@ -429,11 +442,17 @@ class ObjectIdLayout(BaseLayout):
         schema_type = trac_schema.schemaType.name.lower()
         version_suffix = self.__random.randint(0, 1 << 24)
 
-        return self.__DATA_STORAGE_TEMPLATE.format(
+        base_path = self.__DATA_STORAGE_TEMPLATE.format(
             schema_type, data_id.objectId,
             part_key.opaqueKey, snap_index, delta_index,
             version_suffix)
 
+        # STRUCT stored as a single file, not directory layout
+        if trac_schema.schemaType == _meta.SchemaType.STRUCT_SCHEMA:
+            return base_path + ".json"
+        else:
+            return base_path
+
     def _file_storage_path(self, file_id, file_def, prior_copy):
 
         version_suffix = self.__random.randint(0, 1 << 24)
@@ -494,37 +513,44 @@ class DevelopmentLayout(BaseLayout):
 def build_data_spec(
         data_id: _meta.TagHeader, storage_id: _meta.TagHeader,
         context_key: tp.Optional[str], trac_schema: _meta.SchemaDefinition,
-        storage_config: _cfg.StorageConfig,
-        prior_spec: tp.Optional[DataSpec] = None) \
+        sys_config: _cfg.RuntimeConfig,
+        prior_spec: tp.Optional[DataSpec] = None,
+        metadata: tp.Optional[_api.RuntimeMetadata] = None) \
         -> DataSpec:
 
     if prior_spec is None:
-        layout_key = storage_config.defaultLayout
+        layout_key = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LAYOUT, _cfg_p.ConfigKDefaults.STORAGE_DEFAULT_LAYOUT)
         layout = StorageLayout.select(layout_key)
-        return layout.new_data_spec(data_id, storage_id, context_key, trac_schema, storage_config)
-
+        spec = layout.new_data_spec(data_id, storage_id, context_key, trac_schema, sys_config)
     else:
         layout_key = prior_spec.storage.layout
         layout = StorageLayout.select(layout_key)
-        return layout.new_data_version(data_id, storage_id, context_key, trac_schema, prior_spec)
+        spec = layout.new_data_version(data_id, storage_id, context_key, trac_schema, prior_spec)
+
+    # Attach metadata if it is available
+    return spec.with_metadata(metadata) if metadata is not None else spec
 
 
 def build_file_spec(
         file_id: _meta.TagHeader, storage_id: _meta.TagHeader,
         context_key: tp.Optional[str],  file_type: _meta.FileType,
-        storage_config: _cfg.StorageConfig,
-        prior_spec: tp.Optional[DataSpec] = None) \
+        sys_config: _cfg.RuntimeConfig,
+        prior_spec: tp.Optional[DataSpec] = None,
+        metadata: tp.Optional[_api.RuntimeMetadata] = None) \
         -> DataSpec:
 
     if prior_spec is None:
-        layout_key = storage_config.defaultLayout
+        layout_key = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LAYOUT, _cfg_p.ConfigKDefaults.STORAGE_DEFAULT_LAYOUT)
         layout = StorageLayout.select(layout_key)
-        return layout.new_file_spec(file_id, storage_id, context_key, file_type, storage_config)
+        spec = layout.new_file_spec(file_id, storage_id, context_key, file_type, sys_config)
 
     else:
         layout_key = prior_spec.storage.layout
         layout = StorageLayout.select(layout_key)
-        return layout.new_file_version(file_id, storage_id, context_key, file_type, prior_spec)
+        spec = layout.new_file_version(file_id, storage_id, context_key, file_type, prior_spec)
+
+    # Attach metadata if it is available
+    return spec.with_metadata(metadata) if metadata is not None else spec
 
 
 @dc.dataclass(frozen=True)
@@ -544,6 +570,8 @@ class DataItem:
     schema: pa.Schema = None
     table: tp.Optional[pa.Table] = None
 
+    metadata: tp.Optional[_api.RuntimeMetadata] = None
+
     def is_empty(self) -> bool:
         return self.content is None
 
@@ -561,7 +589,7 @@ class DataItem:
     def for_table(table: pa.Table, schema: pa.Schema, trac_schema: _meta.SchemaDefinition) -> "DataItem":
 
         return DataItem(
-            _meta.ObjectType.DATA, _meta.SchemaType.TABLE,
+            _meta.ObjectType.DATA, _meta.SchemaType.TABLE_SCHEMA,
             content=table, content_type=pa.Table,
             trac_schema=trac_schema, native_schema=schema,
             table=table, schema=schema)
@@ -570,7 +598,7 @@ class DataItem:
     def for_struct(content: tp.Any):
 
         return DataItem(
-            _meta.ObjectType.DATA, _meta.SchemaType.STRUCT,
+            _meta.ObjectType.DATA, _meta.SchemaType.STRUCT_SCHEMA,
             content=content, content_type=type(content))
 
     @staticmethod
@@ -580,6 +608,9 @@ class DataItem:
             _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET,
             content=content, content_type=bytes)
 
+    def with_metadata(self, metadata: _api.RuntimeMetadata) -> "DataItem":
+        return dc.replace(self, metadata=metadata)
+
 
 @dc.dataclass(frozen=True)
 class DataView:
@@ -592,6 +623,8 @@ class DataView:
     parts: tp.Dict[DataPartKey, tp.List[DataItem]] = None
     file_item: tp.Optional[DataItem] = None
 
+    metadata: tp.Optional[_api.RuntimeMetadata] = None
+
     @staticmethod
     def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataView":
         if object_type == _meta.ObjectType.DATA:
@@ -618,15 +651,30 @@ class DataView:
 
     def with_trac_schema(self, trac_schema: _meta.SchemaDefinition):
         arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, self.parts)
+        return dc.replace(self, trac_schema=trac_schema, arrow_schema=arrow_schema)
 
     def with_part(self, part_key: DataPartKey, part: DataItem):
-        new_parts = copy.copy(self.parts)
+        new_parts = copy.copy(self.parts) if self.parts is not None else {}
         new_parts[part_key] = [part]
-        return DataView(self.object_type, self.trac_schema, self.arrow_schema, new_parts)
+        return dc.replace(self, parts=new_parts)
 
     def with_file_item(self, file_item: DataItem):
-        return DataView(self.object_type, file_item=file_item)
+        return dc.replace(self, file_item=file_item)
+
+    def with_metadata(self, metadata: _api.RuntimeMetadata) -> "DataView":
+        return dc.replace(self, metadata=metadata)
+
+    def get_metadata(self) -> tp.Optional[_api.RuntimeMetadata]:
+        if self.metadata:
+            return self.metadata
+        if self.object_type == _meta.ObjectType.FILE and self.file_item:
+            return self.file_item.metadata
+        if self.parts:
+            for items in self.parts.values():
+                for item in items:
+                    if item and item.metadata:
+                        return item.metadata
+        return None
 
     def is_empty(self) -> bool:
         if self.object_type == _meta.ObjectType.FILE:
@@ -687,15 +735,6 @@ class DataMapping:
         pa.date64(): _meta.BasicType.DATE
     }
 
-    # For now, categorical handling is disabled by default and enabled by this setting
-    # The default will change to "true" for the 0.9 release
-    CATEGORICAL_CONFIG_KEY = "trac.runtime.categorical"
-    __categorical_enabled = False
-
-    @classmethod
-    def enable_categorical(cls, enabled: bool):
-        cls.__categorical_enabled = enabled
-
     @classmethod
     def arrow_to_python_type(cls, arrow_type: pa.DataType) -> type:
 
@@ -785,7 +824,7 @@ class DataMapping:
 
         # Categorical data uses an unordered dictionary with int32 index, ordered encoding not (currently) supported
         # For legacy compatability, only use dictionary encoding if the categorical feature is enabled
-        if trac_field.categorical and cls.__categorical_enabled:
+        if trac_field.categorical:
             arrow_type = pa.dictionary(pa.int32(), arrow_type, False)
 
         nullable = not (trac_field.notNull or trac_field.businessKey)
@@ -926,7 +965,7 @@ T_INTERNAL_DATA = tp.TypeVar("T_INTERNAL_DATA")
 T_INTERNAL_SCHEMA = tp.TypeVar("T_INTERNAL_SCHEMA")
 
 
-class DataConverter(tp.Generic[T_DATA_API, T_INTERNAL_DATA, T_INTERNAL_SCHEMA]):
+class DataConverter(tp.Generic[T_DATA_API, T_INTERNAL_DATA, T_INTERNAL_SCHEMA], metaclass=abc.ABCMeta):
 
     # Available per-framework args, to enable framework-specific type-checking in public APIs
     # These should (for a purist point of view) be in the individual converter classes
@@ -1702,8 +1741,11 @@ class DataConformance:
 
                 # Special handling for chunked dictionaries
                 elif isinstance(vector, pa.ChunkedArray):
-                    chunks = [cls._coerce_dictionary(chunk, field) for chunk in vector.chunks]
-                    return pa.chunked_array(chunks)
+                    if any(vector.chunks):
+                        chunks = [cls._coerce_dictionary(chunk, field) for chunk in vector.chunks]
+                        return pa.chunked_array(chunks)
+                    else:
+                        return pa.array([], type=field.type, size=0)  # noqa
 
                 # Vector type not recognized, coercion is not possible
                 else:

{tracdap_runtime-0.9.0b1 → tracdap_runtime-0.9.0b2}/src/tracdap/rt/_impl/core/repos.py

@@ -17,6 +17,7 @@ import typing as _tp
 
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
+import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
 import tracdap.rt._impl.core.logging as _logging
 
@@ -34,26 +35,27 @@ class RepositoryManager:
         # Initialize all repos in the system config
         # Any errors for missing repo types (plugins) will be raised during startup
 
-        for repo_name, repo_config in sys_config.repositories.items():
+        for resource_key, resource in sys_config.resources.items():
+            if resource.resourceType == meta.ResourceType.MODEL_REPOSITORY:
 
-            try:
+                try:
 
-                # Add global properties related to the repo protocol
-                related_props = {
-                    k: v for (k, v) in sys_config.properties.items()
-                    if k.startswith(f"{repo_config.protocol}.")}
+                    # Add global properties related to the repo protocol
+                    related_props = {
+                        k: v for (k, v) in sys_config.properties.items()
+                        if k.startswith(f"{resource.protocol}.")}
 
-                repo_config.properties.update(related_props)
+                    resource.properties.update(related_props)
 
-                self._repos[repo_name] = plugins.PluginManager.load_plugin(IModelRepository, repo_config)
+                    self._repos[resource_key] = plugins.PluginManager.load_plugin(IModelRepository, resource)
 
-            except ex.EPluginNotAvailable as e:
+                except ex.EPluginNotAvailable as e:
 
-                msg = f"Model repository type [{repo_config.protocol}] is not recognised" \
-                      + " (this could indicate a missing model repository plugin)"
+                    msg = f"Model repository type [{resource.protocol}] is not recognised" \
+                          + " (this could indicate a missing model repository plugin)"
 
-                self._log.error(msg)
-                raise ex.EStartup(msg) from e
+                    self._log.error(msg)
+                    raise ex.EStartup(msg) from e
 
     def get_repository(self, repo_name: str) -> IModelRepository:
 

{tracdap_runtime-0.9.0b1 → tracdap_runtime-0.9.0b2}/src/tracdap/rt/_impl/core/storage.py

@@ -29,6 +29,7 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
+import tracdap.rt._impl.core.config_parser as _cfg_p
 import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.logging as _logging
 import tracdap.rt._impl.core.util as _util
@@ -83,20 +84,23 @@ class StorageManager:
         self.__external: tp.List[str] = list()
         self.__sys_config = sys_config
 
-        for storage_key, storage_config in sys_config.storage.buckets.items():
-            self.create_storage(storage_key, storage_config)
+        self.__default_location = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_LOCATION)
+        self.__default_format = _util.read_property(sys_config.properties, _cfg_p.ConfigKeys.STORAGE_DEFAULT_FORMAT, "CSV")
 
-        for storage_key, storage_config in sys_config.storage.external.items():
-            if storage_key in self.__file_storage or storage_key in self.__data_storage:
-                raise _ex.EConfig(f"Storage key [{storage_key}] is defined as both internal and external storage")
-            self.__external.append(storage_key)
-            self.create_storage(storage_key, storage_config)
+        for storage_key, storage_config in sys_config.resources.items():
+
+            if storage_config.resourceType == _meta.ResourceType.INTERNAL_STORAGE:
+                self.create_storage(storage_key, storage_config)
+
+            elif storage_config.resourceType == _meta.ResourceType.EXTERNAL_STORAGE:
+                self.__external.append(storage_key)
+                self.create_storage(storage_key, storage_config)
 
     def default_storage_key(self):
-        return self.__sys_config.storage.defaultBucket
+        return self.__default_location
 
     def default_storage_format(self):
-        return self.__sys_config.storage.defaultFormat
+        return self.__default_format
 
     def create_storage(self, storage_key: str, storage_config: _cfg.PluginConfig):
 
@@ -116,11 +120,12 @@ class StorageManager:
 
         provider = plugins.PluginManager.load_plugin(IStorageProvider, storage_config)
 
-        if provider.has_file_storage():
-            file_storage = provider.get_file_storage()
-        elif provider.has_arrow_native():
+        # Prefer arrow native FS if it is available
+        if provider.has_arrow_native():
             fs = provider.get_arrow_native()
             file_storage = CommonFileStorage(storage_key, storage_config, fs)
+        elif provider.has_file_storage():
+            file_storage = provider.get_file_storage()
         else:
             file_storage = None