tracdap-runtime 0.6.4__py3-none-any.whl → 0.6.5__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.pyi

@@ -1,12 +1,22 @@
 from tracdap.rt._impl.grpc.tracdap.metadata import type_pb2 as _type_pb2
 from tracdap.rt._impl.grpc.tracdap.metadata import data_pb2 as _data_pb2
 from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import message as _message
 from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
 
 DESCRIPTOR: _descriptor.FileDescriptor
 
+class ModelType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    STANDARD_MODEL: _ClassVar[ModelType]
+    DATA_IMPORT_MODEL: _ClassVar[ModelType]
+    DATA_EXPORT_MODEL: _ClassVar[ModelType]
+STANDARD_MODEL: ModelType
+DATA_IMPORT_MODEL: ModelType
+DATA_EXPORT_MODEL: ModelType
+
 class ModelParameter(_message.Message):
     __slots__ = ("paramType", "label", "defaultValue", "paramProps")
     class ParamPropsEntry(_message.Message):
@@ -69,7 +79,7 @@ class ModelOutputSchema(_message.Message):
     def __init__(self, schema: _Optional[_Union[_data_pb2.SchemaDefinition, _Mapping]] = ..., label: _Optional[str] = ..., optional: bool = ..., dynamic: bool = ..., outputProps: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
 
 class ModelDefinition(_message.Message):
-    __slots__ = ("language", "repository", "packageGroup", "package", "version", "entryPoint", "path", "parameters", "inputs", "outputs", "staticAttributes")
+    __slots__ = ("language", "repository", "packageGroup", "package", "version", "entryPoint", "path", "parameters", "inputs", "outputs", "staticAttributes", "modelType")
     class ParametersEntry(_message.Message):
         __slots__ = ("key", "value")
         KEY_FIELD_NUMBER: _ClassVar[int]
@@ -109,6 +119,7 @@ class ModelDefinition(_message.Message):
     INPUTS_FIELD_NUMBER: _ClassVar[int]
     OUTPUTS_FIELD_NUMBER: _ClassVar[int]
     STATICATTRIBUTES_FIELD_NUMBER: _ClassVar[int]
+    MODELTYPE_FIELD_NUMBER: _ClassVar[int]
     language: str
     repository: str
     packageGroup: str
@@ -120,4 +131,5 @@ class ModelDefinition(_message.Message):
     inputs: _containers.MessageMap[str, ModelInputSchema]
     outputs: _containers.MessageMap[str, ModelOutputSchema]
     staticAttributes: _containers.MessageMap[str, _type_pb2.Value]
-    def __init__(self, language: _Optional[str] = ..., repository: _Optional[str] = ..., packageGroup: _Optional[str] = ..., package: _Optional[str] = ..., version: _Optional[str] = ..., entryPoint: _Optional[str] = ..., path: _Optional[str] = ..., parameters: _Optional[_Mapping[str, ModelParameter]] = ..., inputs: _Optional[_Mapping[str, ModelInputSchema]] = ..., outputs: _Optional[_Mapping[str, ModelOutputSchema]] = ..., staticAttributes: _Optional[_Mapping[str, _type_pb2.Value]] = ...) -> None: ...
+    modelType: ModelType
+    def __init__(self, language: _Optional[str] = ..., repository: _Optional[str] = ..., packageGroup: _Optional[str] = ..., package: _Optional[str] = ..., version: _Optional[str] = ..., entryPoint: _Optional[str] = ..., path: _Optional[str] = ..., parameters: _Optional[_Mapping[str, ModelParameter]] = ..., inputs: _Optional[_Mapping[str, ModelInputSchema]] = ..., outputs: _Optional[_Mapping[str, ModelOutputSchema]] = ..., staticAttributes: _Optional[_Mapping[str, _type_pb2.Value]] = ..., modelType: _Optional[_Union[ModelType, str]] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto
+# Protobuf Python Version: 4.25.3
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto*m\n\x0cResourceType\x12\x19\n\x15RESOURCE_TYPE_NOT_SET\x10\x00\x12\x14\n\x10MODEL_REPOSITORY\x10\x01\x12\x14\n\x10INTERNAL_STORAGE\x10\x02\"\x04\x08\x03\x10\x03*\x10\x45XTERNAL_STORAGEB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.metadata.resource_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_RESOURCETYPE']._serialized_start=184
+  _globals['_RESOURCETYPE']._serialized_end=293
+# @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.pyi

@@ -0,0 +1,16 @@
+from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as _object_id_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as _object_pb2
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from typing import ClassVar as _ClassVar
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class ResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    RESOURCE_TYPE_NOT_SET: _ClassVar[ResourceType]
+    MODEL_REPOSITORY: _ClassVar[ResourceType]
+    INTERNAL_STORAGE: _ClassVar[ResourceType]
+RESOURCE_TYPE_NOT_SET: ResourceType
+MODEL_REPOSITORY: ResourceType
+INTERNAL_STORAGE: ResourceType

tracdap/rt/_impl/models.py

@@ -19,6 +19,7 @@ import pathlib
 import copy
 
 import tracdap.rt.api as _api
+import tracdap.rt.api.experimental as _eapi
 import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
@@ -208,6 +209,13 @@ class ModelLoader:
             model_def.inputs = inputs
             model_def.outputs = outputs
 
+            if isinstance(model, _eapi.TracDataImport):
+                model_def.modelType = _meta.ModelType.DATA_IMPORT_MODEL
+            elif isinstance(model, _eapi.TracDataExport):
+                model_def.modelType = _meta.ModelType.DATA_EXPORT_MODEL
+            else:
+                model_def.modelType = _meta.ModelType.STANDARD_MODEL
+
             _val.quick_validate_model_def(model_def)
 
             for attr_name, attr_value in attributes.items():

tracdap/rt/_impl/static_api.py

@@ -17,6 +17,7 @@ import types as _ts
 
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
+import tracdap.rt._impl.data as _data
 import tracdap.rt._impl.schemas as _schemas
 import tracdap.rt._impl.type_system as _type_system
 import tracdap.rt._impl.validation as _val
@@ -152,6 +153,21 @@ class StaticApiImpl(_StaticApiHook):
 
         return _schemas.SchemaLoader.load_schema(package, schema_file)
 
+    def infer_schema(self, dataset: _tp.Any) -> _meta.SchemaDefinition:
+
+        if _data.pandas and isinstance(dataset, _data.pandas.DataFrame):
+            arrow_schema = _data.DataMapping.pandas_to_arrow_schema(dataset)
+
+        elif _data.polars and isinstance(dataset, _data.polars.DataFrame):
+            arrow_schema = _data.DataMapping.polars_to_arrow_schema(dataset)
+
+        else:
+            dataset_type = f"{type(dataset).__module__}.{type(dataset).__name__}"
+            message = f"Schema inference is not available for dataset type [{dataset_type}]"
+            raise _ex.ERuntimeValidation(message)
+
+        return _data.DataMapping.arrow_to_trac_schema(arrow_schema)
+
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
             label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,

tracdap/rt/_impl/storage.py

@@ -30,6 +30,7 @@ import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt._impl.data as _data
 import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.validation as _val
 
 # Import storage interfaces
 from tracdap.rt.ext.storage import *
@@ -41,7 +42,11 @@ class FormatManager:
     def get_data_format(cls, format_code: str, format_options: tp.Dict[str, tp.Any]) -> IDataFormat:
 
         try:
-            config = _cfg.PluginConfig(format_code, format_options)
+
+            config = _cfg.PluginConfig(
+                protocol=format_code,
+                properties=format_options)
+
             return plugins.PluginManager.load_plugin(IDataFormat, config)
 
         except _ex.EPluginNotAvailable as e:
@@ -73,11 +78,18 @@ class StorageManager:
         self.__log = _util.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
+        self.__external: tp.List[str] = list()
         self.__settings = sys_config.storage
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
 
+        for storage_key, storage_config in sys_config.storage.external.items():
+            if storage_key in self.__file_storage or storage_key in self.__data_storage:
+                raise _ex.EConfig(f"Storage key [{storage_key}] is defined as both internal and external storage")
+            self.__external.append(storage_key)
+            self.create_storage(storage_key, storage_config)
+
     def default_storage_key(self):
         return self.__settings.defaultBucket
 
@@ -147,26 +159,32 @@ class StorageManager:
         self.__file_storage[storage_key] = file_storage
         self.__data_storage[storage_key] = data_storage
 
-    def has_file_storage(self, storage_key: str) -> bool:
+    def has_file_storage(self, storage_key: str, external: bool = False) -> bool:
+
+        if external ^ (storage_key in self.__external):
+            return False
 
         return storage_key in self.__file_storage
 
-    def get_file_storage(self, storage_key: str) -> IFileStorage:
+    def get_file_storage(self, storage_key: str, external: bool = False) -> IFileStorage:
 
-        if not self.has_file_storage(storage_key):
+        if not self.has_file_storage(storage_key, external):
             err = f"File storage is not configured for storage key [{storage_key}]"
             self.__log.error(err)
             raise _ex.EStorageConfig(err)
 
         return self.__file_storage[storage_key]
 
-    def has_data_storage(self, storage_key: str) -> bool:
+    def has_data_storage(self, storage_key: str, external: bool = False) -> bool:
+
+        if external ^ (storage_key in self.__external):
+            return False
 
         return storage_key in self.__data_storage
 
-    def get_data_storage(self, storage_key: str) -> IDataStorage:
+    def get_data_storage(self, storage_key: str, external: bool = False) -> IDataStorage:
 
-        if not self.has_data_storage(storage_key):
+        if not self.has_data_storage(storage_key, external):
             err = f"Data storage is not configured for storage key [{storage_key}]"
             self.__log.error(err)
             raise _ex.EStorageConfig(err)
@@ -587,29 +605,27 @@ class CommonFileStorage(IFileStorage):
 
         try:
 
-            if storage_path is None or len(storage_path.strip()) == 0:
+            if _val.StorageValidator.storage_path_is_empty(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_NULL_OR_BLANK, operation_name, storage_path)
 
-            if self._ILLEGAL_PATH_CHARS.match(storage_path):
+            if _val.StorageValidator.storage_path_invalid(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_INVALID, operation_name, storage_path)
-    
-            relative_path = pathlib.Path(storage_path)
-    
-            if relative_path.is_absolute():
+
+            if _val.StorageValidator.storage_path_not_relative(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_NOT_RELATIVE, operation_name, storage_path)
 
+            if _val.StorageValidator.storage_path_outside_root(storage_path):
+                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_OUTSIDE_ROOT, operation_name, storage_path)
+
+            if not allow_root_dir and _val.StorageValidator.storage_path_is_root(storage_path):
+                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_IS_ROOT, operation_name, storage_path)
+
             root_path = pathlib.Path("C:\\root") if _util.is_windows() else pathlib.Path("/root")
+            relative_path = pathlib.Path(storage_path)
             absolute_path = root_path.joinpath(relative_path).resolve(False)
 
             if absolute_path == root_path:
-                if not allow_root_dir:
-                    raise self._explicit_error(self.ExplicitError.STORAGE_PATH_IS_ROOT, operation_name, storage_path)
-                else:
-                    return ""
-
-            # is_relative_to only supported in Python 3.9+, we need to support 3.7
-            if root_path not in absolute_path.parents:
-                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_OUTSIDE_ROOT, operation_name, storage_path)
+                return ""
             else:
                 return absolute_path.relative_to(root_path).as_posix()
 
@@ -639,10 +655,6 @@ class CommonFileStorage(IFileStorage):
 
         return err
 
-    _ILLEGAL_PATH_CHARS_WINDOWS = re.compile(r".*[\x00<>:\"\'|?*].*")
-    _ILLEGAL_PATH_CHARS_POSIX = re.compile(r".*[\x00<>:\"\'|?*\\].*")
-    _ILLEGAL_PATH_CHARS = _ILLEGAL_PATH_CHARS_WINDOWS if _util.is_windows() else _ILLEGAL_PATH_CHARS_POSIX
-
     class ExplicitError(enum.Enum):
     
         # Validation failures

tracdap/rt/_impl/validation.py

@@ -15,7 +15,9 @@
 import inspect
 import logging
 import re
+import types
 import typing as tp
+import pathlib
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -25,6 +27,11 @@ import tracdap.rt._impl.util as util
 from tracdap.rt.api.hook import _Named  # noqa
 
 
+def require_package(module_name: str, module_obj: types.ModuleType):
+    if module_obj is None:
+        raise ex.ERuntimeValidation(f"Optional package [{module_name}] is not installed")
+
+
 def validate_signature(method: tp.Callable, *args, **kwargs):
     _TypeValidator.validate_signature(method, *args, **kwargs)
 
@@ -62,15 +69,19 @@ class _TypeValidator:
             signature = inspect.signature(method)
             cls.__method_cache[method.__name__] = signature
 
+        hints = tp.get_type_hints(method)
+
         positional_index = 0
 
         for param_name, param in signature.parameters.items():
 
+            param_type = hints.get(param_name)
+
             values = cls._select_arg(method.__name__, param, positional_index, *args, **kwargs)
             positional_index += len(values)
 
             for value in values:
-                cls._validate_arg(method.__name__, param, value)
+                cls._validate_arg(method.__name__, param_name, param_type, value)
 
     @classmethod
     def validate_return_type(cls, method: tp.Callable, value: tp.Any):
@@ -146,14 +157,18 @@ class _TypeValidator:
         raise ex.EUnexpected("Invalid method signature in runtime API (this is a bug)")
 
     @classmethod
-    def _validate_arg(cls, method_name: str, parameter: inspect.Parameter, value: tp.Any):
+    def _validate_arg(cls, method_name: str, param_name: str, param_type: tp.Type, value: tp.Any):
 
-        if not cls._validate_type(parameter.annotation, value):
+        if not cls._validate_type(param_type, value):
 
-            expected_type = cls._type_name(parameter.annotation)
+            expected_type = cls._type_name(param_type)
             actual_type = cls._type_name(type(value)) if value is not None else str(None)
 
-            err = f"Invalid API call [{method_name}()]: Wrong type for [{parameter.name}]" \
+            if expected_type == actual_type:
+                expected_type = cls._type_name(param_type, qualified=True)
+                actual_type = cls._type_name(type(value), qualified=True)
+
+            err = f"Invalid API call [{method_name}()]: Wrong type for [{param_name}]" \
                   + f" (expected [{expected_type}], got [{actual_type}])"
 
             cls._log.error(err)
@@ -210,7 +225,7 @@ class _TypeValidator:
         return isinstance(value, expected_type)
 
     @classmethod
-    def _type_name(cls, type_var: tp.Type) -> str:
+    def _type_name(cls, type_var: tp.Type, qualified: bool = False) -> str:
 
         if isinstance(type_var, cls.__generic_metaclass):
 
@@ -230,7 +245,10 @@ class _TypeValidator:
 
             raise ex.ETracInternal(f"Validation of [{origin.__name__}] generic parameters is not supported yet")
 
-        return type_var.__name__
+        if qualified:
+            return f"{type_var.__module__}.{type_var.__name__}"
+        else:
+            return type_var.__name__
 
 
 class StaticValidator:
@@ -458,3 +476,54 @@ class StaticValidator:
     def _fail(cls, message: str):
         cls._log.error(message)
         raise ex.EModelValidation(message)
+
+
+class StorageValidator:
+
+    __ILLEGAL_PATH_CHARS_WINDOWS = re.compile(r".*[\x00<>:\"\'|?*].*")
+    __ILLEGAL_PATH_CHARS_POSIX = re.compile(r".*[\x00<>:\"\'|?*\\].*")
+    __ILLEGAL_PATH_CHARS = __ILLEGAL_PATH_CHARS_WINDOWS if util.is_windows() else __ILLEGAL_PATH_CHARS_POSIX
+
+    @classmethod
+    def storage_path_is_empty(cls, storage_path: str):
+
+        return storage_path is None or len(storage_path.strip()) == 0
+
+    @classmethod
+    def storage_path_invalid(cls, storage_path: str):
+
+        if cls.__ILLEGAL_PATH_CHARS.match(storage_path):
+            return True
+
+        try:
+            # Make sure the path can be interpreted as a path
+            pathlib.Path(storage_path)
+            return False
+        except ValueError:
+            return True
+
+    @classmethod
+    def storage_path_not_relative(cls, storage_path: str):
+
+        relative_path = pathlib.Path(storage_path)
+        return relative_path.is_absolute()
+
+    @classmethod
+    def storage_path_outside_root(cls, storage_path: str):
+
+        # is_relative_to only supported in Python 3.9+, we need to support 3.8
+
+        root_path = pathlib.Path("C:\\root") if util.is_windows() else pathlib.Path("/root")
+        relative_path = pathlib.Path(storage_path)
+        absolute_path = root_path.joinpath(relative_path).resolve(False)
+
+        return root_path != absolute_path and root_path not in absolute_path.parents
+
+    @classmethod
+    def storage_path_is_root(cls, storage_path: str):
+
+        root_path = pathlib.Path("C:\\root") if util.is_windows() else pathlib.Path("/root")
+        relative_path = pathlib.Path(storage_path)
+        absolute_path = root_path.joinpath(relative_path).resolve(False)
+
+        return root_path == absolute_path

tracdap/rt/_plugins/repo_git.py

@@ -38,7 +38,7 @@ class GitRepository(IModelRepository):
 
     REPO_URL_KEY = "repoUrl"
     NATIVE_GIT_KEY = "nativeGit"
-    NATIVE_GIT_DEFAULT = True
+    NATIVE_GIT_DEFAULT = False
 
     GIT_TIMEOUT_SECONDS = 30
 

tracdap/rt/_version.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.6.4"
+__version__ = "0.6.5"

tracdap/rt/api/experimental.py

@@ -0,0 +1,220 @@
+#  Copyright 2024 Accenture Global Solutions Limited
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import abc as _abc
+import dataclasses as _dc
+import datetime as _dt
+import enum as _enum
+import typing as _tp
+
+from tracdap.rt.api import *
+from .hook import _StaticApiHook
+
+
+_DATA_FRAMEWORK = _tp.TypeVar('_DATA_FRAMEWORK')
+
+
+class _DataFramework(_tp.Generic[_DATA_FRAMEWORK]):
+
+    PANDAS: "_DataFramework"
+    POLARS: "_DataFramework"
+
+    def __init__(self, framework_name, framework_type: _DATA_FRAMEWORK):
+        self.__framework_name = framework_name
+        self.__framework_type = framework_type
+
+    def __str__(self):
+        return self.__framework_name
+
+
+if _tp.TYPE_CHECKING:
+
+    if pandas:
+        _DataFramework.PANDAS = _DataFramework('pandas', pandas.DataFrame)
+        """The original Python dataframe library, most widely used"""
+    else:
+        _DataFramework.PANDAS = _DataFramework('pandas', None)
+        """Pandas data framework is not installed"""
+
+    if polars:
+        _DataFramework.POLARS = _DataFramework('polars', polars.DataFrame)
+        """A modern, fast and simple alternative to Pandas"""
+    else:
+        _DataFramework.POLARS = _DataFramework('polars', None)
+        """Polars data framework is not installed"""
+
+else:
+
+    _DataFramework.PANDAS = _DataFramework('pandas', None)
+    _DataFramework.POLARS = _DataFramework('polars', None)
+
+PANDAS = _DataFramework.PANDAS
+POLARS = _DataFramework.POLARS
+
+
+class TracContext(TracContext):
+
+    @_abc.abstractmethod
+    def get_table(self, dataset_name: str, framework: _DataFramework[_DATA_FRAMEWORK]) -> _DATA_FRAMEWORK:
+
+        pass
+
+    @_abc.abstractmethod
+    def put_table(self, dataset_name: str, dataset: _DATA_FRAMEWORK):
+
+        pass
+
+
+def init_static():
+    import tracdap.rt._impl.static_api as _static_impl  # noqa
+    _static_impl.StaticApiImpl.register_impl()
+
+
+def infer_schema(dataset: _tp.Any) -> SchemaDefinition:
+    sa = _StaticApiHook.get_instance()
+    return sa.infer_schema(dataset)
+
+
+class FileType(_enum.Enum):
+
+    FILE = 1
+    DIRECTORY = 2
+
+
+@_dc.dataclass
+class FileStat:
+
+    """
+    Dataclass to represent some basic  file stat info independent of the storage technology used
+    I.e. do not depend on Python stat_result class that refers to locally-mounted filesystems
+    Timestamps are held in UTC
+    """
+
+    file_name: str
+    file_type: FileType
+    storage_path: str
+    size: int
+
+    mtime: _tp.Optional[_dt.datetime] = None
+    atime: _tp.Optional[_dt.datetime] = None
+
+
+class TracFileStorage:
+
+    @_abc.abstractmethod
+    def get_storage_key(self) -> str:
+        pass
+
+    @_abc.abstractmethod
+    def exists(self, storage_path: str) -> bool:
+        """The exists method can be used for both files and directories"""
+        pass
+
+    @_abc.abstractmethod
+    def size(self, storage_path: str) -> int:
+        """The rm method only works on regular files, it cannot be used for directories"""
+        pass
+
+    @_abc.abstractmethod
+    def stat(self, storage_path: str) -> FileStat:
+        """The stat method can be used for both files and directories, so long as they exist"""
+        pass
+
+    @_abc.abstractmethod
+    def ls(self, storage_path: str, recursive: bool = False) -> _tp.List[FileStat]:
+        """The ls method only works on directories, it cannot be used for regular files"""
+        pass
+
+    @_abc.abstractmethod
+    def mkdir(self, storage_path: str, recursive: bool = False):
+        """The mkdir method will succeed silently if the directory already exists"""
+        pass
+
+    @_abc.abstractmethod
+    def rm(self, storage_path: str):
+        """The rm method only works on regular files, it cannot be used for directories and is not recursive"""
+        pass
+
+    @_abc.abstractmethod
+    def rmdir(self, storage_path: str):
+        """The rmdir method only works on directories and is always recursive"""
+        pass
+
+    @_abc.abstractmethod
+    def read_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The read_byte_stream method only works for existing files"""
+        pass
+
+    @_abc.abstractmethod
+    def write_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The write_byte_stream method will always overwrite an existing file if it exists"""
+        pass
+
+    def read_bytes(self, storage_path: str) -> bytes:
+        """The read_bytes method only works for existing files"""
+        with self.read_byte_stream(storage_path) as stream:
+            return stream.read()
+
+    def write_bytes(self, storage_path: str, data: bytes):
+        """The write_bytes method will always overwrite an existing file if it exists"""
+        with self.write_byte_stream(storage_path) as stream:
+            stream.write(data)
+
+
+
+class TracDataContext(TracContext):
+
+    @_abc.abstractmethod
+    def get_file_storage(self, storage_key: str) -> TracFileStorage:
+        pass
+
+    @_abc.abstractmethod
+    def get_data_storage(self, storage_key: str) -> None:
+        pass
+
+    @_abc.abstractmethod
+    def add_data_import(self, dataset_key: str):
+        pass
+
+    @_abc.abstractmethod
+    def set_source_metadata(self, dataset_key: str, storage_key: str, source_info: FileStat):
+        pass
+
+    @_abc.abstractmethod
+    def set_attribute(self, dataset_key: str, attribute_name: str, value: _tp.Any):
+        pass
+
+    @_abc.abstractmethod
+    def set_schema(self, dataset_key: str, schema: SchemaDefinition):
+        pass
+
+
+class TracDataImport(TracModel):
+
+    def define_inputs(self) -> _tp.Dict[str, ModelInputSchema]:
+        return dict()
+
+    @_abc.abstractmethod
+    def run_model(self, ctx: TracDataContext):
+        pass
+
+
+class TracDataExport(TracModel):
+
+    def define_outputs(self) -> _tp.Dict[str, ModelOutputSchema]:
+        return dict()
+
+    @_abc.abstractmethod
+    def run_model(self, ctx: TracDataContext):
+        pass

tracdap/rt/api/hook.py

@@ -118,6 +118,10 @@ class _StaticApiHook:
 
         pass
 
+    @_abc.abstractmethod
+    def infer_schema(self, dataset: _tp.Any) -> _meta.SchemaDefinition:
+        pass
+
     @_abc.abstractmethod
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],