tracdap-runtime 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

tracdap/rt/_impl/storage.py

@@ -30,6 +30,7 @@ import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt._impl.data as _data
 import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.validation as _val
 
 # Import storage interfaces
 from tracdap.rt.ext.storage import *
@@ -41,7 +42,11 @@ class FormatManager:
     def get_data_format(cls, format_code: str, format_options: tp.Dict[str, tp.Any]) -> IDataFormat:
 
         try:
-            config = _cfg.PluginConfig(format_code, format_options)
+
+            config = _cfg.PluginConfig(
+                protocol=format_code,
+                properties=format_options)
+
             return plugins.PluginManager.load_plugin(IDataFormat, config)
 
         except _ex.EPluginNotAvailable as e:
@@ -73,11 +78,18 @@ class StorageManager:
         self.__log = _util.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
+        self.__external: tp.List[str] = list()
         self.__settings = sys_config.storage
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
 
+        for storage_key, storage_config in sys_config.storage.external.items():
+            if storage_key in self.__file_storage or storage_key in self.__data_storage:
+                raise _ex.EConfig(f"Storage key [{storage_key}] is defined as both internal and external storage")
+            self.__external.append(storage_key)
+            self.create_storage(storage_key, storage_config)
+
     def default_storage_key(self):
         return self.__settings.defaultBucket
 
@@ -147,26 +159,32 @@ class StorageManager:
         self.__file_storage[storage_key] = file_storage
         self.__data_storage[storage_key] = data_storage
 
-    def has_file_storage(self, storage_key: str) -> bool:
+    def has_file_storage(self, storage_key: str, external: bool = False) -> bool:
+
+        if external ^ (storage_key in self.__external):
+            return False
 
         return storage_key in self.__file_storage
 
-    def get_file_storage(self, storage_key: str) -> IFileStorage:
+    def get_file_storage(self, storage_key: str, external: bool = False) -> IFileStorage:
 
-        if not self.has_file_storage(storage_key):
+        if not self.has_file_storage(storage_key, external):
             err = f"File storage is not configured for storage key [{storage_key}]"
             self.__log.error(err)
             raise _ex.EStorageConfig(err)
 
         return self.__file_storage[storage_key]
 
-    def has_data_storage(self, storage_key: str) -> bool:
+    def has_data_storage(self, storage_key: str, external: bool = False) -> bool:
+
+        if external ^ (storage_key in self.__external):
+            return False
 
         return storage_key in self.__data_storage
 
-    def get_data_storage(self, storage_key: str) -> IDataStorage:
+    def get_data_storage(self, storage_key: str, external: bool = False) -> IDataStorage:
 
-        if not self.has_data_storage(storage_key):
+        if not self.has_data_storage(storage_key, external):
             err = f"Data storage is not configured for storage key [{storage_key}]"
             self.__log.error(err)
             raise _ex.EStorageConfig(err)
@@ -587,29 +605,27 @@ class CommonFileStorage(IFileStorage):
 
         try:
 
-            if storage_path is None or len(storage_path.strip()) == 0:
+            if _val.StorageValidator.storage_path_is_empty(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_NULL_OR_BLANK, operation_name, storage_path)
 
-            if self._ILLEGAL_PATH_CHARS.match(storage_path):
+            if _val.StorageValidator.storage_path_invalid(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_INVALID, operation_name, storage_path)
-    
-            relative_path = pathlib.Path(storage_path)
-    
-            if relative_path.is_absolute():
+
+            if _val.StorageValidator.storage_path_not_relative(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_NOT_RELATIVE, operation_name, storage_path)
 
+            if _val.StorageValidator.storage_path_outside_root(storage_path):
+                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_OUTSIDE_ROOT, operation_name, storage_path)
+
+            if not allow_root_dir and _val.StorageValidator.storage_path_is_root(storage_path):
+                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_IS_ROOT, operation_name, storage_path)
+
             root_path = pathlib.Path("C:\\root") if _util.is_windows() else pathlib.Path("/root")
+            relative_path = pathlib.Path(storage_path)
             absolute_path = root_path.joinpath(relative_path).resolve(False)
 
             if absolute_path == root_path:
-                if not allow_root_dir:
-                    raise self._explicit_error(self.ExplicitError.STORAGE_PATH_IS_ROOT, operation_name, storage_path)
-                else:
-                    return ""
-
-            # is_relative_to only supported in Python 3.9+, we need to support 3.7
-            if root_path not in absolute_path.parents:
-                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_OUTSIDE_ROOT, operation_name, storage_path)
+                return ""
             else:
                 return absolute_path.relative_to(root_path).as_posix()
 
@@ -639,10 +655,6 @@ class CommonFileStorage(IFileStorage):
 
         return err
 
-    _ILLEGAL_PATH_CHARS_WINDOWS = re.compile(r".*[\x00<>:\"\'|?*].*")
-    _ILLEGAL_PATH_CHARS_POSIX = re.compile(r".*[\x00<>:\"\'|?*\\].*")
-    _ILLEGAL_PATH_CHARS = _ILLEGAL_PATH_CHARS_WINDOWS if _util.is_windows() else _ILLEGAL_PATH_CHARS_POSIX
-
     class ExplicitError(enum.Enum):
     
         # Validation failures

tracdap/rt/_impl/validation.py

@@ -15,7 +15,9 @@
 import inspect
 import logging
 import re
+import types
 import typing as tp
+import pathlib
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -25,6 +27,11 @@ import tracdap.rt._impl.util as util
 from tracdap.rt.api.hook import _Named  # noqa
 
 
+def require_package(module_name: str, module_obj: types.ModuleType):
+    if module_obj is None:
+        raise ex.ERuntimeValidation(f"Optional package [{module_name}] is not installed")
+
+
 def validate_signature(method: tp.Callable, *args, **kwargs):
     _TypeValidator.validate_signature(method, *args, **kwargs)
 
@@ -38,7 +45,7 @@ def check_type(expected_type: tp.Type, value: tp.Any) -> bool:
 
 
 def quick_validate_model_def(model_def: meta.ModelDefinition):
-    _StaticValidator.quick_validate_model_def(model_def)
+    StaticValidator.quick_validate_model_def(model_def)
 
 
 class _TypeValidator:
@@ -62,15 +69,19 @@ class _TypeValidator:
             signature = inspect.signature(method)
             cls.__method_cache[method.__name__] = signature
 
+        hints = tp.get_type_hints(method)
+
         positional_index = 0
 
         for param_name, param in signature.parameters.items():
 
+            param_type = hints.get(param_name)
+
             values = cls._select_arg(method.__name__, param, positional_index, *args, **kwargs)
             positional_index += len(values)
 
             for value in values:
-                cls._validate_arg(method.__name__, param, value)
+                cls._validate_arg(method.__name__, param_name, param_type, value)
 
     @classmethod
     def validate_return_type(cls, method: tp.Callable, value: tp.Any):
@@ -146,14 +157,18 @@ class _TypeValidator:
         raise ex.EUnexpected("Invalid method signature in runtime API (this is a bug)")
 
     @classmethod
-    def _validate_arg(cls, method_name: str, parameter: inspect.Parameter, value: tp.Any):
+    def _validate_arg(cls, method_name: str, param_name: str, param_type: tp.Type, value: tp.Any):
 
-        if not cls._validate_type(parameter.annotation, value):
+        if not cls._validate_type(param_type, value):
 
-            expected_type = cls._type_name(parameter.annotation)
+            expected_type = cls._type_name(param_type)
             actual_type = cls._type_name(type(value)) if value is not None else str(None)
 
-            err = f"Invalid API call [{method_name}()]: Wrong type for [{parameter.name}]" \
+            if expected_type == actual_type:
+                expected_type = cls._type_name(param_type, qualified=True)
+                actual_type = cls._type_name(type(value), qualified=True)
+
+            err = f"Invalid API call [{method_name}()]: Wrong type for [{param_name}]" \
                   + f" (expected [{expected_type}], got [{actual_type}])"
 
             cls._log.error(err)
@@ -210,7 +225,7 @@ class _TypeValidator:
         return isinstance(value, expected_type)
 
     @classmethod
-    def _type_name(cls, type_var: tp.Type) -> str:
+    def _type_name(cls, type_var: tp.Type, qualified: bool = False) -> str:
 
         if isinstance(type_var, cls.__generic_metaclass):
 
@@ -230,10 +245,13 @@ class _TypeValidator:
 
             raise ex.ETracInternal(f"Validation of [{origin.__name__}] generic parameters is not supported yet")
 
-        return type_var.__name__
+        if qualified:
+            return f"{type_var.__module__}.{type_var.__name__}"
+        else:
+            return type_var.__name__
 
 
-class _StaticValidator:
+class StaticValidator:
 
     __identifier_pattern = re.compile("\\A[a-zA-Z_]\\w*\\Z", re.ASCII)
     __reserved_identifier_pattern = re.compile("\\A(_|trac_)", re.ASCII)
@@ -301,6 +319,28 @@ class _StaticValidator:
         cls._check_inputs_or_outputs(model_def.inputs)
         cls._check_inputs_or_outputs(model_def.outputs)
 
+    @classmethod
+    def quick_validate_schema(cls, schema: meta.SchemaDefinition):
+
+        if schema.schemaType != meta.SchemaType.TABLE:
+            cls._fail(f"Unsupported schema type [{schema.schemaType}]")
+
+        if schema.partType != meta.PartType.PART_ROOT:
+            cls._fail(f"Unsupported partition type [{schema.partType}]")
+
+        if schema.table is None or schema.table.fields is None or len(schema.table.fields) == 0:
+            cls._fail(f"Table schema does not define any fields")
+
+        fields = schema.table.fields
+        field_names = list(map(lambda f: f.fieldName, fields))
+        property_type = f"field"
+
+        cls._valid_identifiers(field_names, property_type)
+        cls._case_insensitive_duplicates(field_names, property_type)
+
+        for field in fields:
+            cls._check_single_field(field, property_type)
+
     @classmethod
     def _check_label(cls, label, param_name):
         if label is not None:
@@ -330,10 +370,20 @@ class _StaticValidator:
 
             cls._log.info(f"Checking {input_name}")
 
+            if input_schema.dynamic:
+                if input_schema.schema and input_schema.schema.table:
+                    error = "Dynamic schemas must have schema.table = None"
+                    cls._fail(f"Invalid schema for [{input_name}]: {error}")
+                else:
+                    continue
+
             fields = input_schema.schema.table.fields
             field_names = list(map(lambda f: f.fieldName, fields))
             property_type = f"field in [{input_name}]"
 
+            if len(fields) == 0:
+                cls._fail(f"Invalid schema for [{input_name}]: No fields defined")
+
             cls._valid_identifiers(field_names, property_type)
             cls._case_insensitive_duplicates(field_names, property_type)
 
@@ -375,8 +425,9 @@ class _StaticValidator:
         if field.categorical and field.fieldType != meta.BasicType.STRING:
             cls._fail(f"Invalid {property_type}: [{field.fieldName}] fieldType {field.fieldType} used as categorical")
 
-        if field.businessKey and not field.notNull:
-            cls._fail(f"Invalid {property_type}: [{field.fieldName}] is a business key but not_null = False")
+        # Do not require notNull = True for business keys here
+        # Instead setting businessKey = True will cause notNull = True to be set during normalization
+        # This agrees with the semantics in platform API and CSV schema loader
 
     @classmethod
     def _valid_identifiers(cls, keys, property_type):
@@ -425,3 +476,54 @@ class _StaticValidator:
     def _fail(cls, message: str):
         cls._log.error(message)
         raise ex.EModelValidation(message)
+
+
+class StorageValidator:
+
+    __ILLEGAL_PATH_CHARS_WINDOWS = re.compile(r".*[\x00<>:\"\'|?*].*")
+    __ILLEGAL_PATH_CHARS_POSIX = re.compile(r".*[\x00<>:\"\'|?*\\].*")
+    __ILLEGAL_PATH_CHARS = __ILLEGAL_PATH_CHARS_WINDOWS if util.is_windows() else __ILLEGAL_PATH_CHARS_POSIX
+
+    @classmethod
+    def storage_path_is_empty(cls, storage_path: str):
+
+        return storage_path is None or len(storage_path.strip()) == 0
+
+    @classmethod
+    def storage_path_invalid(cls, storage_path: str):
+
+        if cls.__ILLEGAL_PATH_CHARS.match(storage_path):
+            return True
+
+        try:
+            # Make sure the path can be interpreted as a path
+            pathlib.Path(storage_path)
+            return False
+        except ValueError:
+            return True
+
+    @classmethod
+    def storage_path_not_relative(cls, storage_path: str):
+
+        relative_path = pathlib.Path(storage_path)
+        return relative_path.is_absolute()
+
+    @classmethod
+    def storage_path_outside_root(cls, storage_path: str):
+
+        # is_relative_to only supported in Python 3.9+, we need to support 3.8
+
+        root_path = pathlib.Path("C:\\root") if util.is_windows() else pathlib.Path("/root")
+        relative_path = pathlib.Path(storage_path)
+        absolute_path = root_path.joinpath(relative_path).resolve(False)
+
+        return root_path != absolute_path and root_path not in absolute_path.parents
+
+    @classmethod
+    def storage_path_is_root(cls, storage_path: str):
+
+        root_path = pathlib.Path("C:\\root") if util.is_windows() else pathlib.Path("/root")
+        relative_path = pathlib.Path(storage_path)
+        absolute_path = root_path.joinpath(relative_path).resolve(False)
+
+        return root_path == absolute_path

tracdap/rt/_plugins/repo_git.py

@@ -38,7 +38,7 @@ class GitRepository(IModelRepository):
 
     REPO_URL_KEY = "repoUrl"
     NATIVE_GIT_KEY = "nativeGit"
-    NATIVE_GIT_DEFAULT = True
+    NATIVE_GIT_DEFAULT = False
 
     GIT_TIMEOUT_SECONDS = 30
 

tracdap/rt/_version.py

@@ -12,4 +12,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.6.3"
+__version__ = "0.6.5"

tracdap/rt/api/experimental.py

@@ -0,0 +1,220 @@
+#  Copyright 2024 Accenture Global Solutions Limited
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import abc as _abc
+import dataclasses as _dc
+import datetime as _dt
+import enum as _enum
+import typing as _tp
+
+from tracdap.rt.api import *
+from .hook import _StaticApiHook
+
+
+_DATA_FRAMEWORK = _tp.TypeVar('_DATA_FRAMEWORK')
+
+
+class _DataFramework(_tp.Generic[_DATA_FRAMEWORK]):
+
+    PANDAS: "_DataFramework"
+    POLARS: "_DataFramework"
+
+    def __init__(self, framework_name, framework_type: _DATA_FRAMEWORK):
+        self.__framework_name = framework_name
+        self.__framework_type = framework_type
+
+    def __str__(self):
+        return self.__framework_name
+
+
+if _tp.TYPE_CHECKING:
+
+    if pandas:
+        _DataFramework.PANDAS = _DataFramework('pandas', pandas.DataFrame)
+        """The original Python dataframe library, most widely used"""
+    else:
+        _DataFramework.PANDAS = _DataFramework('pandas', None)
+        """Pandas data framework is not installed"""
+
+    if polars:
+        _DataFramework.POLARS = _DataFramework('polars', polars.DataFrame)
+        """A modern, fast and simple alternative to Pandas"""
+    else:
+        _DataFramework.POLARS = _DataFramework('polars', None)
+        """Polars data framework is not installed"""
+
+else:
+
+    _DataFramework.PANDAS = _DataFramework('pandas', None)
+    _DataFramework.POLARS = _DataFramework('polars', None)
+
+PANDAS = _DataFramework.PANDAS
+POLARS = _DataFramework.POLARS
+
+
+class TracContext(TracContext):
+
+    @_abc.abstractmethod
+    def get_table(self, dataset_name: str, framework: _DataFramework[_DATA_FRAMEWORK]) -> _DATA_FRAMEWORK:
+
+        pass
+
+    @_abc.abstractmethod
+    def put_table(self, dataset_name: str, dataset: _DATA_FRAMEWORK):
+
+        pass
+
+
+def init_static():
+    import tracdap.rt._impl.static_api as _static_impl  # noqa
+    _static_impl.StaticApiImpl.register_impl()
+
+
+def infer_schema(dataset: _tp.Any) -> SchemaDefinition:
+    sa = _StaticApiHook.get_instance()
+    return sa.infer_schema(dataset)
+
+
+class FileType(_enum.Enum):
+
+    FILE = 1
+    DIRECTORY = 2
+
+
+@_dc.dataclass
+class FileStat:
+
+    """
+    Dataclass to represent some basic  file stat info independent of the storage technology used
+    I.e. do not depend on Python stat_result class that refers to locally-mounted filesystems
+    Timestamps are held in UTC
+    """
+
+    file_name: str
+    file_type: FileType
+    storage_path: str
+    size: int
+
+    mtime: _tp.Optional[_dt.datetime] = None
+    atime: _tp.Optional[_dt.datetime] = None
+
+
+class TracFileStorage:
+
+    @_abc.abstractmethod
+    def get_storage_key(self) -> str:
+        pass
+
+    @_abc.abstractmethod
+    def exists(self, storage_path: str) -> bool:
+        """The exists method can be used for both files and directories"""
+        pass
+
+    @_abc.abstractmethod
+    def size(self, storage_path: str) -> int:
+        """The rm method only works on regular files, it cannot be used for directories"""
+        pass
+
+    @_abc.abstractmethod
+    def stat(self, storage_path: str) -> FileStat:
+        """The stat method can be used for both files and directories, so long as they exist"""
+        pass
+
+    @_abc.abstractmethod
+    def ls(self, storage_path: str, recursive: bool = False) -> _tp.List[FileStat]:
+        """The ls method only works on directories, it cannot be used for regular files"""
+        pass
+
+    @_abc.abstractmethod
+    def mkdir(self, storage_path: str, recursive: bool = False):
+        """The mkdir method will succeed silently if the directory already exists"""
+        pass
+
+    @_abc.abstractmethod
+    def rm(self, storage_path: str):
+        """The rm method only works on regular files, it cannot be used for directories and is not recursive"""
+        pass
+
+    @_abc.abstractmethod
+    def rmdir(self, storage_path: str):
+        """The rmdir method only works on directories and is always recursive"""
+        pass
+
+    @_abc.abstractmethod
+    def read_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The read_byte_stream method only works for existing files"""
+        pass
+
+    @_abc.abstractmethod
+    def write_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The write_byte_stream method will always overwrite an existing file if it exists"""
+        pass
+
+    def read_bytes(self, storage_path: str) -> bytes:
+        """The read_bytes method only works for existing files"""
+        with self.read_byte_stream(storage_path) as stream:
+            return stream.read()
+
+    def write_bytes(self, storage_path: str, data: bytes):
+        """The write_bytes method will always overwrite an existing file if it exists"""
+        with self.write_byte_stream(storage_path) as stream:
+            stream.write(data)
+
+
+
+class TracDataContext(TracContext):
+
+    @_abc.abstractmethod
+    def get_file_storage(self, storage_key: str) -> TracFileStorage:
+        pass
+
+    @_abc.abstractmethod
+    def get_data_storage(self, storage_key: str) -> None:
+        pass
+
+    @_abc.abstractmethod
+    def add_data_import(self, dataset_key: str):
+        pass
+
+    @_abc.abstractmethod
+    def set_source_metadata(self, dataset_key: str, storage_key: str, source_info: FileStat):
+        pass
+
+    @_abc.abstractmethod
+    def set_attribute(self, dataset_key: str, attribute_name: str, value: _tp.Any):
+        pass
+
+    @_abc.abstractmethod
+    def set_schema(self, dataset_key: str, schema: SchemaDefinition):
+        pass
+
+
+class TracDataImport(TracModel):
+
+    def define_inputs(self) -> _tp.Dict[str, ModelInputSchema]:
+        return dict()
+
+    @_abc.abstractmethod
+    def run_model(self, ctx: TracDataContext):
+        pass
+
+
+class TracDataExport(TracModel):
+
+    def define_outputs(self) -> _tp.Dict[str, ModelOutputSchema]:
+        return dict()
+
+    @_abc.abstractmethod
+    def run_model(self, ctx: TracDataContext):
+        pass

tracdap/rt/api/hook.py

@@ -118,11 +118,14 @@ class _StaticApiHook:
 
         pass
 
+    @_abc.abstractmethod
+    def infer_schema(self, dataset: _tp.Any) -> _meta.SchemaDefinition:
+        pass
+
     @_abc.abstractmethod
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None,
-            optional: bool = False,
+            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
             input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelInputSchema:
 
@@ -131,8 +134,7 @@ class _StaticApiHook:
     @_abc.abstractmethod
     def define_output_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None,
-            optional: bool = False,
+            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
             output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelOutputSchema: