tracdap-runtime 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto
+# Protobuf Python Version: 4.25.3
+"""Generated protocol buffer code."""
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+from google.protobuf.internal import builder as _builder
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__id__pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as tracdap_dot_rt_dot___impl_dot_grpc_dot_tracdap_dot_metadata_dot_object__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n5tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto\x12\x10tracdap.metadata\x1a\x36tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto\x1a\x33tracdap/rt/_impl/grpc/tracdap/metadata/object.proto*m\n\x0cResourceType\x12\x19\n\x15RESOURCE_TYPE_NOT_SET\x10\x00\x12\x14\n\x10MODEL_REPOSITORY\x10\x01\x12\x14\n\x10INTERNAL_STORAGE\x10\x02\"\x04\x08\x03\x10\x03*\x10\x45XTERNAL_STORAGEB\x1e\n\x1aorg.finos.tracdap.metadataP\x01\x62\x06proto3')
+
+_globals = globals()
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'tracdap.rt._impl.grpc.tracdap.metadata.resource_pb2', _globals)
+if _descriptor._USE_C_DESCRIPTORS == False:
+  _globals['DESCRIPTOR']._options = None
+  _globals['DESCRIPTOR']._serialized_options = b'\n\032org.finos.tracdap.metadataP\001'
+  _globals['_RESOURCETYPE']._serialized_start=184
+  _globals['_RESOURCETYPE']._serialized_end=293
+# @@protoc_insertion_point(module_scope)

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.pyi

@@ -0,0 +1,16 @@
+from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as _object_id_pb2
+from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as _object_pb2
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from typing import ClassVar as _ClassVar
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class ResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = ()
+    RESOURCE_TYPE_NOT_SET: _ClassVar[ResourceType]
+    MODEL_REPOSITORY: _ClassVar[ResourceType]
+    INTERNAL_STORAGE: _ClassVar[ResourceType]
+RESOURCE_TYPE_NOT_SET: ResourceType
+MODEL_REPOSITORY: ResourceType
+INTERNAL_STORAGE: ResourceType

tracdap/rt/_impl/models.py

@@ -19,6 +19,7 @@ import pathlib
 import copy
 
 import tracdap.rt.api as _api
+import tracdap.rt.api.experimental as _eapi
 import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
@@ -208,6 +209,13 @@ class ModelLoader:
             model_def.inputs = inputs
             model_def.outputs = outputs
 
+            if isinstance(model, _eapi.TracDataImport):
+                model_def.modelType = _meta.ModelType.DATA_IMPORT_MODEL
+            elif isinstance(model, _eapi.TracDataExport):
+                model_def.modelType = _meta.ModelType.DATA_EXPORT_MODEL
+            else:
+                model_def.modelType = _meta.ModelType.STANDARD_MODEL
+
             _val.quick_validate_model_def(model_def)
 
             for attr_name, attr_value in attributes.items():

tracdap/rt/_impl/static_api.py

@@ -15,8 +15,10 @@
 import typing as _tp
 import types as _ts
 
+import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
+import tracdap.rt._impl.data as _data
 import tracdap.rt._impl.schemas as _schemas
 import tracdap.rt._impl.type_system as _type_system
 import tracdap.rt._impl.validation as _val
@@ -36,6 +38,24 @@ class StaticApiImpl(_StaticApiHook):
         if not _StaticApiHook._is_registered():
             _StaticApiHook._register(StaticApiImpl())
 
+    def array_type(self, item_type: _meta.BasicType) -> _meta.TypeDescriptor:
+
+        _val.validate_signature(self.array_type, item_type)
+
+        if not _val.is_primitive_type(item_type):
+            raise _ex.EModelValidation(f"Arrays can only contain primitive types, [{item_type}] is not primitive")
+
+        return _meta.TypeDescriptor(_meta.BasicType.ARRAY, arrayType=_meta.TypeDescriptor(item_type))
+
+    def map_type(self, entry_type: _meta.BasicType) -> _meta.TypeDescriptor:
+
+        _val.validate_signature(self.map_type, entry_type)
+
+        if not _val.is_primitive_type(entry_type):
+            raise _ex.EModelValidation(f"Maps can only contain primitive types, [{entry_type}] is not primitive")
+
+        return _meta.TypeDescriptor(_meta.BasicType.MAP, arrayType=_meta.TypeDescriptor(entry_type))
+
     def define_attribute(
             self, attr_name: str, attr_value: _tp.Any,
             attr_type: _tp.Optional[_meta.BasicType] = None,
@@ -152,6 +172,15 @@ class StaticApiImpl(_StaticApiHook):
 
         return _schemas.SchemaLoader.load_schema(package, schema_file)
 
+    def infer_schema(self, dataset: _api.DATA_API) -> _meta.SchemaDefinition:
+
+        _val.validate_signature(self.infer_schema, dataset)
+
+        framework = _data.DataConverter.get_framework(dataset)
+        converter = _data.DataConverter.for_framework(framework)
+
+        return converter.infer_schema(dataset)
+
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
             label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,

tracdap/rt/_impl/storage.py

@@ -30,9 +30,10 @@ import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt._impl.data as _data
 import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.validation as _val
 
-# Import storage interfaces
-from tracdap.rt.ext.storage import *
+# Import storage interfaces (using the internal version, it has extra bits that are not public)
+from tracdap.rt._impl.ext.storage import *
 
 
 class FormatManager:
@@ -41,7 +42,11 @@ class FormatManager:
     def get_data_format(cls, format_code: str, format_options: tp.Dict[str, tp.Any]) -> IDataFormat:
 
         try:
-            config = _cfg.PluginConfig(format_code, format_options)
+
+            config = _cfg.PluginConfig(
+                protocol=format_code,
+                properties=format_options)
+
             return plugins.PluginManager.load_plugin(IDataFormat, config)
 
         except _ex.EPluginNotAvailable as e:
@@ -73,11 +78,18 @@ class StorageManager:
         self.__log = _util.logger_for_object(self)
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
+        self.__external: tp.List[str] = list()
         self.__settings = sys_config.storage
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
 
+        for storage_key, storage_config in sys_config.storage.external.items():
+            if storage_key in self.__file_storage or storage_key in self.__data_storage:
+                raise _ex.EConfig(f"Storage key [{storage_key}] is defined as both internal and external storage")
+            self.__external.append(storage_key)
+            self.create_storage(storage_key, storage_config)
+
     def default_storage_key(self):
         return self.__settings.defaultBucket
 
@@ -147,26 +159,32 @@ class StorageManager:
         self.__file_storage[storage_key] = file_storage
         self.__data_storage[storage_key] = data_storage
 
-    def has_file_storage(self, storage_key: str) -> bool:
+    def has_file_storage(self, storage_key: str, external: bool = False) -> bool:
+
+        if external ^ (storage_key in self.__external):
+            return False
 
         return storage_key in self.__file_storage
 
-    def get_file_storage(self, storage_key: str) -> IFileStorage:
+    def get_file_storage(self, storage_key: str, external: bool = False) -> IFileStorage:
 
-        if not self.has_file_storage(storage_key):
+        if not self.has_file_storage(storage_key, external):
             err = f"File storage is not configured for storage key [{storage_key}]"
             self.__log.error(err)
             raise _ex.EStorageConfig(err)
 
         return self.__file_storage[storage_key]
 
-    def has_data_storage(self, storage_key: str) -> bool:
+    def has_data_storage(self, storage_key: str, external: bool = False) -> bool:
+
+        if external ^ (storage_key in self.__external):
+            return False
 
         return storage_key in self.__data_storage
 
-    def get_data_storage(self, storage_key: str) -> IDataStorage:
+    def get_data_storage(self, storage_key: str, external: bool = False) -> IDataStorage:
 
-        if not self.has_data_storage(storage_key):
+        if not self.has_data_storage(storage_key, external):
             err = f"Data storage is not configured for storage key [{storage_key}]"
             self.__log.error(err)
             raise _ex.EStorageConfig(err)
@@ -587,29 +605,27 @@ class CommonFileStorage(IFileStorage):
 
         try:
 
-            if storage_path is None or len(storage_path.strip()) == 0:
+            if _val.StorageValidator.storage_path_is_empty(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_NULL_OR_BLANK, operation_name, storage_path)
 
-            if self._ILLEGAL_PATH_CHARS.match(storage_path):
+            if _val.StorageValidator.storage_path_invalid(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_INVALID, operation_name, storage_path)
-    
-            relative_path = pathlib.Path(storage_path)
-    
-            if relative_path.is_absolute():
+
+            if _val.StorageValidator.storage_path_not_relative(storage_path):
                 raise self._explicit_error(self.ExplicitError.STORAGE_PATH_NOT_RELATIVE, operation_name, storage_path)
 
+            if _val.StorageValidator.storage_path_outside_root(storage_path):
+                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_OUTSIDE_ROOT, operation_name, storage_path)
+
+            if not allow_root_dir and _val.StorageValidator.storage_path_is_root(storage_path):
+                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_IS_ROOT, operation_name, storage_path)
+
             root_path = pathlib.Path("C:\\root") if _util.is_windows() else pathlib.Path("/root")
+            relative_path = pathlib.Path(storage_path)
             absolute_path = root_path.joinpath(relative_path).resolve(False)
 
             if absolute_path == root_path:
-                if not allow_root_dir:
-                    raise self._explicit_error(self.ExplicitError.STORAGE_PATH_IS_ROOT, operation_name, storage_path)
-                else:
-                    return ""
-
-            # is_relative_to only supported in Python 3.9+, we need to support 3.7
-            if root_path not in absolute_path.parents:
-                raise self._explicit_error(self.ExplicitError.STORAGE_PATH_OUTSIDE_ROOT, operation_name, storage_path)
+                return ""
             else:
                 return absolute_path.relative_to(root_path).as_posix()
 
@@ -639,10 +655,6 @@ class CommonFileStorage(IFileStorage):
 
         return err
 
-    _ILLEGAL_PATH_CHARS_WINDOWS = re.compile(r".*[\x00<>:\"\'|?*].*")
-    _ILLEGAL_PATH_CHARS_POSIX = re.compile(r".*[\x00<>:\"\'|?*\\].*")
-    _ILLEGAL_PATH_CHARS = _ILLEGAL_PATH_CHARS_WINDOWS if _util.is_windows() else _ILLEGAL_PATH_CHARS_POSIX
-
     class ExplicitError(enum.Enum):
     
         # Validation failures

tracdap/rt/_impl/util.py

@@ -262,6 +262,16 @@ def get_args(metaclass: type):
         return None
 
 
+def get_constraints(metaclass: tp.TypeVar):
+
+    return metaclass.__constraints__ or None
+
+
+def get_bound(metaclass: tp.TypeVar):
+
+    return metaclass.__bound__ or None
+
+
 def try_clean_dir(dir_path: pathlib.Path, remove: bool = False) -> bool:
 
     normalized_path = windows_unc_path(dir_path)

tracdap/rt/_impl/validation.py

@@ -15,7 +15,9 @@
 import inspect
 import logging
 import re
+import types
 import typing as tp
+import pathlib
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -25,6 +27,11 @@ import tracdap.rt._impl.util as util
 from tracdap.rt.api.hook import _Named  # noqa
 
 
+def require_package(module_name: str, module_obj: types.ModuleType):
+    if module_obj is None:
+        raise ex.ERuntimeValidation(f"Optional package [{module_name}] is not installed")
+
+
 def validate_signature(method: tp.Callable, *args, **kwargs):
     _TypeValidator.validate_signature(method, *args, **kwargs)
 
@@ -37,10 +44,25 @@ def check_type(expected_type: tp.Type, value: tp.Any) -> bool:
     return _TypeValidator.check_type(expected_type, value)
 
 
+def type_name(type_: tp.Type, qualified: bool) -> str:
+    return _TypeValidator._type_name(type_, qualified)  # noqa
+
+
 def quick_validate_model_def(model_def: meta.ModelDefinition):
     StaticValidator.quick_validate_model_def(model_def)
 
 
+def is_primitive_type(basic_type: meta.BasicType) -> bool:
+    return StaticValidator.is_primitive_type(basic_type)
+
+
+T_SKIP_VAL = tp.TypeVar("T_SKIP_VAL")
+
+class SkipValidation(tp.Generic[T_SKIP_VAL]):
+    def __init__(self, skip_type: tp.Type[T_SKIP_VAL]):
+        self.skip_type = skip_type
+
+
 class _TypeValidator:
 
     # The metaclass for generic types varies between versions of the typing library
@@ -49,37 +71,42 @@ class _TypeValidator:
 
     # Cache method signatures to avoid inspection on every call
     # Inspecting a function signature can take ~ half a second in Python 3.7
-    __method_cache: tp.Dict[str, inspect.Signature] = dict()
+    __method_cache: tp.Dict[str, tp.Tuple[inspect.Signature, tp.Any]] = dict()
 
     _log: logging.Logger = util.logger_for_namespace(__name__)
 
     @classmethod
     def validate_signature(cls, method: tp.Callable, *args, **kwargs):
 
-        if method.__name__ in cls.__method_cache:
-            signature = cls.__method_cache[method.__name__]
+        if method.__qualname__ in cls.__method_cache:
+            signature, hints = cls.__method_cache[method.__qualname__]
         else:
             signature = inspect.signature(method)
-            cls.__method_cache[method.__name__] = signature
+            hints = tp.get_type_hints(method)
+            cls.__method_cache[method.__qualname__] = signature, hints
 
+        named_params = list(signature.parameters.keys())
         positional_index = 0
 
         for param_name, param in signature.parameters.items():
 
-            values = cls._select_arg(method.__name__, param, positional_index, *args, **kwargs)
+            param_type = hints.get(param_name)
+
+            values = cls._select_arg(method.__name__, param, positional_index, named_params, *args, **kwargs)
             positional_index += len(values)
 
             for value in values:
-                cls._validate_arg(method.__name__, param, value)
+                cls._validate_arg(method.__name__, param_name, param_type, value)
 
     @classmethod
     def validate_return_type(cls, method: tp.Callable, value: tp.Any):
 
-        if method.__name__ in cls.__method_cache:
-            signature = cls.__method_cache[method.__name__]
+        if method.__qualname__ in cls.__method_cache:
+            signature, hints = cls.__method_cache[method.__qualname__]
         else:
             signature = inspect.signature(method)
-            cls.__method_cache[method.__name__] = signature
+            hints = tp.get_type_hints(method)
+            cls.__method_cache[method.__qualname__] = signature, hints
 
         correct_type = cls._validate_type(signature.return_annotation, value)
 
@@ -96,7 +123,7 @@ class _TypeValidator:
 
     @classmethod
     def _select_arg(
-            cls, method_name: str, parameter: inspect.Parameter, positional_index,
+            cls, method_name: str, parameter: inspect.Parameter, positional_index, named_params,
             *args, **kwargs) -> tp.List[tp.Any]:
 
         if parameter.kind == inspect.Parameter.POSITIONAL_ONLY:
@@ -141,19 +168,23 @@ class _TypeValidator:
 
         if parameter.kind == inspect.Parameter.VAR_KEYWORD:
 
-            raise ex.ETracInternal("Validation of VAR_KEYWORD params is not supported yet")
+            return [arg for kw, arg in kwargs.items() if kw not in named_params]
 
         raise ex.EUnexpected("Invalid method signature in runtime API (this is a bug)")
 
     @classmethod
-    def _validate_arg(cls, method_name: str, parameter: inspect.Parameter, value: tp.Any):
+    def _validate_arg(cls, method_name: str, param_name: str, param_type: tp.Type, value: tp.Any):
 
-        if not cls._validate_type(parameter.annotation, value):
+        if not cls._validate_type(param_type, value):
 
-            expected_type = cls._type_name(parameter.annotation)
+            expected_type = cls._type_name(param_type)
             actual_type = cls._type_name(type(value)) if value is not None else str(None)
 
-            err = f"Invalid API call [{method_name}()]: Wrong type for [{parameter.name}]" \
+            if expected_type == actual_type:
+                expected_type = cls._type_name(param_type, qualified=True)
+                actual_type = cls._type_name(type(value), qualified=True)
+
+            err = f"Invalid API call [{method_name}()]: Wrong type for [{param_name}]" \
                   + f" (expected [{expected_type}], got [{actual_type}])"
 
             cls._log.error(err)
@@ -165,6 +196,12 @@ class _TypeValidator:
         if expected_type == tp.Any:
             return True
 
+        # Sometimes we need to validate a partial set of arguments
+        # Explicitly passing a SkipValidation value allows for this
+        if isinstance(value, SkipValidation):
+            if value.skip_type == expected_type:
+                return True
+
         if isinstance(expected_type, cls.__generic_metaclass):
 
             origin = util.get_origin(expected_type)
@@ -201,8 +238,31 @@ class _TypeValidator:
                     all(map(lambda k: cls._validate_type(key_type, k), value.keys())) and \
                     all(map(lambda v: cls._validate_type(value_type, v), value.values()))
 
+            if origin.__module__.startswith("tracdap.rt.api."):
+                return isinstance(value, origin)
+
             raise ex.ETracInternal(f"Validation of [{origin.__name__}] generic parameters is not supported yet")
 
+        # Support for generic type variables
+        if isinstance(expected_type, tp.TypeVar):
+
+            # If there are any constraints or a bound, those must be honoured
+
+            constraints =  util.get_constraints(expected_type)
+            bound = util.get_bound(expected_type)
+
+            if constraints:
+                if not any(map(lambda c: type(value) == c, constraints)):
+                    return False
+
+            if bound:
+                if not isinstance(value, bound):
+                    return False
+
+            # So long as constraints / bound are ok, any type matches a generic type var
+            return True
+
+
         # Validate everything else as a concrete type
 
         # TODO: Recursive validation of types for class members using field annotations
@@ -210,7 +270,7 @@ class _TypeValidator:
         return isinstance(value, expected_type)
 
     @classmethod
-    def _type_name(cls, type_var: tp.Type) -> str:
+    def _type_name(cls, type_var: tp.Type, qualified: bool = False) -> str:
 
         if isinstance(type_var, cls.__generic_metaclass):
 
@@ -222,7 +282,10 @@ class _TypeValidator:
                 return f"Named[{named_type}]"
 
             if origin is tp.Union:
-                return "|".join(map(cls._type_name, args))
+                if len(args) == 2 and args[1] == type(None):
+                    return f"Optional[{cls._type_name(args[0])}]"
+                else:
+                    return "|".join(map(cls._type_name, args))
 
             if origin is list:
                 list_type = cls._type_name(args[0])
@@ -230,7 +293,10 @@ class _TypeValidator:
 
             raise ex.ETracInternal(f"Validation of [{origin.__name__}] generic parameters is not supported yet")
 
-        return type_var.__name__
+        if qualified:
+            return f"{type_var.__module__}.{type_var.__name__}"
+        else:
+            return type_var.__name__
 
 
 class StaticValidator:
@@ -256,6 +322,11 @@ class StaticValidator:
 
     _log: logging.Logger = util.logger_for_namespace(__name__)
 
+    @classmethod
+    def is_primitive_type(cls, basic_type: meta.BasicType) -> bool:
+
+        return basic_type in cls.__PRIMITIVE_TYPES
+
     @classmethod
     def quick_validate_model_def(cls, model_def: meta.ModelDefinition):
 
@@ -458,3 +529,54 @@ class StaticValidator:
     def _fail(cls, message: str):
         cls._log.error(message)
         raise ex.EModelValidation(message)
+
+
+class StorageValidator:
+
+    __ILLEGAL_PATH_CHARS_WINDOWS = re.compile(r".*[\x00<>:\"\'|?*].*")
+    __ILLEGAL_PATH_CHARS_POSIX = re.compile(r".*[\x00<>:\"\'|?*\\].*")
+    __ILLEGAL_PATH_CHARS = __ILLEGAL_PATH_CHARS_WINDOWS if util.is_windows() else __ILLEGAL_PATH_CHARS_POSIX
+
+    @classmethod
+    def storage_path_is_empty(cls, storage_path: str):
+
+        return storage_path is None or len(storage_path.strip()) == 0
+
+    @classmethod
+    def storage_path_invalid(cls, storage_path: str):
+
+        if cls.__ILLEGAL_PATH_CHARS.match(storage_path):
+            return True
+
+        try:
+            # Make sure the path can be interpreted as a path
+            pathlib.Path(storage_path)
+            return False
+        except ValueError:
+            return True
+
+    @classmethod
+    def storage_path_not_relative(cls, storage_path: str):
+
+        relative_path = pathlib.Path(storage_path)
+        return relative_path.is_absolute()
+
+    @classmethod
+    def storage_path_outside_root(cls, storage_path: str):
+
+        # is_relative_to only supported in Python 3.9+, we need to support 3.8
+
+        root_path = pathlib.Path("C:\\root") if util.is_windows() else pathlib.Path("/root")
+        relative_path = pathlib.Path(storage_path)
+        absolute_path = root_path.joinpath(relative_path).resolve(False)
+
+        return root_path != absolute_path and root_path not in absolute_path.parents
+
+    @classmethod
+    def storage_path_is_root(cls, storage_path: str):
+
+        root_path = pathlib.Path("C:\\root") if util.is_windows() else pathlib.Path("/root")
+        relative_path = pathlib.Path(storage_path)
+        absolute_path = root_path.joinpath(relative_path).resolve(False)
+
+        return root_path == absolute_path

tracdap/rt/_plugins/repo_git.py

@@ -38,7 +38,7 @@ class GitRepository(IModelRepository):
 
     REPO_URL_KEY = "repoUrl"
     NATIVE_GIT_KEY = "nativeGit"
-    NATIVE_GIT_DEFAULT = True
+    NATIVE_GIT_DEFAULT = False
 
     GIT_TIMEOUT_SECONDS = 30