tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/_impl/static_api.py

@@ -19,10 +19,11 @@ import types as _ts
 import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.schemas as _schemas
-import tracdap.rt._impl.type_system as _type_system
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.schemas as _schemas
+import tracdap.rt._impl.core.struct as _struct
+import tracdap.rt._impl.core.type_system as _type_system
+import tracdap.rt._impl.core.validation as _val
 
 # Import hook interfaces into this module namespace
 from tracdap.rt.api.hook import _StaticApiHook  # noqa
@@ -55,7 +56,7 @@ class StaticApiImpl(_StaticApiHook):
         if not _val.is_primitive_type(entry_type):
             raise _ex.EModelValidation(f"Maps can only contain primitive types, [{entry_type}] is not primitive")
 
-        return _meta.TypeDescriptor(_meta.BasicType.MAP, arrayType=_meta.TypeDescriptor(entry_type))
+        return _meta.TypeDescriptor(_meta.BasicType.MAP, mapType=_meta.TypeDescriptor(entry_type))
 
     def define_attribute(
             self, attr_name: str, attr_value: _tp.Any,
@@ -150,16 +151,27 @@ class StaticApiImpl(_StaticApiHook):
             notNull=not_null,
             formatCode=format_code)
 
+    def define_struct(self, python_type: type[_api.STRUCT_TYPE]):
+
+        _val.validate_signature(self.define_struct, python_type)
+
+        struct_schema = _struct.StructProcessor.define_struct(python_type)
+        return _meta.SchemaDefinition(schemaType=_meta.SchemaType.STRUCT, struct=struct_schema)
+
     def define_schema(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) \
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE, dynamic: bool = False) \
             -> _meta.SchemaDefinition:
 
-        _val.validate_signature(self.define_schema, *fields, schema_type=schema_type)
+        _val.validate_signature(self.define_schema, *fields, schema_type=schema_type, dynamic=dynamic)
 
         if schema_type == _meta.SchemaType.TABLE:
 
-            table_schema = self._build_table_schema(*fields)
+            if dynamic and not fields:
+                table_schema = None
+            else:
+                table_schema = self._build_table_schema(*fields)
+
             return _meta.SchemaDefinition(_meta.SchemaType.TABLE, table=table_schema)
 
         raise _ex.ERuntimeValidation(f"Invalid schema type [{schema_type.name}]")
@@ -182,51 +194,67 @@ class StaticApiImpl(_StaticApiHook):
 
         return converter.infer_schema(dataset)
 
-    def define_input_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
-            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
-            -> _meta.ModelInputSchema:
+    def define_file_type(self, extension: str, mime_type: str) -> _meta.FileType:
+
+        _val.validate_signature(self.define_file_type, extension, mime_type)
+
+        return _meta.FileType(extension=extension, mimeType=mime_type)
+
+    def define_input(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
+            input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None):
 
         _val.validate_signature(
-            self.define_input_table, *fields,
+            self.define_input, requirement,
             label=label, optional=optional, dynamic=dynamic,
             input_props=input_props)
 
-        # Do not define details for dynamic schemas
+        if isinstance(requirement, _meta.SchemaDefinition):
 
-        if dynamic:
-            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
-        else:
-            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
+            return _meta.ModelInputSchema(
+                objectType=_meta.ObjectType.DATA, schema=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                inputProps=input_props)
 
-        return _meta.ModelInputSchema(
-            schema=schema_def, label=label,
-            optional=optional, dynamic=dynamic,
-            inputProps=input_props)
+        elif isinstance(requirement, _meta.FileType):
 
-    def define_output_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
-            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
-            -> _meta.ModelOutputSchema:
+            return _meta.ModelInputSchema(
+                objectType=_meta.ObjectType.FILE, fileType=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                inputProps=input_props)
+
+        else:
+            raise _ex.EUnexpected()
+
+    def define_output(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
+            output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None):
 
         _val.validate_signature(
-            self.define_output_table, *fields,
+            self.define_output, requirement,
             label=label, optional=optional, dynamic=dynamic,
             output_props=output_props)
 
-        # Do not define details for dynamic schemas
+        if isinstance(requirement, _meta.SchemaDefinition):
 
-        if dynamic:
-            schema_def = _meta.SchemaDefinition(_meta.SchemaType.TABLE)
-        else:
-            schema_def = self.define_schema(*fields, schema_type=_meta.SchemaType.TABLE)
+            return _meta.ModelOutputSchema(
+                objectType=_meta.ObjectType.DATA, schema=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                outputProps=output_props)
+
+        elif isinstance(requirement, _meta.FileType):
 
-        return _meta.ModelOutputSchema(
-            schema=schema_def, label=label,
-            optional=optional, dynamic=dynamic,
-            outputProps=output_props)
+            return _meta.ModelOutputSchema(
+                objectType=_meta.ObjectType.FILE, fileType=requirement,
+                label=label, optional=optional, dynamic=dynamic,
+                outputProps=output_props)
+
+        else:
+            raise _ex.EUnexpected()
 
     @staticmethod
     def _build_named_dict(

tracdap/rt/_plugins/format_csv.py

@@ -34,7 +34,7 @@ from tracdap.rt.ext.storage import IDataFormat
 from . import _helpers
 
 # TODO: Remove dependency on internal code
-import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.core.data as _data
 
 
 class CsvStorageFormat(IDataFormat):

tracdap/rt/_plugins/repo_git.py

@@ -23,6 +23,7 @@ import time
 import dulwich.repo as git_repo
 import dulwich.client as git_client
 import dulwich.index as git_index
+import urllib3.exceptions  # noqa
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -75,20 +76,45 @@ class GitRepository(IModelRepository):
 
     def do_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
-        self._log.info(
-            f"Git checkout: repo = [{model_def.repository}], " +
-            f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
+        try:
 
-        self._log.info(f"Checkout location: [{checkout_dir}]")
+            self._log.info(
+                f"Git checkout: repo = [{model_def.repository}], " +
+                f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
 
-        if self._native_git:
-            package_path = self._do_native_checkout(model_def, checkout_dir)
-        else:
-            package_path = self._do_python_checkout(model_def, checkout_dir)
+            self._log.info(f"Checkout location: [{checkout_dir}]")
+
+            if self._native_git:
+                package_path = self._do_native_checkout(model_def, checkout_dir)
+            else:
+                package_path = self._do_python_checkout(model_def, checkout_dir)
+
+            self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+
+            return package_path
+
+        except Exception as e:
+
+            error = e
+
+            # For retry failures, try to find the original cause
+            while e.__cause__ is not None:
+                if isinstance(e, urllib3.exceptions.MaxRetryError):
+                    error = e.__cause__
+                    break
+                else:
+                    e = e.__cause__
+
+            # Try to sanitize error messages from urllib3
+            if isinstance(error, urllib3.exceptions.HTTPError):
+                detail = self._clean_urllib3_error(error)
+            else:
+                detail = str(error)
 
-        self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+            message = f"Failed to check out [{model_def.repository}]: {detail}"
 
-        return package_path
+            self._log.error(message)
+            raise ex.EModelRepo(message) from error
 
     def _do_native_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
@@ -150,10 +176,15 @@ class GitRepository(IModelRepository):
                 for line in cmd_err:
                     self._log.info(line)
 
-            else:
+            elif cmd_err:
+
                 for line in cmd_err:
                     self._log.error(line)
 
+                raise ex.EModelRepo(cmd_err[-1])
+
+            else:
+
                 error_msg = f"Git checkout failed for {model_def.package} {model_def.version}"
                 self._log.error(error_msg)
                 raise ex.EModelRepo(error_msg)
@@ -265,6 +296,20 @@ class GitRepository(IModelRepository):
     def _ref_key(key):
         return bytes(key, "ascii")
 
+    @classmethod
+    def _clean_urllib3_error(cls, error: urllib3.exceptions.HTTPError):
+
+        match = cls._URLLIB3_ERROR_PATTERN.match(str(error))
+
+        # Best efforts to clean up the message, fall back on str(error)
+        if match:
+            return match.group(1)
+        else:
+            return str(error)
+
+    # Error message format is like this:
+    # <pkg.ClassName object at 0xXXXXXXX>: Message
+    _URLLIB3_ERROR_PATTERN = re.compile(r"<[^>]*>: (.*)")
 
 # Register plugin
 plugins.PluginManager.register_plugin(IModelRepository, GitRepository, ["git"])

tracdap/rt/_plugins/storage_sql.py

@@ -30,7 +30,7 @@ from tracdap.rt._impl.ext.sql import *  # noqa
 import tracdap.rt._plugins._helpers as _helpers
 
 # TODO: Remove internal references
-import tracdap.rt._impl.data as _data
+import tracdap.rt._impl.core.data as _data
 
 
 class SqlDataStorage(IDataStorageBase[pa.Table, pa.Schema]):
@@ -268,10 +268,18 @@ plugins.PluginManager.register_plugin(IStorageProvider, SqlStorageProvider, ["SQ
 
 
 try:
-
     import sqlalchemy as sqla  # noqa
     import sqlalchemy.exc as sqla_exc  # noqa
 
+    # Only 2.x versions of SQL Alchemy are currently supported
+    sqla_supported = sqla.__version__.startswith("2.")
+
+except ModuleNotFoundError:
+    sqla = None
+    sqla_supported = False
+
+if sqla_supported:
+
     class SqlAlchemyDriver(ISqlDriver):
 
         def __init__(self, properties: tp.Dict[str, str]):
@@ -336,7 +344,7 @@ try:
 
         class ConnectionWrapper(DbApiWrapper.Connection):
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
 
             def close(self):
@@ -355,7 +363,7 @@ try:
 
             arraysize: int = 1000
 
-            def __init__(self, conn: sqla.Connection):
+            def __init__(self, conn: "sqla.Connection"):
                 self.__conn = conn
                 self.__result: tp.Optional[sqla.CursorResult] = None
 
@@ -414,5 +422,4 @@ try:
 
     plugins.PluginManager.register_plugin(ISqlDriver, SqlAlchemyDriver, ["alchemy"])
 
-except ModuleNotFoundError:
-    pass
+

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.7.0rc1"
+__version__ = "0.8.0"

tracdap/rt/api/__init__.py

@@ -17,32 +17,13 @@
 TRAC model API for Python
 """
 
-from .model_api import *
-from .static_api import *
-
 # Make metadata classes available to client code when importing the API package
 # Remove this import when generating docs, so metadata classes are only documented once
 from tracdap.rt.metadata import *  # noqa DOCGEN_REMOVE
 
-# Map basic types into the root of the API package
-
-BOOLEAN = BasicType.BOOLEAN
-"""Synonym for :py:attr:`BasicType.BOOLEAN <tracdap.rt.metadata.BasicType.BOOLEAN>`"""
-
-INTEGER = BasicType.INTEGER
-"""Synonym for :py:attr:`BasicType.INTEGER <tracdap.rt.metadata.BasicType.INTEGER>`"""
-
-FLOAT = BasicType.FLOAT
-"""Synonym for :py:attr:`BasicType.FLOAT <tracdap.rt.metadata.BasicType.FLOAT>`"""
-
-DECIMAL = BasicType.DECIMAL
-"""Synonym for :py:attr:`BasicType.DECIMAL <tracdap.rt.metadata.BasicType.DECIMAL>`"""
-
-STRING = BasicType.STRING
-"""Synonym for :py:attr:`BasicType.STRING <tracdap.rt.metadata.BasicType.STRING>`"""
-
-DATE = BasicType.DATE
-"""Synonym for :py:attr:`BasicType.DATE <tracdap.rt.metadata.BasicType.DATE>`"""
+# static_api overrides some metadata types for backwards compatibility with pre-0.8 versions
+# Make sure it is last in the list
+from .constants import *
+from .model_api import *
+from .static_api import *
 
-DATETIME = BasicType.DATETIME
-"""Synonym for :py:attr:`BasicType.DATETIME <tracdap.rt.metadata.BasicType.DATETIME>`"""

tracdap/rt/api/constants.py

@@ -0,0 +1,57 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import tracdap.rt.metadata
+
+
+class CommonFileTypes:
+
+    """
+    A collection of common :py:class:`FileTypes <tracdap.rt.metadata.FileType>` to use as model inputs and outputs
+    """
+
+    TXT = tracdap.rt.metadata.FileType("txt", "text/plain")
+
+    JPG = tracdap.rt.metadata.FileType("jpg", "image/jpeg")
+    PNG = tracdap.rt.metadata.FileType("png", "image/png")
+    SVG  = tracdap.rt.metadata.FileType("svg", "image/svg+xml")
+
+    WORD = tracdap.rt.metadata.FileType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document")
+    EXCEL = tracdap.rt.metadata.FileType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+    POWERPOINT = tracdap.rt.metadata.FileType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation")
+
+
+# Map basic types into the root of the API package
+
+BOOLEAN = tracdap.rt.metadata.BasicType.BOOLEAN
+"""Synonym for :py:attr:`BasicType.BOOLEAN <tracdap.rt.metadata.BasicType.BOOLEAN>`"""
+
+INTEGER = tracdap.rt.metadata.BasicType.INTEGER
+"""Synonym for :py:attr:`BasicType.INTEGER <tracdap.rt.metadata.BasicType.INTEGER>`"""
+
+FLOAT = tracdap.rt.metadata.BasicType.FLOAT
+"""Synonym for :py:attr:`BasicType.FLOAT <tracdap.rt.metadata.BasicType.FLOAT>`"""
+
+DECIMAL = tracdap.rt.metadata.BasicType.DECIMAL
+"""Synonym for :py:attr:`BasicType.DECIMAL <tracdap.rt.metadata.BasicType.DECIMAL>`"""
+
+STRING = tracdap.rt.metadata.BasicType.STRING
+"""Synonym for :py:attr:`BasicType.STRING <tracdap.rt.metadata.BasicType.STRING>`"""
+
+DATE = tracdap.rt.metadata.BasicType.DATE
+"""Synonym for :py:attr:`BasicType.DATE <tracdap.rt.metadata.BasicType.DATE>`"""
+
+DATETIME = tracdap.rt.metadata.BasicType.DATETIME
+"""Synonym for :py:attr:`BasicType.DATETIME <tracdap.rt.metadata.BasicType.DATETIME>`"""

tracdap/rt/api/experimental.py

@@ -70,6 +70,8 @@ PANDAS = DataFramework.pandas()
 POLARS = DataFramework.polars()
 """Data framework constant for the Polars data library"""
 
+STRUCT_TYPE = _tp.TypeVar('STRUCT_TYPE')
+
 
 class TracContext(TracContext):
 
@@ -83,6 +85,14 @@ class TracContext(TracContext):
 
         pass
 
+    def get_struct(self, struct_name: str, python_class: _tp.Type[STRUCT_TYPE]) -> STRUCT_TYPE:
+
+        pass
+
+    def put_struct(self, struct_name: str, struct_data: STRUCT_TYPE):
+
+        pass
+
 
 def init_static():
     """Ensure TRAC's static model API is available to use (for static definitions at module or class scope)"""
@@ -108,6 +118,28 @@ def map_type(entry_type: BasicType) -> TypeDescriptor:
     return sa.map_type(entry_type)
 
 
+def define_struct(python_type: _tp.Type[STRUCT_TYPE]) -> SchemaDefinition:
+    """Build schema definition for a STRUCT"""
+    sa = _StaticApiHook.get_instance()
+    return sa.define_struct(python_type)
+
+def define_input_struct(
+        python_type: _tp.Type[STRUCT_TYPE], *,
+        label: _tp.Optional[str] = None, optional: bool = False,
+        input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) -> ModelInputSchema:
+
+    schema = define_struct(python_type)
+    return define_input(schema, label=label, optional=optional, input_props=input_props)
+
+def define_output_struct(
+        python_type: _tp.Type[STRUCT_TYPE], *,
+        label: _tp.Optional[str] = None, optional: bool = False,
+        output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) -> ModelOutputSchema:
+
+    schema = define_struct(python_type)
+    return define_output(schema, label=label, optional=optional, output_props=output_props)
+
+
 class FileType(_enum.Enum):
 
     FILE = 1

tracdap/rt/api/hook.py

@@ -116,11 +116,16 @@ class _StaticApiHook:
     @_abc.abstractmethod
     def define_schema(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) \
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE, dynamic: bool = False) \
             -> _meta.SchemaDefinition:
 
         pass
 
+    @_abc.abstractmethod
+    def define_struct(self, python_type: type) -> _meta.SchemaDefinition:
+
+        pass
+
     @_abc.abstractmethod
     def load_schema(
             self, package: _tp.Union[_ts.ModuleType, str], schema_file: str,
@@ -131,22 +136,36 @@ class _StaticApiHook:
 
     @_abc.abstractmethod
     def infer_schema(self, dataset: _tp.Any) -> _meta.SchemaDefinition:
+
         pass
 
     @_abc.abstractmethod
-    def define_input_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
+    def define_file_type(self, extension: str, mime_type: str) -> _meta.FileType:
+
+        pass
+
+    @_abc.abstractmethod
+    def define_input(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
             input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelInputSchema:
 
         pass
 
     @_abc.abstractmethod
-    def define_output_table(
-            self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],
-            label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
+    def define_output(
+            self, requirement: _tp.Union[_meta.SchemaDefinition, _meta.FileType], *,
+            label: _tp.Optional[str] = None,
+            optional: bool = False, dynamic: bool = False,
             output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
             -> _meta.ModelOutputSchema:
 
         pass
+
+X = _tp.TypeVar("X")
+
+def do_x(x: type[X]) -> X:
+
+    return x.__new__(x)

tracdap/rt/api/model_api.py

@@ -194,6 +194,14 @@ class TracContext(metaclass=_abc.ABCMeta):
 
         pass
 
+    def get_file(self, file_name: str) -> bytes:
+
+        pass
+
+    def get_file_stream(self, file_name: str) -> _tp.ContextManager[_tp.BinaryIO]:
+
+        pass
+
     def put_schema(self, dataset_name: str, schema: SchemaDefinition):
 
         """
@@ -283,6 +291,14 @@ class TracContext(metaclass=_abc.ABCMeta):
 
         pass
 
+    def put_file(self, file_name: str, file_content: _tp.Union[bytes, bytearray]):
+
+        pass
+
+    def put_file_stream(self, file_name: str) -> _tp.ContextManager[_tp.BinaryIO]:
+
+        pass
+
     def log(self) -> _logging.Logger:
 
         """