tracdap-runtime 0.8.0b2__py3-none-any.whl → 0.8.0b4__py3-none-any.whl

tracdap/rt/_impl/core/__init__.py

@@ -0,0 +1,14 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

tracdap/rt/_impl/{config_parser.py → core/config_parser.py}

@@ -13,8 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import dataclasses as _dc
 import decimal
 import enum
@@ -24,6 +22,7 @@ import json
 import os
 import pathlib
 import re
+import types as ts
 import typing as tp
 import urllib.parse as _urlp
 import uuid
@@ -32,19 +31,24 @@ import tracdap.rt.config as _config
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
 import tracdap.rt.ext.config as _config_ext
-import tracdap.rt._impl.logging as _logging
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.util as _util
 
 import yaml
 import yaml.parser
 
+try:
+    import pydantic as _pyd  # noqa
+except ModuleNotFoundError:
+    _pyd = None
+
 _T = tp.TypeVar('_T')
 
 
 class ConfigManager:
 
     @classmethod
-    def for_root_config(cls, root_config_file: tp.Union[str, pathlib.Path, None]) -> ConfigManager:
+    def for_root_config(cls, root_config_file: tp.Union[str, pathlib.Path, None]) -> "ConfigManager":
 
         if isinstance(root_config_file, pathlib.Path):
             root_file_path = cls._resolve_scheme(root_config_file)
@@ -70,7 +74,7 @@ class ConfigManager:
             return ConfigManager(working_dir_url, None)
 
     @classmethod
-    def for_root_dir(cls, root_config_dir: tp.Union[str, pathlib.Path]) -> ConfigManager:
+    def for_root_dir(cls, root_config_dir: tp.Union[str, pathlib.Path]) -> "ConfigManager":
 
         if isinstance(root_config_dir, pathlib.Path):
             root_dir_path = cls._resolve_scheme(root_config_dir)
@@ -280,9 +284,17 @@ class ConfigManager:
 
 class ConfigParser(tp.Generic[_T]):
 
-    # The metaclass for generic types varies between versions of the typing library
-    # To work around this, detect the correct metaclass by inspecting a generic type variable
-    __generic_metaclass = type(tp.List[object])
+    # Support both new and old styles for generic, union and optional types
+    # Old-style annotations are still valid, even when the new style is fully supported
+    __generic_types: list[type] = [
+        ts.GenericAlias,
+        type(tp.List[int]),
+        type(tp.Optional[int])
+    ]
+
+    # UnionType was added to the types module in Python 3.10, we support 3.9 (Jan 2025)
+    if hasattr(ts, "UnionType"):
+        __generic_types.append(ts.UnionType)
 
     __primitive_types: tp.Dict[type, callable] = {
         bool: bool,
@@ -322,6 +334,23 @@ class ConfigParser(tp.Generic[_T]):
 
     def _parse_value(self, location: str, raw_value: tp.Any, annotation: type):
 
+        if self._is_dev_mode_location(location):
+
+            if type(raw_value) in ConfigParser.__primitive_types:
+                return self._parse_primitive(location, raw_value, type(raw_value))
+
+            if isinstance(raw_value, list):
+                if len(raw_value) == 0:
+                    return []
+                items = iter((self._child_location(location, i), x) for i, x in enumerate(raw_value))
+                return list(self._parse_value(loc, x, tp.Any) for loc, x in items)
+
+            if isinstance(raw_value, dict):
+                if len(raw_value) == 0:
+                    return {}
+                items = iter((self._child_location(location, k), k, v) for k, v in raw_value.items())
+                return dict((k, self._parse_value(loc, v, tp.Any)) for loc, k, v in items)
+
         if raw_value is None:
             return None
 
@@ -340,24 +369,17 @@ class ConfigParser(tp.Generic[_T]):
             return self._parse_enum(location, raw_value, annotation)
 
         if _dc.is_dataclass(annotation):
+            return self._parse_simple_class(location, raw_value, annotation)
 
-            if isinstance(raw_value, tp.Dict):
-                return self._parse_simple_class(location, raw_value, annotation)
+        # Basic support for Pydantic, if it is installed
+        if _pyd and isinstance(annotation, type) and issubclass(annotation, _pyd.BaseModel):
+            return self._parse_simple_class(location, raw_value, annotation)
 
-            if self._is_dev_mode_location(location):
-                if type(raw_value) in ConfigParser.__primitive_types:
-                    return self._parse_primitive(location, raw_value, type(raw_value))
-                if isinstance(raw_value, list):
-                    if len(raw_value) == 0:
-                        return []
-                    list_type = type(raw_value[0])
-                    return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
-
-            return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
-
-        if isinstance(annotation, self.__generic_metaclass):
+        if any(map(lambda _t: isinstance(annotation, _t), self.__generic_types)):
             return self._parse_generic_class(location, raw_value, annotation)  # noqa
 
+        return self._error(location, f"Cannot parse value of type {annotation.__name__}")
+
     def _is_dev_mode_location(self, location):
 
         return any(map(lambda pattern: re.match(pattern, location), self._dev_mode_locations))
@@ -417,14 +439,14 @@ class ConfigParser(tp.Generic[_T]):
     def _parse_simple_class(self, location: str, raw_dict: tp.Any, metaclass: type) -> object:
 
         if raw_dict is not None and not isinstance(raw_dict, dict):
-            pass
+            return self._error(location, f"Expected type {metaclass.__name__}, got '{str(raw_dict)}'")
 
         obj = metaclass.__new__(metaclass, object())  # noqa
 
         init_signature = inspect.signature(metaclass.__init__)
         init_types = tp.get_type_hints(metaclass.__init__)
         init_params = iter(init_signature.parameters.items())
-        init_values: tp.List[tp.Any] = list()
+        init_values: tp.Dict[str, tp.Any] = dict()
 
         # Do not process 'self'
         next(init_params)
@@ -438,20 +460,20 @@ class ConfigParser(tp.Generic[_T]):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Missing type information for init parameter '{param_name}'"
                 self._error(location, message)
-                init_values.append(None)
+                init_values[param_name] = None
 
             elif param_name in raw_dict and raw_dict[param_name] is not None:
                 param_value = self._parse_value(param_location, raw_dict[param_name], param_type)
-                init_values.append(param_value)
+                init_values[param_name] = param_value
 
             elif param.default != inspect._empty:  # noqa
-                init_values.append(param.default)
+                init_values[param_name] = param.default
 
             else:
                 self._error(location, f"Missing required value '{param_name}'")
-                init_values.append(None)
+                init_values[param_name] = None
 
-        binding = init_signature.bind(obj, *init_values)
+        binding = init_signature.bind(obj, **init_values)
         metaclass.__init__(*binding.args, **binding.kwargs)
 
         # Now go back over the members and look for any that weren't declared in __init__
@@ -472,7 +494,7 @@ class ConfigParser(tp.Generic[_T]):
                 self._error(location, message)
 
             # Generic members must be declared in __init__ since that is the only way to get the full annotation
-            if isinstance(type(default_value), self.__generic_metaclass):
+            if any(map(lambda _t: isinstance(type(default_value), _t), self.__generic_types)):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Members with no default value must be declared in __init__: '{member_name}'"
                 self._error(location, message)
@@ -498,7 +520,7 @@ class ConfigParser(tp.Generic[_T]):
 
         return obj
 
-    def _parse_generic_class(self, location: str, raw_value: tp.Any, metaclass:  __generic_metaclass):
+    def _parse_generic_class(self, location: str, raw_value: tp.Any,  metaclass: type):
 
         origin = _util.get_origin(metaclass)
         args = _util.get_args(metaclass)
@@ -511,7 +533,7 @@ class ConfigParser(tp.Generic[_T]):
                 return self._error(location, f"Expected a list, got {type(raw_value)}")
 
             return [
-                self._parse_value(self._child_location(location, str(idx)), item, list_type)
+                self._parse_value(self._child_location(location, idx), item, list_type)
                 for (idx, item) in enumerate(raw_value)]
 
         if origin == tp.Dict or origin == dict:
@@ -542,12 +564,14 @@ class ConfigParser(tp.Generic[_T]):
         return None
 
     @staticmethod
-    def _child_location(parent_location: str, item: str):
+    def _child_location(parent_location: str, item: tp.Union[str, int]):
 
         if parent_location is None or parent_location == "":
             return item
+        elif isinstance(item, int):
+            return f"{parent_location}[{item}]"
         else:
-            return parent_location + "." + item
+            return f"{parent_location}.{item}"
 
 
 class ConfigQuoter:

tracdap/rt/_impl/{data.py → core/data.py}

@@ -37,13 +37,14 @@ except ModuleNotFoundError:
 import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.logging as _log
+import tracdap.rt._impl.core.logging as _log
 
 
 @dc.dataclass(frozen=True)
 class DataSpec:
 
     object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
     data_item: str
 
     data_def: _meta.DataDefinition
@@ -58,8 +59,15 @@ class DataSpec:
             storage_def: _meta.StorageDefinition,
             schema_def: tp.Optional[_meta.SchemaDefinition] = None) -> "DataSpec":
 
+        if schema_def:
+            schema_type = schema_def.schemaType
+        elif data_def.schema:
+            schema_type = data_def.schema.schemaType
+        else:
+            schema_type = _meta.SchemaType.SCHEMA_TYPE_NOT_SET
+
         return DataSpec(
-            _meta.ObjectType.DATA, data_item,
+            _meta.ObjectType.DATA, schema_type, data_item,
             data_def,
             storage_def=storage_def,
             schema_def=schema_def,
@@ -72,15 +80,15 @@ class DataSpec:
             storage_def: _meta.StorageDefinition) -> "DataSpec":
 
         return DataSpec(
-            _meta.ObjectType.FILE, data_item,
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET, data_item,
             file_def=file_def,
             storage_def=storage_def,
             data_def=None,
             schema_def=None)
 
     @staticmethod
-    def create_empty_spec(object_type: _meta.ObjectType):
-        return DataSpec(object_type, None, None, None, None, None)
+    def create_empty_spec(object_type: _meta.ObjectType, schema_type: _meta.SchemaType):
+        return DataSpec(object_type, schema_type, None, None, None, None, None)
 
     def is_empty(self):
         return self.data_item is None or len(self.data_item) == 0
@@ -100,32 +108,54 @@ class DataPartKey:
 class DataItem:
 
     object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
+
+    content: tp.Any = None
+    content_type: tp.Type = None
+    content_func: tp.Callable[[], tp.Any] = None
+
+    trac_schema: _meta.SchemaDefinition = None
+    native_schema: tp.Any = None
 
+    # TODO: Remove legacy API and use content / native_schema instead
     schema: pa.Schema = None
     table: tp.Optional[pa.Table] = None
-    batches: tp.Optional[tp.List[pa.RecordBatch]] = None
 
-    pandas: "tp.Optional[pandas.DataFrame]" = None
-    pyspark: tp.Any = None
+    def is_empty(self) -> bool:
+        return self.content is None
 
-    raw_bytes: bytes = None
+    @staticmethod
+    def create_empty(
+            object_type: _meta.ObjectType = _meta.ObjectType.DATA,
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) -> "DataItem":
 
-    def is_empty(self) -> bool:
-        if self.object_type == _meta.ObjectType.FILE:
-            return self.raw_bytes is None or len(self.raw_bytes) == 0
+        if object_type == _meta.ObjectType.DATA and schema_type == _meta.SchemaType.TABLE:
+            return DataItem(_meta.ObjectType.DATA, _meta.SchemaType.TABLE, schema=pa.schema([]))
         else:
-            return self.table is None and (self.batches is None or len(self.batches) == 0)
+            return DataItem(object_type, schema_type)
 
     @staticmethod
-    def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataItem":
-        if object_type == _meta.ObjectType.DATA:
-            return DataItem(_meta.ObjectType.DATA, pa.schema([]))
-        else:
-            return DataItem(object_type)
+    def for_table(table: pa.Table, schema: pa.Schema, trac_schema: _meta.SchemaDefinition) -> "DataItem":
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.TABLE,
+            content=table, content_type=pa.Table,
+            trac_schema=trac_schema, native_schema=schema,
+            table=table, schema=schema)
+
+    @staticmethod
+    def for_struct(content: tp.Any):
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.STRUCT,
+            content=content, content_type=type(content))
 
     @staticmethod
-    def for_file_content(raw_bytes: bytes):
-        return DataItem(_meta.ObjectType.FILE, raw_bytes=raw_bytes)
+    def for_file_content(content: bytes):
+
+        return DataItem(
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET,
+            content=content, content_type=bytes)
 
 
 @dc.dataclass(frozen=True)
@@ -148,8 +178,11 @@ class DataView:
 
     @staticmethod
     def for_trac_schema(trac_schema: _meta.SchemaDefinition):
-        arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+        if trac_schema.schemaType == _meta.SchemaType.TABLE:
+            arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
+            return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+        else:
+            return DataView(_meta.ObjectType.DATA, trac_schema, parts = dict())
 
     @staticmethod
     def for_file_item(file_item: DataItem):
@@ -381,29 +414,27 @@ class DataMapping:
         if not deltas:
             raise _ex.ETracInternal(f"Data view for part [{part.opaque_key}] does not contain any items")
 
+        # For a single delta, use the existing Arrow content
         if len(deltas) == 1:
             return cls.item_to_arrow(deltas[0])
 
-        batches = {
+        # For multiple deltas, construct a new table by assembling the record batches
+        # Atm no consideration is given to overwriting records based on business key
+        batches = iter(
             batch
             for delta in deltas
-            for batch in (
-                delta.batches
-                if delta.batches
-                else delta.table.to_batches())}
+            for batch in cls.item_to_arrow(delta).to_batches())
 
         return pa.Table.from_batches(batches) # noqa
 
     @classmethod
     def item_to_arrow(cls, item: DataItem) -> pa.Table:
 
-        if item.table is not None:
-            return item.table
-
-        if item.batches is not None:
-            return pa.Table.from_batches(item.batches, item.schema)  # noqa
+        if item.content_type != pa.Table:
+            detail = f"expected Arrow table, got [{item.content_type}]"
+            raise _ex.ETracInternal(f"Data item does not contain tabular data ({detail})")
 
-        raise _ex.ETracInternal(f"Data item does not contain any usable data")
+        return item.content
 
     @classmethod
     def arrow_to_pandas(

tracdap/rt/_impl/{models.py → core/models.py}

@@ -25,12 +25,12 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 
-import tracdap.rt._impl.logging as _logging
-import tracdap.rt._impl.repos as _repos
-import tracdap.rt._impl.shim as _shim
-import tracdap.rt._impl.type_system as _types
-import tracdap.rt._impl.util as _util
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.repos as _repos
+import tracdap.rt._impl.core.shim as _shim
+import tracdap.rt._impl.core.type_system as _types
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.core.validation as _val
 
 
 class ModelLoader:

tracdap/rt/_impl/{repos.py → core/repos.py}

@@ -18,7 +18,7 @@ import typing as _tp
 import tracdap.rt.ext.plugins as plugins
 import tracdap.rt.config as cfg
 import tracdap.rt.exceptions as ex
-import tracdap.rt._impl.logging as _logging
+import tracdap.rt._impl.core.logging as _logging
 
 # Import repo interfaces
 from tracdap.rt.ext.repos import *

tracdap/rt/_impl/{schemas.py → core/schemas.py}

@@ -21,10 +21,10 @@ import pyarrow as pa
 
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.logging as _log
-import tracdap.rt._impl.storage as _storage
-import tracdap.rt._impl.shim as _shim
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _log
+import tracdap.rt._impl.core.storage as _storage
+import tracdap.rt._impl.core.shim as _shim
 
 
 class SchemaLoader:

tracdap/rt/_impl/{shim.py → core/shim.py}

@@ -30,9 +30,9 @@ import importlib.machinery as _ilm
 import importlib.resources as _ilr
 
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.guard_rails as _guard
-import tracdap.rt._impl.logging as _log
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.guard_rails as _guard
+import tracdap.rt._impl.core.logging as _log
+import tracdap.rt._impl.core.util as _util
 
 
 class _Shim:

tracdap/rt/_impl/{storage.py → core/storage.py}

@@ -29,10 +29,10 @@ import tracdap.rt.metadata as _meta
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as plugins
-import tracdap.rt._impl.data as _data
-import tracdap.rt._impl.logging as _logging
-import tracdap.rt._impl.util as _util
-import tracdap.rt._impl.validation as _val
+import tracdap.rt._impl.core.data as _data
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.util as _util
+import tracdap.rt._impl.core.validation as _val
 
 # Import storage interfaces (using the internal version, it has extra bits that are not public)
 from tracdap.rt._impl.ext.storage import *
@@ -496,7 +496,10 @@ class CommonFileStorage(IFileStorage):
 
         # For successful write streams, log the total size written
         if is_write and not error:
-            file_size = _util.format_file_size(stream.tell())
+            if not stream.closed:
+                file_size = _util.format_file_size(stream.tell())
+            else:
+                file_size = self._fs.get_file_info(storage_path).size
             self._log.info(f"File size [{self._key}]: {file_size} [{storage_path}]")
 
         # Close the stream - this may take time for write streams that are not flushed