tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/_impl/core/__init__.py

@@ -0,0 +1,14 @@
+#  Licensed to the Fintech Open Source Foundation (FINOS) under one or
+#  more contributor license agreements. See the NOTICE file distributed
+#  with this work for additional information regarding copyright ownership.
+#  FINOS licenses this file to you under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with the
+#  License. You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.

tracdap/rt/_impl/{config_parser.py → core/config_parser.py}

@@ -13,8 +13,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import dataclasses as _dc
 import decimal
 import enum
@@ -24,6 +22,7 @@ import json
 import os
 import pathlib
 import re
+import types as ts
 import typing as tp
 import urllib.parse as _urlp
 import uuid
@@ -32,18 +31,24 @@ import tracdap.rt.config as _config
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
 import tracdap.rt.ext.config as _config_ext
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.logging as _logging
+import tracdap.rt._impl.core.util as _util
 
 import yaml
 import yaml.parser
 
+try:
+    import pydantic as _pyd  # noqa
+except ModuleNotFoundError:
+    _pyd = None
+
 _T = tp.TypeVar('_T')
 
 
 class ConfigManager:
 
     @classmethod
-    def for_root_config(cls, root_config_file: tp.Union[str, pathlib.Path, None]) -> ConfigManager:
+    def for_root_config(cls, root_config_file: tp.Union[str, pathlib.Path, None]) -> "ConfigManager":
 
         if isinstance(root_config_file, pathlib.Path):
             root_file_path = cls._resolve_scheme(root_config_file)
@@ -69,7 +74,7 @@ class ConfigManager:
             return ConfigManager(working_dir_url, None)
 
     @classmethod
-    def for_root_dir(cls, root_config_dir: tp.Union[str, pathlib.Path]) -> ConfigManager:
+    def for_root_dir(cls, root_config_dir: tp.Union[str, pathlib.Path]) -> "ConfigManager":
 
         if isinstance(root_config_dir, pathlib.Path):
             root_dir_path = cls._resolve_scheme(root_config_dir)
@@ -103,7 +108,7 @@ class ConfigManager:
             return raw_url
 
     def __init__(self, root_dir_url: _urlp.ParseResult, root_file_url: tp.Optional[_urlp.ParseResult]):
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._root_dir_url = root_dir_url
         self._root_file_url = root_file_url
 
@@ -279,9 +284,17 @@ class ConfigManager:
 
 class ConfigParser(tp.Generic[_T]):
 
-    # The metaclass for generic types varies between versions of the typing library
-    # To work around this, detect the correct metaclass by inspecting a generic type variable
-    __generic_metaclass = type(tp.List[object])
+    # Support both new and old styles for generic, union and optional types
+    # Old-style annotations are still valid, even when the new style is fully supported
+    __generic_types: list[type] = [
+        ts.GenericAlias,
+        type(tp.List[int]),
+        type(tp.Optional[int])
+    ]
+
+    # UnionType was added to the types module in Python 3.10, we support 3.9 (Jan 2025)
+    if hasattr(ts, "UnionType"):
+        __generic_types.append(ts.UnionType)
 
     __primitive_types: tp.Dict[type, callable] = {
         bool: bool,
@@ -294,7 +307,7 @@ class ConfigParser(tp.Generic[_T]):
     }
 
     def __init__(self, config_class: _T.__class__, dev_mode_locations: tp.List[str] = None):
-        self._log = _util.logger_for_object(self)
+        self._log = _logging.logger_for_object(self)
         self._config_class = config_class
         self._dev_mode_locations = dev_mode_locations or []
         self._errors = []
@@ -321,6 +334,23 @@ class ConfigParser(tp.Generic[_T]):
 
     def _parse_value(self, location: str, raw_value: tp.Any, annotation: type):
 
+        if self._is_dev_mode_location(location):
+
+            if type(raw_value) in ConfigParser.__primitive_types:
+                return self._parse_primitive(location, raw_value, type(raw_value))
+
+            if isinstance(raw_value, list):
+                if len(raw_value) == 0:
+                    return []
+                items = iter((self._child_location(location, i), x) for i, x in enumerate(raw_value))
+                return list(self._parse_value(loc, x, tp.Any) for loc, x in items)
+
+            if isinstance(raw_value, dict):
+                if len(raw_value) == 0:
+                    return {}
+                items = iter((self._child_location(location, k), k, v) for k, v in raw_value.items())
+                return dict((k, self._parse_value(loc, v, tp.Any)) for loc, k, v in items)
+
         if raw_value is None:
             return None
 
@@ -339,24 +369,17 @@ class ConfigParser(tp.Generic[_T]):
             return self._parse_enum(location, raw_value, annotation)
 
         if _dc.is_dataclass(annotation):
+            return self._parse_simple_class(location, raw_value, annotation)
 
-            if isinstance(raw_value, tp.Dict):
-                return self._parse_simple_class(location, raw_value, annotation)
+        # Basic support for Pydantic, if it is installed
+        if _pyd and isinstance(annotation, type) and issubclass(annotation, _pyd.BaseModel):
+            return self._parse_simple_class(location, raw_value, annotation)
 
-            if self._is_dev_mode_location(location):
-                if type(raw_value) in ConfigParser.__primitive_types:
-                    return self._parse_primitive(location, raw_value, type(raw_value))
-                if isinstance(raw_value, list):
-                    if len(raw_value) == 0:
-                        return []
-                    list_type = type(raw_value[0])
-                    return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
-
-            return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
-
-        if isinstance(annotation, self.__generic_metaclass):
+        if any(map(lambda _t: isinstance(annotation, _t), self.__generic_types)):
             return self._parse_generic_class(location, raw_value, annotation)  # noqa
 
+        return self._error(location, f"Cannot parse value of type {annotation.__name__}")
+
     def _is_dev_mode_location(self, location):
 
         return any(map(lambda pattern: re.match(pattern, location), self._dev_mode_locations))
@@ -416,14 +439,14 @@ class ConfigParser(tp.Generic[_T]):
     def _parse_simple_class(self, location: str, raw_dict: tp.Any, metaclass: type) -> object:
 
         if raw_dict is not None and not isinstance(raw_dict, dict):
-            pass
+            return self._error(location, f"Expected type {metaclass.__name__}, got '{str(raw_dict)}'")
 
         obj = metaclass.__new__(metaclass, object())  # noqa
 
         init_signature = inspect.signature(metaclass.__init__)
         init_types = tp.get_type_hints(metaclass.__init__)
         init_params = iter(init_signature.parameters.items())
-        init_values: tp.List[tp.Any] = list()
+        init_values: tp.Dict[str, tp.Any] = dict()
 
         # Do not process 'self'
         next(init_params)
@@ -437,20 +460,20 @@ class ConfigParser(tp.Generic[_T]):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Missing type information for init parameter '{param_name}'"
                 self._error(location, message)
-                init_values.append(None)
+                init_values[param_name] = None
 
             elif param_name in raw_dict and raw_dict[param_name] is not None:
                 param_value = self._parse_value(param_location, raw_dict[param_name], param_type)
-                init_values.append(param_value)
+                init_values[param_name] = param_value
 
             elif param.default != inspect._empty:  # noqa
-                init_values.append(param.default)
+                init_values[param_name] = param.default
 
             else:
                 self._error(location, f"Missing required value '{param_name}'")
-                init_values.append(None)
+                init_values[param_name] = None
 
-        binding = init_signature.bind(obj, *init_values)
+        binding = init_signature.bind(obj, **init_values)
         metaclass.__init__(*binding.args, **binding.kwargs)
 
         # Now go back over the members and look for any that weren't declared in __init__
@@ -471,7 +494,7 @@ class ConfigParser(tp.Generic[_T]):
                 self._error(location, message)
 
             # Generic members must be declared in __init__ since that is the only way to get the full annotation
-            if isinstance(type(default_value), self.__generic_metaclass):
+            if any(map(lambda _t: isinstance(type(default_value), _t), self.__generic_types)):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Members with no default value must be declared in __init__: '{member_name}'"
                 self._error(location, message)
@@ -497,7 +520,7 @@ class ConfigParser(tp.Generic[_T]):
 
         return obj
 
-    def _parse_generic_class(self, location: str, raw_value: tp.Any, metaclass:  __generic_metaclass):
+    def _parse_generic_class(self, location: str, raw_value: tp.Any,  metaclass: type):
 
         origin = _util.get_origin(metaclass)
         args = _util.get_args(metaclass)
@@ -510,7 +533,7 @@ class ConfigParser(tp.Generic[_T]):
                 return self._error(location, f"Expected a list, got {type(raw_value)}")
 
             return [
-                self._parse_value(self._child_location(location, str(idx)), item, list_type)
+                self._parse_value(self._child_location(location, idx), item, list_type)
                 for (idx, item) in enumerate(raw_value)]
 
         if origin == tp.Dict or origin == dict:
@@ -541,12 +564,14 @@ class ConfigParser(tp.Generic[_T]):
         return None
 
     @staticmethod
-    def _child_location(parent_location: str, item: str):
+    def _child_location(parent_location: str, item: tp.Union[str, int]):
 
         if parent_location is None or parent_location == "":
             return item
+        elif isinstance(item, int):
+            return f"{parent_location}[{item}]"
         else:
-            return parent_location + "." + item
+            return f"{parent_location}.{item}"
 
 
 class ConfigQuoter:

tracdap/rt/_impl/{data.py → core/data.py}

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 import abc
+import copy
 import dataclasses as dc
 import typing as tp
 import datetime as dt
@@ -36,17 +37,62 @@ except ModuleNotFoundError:
 import tracdap.rt.api.experimental as _api
 import tracdap.rt.metadata as _meta
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.util as _util
+import tracdap.rt._impl.core.logging as _log
 
 
 @dc.dataclass(frozen=True)
 class DataSpec:
 
+    object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
     data_item: str
+
     data_def: _meta.DataDefinition
+    file_def: _meta.FileDefinition
     storage_def: _meta.StorageDefinition
     schema_def: tp.Optional[_meta.SchemaDefinition]
 
+    @staticmethod
+    def create_data_spec(
+            data_item: str,
+            data_def: _meta.DataDefinition,
+            storage_def: _meta.StorageDefinition,
+            schema_def: tp.Optional[_meta.SchemaDefinition] = None) -> "DataSpec":
+
+        if schema_def:
+            schema_type = schema_def.schemaType
+        elif data_def.schema:
+            schema_type = data_def.schema.schemaType
+        else:
+            schema_type = _meta.SchemaType.SCHEMA_TYPE_NOT_SET
+
+        return DataSpec(
+            _meta.ObjectType.DATA, schema_type, data_item,
+            data_def,
+            storage_def=storage_def,
+            schema_def=schema_def,
+            file_def=None)
+
+    @staticmethod
+    def create_file_spec(
+            data_item: str,
+            file_def: _meta.FileDefinition,
+            storage_def: _meta.StorageDefinition) -> "DataSpec":
+
+        return DataSpec(
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET, data_item,
+            file_def=file_def,
+            storage_def=storage_def,
+            data_def=None,
+            schema_def=None)
+
+    @staticmethod
+    def create_empty_spec(object_type: _meta.ObjectType, schema_type: _meta.SchemaType):
+        return DataSpec(object_type, schema_type, None, None, None, None, None)
+
+    def is_empty(self):
+        return self.data_item is None or len(self.data_item) == 0
+
 
 @dc.dataclass(frozen=True)
 class DataPartKey:
@@ -61,44 +107,104 @@ class DataPartKey:
 @dc.dataclass(frozen=True)
 class DataItem:
 
-    schema: pa.Schema
-    table: tp.Optional[pa.Table] = None
-    batches: tp.Optional[tp.List[pa.RecordBatch]] = None
+    object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
+
+    content: tp.Any = None
+    content_type: tp.Type = None
+    content_func: tp.Callable[[], tp.Any] = None
 
-    pandas: "tp.Optional[pandas.DataFrame]" = None
-    pyspark: tp.Any = None
+    trac_schema: _meta.SchemaDefinition = None
+    native_schema: tp.Any = None
+
+    # TODO: Remove legacy API and use content / native_schema instead
+    schema: pa.Schema = None
+    table: tp.Optional[pa.Table] = None
 
     def is_empty(self) -> bool:
-        return self.table is None and (self.batches is None or len(self.batches) == 0)
+        return self.content is None
+
+    @staticmethod
+    def create_empty(
+            object_type: _meta.ObjectType = _meta.ObjectType.DATA,
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) -> "DataItem":
+
+        if object_type == _meta.ObjectType.DATA and schema_type == _meta.SchemaType.TABLE:
+            return DataItem(_meta.ObjectType.DATA, _meta.SchemaType.TABLE, schema=pa.schema([]))
+        else:
+            return DataItem(object_type, schema_type)
 
     @staticmethod
-    def create_empty() -> "DataItem":
-        return DataItem(pa.schema([]))
+    def for_table(table: pa.Table, schema: pa.Schema, trac_schema: _meta.SchemaDefinition) -> "DataItem":
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.TABLE,
+            content=table, content_type=pa.Table,
+            trac_schema=trac_schema, native_schema=schema,
+            table=table, schema=schema)
+
+    @staticmethod
+    def for_struct(content: tp.Any):
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.STRUCT,
+            content=content, content_type=type(content))
+
+    @staticmethod
+    def for_file_content(content: bytes):
+
+        return DataItem(
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET,
+            content=content, content_type=bytes)
 
 
 @dc.dataclass(frozen=True)
 class DataView:
 
-    trac_schema: _meta.SchemaDefinition
-    arrow_schema: pa.Schema
+    object_type: _meta.ObjectType
 
-    parts: tp.Dict[DataPartKey, tp.List[DataItem]]
+    trac_schema: _meta.SchemaDefinition = None
+    arrow_schema: pa.Schema = None
+
+    parts: tp.Dict[DataPartKey, tp.List[DataItem]] = None
+    file_item: tp.Optional[DataItem] = None
 
     @staticmethod
-    def create_empty() -> "DataView":
-        return DataView(_meta.SchemaDefinition(), pa.schema([]), dict())
+    def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataView":
+        if object_type == _meta.ObjectType.DATA:
+            return DataView(object_type, _meta.SchemaDefinition(), pa.schema([]), dict())
+        else:
+            return DataView(object_type)
 
     @staticmethod
     def for_trac_schema(trac_schema: _meta.SchemaDefinition):
-        arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(trac_schema, arrow_schema, dict())
+        if trac_schema.schemaType == _meta.SchemaType.TABLE:
+            arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
+            return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+        else:
+            return DataView(_meta.ObjectType.DATA, trac_schema, parts = dict())
+
+    @staticmethod
+    def for_file_item(file_item: DataItem):
+        return DataView(file_item.object_type, file_item=file_item)
 
     def with_trac_schema(self, trac_schema: _meta.SchemaDefinition):
         arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(trac_schema, arrow_schema, self.parts)
+        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, self.parts)
+
+    def with_part(self, part_key: DataPartKey, part: DataItem):
+        new_parts = copy.copy(self.parts)
+        new_parts[part_key] = [part]
+        return DataView(self.object_type, self.trac_schema, self.arrow_schema, new_parts)
+
+    def with_file_item(self, file_item: DataItem):
+        return DataView(self.object_type, file_item=file_item)
 
     def is_empty(self) -> bool:
-        return self.parts is None or not any(self.parts.values())
+        if self.object_type == _meta.ObjectType.FILE:
+            return self.file_item is None
+        else:
+            return self.parts is None or not any(self.parts.values())
 
 
 class _DataInternal:
@@ -115,7 +221,7 @@ class DataMapping:
     :py:class:`TypeMapping <tracdap.rt.impl.type_system.MetadataCodec>`.
     """
 
-    __log = _util.logger_for_namespace(_DataInternal.__module__ + ".DataMapping")
+    __log = _log.logger_for_namespace(_DataInternal.__module__ + ".DataMapping")
 
     # Matches TRAC_ARROW_TYPE_MAPPING in ArrowSchema, tracdap-lib-data
 
@@ -293,7 +399,7 @@ class DataMapping:
         deltas = [*prior_deltas, item]
         parts = {**view.parts, part: deltas}
 
-        return DataView(view.trac_schema, view.arrow_schema, parts)
+        return DataView(view.object_type, view.trac_schema, view.arrow_schema, parts=parts)
 
     @classmethod
     def view_to_arrow(cls, view: DataView, part: DataPartKey) -> pa.Table:
@@ -308,29 +414,27 @@ class DataMapping:
         if not deltas:
             raise _ex.ETracInternal(f"Data view for part [{part.opaque_key}] does not contain any items")
 
+        # For a single delta, use the existing Arrow content
         if len(deltas) == 1:
             return cls.item_to_arrow(deltas[0])
 
-        batches = {
+        # For multiple deltas, construct a new table by assembling the record batches
+        # Atm no consideration is given to overwriting records based on business key
+        batches = iter(
             batch
             for delta in deltas
-            for batch in (
-                delta.batches
-                if delta.batches
-                else delta.table.to_batches())}
+            for batch in cls.item_to_arrow(delta).to_batches())
 
         return pa.Table.from_batches(batches) # noqa
 
     @classmethod
     def item_to_arrow(cls, item: DataItem) -> pa.Table:
 
-        if item.table is not None:
-            return item.table
-
-        if item.batches is not None:
-            return pa.Table.from_batches(item.batches, item.schema)  # noqa
+        if item.content_type != pa.Table:
+            detail = f"expected Arrow table, got [{item.content_type}]"
+            raise _ex.ETracInternal(f"Data item does not contain tabular data ({detail})")
 
-        raise _ex.ETracInternal(f"Data item does not contain any usable data")
+        return item.content
 
     @classmethod
     def arrow_to_pandas(
@@ -642,7 +746,7 @@ class DataConformance:
     Check and/or apply conformance between datasets and schemas.
     """
 
-    __log = _util.logger_for_namespace(_DataInternal.__module__ + ".DataConformance")
+    __log = _log.logger_for_namespace(_DataInternal.__module__ + ".DataConformance")
 
     __E_FIELD_MISSING = \
         "Field [{field_name}] is missing from the data"