tracdap-runtime 0.8.0b3__tar.gz → 0.8.0b4__tar.gz

{tracdap_runtime-0.8.0b3/tracdap_runtime.egg-info → tracdap_runtime-0.8.0b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: tracdap-runtime
-Version: 0.8.0b3
+Version: 0.8.0b4
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -13,19 +13,20 @@ Platform: any
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: <3.13,>=3.8
+Requires-Python: <3.14,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: protobuf==5.28.2
-Requires-Dist: pyarrow==16.1.0
+Requires-Dist: protobuf==5.29.3
+Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
-Requires-Dist: dulwich==0.22.1
+Requires-Dist: dulwich==0.22.7
 Requires-Dist: requests==2.32.3
-Requires-Dist: pandas<2.3.0,>=1.2.0
-Requires-Dist: numpy<2.0.0
 Provides-Extra: grpc
-Requires-Dist: grpcio==1.66.1; extra == "grpc"
-Requires-Dist: grpcio-status==1.66.1; extra == "grpc"
+Requires-Dist: grpcio==1.70.0; extra == "grpc"
+Requires-Dist: grpcio-status==1.70.0; extra == "grpc"
+Provides-Extra: pandas
+Requires-Dist: pandas<2.3.0,>=1.2.0; extra == "pandas"
+Requires-Dist: numpy<2.3.0,>=1.20; extra == "pandas"
 Provides-Extra: polars
 Requires-Dist: polars<2.0.0,>=1.0.0; extra == "polars"
 Provides-Extra: pyspark
@@ -64,12 +65,13 @@ Documentation for the TRAC platform is available on our website at
 
 The TRAC runtime for Python has these requirements:
 
-* Python: 3.8 up to 3.12
+* Python: 3.9 up to 3.13
 * Pandas: 1.2 up to 2.2
 * PySpark 3.0 up to 3.5 (optional)
 
-3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
-As of February 2024, the Python libraries for GCP do not yet support Python 3.12.
+3rd party libraries may impose additional constraints on supported versions of key libraries.
+For example, Pandas 1.5 is not available for Python 3.12 or 3.13, while NumPy 2.0 is only
+compatible with Pandas 2.1 and later.
 
 ## Installing the runtime
 

{tracdap_runtime-0.8.0b3 → tracdap_runtime-0.8.0b4}/README.md

@@ -16,12 +16,13 @@ Documentation for the TRAC platform is available on our website at
 
 The TRAC runtime for Python has these requirements:
 
-* Python: 3.8 up to 3.12
+* Python: 3.9 up to 3.13
 * Pandas: 1.2 up to 2.2
 * PySpark 3.0 up to 3.5 (optional)
 
-3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
-As of February 2024, the Python libraries for GCP do not yet support Python 3.12.
+3rd party libraries may impose additional constraints on supported versions of key libraries.
+For example, Pandas 1.5 is not available for Python 3.12 or 3.13, while NumPy 2.0 is only
+compatible with Pandas 2.1 and later.
 
 ## Installing the runtime
 

{tracdap_runtime-0.8.0b3 → tracdap_runtime-0.8.0b4}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tracdap-runtime
-version = 0.8.0b3
+version = 0.8.0b4
 description = Runtime package for building models on the TRAC Data & Analytics Platform
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -38,21 +38,21 @@ packages =
 package_dir = 
 	tracdap = src/tracdap
 	tracdap.rt = src/tracdap/rt
-python_requires = >= 3.8, < 3.13
+python_requires = >= 3.9, < 3.14
 install_requires = 
-	protobuf == 5.28.2
-	pyarrow == 16.1.0
+	protobuf == 5.29.3
+	pyarrow == 18.1.0
 	pyyaml == 6.0.2
-	dulwich == 0.22.1
+	dulwich == 0.22.7
 	requests == 2.32.3
-	
-	pandas >= 1.2.0, < 2.3.0
-	numpy < 2.0.0
 
 [options.extras_require]
 grpc = 
-	grpcio == 1.66.1
-	grpcio-status == 1.66.1
+	grpcio == 1.70.0
+	grpcio-status == 1.70.0
+pandas = 
+	pandas >= 1.2.0, < 2.3.0
+	numpy >= 1.20, < 2.3.0
 polars = 
 	polars >= 1.0.0, < 2.0.0
 pyspark = 

{tracdap_runtime-0.8.0b3 → tracdap_runtime-0.8.0b4}/src/tracdap/rt/_impl/core/config_parser.py

@@ -12,14 +12,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
 
 import dataclasses as _dc
 import decimal
@@ -30,6 +22,7 @@ import json
 import os
 import pathlib
 import re
+import types as ts
 import typing as tp
 import urllib.parse as _urlp
 import uuid
@@ -44,6 +37,11 @@ import tracdap.rt._impl.core.util as _util
 import yaml
 import yaml.parser
 
+try:
+    import pydantic as _pyd  # noqa
+except ModuleNotFoundError:
+    _pyd = None
+
 _T = tp.TypeVar('_T')
 
 
@@ -286,9 +284,17 @@ class ConfigManager:
 
 class ConfigParser(tp.Generic[_T]):
 
-    # The metaclass for generic types varies between versions of the typing library
-    # To work around this, detect the correct metaclass by inspecting a generic type variable
-    __generic_metaclass = type(tp.List[object])
+    # Support both new and old styles for generic, union and optional types
+    # Old-style annotations are still valid, even when the new style is fully supported
+    __generic_types: list[type] = [
+        ts.GenericAlias,
+        type(tp.List[int]),
+        type(tp.Optional[int])
+    ]
+
+    # UnionType was added to the types module in Python 3.10, we support 3.9 (Jan 2025)
+    if hasattr(ts, "UnionType"):
+        __generic_types.append(ts.UnionType)
 
     __primitive_types: tp.Dict[type, callable] = {
         bool: bool,
@@ -328,6 +334,23 @@ class ConfigParser(tp.Generic[_T]):
 
     def _parse_value(self, location: str, raw_value: tp.Any, annotation: type):
 
+        if self._is_dev_mode_location(location):
+
+            if type(raw_value) in ConfigParser.__primitive_types:
+                return self._parse_primitive(location, raw_value, type(raw_value))
+
+            if isinstance(raw_value, list):
+                if len(raw_value) == 0:
+                    return []
+                items = iter((self._child_location(location, i), x) for i, x in enumerate(raw_value))
+                return list(self._parse_value(loc, x, tp.Any) for loc, x in items)
+
+            if isinstance(raw_value, dict):
+                if len(raw_value) == 0:
+                    return {}
+                items = iter((self._child_location(location, k), k, v) for k, v in raw_value.items())
+                return dict((k, self._parse_value(loc, v, tp.Any)) for loc, k, v in items)
+
         if raw_value is None:
             return None
 
@@ -346,24 +369,17 @@ class ConfigParser(tp.Generic[_T]):
             return self._parse_enum(location, raw_value, annotation)
 
         if _dc.is_dataclass(annotation):
+            return self._parse_simple_class(location, raw_value, annotation)
 
-            if isinstance(raw_value, tp.Dict):
-                return self._parse_simple_class(location, raw_value, annotation)
-
-            if self._is_dev_mode_location(location):
-                if type(raw_value) in ConfigParser.__primitive_types:
-                    return self._parse_primitive(location, raw_value, type(raw_value))
-                if isinstance(raw_value, list):
-                    if len(raw_value) == 0:
-                        return []
-                    list_type = type(raw_value[0])
-                    return list(map(lambda x: self._parse_primitive(location, x, list_type), raw_value))
+        # Basic support for Pydantic, if it is installed
+        if _pyd and isinstance(annotation, type) and issubclass(annotation, _pyd.BaseModel):
+            return self._parse_simple_class(location, raw_value, annotation)
 
-            return self._error(location, f"Expected type {annotation.__name__}, got '{str(raw_value)}'")
-
-        if isinstance(annotation, self.__generic_metaclass):
+        if any(map(lambda _t: isinstance(annotation, _t), self.__generic_types)):
             return self._parse_generic_class(location, raw_value, annotation)  # noqa
 
+        return self._error(location, f"Cannot parse value of type {annotation.__name__}")
+
     def _is_dev_mode_location(self, location):
 
         return any(map(lambda pattern: re.match(pattern, location), self._dev_mode_locations))
@@ -423,14 +439,14 @@ class ConfigParser(tp.Generic[_T]):
     def _parse_simple_class(self, location: str, raw_dict: tp.Any, metaclass: type) -> object:
 
         if raw_dict is not None and not isinstance(raw_dict, dict):
-            pass
+            return self._error(location, f"Expected type {metaclass.__name__}, got '{str(raw_dict)}'")
 
         obj = metaclass.__new__(metaclass, object())  # noqa
 
         init_signature = inspect.signature(metaclass.__init__)
         init_types = tp.get_type_hints(metaclass.__init__)
         init_params = iter(init_signature.parameters.items())
-        init_values: tp.List[tp.Any] = list()
+        init_values: tp.Dict[str, tp.Any] = dict()
 
         # Do not process 'self'
         next(init_params)
@@ -444,20 +460,20 @@ class ConfigParser(tp.Generic[_T]):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Missing type information for init parameter '{param_name}'"
                 self._error(location, message)
-                init_values.append(None)
+                init_values[param_name] = None
 
             elif param_name in raw_dict and raw_dict[param_name] is not None:
                 param_value = self._parse_value(param_location, raw_dict[param_name], param_type)
-                init_values.append(param_value)
+                init_values[param_name] = param_value
 
             elif param.default != inspect._empty:  # noqa
-                init_values.append(param.default)
+                init_values[param_name] = param.default
 
             else:
                 self._error(location, f"Missing required value '{param_name}'")
-                init_values.append(None)
+                init_values[param_name] = None
 
-        binding = init_signature.bind(obj, *init_values)
+        binding = init_signature.bind(obj, **init_values)
         metaclass.__init__(*binding.args, **binding.kwargs)
 
         # Now go back over the members and look for any that weren't declared in __init__
@@ -478,7 +494,7 @@ class ConfigParser(tp.Generic[_T]):
                 self._error(location, message)
 
             # Generic members must be declared in __init__ since that is the only way to get the full annotation
-            if isinstance(type(default_value), self.__generic_metaclass):
+            if any(map(lambda _t: isinstance(type(default_value), _t), self.__generic_types)):
                 message = f"Class {metaclass.__name__} does not support config decoding: " + \
                           f"Members with no default value must be declared in __init__: '{member_name}'"
                 self._error(location, message)
@@ -504,7 +520,7 @@ class ConfigParser(tp.Generic[_T]):
 
         return obj
 
-    def _parse_generic_class(self, location: str, raw_value: tp.Any, metaclass:  __generic_metaclass):
+    def _parse_generic_class(self, location: str, raw_value: tp.Any,  metaclass: type):
 
         origin = _util.get_origin(metaclass)
         args = _util.get_args(metaclass)
@@ -517,7 +533,7 @@ class ConfigParser(tp.Generic[_T]):
                 return self._error(location, f"Expected a list, got {type(raw_value)}")
 
             return [
-                self._parse_value(self._child_location(location, str(idx)), item, list_type)
+                self._parse_value(self._child_location(location, idx), item, list_type)
                 for (idx, item) in enumerate(raw_value)]
 
         if origin == tp.Dict or origin == dict:
@@ -548,12 +564,14 @@ class ConfigParser(tp.Generic[_T]):
         return None
 
     @staticmethod
-    def _child_location(parent_location: str, item: str):
+    def _child_location(parent_location: str, item: tp.Union[str, int]):
 
         if parent_location is None or parent_location == "":
             return item
+        elif isinstance(item, int):
+            return f"{parent_location}[{item}]"
         else:
-            return parent_location + "." + item
+            return f"{parent_location}.{item}"
 
 
 class ConfigQuoter:

{tracdap_runtime-0.8.0b3 → tracdap_runtime-0.8.0b4}/src/tracdap/rt/_impl/core/data.py

@@ -44,6 +44,7 @@ import tracdap.rt._impl.core.logging as _log
 class DataSpec:
 
     object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
     data_item: str
 
     data_def: _meta.DataDefinition
@@ -58,8 +59,15 @@ class DataSpec:
             storage_def: _meta.StorageDefinition,
             schema_def: tp.Optional[_meta.SchemaDefinition] = None) -> "DataSpec":
 
+        if schema_def:
+            schema_type = schema_def.schemaType
+        elif data_def.schema:
+            schema_type = data_def.schema.schemaType
+        else:
+            schema_type = _meta.SchemaType.SCHEMA_TYPE_NOT_SET
+
         return DataSpec(
-            _meta.ObjectType.DATA, data_item,
+            _meta.ObjectType.DATA, schema_type, data_item,
             data_def,
             storage_def=storage_def,
             schema_def=schema_def,
@@ -72,15 +80,15 @@ class DataSpec:
             storage_def: _meta.StorageDefinition) -> "DataSpec":
 
         return DataSpec(
-            _meta.ObjectType.FILE, data_item,
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET, data_item,
             file_def=file_def,
             storage_def=storage_def,
             data_def=None,
             schema_def=None)
 
     @staticmethod
-    def create_empty_spec(object_type: _meta.ObjectType):
-        return DataSpec(object_type, None, None, None, None, None)
+    def create_empty_spec(object_type: _meta.ObjectType, schema_type: _meta.SchemaType):
+        return DataSpec(object_type, schema_type, None, None, None, None, None)
 
     def is_empty(self):
         return self.data_item is None or len(self.data_item) == 0
@@ -100,32 +108,54 @@ class DataPartKey:
 class DataItem:
 
     object_type: _meta.ObjectType
+    schema_type: _meta.SchemaType
+
+    content: tp.Any = None
+    content_type: tp.Type = None
+    content_func: tp.Callable[[], tp.Any] = None
+
+    trac_schema: _meta.SchemaDefinition = None
+    native_schema: tp.Any = None
 
+    # TODO: Remove legacy API and use content / native_schema instead
     schema: pa.Schema = None
     table: tp.Optional[pa.Table] = None
-    batches: tp.Optional[tp.List[pa.RecordBatch]] = None
 
-    pandas: "tp.Optional[pandas.DataFrame]" = None
-    pyspark: tp.Any = None
+    def is_empty(self) -> bool:
+        return self.content is None
 
-    raw_bytes: bytes = None
+    @staticmethod
+    def create_empty(
+            object_type: _meta.ObjectType = _meta.ObjectType.DATA,
+            schema_type: _meta.SchemaType = _meta.SchemaType.TABLE) -> "DataItem":
 
-    def is_empty(self) -> bool:
-        if self.object_type == _meta.ObjectType.FILE:
-            return self.raw_bytes is None or len(self.raw_bytes) == 0
+        if object_type == _meta.ObjectType.DATA and schema_type == _meta.SchemaType.TABLE:
+            return DataItem(_meta.ObjectType.DATA, _meta.SchemaType.TABLE, schema=pa.schema([]))
         else:
-            return self.table is None and (self.batches is None or len(self.batches) == 0)
+            return DataItem(object_type, schema_type)
 
     @staticmethod
-    def create_empty(object_type: _meta.ObjectType = _meta.ObjectType.DATA) -> "DataItem":
-        if object_type == _meta.ObjectType.DATA:
-            return DataItem(_meta.ObjectType.DATA, pa.schema([]))
-        else:
-            return DataItem(object_type)
+    def for_table(table: pa.Table, schema: pa.Schema, trac_schema: _meta.SchemaDefinition) -> "DataItem":
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.TABLE,
+            content=table, content_type=pa.Table,
+            trac_schema=trac_schema, native_schema=schema,
+            table=table, schema=schema)
+
+    @staticmethod
+    def for_struct(content: tp.Any):
+
+        return DataItem(
+            _meta.ObjectType.DATA, _meta.SchemaType.STRUCT,
+            content=content, content_type=type(content))
 
     @staticmethod
-    def for_file_content(raw_bytes: bytes):
-        return DataItem(_meta.ObjectType.FILE, raw_bytes=raw_bytes)
+    def for_file_content(content: bytes):
+
+        return DataItem(
+            _meta.ObjectType.FILE, _meta.SchemaType.SCHEMA_TYPE_NOT_SET,
+            content=content, content_type=bytes)
 
 
 @dc.dataclass(frozen=True)
@@ -148,8 +178,11 @@ class DataView:
 
     @staticmethod
     def for_trac_schema(trac_schema: _meta.SchemaDefinition):
-        arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
-        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+        if trac_schema.schemaType == _meta.SchemaType.TABLE:
+            arrow_schema = DataMapping.trac_to_arrow_schema(trac_schema)
+            return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+        else:
+            return DataView(_meta.ObjectType.DATA, trac_schema, parts = dict())
 
     @staticmethod
     def for_file_item(file_item: DataItem):
@@ -381,29 +414,27 @@ class DataMapping:
         if not deltas:
             raise _ex.ETracInternal(f"Data view for part [{part.opaque_key}] does not contain any items")
 
+        # For a single delta, use the existing Arrow content
         if len(deltas) == 1:
             return cls.item_to_arrow(deltas[0])
 
-        batches = {
+        # For multiple deltas, construct a new table by assembling the record batches
+        # Atm no consideration is given to overwriting records based on business key
+        batches = iter(
             batch
             for delta in deltas
-            for batch in (
-                delta.batches
-                if delta.batches
-                else delta.table.to_batches())}
+            for batch in cls.item_to_arrow(delta).to_batches())
 
         return pa.Table.from_batches(batches) # noqa
 
     @classmethod
     def item_to_arrow(cls, item: DataItem) -> pa.Table:
 
-        if item.table is not None:
-            return item.table
-
-        if item.batches is not None:
-            return pa.Table.from_batches(item.batches, item.schema)  # noqa
+        if item.content_type != pa.Table:
+            detail = f"expected Arrow table, got [{item.content_type}]"
+            raise _ex.ETracInternal(f"Data item does not contain tabular data ({detail})")
 
-        raise _ex.ETracInternal(f"Data item does not contain any usable data")
+        return item.content
 
     @classmethod
     def arrow_to_pandas(

{tracdap_runtime-0.8.0b3 → tracdap_runtime-0.8.0b4}/src/tracdap/rt/_impl/core/storage.py

@@ -496,7 +496,10 @@ class CommonFileStorage(IFileStorage):
 
         # For successful write streams, log the total size written
         if is_write and not error:
-            file_size = _util.format_file_size(stream.tell())
+            if not stream.closed:
+                file_size = _util.format_file_size(stream.tell())
+            else:
+                file_size = self._fs.get_file_info(storage_path).size
             self._log.info(f"File size [{self._key}]: {file_size} [{storage_path}]")
 
         # Close the stream - this may take time for write streams that are not flushed