tracdap-runtime 0.8.0rc2__tar.gz → 0.9.0rc1__tar.gz

{tracdap_runtime-0.8.0rc2/tracdap_runtime.egg-info → tracdap_runtime-0.9.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracdap-runtime
-Version: 0.8.0rc2
+Version: 0.9.0rc1
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -21,6 +21,8 @@ Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: dulwich==0.22.7
 Requires-Dist: requests==2.32.3
+Requires-Dist: typing_extensions<4.13
+Requires-Dist: urllib3<2.4.0
 Provides-Extra: grpc
 Requires-Dist: grpcio==1.70.0; extra == "grpc"
 Requires-Dist: grpcio-status==1.70.0; extra == "grpc"

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = tracdap-runtime
-version = 0.8.0rc2
+version = 0.9.0rc1
 description = Runtime package for building models on the TRAC Data & Analytics Platform
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -45,6 +45,8 @@ install_requires =
 	pyyaml == 6.0.2
 	dulwich == 0.22.7
 	requests == 2.32.3
+	typing_extensions < 4.13
+	urllib3 < 2.4.0
 
 [options.extras_require]
 grpc = 

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/core/data.py

@@ -184,6 +184,11 @@ class DataView:
         else:
             return DataView(_meta.ObjectType.DATA, trac_schema, parts = dict())
 
+    @staticmethod
+    def for_arrow_schema(arrow_schema: pa.Schema):
+        trac_schema = DataMapping.arrow_to_trac_schema(arrow_schema)
+        return DataView(_meta.ObjectType.DATA, trac_schema, arrow_schema, dict())
+
     @staticmethod
     def for_file_item(file_item: DataItem):
         return DataView(file_item.object_type, file_item=file_item)
@@ -259,8 +264,17 @@ class DataMapping:
         pa.date64(): _meta.BasicType.DATE
     }
 
-    @staticmethod
-    def arrow_to_python_type(arrow_type: pa.DataType) -> type:
+    # For now, categorical handling is disabled by default and enabled by this setting
+    # The default will change to "true" for the 0.9 release
+    CATEGORICAL_CONFIG_KEY = "trac.runtime.categorical"
+    __categorical_enabled = False
+
+    @classmethod
+    def enable_categorical(cls, enabled: bool):
+        cls.__categorical_enabled = enabled
+
+    @classmethod
+    def arrow_to_python_type(cls, arrow_type: pa.DataType) -> type:
 
         if pa.types.is_boolean(arrow_type):
             return bool
@@ -283,6 +297,11 @@ class DataMapping:
         if pa.types.is_timestamp(arrow_type):
             return dt.datetime
 
+        # The python type for a dictionary-encoded field is its value type
+        if pa.types.is_dictionary(arrow_type):
+            if isinstance(arrow_type, pa.DictionaryType):
+                return cls.arrow_to_python_type(arrow_type.value_type)
+
         raise _ex.ETracInternal(f"No Python type mapping available for Arrow type [{arrow_type}]")
 
     @classmethod
@@ -340,7 +359,13 @@ class DataMapping:
     def trac_to_arrow_field(cls, trac_field: _meta.FieldSchema):
 
         arrow_type = cls.trac_to_arrow_basic_type(trac_field.fieldType)
-        nullable = not trac_field.notNull if trac_field.notNull is not None else not trac_field.businessKey
+
+        # Categorical data uses an unordered dictionary with int32 index, ordered encoding not (currently) supported
+        # For legacy compatability, only use dictionary encoding if the categorical feature is enabled
+        if trac_field.categorical and cls.__categorical_enabled:
+            arrow_type = pa.dictionary(pa.int32(), arrow_type, False)
+
+        nullable = not (trac_field.notNull or trac_field.businessKey)
 
         return pa.field(trac_field.fieldName, arrow_type, nullable)
 
@@ -369,12 +394,15 @@ class DataMapping:
         field_type = cls.arrow_to_trac_type(field.type)
         label = field.metadata["label"] if field.metadata and "label" in field.metadata else field.name
 
+        # When converting Arrow -> TRAC, always set the categorical flag for dictionary encoded fields
+        # This affects dynamic imports and is informational only (physical layout is controlled by Arrow schemas)
+
         return _meta.FieldSchema(
             field.name, field_index, field_type,
             label=label,
             businessKey=False,
             notNull=not field.nullable,
-            categorical=False)
+            categorical=pa.types.is_dictionary(field.type))
 
     @classmethod
     def arrow_to_trac_type(cls, arrow_type: pa.DataType) -> _meta.BasicType:
@@ -390,6 +418,11 @@ class DataMapping:
         if pa.types.is_timestamp(arrow_type):
             return _meta.BasicType.DATETIME
 
+        # The basic type for a dictionary-encoded field is its value type
+        if pa.types.is_dictionary(arrow_type):
+            if isinstance(arrow_type, pa.DictionaryType):
+                return cls.arrow_to_trac_type(arrow_type.value_type)
+
         raise _ex.ETracInternal(f"No data type mapping available for Arrow type [{arrow_type}]")
 
     @classmethod
@@ -766,6 +799,10 @@ class DataConformance:
         "Field [{field_name}] cannot be converted from {vector_type} to {field_type}, " + \
         "source and target have different time zones"
 
+    __E_WRONG_CATEGORICAL_TYPE = \
+        "Field [{field_name}] categorical types do not match" + \
+        "(expected {field_type}, got {vector_type})"
+
     @classmethod
     def column_filter(cls, columns: tp.List[str], schema: tp.Optional[pa.Schema]) -> tp.Optional[tp.List[str]]:
 
@@ -918,13 +955,19 @@ class DataConformance:
     @classmethod
     def _coerce_vector(cls, vector: pa.Array, field: pa.Field, pandas_type=None) -> pa.Array:
 
+        # Handle null vector
         if pa.types.is_null(vector.type):
-
             if field.nullable:
                 return pa.nulls(size=len(vector), type=field.type)
             else:
                 raise _ex.EDataConformance(f"All null values in non-null field [{field.name}]")
 
+        # If the vector is dict-encoded but the expected result is not, decode the dictionary
+        if pa.types.is_dictionary(vector.type) and not pa.types.is_dictionary(field.type):
+            if isinstance(vector, pa.DictionaryArray):
+                dict_vector: pa.DictionaryArray = vector
+                vector = dict_vector.dictionary_decode()
+
         if pa.types.is_boolean(field.type):
             return cls._coerce_boolean(vector, field)
 
@@ -946,6 +989,9 @@ class DataConformance:
         if pa.types.is_timestamp(field.type):
             return cls._coerce_timestamp(vector, field)
 
+        if pa.types.is_dictionary(field.type):
+            return cls._coerce_dictionary(vector, field)
+
         error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)
         cls.__log.error(error_message)
         raise _ex.EDataConformance(error_message)
@@ -1188,6 +1234,82 @@ class DataConformance:
 
         return pc.cast(scaled_vector, field.type)
 
+    @classmethod
+    def _coerce_dictionary(cls, vector: pa.Array, field: pa.Field):
+
+        try:
+
+            if not isinstance(field.type, pa.DictionaryType):
+                raise _ex.EUnexpected()
+
+            field_type: pa.DictionaryType = field.type
+
+            # Supplied vector is a dictionary (but the dictionary type is not an exact match)
+            if pa.types.is_dictionary(vector.type):
+
+                if not isinstance(vector.type, pa.DictionaryType):
+                    raise _ex.EUnexpected()
+
+                vector_type: pa.DictionaryType = vector.type
+
+                # Do not allow coercion to a smaller index type or from unordered to ordered
+                if (vector_type.index_type.bit_width > field_type.index_type.bit_width) or \
+                   (field_type.ordered and not vector_type.ordered):
+
+                    error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)
+                    cls.__log.error(error_message)
+                    raise _ex.EDataConformance(error_message)
+
+                # Value types are the same - basic cast should succeed
+                if vector_type.value_type == field_type.value_type:
+                    return pc.cast(vector, field.type)
+
+                # Value types differ - try to coerce the underlying dictionary
+                elif isinstance(vector, pa.DictionaryArray):
+                    try:
+                        values_field = pa.field(field.name, field_type.value_type, field.nullable)
+                        values_vector = cls._coerce_vector(vector.dictionary, values_field)
+                        dict_vector = pa.DictionaryArray.from_arrays(vector.indices, values_vector, ordered=field_type.ordered)  # noqa
+                        return pc.cast(dict_vector, field.type)
+                    # Handle errors converting the value type
+                    except _ex.EDataConformance as e:
+                        error_message = cls._format_error(cls.__E_WRONG_CATEGORICAL_TYPE, vector, field)
+                        cls.__log.error(error_message)
+                        raise _ex.EDataConformance(error_message) from e
+
+                # Special handling for chunked dictionaries
+                elif isinstance(vector, pa.ChunkedArray):
+                    chunks = [cls._coerce_dictionary(chunk, field) for chunk in vector.chunks]
+                    return pa.chunked_array(chunks)
+
+                # Vector type not recognized, coercion is not possible
+                else:
+                    error_message = cls._format_error(cls.__E_WRONG_DATA_TYPE, vector, field)
+                    cls.__log.error(error_message)
+                    raise _ex.EDataConformance(error_message)
+
+            # Supplied vector matches the dictionary value type - perform dictionary encoding
+            elif vector.type == field_type.value_type and not field_type.ordered:
+                return vector.dictionary_encode().cast(field.type)
+
+            # Fallback option - try to coerce the value type first, then perform dictionary encoding
+            else:
+                try:
+                    values_field = pa.field(field.name, field_type.value_type, field.nullable)
+                    values_vector = cls._coerce_vector(vector, values_field)
+                    return values_vector.dictionary_encode().cast(field.type)
+                # Handle errors converting the value type
+                except _ex.EDataConformance as e:
+                    error_message = cls._format_error(cls.__E_WRONG_CATEGORICAL_TYPE, vector, field)
+                    cls.__log.error(error_message)
+                    raise _ex.EDataConformance(error_message) from e
+
+        except pa.ArrowInvalid as e:
+
+            error_message = cls._format_error(cls.__E_DATA_LOSS_DID_OCCUR, vector, field, e)
+            cls.__log.error(error_message)
+            raise _ex.EDataConformance(error_message) from e
+
     @classmethod
     def _format_error(cls, error_template: str, vector: pa.Array, field: pa.Field, e: Exception = None):
 

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/core/repos.py

@@ -38,6 +38,13 @@ class RepositoryManager:
 
             try:
 
+                # Add global properties related to the repo protocol
+                related_props = {
+                    k: v for (k, v) in sys_config.properties.items()
+                    if k.startswith(f"{repo_config.protocol}.")}
+
+                repo_config.properties.update(related_props)
+
                 self._repos[repo_name] = plugins.PluginManager.load_plugin(IModelRepository, repo_config)
 
             except ex.EPluginNotAvailable as e:

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/core/storage.py

@@ -81,7 +81,7 @@ class StorageManager:
         self.__file_storage: tp.Dict[str, IFileStorage] = dict()
         self.__data_storage: tp.Dict[str, IDataStorage] = dict()
         self.__external: tp.List[str] = list()
-        self.__settings = sys_config.storage
+        self.__sys_config = sys_config
 
         for storage_key, storage_config in sys_config.storage.buckets.items():
             self.create_storage(storage_key, storage_config)
@@ -93,13 +93,20 @@ class StorageManager:
             self.create_storage(storage_key, storage_config)
 
     def default_storage_key(self):
-        return self.__settings.defaultBucket
+        return self.__sys_config.storage.defaultBucket
 
     def default_storage_format(self):
-        return self.__settings.defaultFormat
+        return self.__sys_config.storage.defaultFormat
 
     def create_storage(self, storage_key: str, storage_config: _cfg.PluginConfig):
 
+        # Add global properties related to the storage protocol
+        related_props = {
+            k: v for (k, v) in self.__sys_config.properties.items()
+            if k.startswith(f"{storage_config.protocol}.")}
+
+        storage_config.properties.update(related_props)
+
         if plugins.PluginManager.is_plugin_available(IStorageProvider, storage_config.protocol):
             self._create_storage_from_provider(storage_key, storage_config)
         else:

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/exec/functions.py

@@ -294,11 +294,11 @@ class DataViewFunc(NodeFunction[_data.DataView]):
         # Everything else is a regular data view
         if self.node.schema is not None and len(self.node.schema.table.fields) > 0:
             trac_schema = self.node.schema
+            data_view = _data.DataView.for_trac_schema(trac_schema)
         else:
             arrow_schema = root_item.schema
-            trac_schema = _data.DataMapping.arrow_to_trac_schema(arrow_schema)
+            data_view = _data.DataView.for_arrow_schema(arrow_schema)
 
-        data_view = _data.DataView.for_trac_schema(trac_schema)
         data_view = _data.DataMapping.add_item_to_view(data_view, root_part_key, root_item)
 
         return data_view

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/api/internal/runtime_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/api/internal/runtime.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/common_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/common.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/config_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/config.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/custom_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/custom.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/data_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/data.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/file_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/file.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/flow_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/flow.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/job_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/job.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/model_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/model.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_id_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/object_id.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/object_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/object.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/resource.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/search.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/storage_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/tag.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/tag_update.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/type.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_impl/runtime.py

@@ -30,6 +30,7 @@ import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
 import tracdap.rt._impl.core.config_parser as _cparse
+import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.guard_rails as _guard
 import tracdap.rt._impl.core.logging as _logging
 import tracdap.rt._impl.core.models as _models
@@ -167,6 +168,13 @@ class TracRuntime:
             else:
                 self._log.info("Using embedded system config")
 
+            # Check whether to enable categorical processing in the data layer
+
+            if _data.DataMapping.CATEGORICAL_CONFIG_KEY in self._sys_config.properties:
+                categorical_prop = self._sys_config.properties[_data.DataMapping.CATEGORICAL_CONFIG_KEY]
+                categorical_flag = categorical_prop and categorical_prop.lower() == "true"
+                _data.DataMapping.enable_categorical(categorical_flag)
+
             # Dev mode translation is controlled by the dev mode flag
             # I.e. it can be applied to embedded configs
 

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_plugins/repo_git.py

@@ -23,6 +23,7 @@ import time
 import dulwich.repo as git_repo
 import dulwich.client as git_client
 import dulwich.index as git_index
+import urllib3.exceptions  # noqa
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -75,20 +76,45 @@ class GitRepository(IModelRepository):
 
     def do_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
-        self._log.info(
-            f"Git checkout: repo = [{model_def.repository}], " +
-            f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
+        try:
 
-        self._log.info(f"Checkout location: [{checkout_dir}]")
+            self._log.info(
+                f"Git checkout: repo = [{model_def.repository}], " +
+                f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
 
-        if self._native_git:
-            package_path = self._do_native_checkout(model_def, checkout_dir)
-        else:
-            package_path = self._do_python_checkout(model_def, checkout_dir)
+            self._log.info(f"Checkout location: [{checkout_dir}]")
+
+            if self._native_git:
+                package_path = self._do_native_checkout(model_def, checkout_dir)
+            else:
+                package_path = self._do_python_checkout(model_def, checkout_dir)
+
+            self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+
+            return package_path
+
+        except Exception as e:
+
+            error = e
+
+            # For retry failures, try to find the original cause
+            while e.__cause__ is not None:
+                if isinstance(e, urllib3.exceptions.MaxRetryError):
+                    error = e.__cause__
+                    break
+                else:
+                    e = e.__cause__
+
+            # Try to sanitize error messages from urllib3
+            if isinstance(error, urllib3.exceptions.HTTPError):
+                detail = self._clean_urllib3_error(error)
+            else:
+                detail = str(error)
 
-        self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+            message = f"Failed to check out [{model_def.repository}]: {detail}"
 
-        return package_path
+            self._log.error(message)
+            raise ex.EModelRepo(message) from error
 
     def _do_native_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
@@ -150,10 +176,15 @@ class GitRepository(IModelRepository):
                 for line in cmd_err:
                     self._log.info(line)
 
-            else:
+            elif cmd_err:
+
                 for line in cmd_err:
                     self._log.error(line)
 
+                raise ex.EModelRepo(cmd_err[-1])
+
+            else:
+
                 error_msg = f"Git checkout failed for {model_def.package} {model_def.version}"
                 self._log.error(error_msg)
                 raise ex.EModelRepo(error_msg)
@@ -265,6 +296,20 @@ class GitRepository(IModelRepository):
     def _ref_key(key):
         return bytes(key, "ascii")
 
+    @classmethod
+    def _clean_urllib3_error(cls, error: urllib3.exceptions.HTTPError):
+
+        match = cls._URLLIB3_ERROR_PATTERN.match(str(error))
+
+        # Best efforts to clean up the message, fall back on str(error)
+        if match:
+            return match.group(1)
+        else:
+            return str(error)
+
+    # Error message format is like this:
+    # <pkg.ClassName object at 0xXXXXXXX>: Message
+    _URLLIB3_ERROR_PATTERN = re.compile(r"<[^>]*>: (.*)")
 
 # Register plugin
 plugins.PluginManager.register_plugin(IModelRepository, GitRepository, ["git"])

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0rc2"
+__version__ = "0.9.0rc1"

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/config/runtime.py

@@ -21,6 +21,8 @@ class RuntimeConfig:
 
     runtimeApi: "ServiceConfig" = _dc.field(default_factory=lambda: ServiceConfig())
 
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
 
 @_dc.dataclass
 class SparkSettings:

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1/tracdap_runtime.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracdap-runtime
-Version: 0.8.0rc2
+Version: 0.9.0rc1
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -21,6 +21,8 @@ Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: dulwich==0.22.7
 Requires-Dist: requests==2.32.3
+Requires-Dist: typing_extensions<4.13
+Requires-Dist: urllib3<2.4.0
 Provides-Extra: grpc
 Requires-Dist: grpcio==1.70.0; extra == "grpc"
 Requires-Dist: grpcio-status==1.70.0; extra == "grpc"

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/tracdap_runtime.egg-info/requires.txt

@@ -3,6 +3,8 @@ pyarrow==18.1.0
 pyyaml==6.0.2
 dulwich==0.22.7
 requests==2.32.3
+typing_extensions<4.13
+urllib3<2.4.0
 
 [aws]
 botocore==1.34.93

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/config/__init__.py

@@ -1,6 +1,7 @@
 # Code generated by TRAC
 
-from .job import JobConfig
+from .result import TagUpdateList
+from .result import JobResult
 
 from .common import _ConfigFile
 from .common import PluginConfig
@@ -8,11 +9,6 @@ from .common import PlatformInfo
 from .common import StorageConfig
 from .common import ServiceConfig
 
-from .dynamic import DynamicConfig
-
-from .result import TagUpdateList
-from .result import JobResult
-
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
@@ -28,3 +24,7 @@ from .platform import ClientConfig
 
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
+
+from .job import JobConfig
+
+from .dynamic import DynamicConfig

{tracdap_runtime-0.8.0rc2 → tracdap_runtime-0.9.0rc1}/src/tracdap/rt/metadata/__init__.py

@@ -9,40 +9,16 @@ from .type import Value
 from .type import ArrayValue
 from .type import MapValue
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
 from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
 
-from .job import JobType
-from .job import JobStatusCode
-from .job import JobGroupType
-from .job import JobDefinition
-from .job import ResultDefinition
-from .job import RunModelJob
-from .job import RunFlowJob
-from .job import ImportModelJob
-from .job import ImportDataJob
-from .job import ExportDataJob
-from .job import JobGroup
-from .job import SequentialJobGroup
-from .job import ParallelJobGroup
-
-from .common import MetadataFormat
-from .common import MetadataVersion
-from .common import TenantInfo
-
-from .resource import ResourceType
-from .resource import ResourceDefinition
-
-from .config import ConfigType
-from .config import ConfigEntry
-from .config import ConfigDetails
-from .config import ConfigDefinition
-
-from .custom import CustomDefinition
+from .search import SearchOperator
+from .search import LogicalOperator
+from .search import SearchTerm
+from .search import LogicalExpression
+from .search import SearchExpression
+from .search import SearchParameters
 
 from .data import SchemaType
 from .data import PartType
@@ -63,12 +39,8 @@ from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .search import SearchOperator
-from .search import LogicalOperator
-from .search import SearchTerm
-from .search import LogicalExpression
-from .search import SearchExpression
-from .search import SearchParameters
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
 
 from .flow import FlowNodeType
 from .flow import FlowNode
@@ -76,6 +48,34 @@ from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
+from .custom import CustomDefinition
+
+from .resource import ResourceType
+from .resource import ResourceDefinition
+
+from .job import JobType
+from .job import JobStatusCode
+from .job import JobGroupType
+from .job import JobDefinition
+from .job import ResultDefinition
+from .job import RunModelJob
+from .job import RunFlowJob
+from .job import ImportModelJob
+from .job import ImportDataJob
+from .job import ExportDataJob
+from .job import JobGroup
+from .job import SequentialJobGroup
+from .job import ParallelJobGroup
+
+from .common import MetadataFormat
+from .common import MetadataVersion
+from .common import TenantInfo
+
+from .config import ConfigType
+from .config import ConfigEntry
+from .config import ConfigDetails
+from .config import ConfigDefinition
+
 from .storage import CopyStatus
 from .storage import IncarnationStatus
 from .storage import StorageCopy