tracdap-runtime 0.8.0rc1__py3-none-any.whl → 0.9.0rc1__py3-none-any.whl

tracdap/rt/_impl/grpc/tracdap/metadata/resource_pb2.pyi

@@ -1,8 +1,9 @@
 from tracdap.rt._impl.grpc.tracdap.metadata import object_id_pb2 as _object_id_pb2
-from tracdap.rt._impl.grpc.tracdap.metadata import object_pb2 as _object_pb2
+from google.protobuf.internal import containers as _containers
 from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
 from google.protobuf import descriptor as _descriptor
-from typing import ClassVar as _ClassVar
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Mapping as _Mapping, Optional as _Optional, Union as _Union
 
 DESCRIPTOR: _descriptor.FileDescriptor
 
@@ -11,6 +12,45 @@ class ResourceType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
     RESOURCE_TYPE_NOT_SET: _ClassVar[ResourceType]
     MODEL_REPOSITORY: _ClassVar[ResourceType]
     INTERNAL_STORAGE: _ClassVar[ResourceType]
+    EXTERNAL_STORAGE: _ClassVar[ResourceType]
 RESOURCE_TYPE_NOT_SET: ResourceType
 MODEL_REPOSITORY: ResourceType
 INTERNAL_STORAGE: ResourceType
+EXTERNAL_STORAGE: ResourceType
+
+class ResourceDefinition(_message.Message):
+    __slots__ = ("resourceType", "protocol", "subProtocol", "publicProperties", "properties", "secrets")
+    class PublicPropertiesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    class PropertiesEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    class SecretsEntry(_message.Message):
+        __slots__ = ("key", "value")
+        KEY_FIELD_NUMBER: _ClassVar[int]
+        VALUE_FIELD_NUMBER: _ClassVar[int]
+        key: str
+        value: str
+        def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ...
+    RESOURCETYPE_FIELD_NUMBER: _ClassVar[int]
+    PROTOCOL_FIELD_NUMBER: _ClassVar[int]
+    SUBPROTOCOL_FIELD_NUMBER: _ClassVar[int]
+    PUBLICPROPERTIES_FIELD_NUMBER: _ClassVar[int]
+    PROPERTIES_FIELD_NUMBER: _ClassVar[int]
+    SECRETS_FIELD_NUMBER: _ClassVar[int]
+    resourceType: ResourceType
+    protocol: str
+    subProtocol: str
+    publicProperties: _containers.ScalarMap[str, str]
+    properties: _containers.ScalarMap[str, str]
+    secrets: _containers.ScalarMap[str, str]
+    def __init__(self, resourceType: _Optional[_Union[ResourceType, str]] = ..., protocol: _Optional[str] = ..., subProtocol: _Optional[str] = ..., publicProperties: _Optional[_Mapping[str, str]] = ..., properties: _Optional[_Mapping[str, str]] = ..., secrets: _Optional[_Mapping[str, str]] = ...) -> None: ...

tracdap/rt/_impl/grpc/tracdap/metadata/search_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/search.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/storage_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/storage.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/tag_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/tag.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/tag_update_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/tag_update.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/grpc/tracdap/metadata/type_pb2.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Generated by the protocol buffer compiler.  DO NOT EDIT!
 # source: tracdap/rt/_impl/grpc/tracdap/metadata/type.proto
-# Protobuf Python Version: 4.25.3
+# Protobuf Python Version: 4.25.5
 """Generated protocol buffer code."""
 from google.protobuf import descriptor as _descriptor
 from google.protobuf import descriptor_pool as _descriptor_pool

tracdap/rt/_impl/runtime.py

@@ -30,6 +30,7 @@ import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
 import tracdap.rt.ext.plugins as _plugins
 import tracdap.rt._impl.core.config_parser as _cparse
+import tracdap.rt._impl.core.data as _data
 import tracdap.rt._impl.core.guard_rails as _guard
 import tracdap.rt._impl.core.logging as _logging
 import tracdap.rt._impl.core.models as _models
@@ -167,6 +168,13 @@ class TracRuntime:
             else:
                 self._log.info("Using embedded system config")
 
+            # Check whether to enable categorical processing in the data layer
+
+            if _data.DataMapping.CATEGORICAL_CONFIG_KEY in self._sys_config.properties:
+                categorical_prop = self._sys_config.properties[_data.DataMapping.CATEGORICAL_CONFIG_KEY]
+                categorical_flag = categorical_prop and categorical_prop.lower() == "true"
+                _data.DataMapping.enable_categorical(categorical_flag)
+
             # Dev mode translation is controlled by the dev mode flag
             # I.e. it can be applied to embedded configs
 

tracdap/rt/_plugins/repo_git.py

@@ -23,6 +23,7 @@ import time
 import dulwich.repo as git_repo
 import dulwich.client as git_client
 import dulwich.index as git_index
+import urllib3.exceptions  # noqa
 
 import tracdap.rt.metadata as meta
 import tracdap.rt.exceptions as ex
@@ -75,20 +76,45 @@ class GitRepository(IModelRepository):
 
     def do_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
-        self._log.info(
-            f"Git checkout: repo = [{model_def.repository}], " +
-            f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
+        try:
 
-        self._log.info(f"Checkout location: [{checkout_dir}]")
+            self._log.info(
+                f"Git checkout: repo = [{model_def.repository}], " +
+                f"group = [{model_def.packageGroup}], package = [{model_def.package}], version = [{model_def.version}]")
 
-        if self._native_git:
-            package_path = self._do_native_checkout(model_def, checkout_dir)
-        else:
-            package_path = self._do_python_checkout(model_def, checkout_dir)
+            self._log.info(f"Checkout location: [{checkout_dir}]")
+
+            if self._native_git:
+                package_path = self._do_native_checkout(model_def, checkout_dir)
+            else:
+                package_path = self._do_python_checkout(model_def, checkout_dir)
+
+            self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+
+            return package_path
+
+        except Exception as e:
+
+            error = e
+
+            # For retry failures, try to find the original cause
+            while e.__cause__ is not None:
+                if isinstance(e, urllib3.exceptions.MaxRetryError):
+                    error = e.__cause__
+                    break
+                else:
+                    e = e.__cause__
+
+            # Try to sanitize error messages from urllib3
+            if isinstance(error, urllib3.exceptions.HTTPError):
+                detail = self._clean_urllib3_error(error)
+            else:
+                detail = str(error)
 
-        self._log.info(f"Git checkout succeeded for {model_def.package} {model_def.version}")
+            message = f"Failed to check out [{model_def.repository}]: {detail}"
 
-        return package_path
+            self._log.error(message)
+            raise ex.EModelRepo(message) from error
 
     def _do_native_checkout(self, model_def: meta.ModelDefinition, checkout_dir: pathlib.Path) -> pathlib.Path:
 
@@ -150,10 +176,15 @@ class GitRepository(IModelRepository):
                 for line in cmd_err:
                     self._log.info(line)
 
-            else:
+            elif cmd_err:
+
                 for line in cmd_err:
                     self._log.error(line)
 
+                raise ex.EModelRepo(cmd_err[-1])
+
+            else:
+
                 error_msg = f"Git checkout failed for {model_def.package} {model_def.version}"
                 self._log.error(error_msg)
                 raise ex.EModelRepo(error_msg)
@@ -265,6 +296,20 @@ class GitRepository(IModelRepository):
     def _ref_key(key):
         return bytes(key, "ascii")
 
+    @classmethod
+    def _clean_urllib3_error(cls, error: urllib3.exceptions.HTTPError):
+
+        match = cls._URLLIB3_ERROR_PATTERN.match(str(error))
+
+        # Best efforts to clean up the message, fall back on str(error)
+        if match:
+            return match.group(1)
+        else:
+            return str(error)
+
+    # Error message format is like this:
+    # <pkg.ClassName object at 0xXXXXXXX>: Message
+    _URLLIB3_ERROR_PATTERN = re.compile(r"<[^>]*>: (.*)")
 
 # Register plugin
 plugins.PluginManager.register_plugin(IModelRepository, GitRepository, ["git"])

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0rc1"
+__version__ = "0.9.0rc1"

tracdap/rt/config/__init__.py

@@ -1,6 +1,7 @@
 # Code generated by TRAC
 
-from .job import JobConfig
+from .result import TagUpdateList
+from .result import JobResult
 
 from .common import _ConfigFile
 from .common import PluginConfig
@@ -8,9 +9,6 @@ from .common import PlatformInfo
 from .common import StorageConfig
 from .common import ServiceConfig
 
-from .result import TagUpdateList
-from .result import JobResult
-
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
@@ -26,3 +24,7 @@ from .platform import ClientConfig
 
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
+
+from .job import JobConfig
+
+from .dynamic import DynamicConfig

tracdap/rt/config/dynamic.py

@@ -0,0 +1,28 @@
+# Code generated by TRAC
+
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+import tracdap.rt.metadata as metadata
+
+
+
+@_dc.dataclass
+class DynamicConfig:
+
+    """
+    Represent config that is normally held in the metadata store
+    
+    This can be used to supply config to isolated processes, including
+    the runtime when operating in batch mode. It can also be used to
+    to supply config for testing or to bootstrap the metadata store.
+    """
+
+    configClass: "str" = ""
+
+    resourceClass: "str" = ""
+
+    config: "_tp.Dict[str, metadata.ConfigDefinition]" = _dc.field(default_factory=dict)
+
+    resources: "_tp.Dict[str, metadata.ResourceDefinition]" = _dc.field(default_factory=dict)

tracdap/rt/config/runtime.py

@@ -21,6 +21,8 @@ class RuntimeConfig:
 
     runtimeApi: "ServiceConfig" = _dc.field(default_factory=lambda: ServiceConfig())
 
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
 
 @_dc.dataclass
 class SparkSettings:

tracdap/rt/metadata/__init__.py

@@ -9,32 +9,16 @@ from .type import Value
 from .type import ArrayValue
 from .type import MapValue
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
 from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
 
-from .job import JobType
-from .job import JobStatusCode
-from .job import JobGroupType
-from .job import JobDefinition
-from .job import ResultDefinition
-from .job import RunModelJob
-from .job import RunFlowJob
-from .job import ImportModelJob
-from .job import ImportDataJob
-from .job import ExportDataJob
-from .job import JobGroup
-from .job import SequentialJobGroup
-from .job import ParallelJobGroup
-
-from .common import MetadataFormat
-from .common import MetadataVersion
-from .common import TenantInfo
-
-from .custom import CustomDefinition
+from .search import SearchOperator
+from .search import LogicalOperator
+from .search import SearchTerm
+from .search import LogicalExpression
+from .search import SearchExpression
+from .search import SearchParameters
 
 from .data import SchemaType
 from .data import PartType
@@ -55,12 +39,8 @@ from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .search import SearchOperator
-from .search import LogicalOperator
-from .search import SearchTerm
-from .search import LogicalExpression
-from .search import SearchExpression
-from .search import SearchParameters
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
 
 from .flow import FlowNodeType
 from .flow import FlowNode
@@ -68,6 +48,34 @@ from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
+from .custom import CustomDefinition
+
+from .resource import ResourceType
+from .resource import ResourceDefinition
+
+from .job import JobType
+from .job import JobStatusCode
+from .job import JobGroupType
+from .job import JobDefinition
+from .job import ResultDefinition
+from .job import RunModelJob
+from .job import RunFlowJob
+from .job import ImportModelJob
+from .job import ImportDataJob
+from .job import ExportDataJob
+from .job import JobGroup
+from .job import SequentialJobGroup
+from .job import ParallelJobGroup
+
+from .common import MetadataFormat
+from .common import MetadataVersion
+from .common import TenantInfo
+
+from .config import ConfigType
+from .config import ConfigEntry
+from .config import ConfigDetails
+from .config import ConfigDefinition
+
 from .storage import CopyStatus
 from .storage import IncarnationStatus
 from .storage import StorageCopy
@@ -78,5 +86,3 @@ from .storage import StorageDefinition
 from .object import ObjectDefinition
 
 from .tag import Tag
-
-from .resource import ResourceType

tracdap/rt/metadata/config.py

@@ -0,0 +1,95 @@
+# Code generated by TRAC
+
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+from .type import *  # noqa
+from .object_id import *  # noqa
+from .resource import *  # noqa
+
+
+class ConfigType(_enum.Enum):
+
+    """Enumeration of available config types"""
+
+    CONFIG_TYPE_NOT_SET = 0
+
+    PROPERTIES = 1
+
+    """A key-value properties map, follows the convention used for .properties files"""
+
+
+@_dc.dataclass
+class ConfigEntry:
+
+    """Represents the mapping of a config key to a config object."""
+
+    configClass: "str" = ""
+
+    """Config class for this config entry."""
+
+    configKey: "str" = ""
+
+    """Config key for this config entry."""
+
+    configVersion: "int" = 0
+
+    """Version of the config entry."""
+
+    configTimestamp: "DatetimeValue" = _dc.field(default_factory=lambda: DatetimeValue())
+
+    """Timestamp for when this version of the config entry was created."""
+
+    isLatestConfig: "bool" = False
+
+    """Flag indicating whether this is the latest version of the config entry."""
+
+    configDeleted: "bool" = False
+
+    """Flag indicating whether this config entry has been deleted."""
+
+    details: "_tp.Optional[ConfigDetails]" = None
+
+    """Describes the content of this config entry, can be used for list / summary views (optional)."""
+
+
+@_dc.dataclass
+class ConfigDetails:
+
+    """
+    Summary content of a config entry.
+    
+    .. seealso::
+        :py:obj:`ConfigEntry <ConfigEntry>`
+    """
+
+    objectSelector: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
+    """Selector for the metadata object holding the content of this config entry."""
+
+    objectType: "ObjectType" = ObjectType.OBJECT_TYPE_NOT_SET
+
+    """Type of the metadata object referred to by this config entry."""
+
+    configType: "_tp.Optional[ConfigType]" = None
+
+    """Indicate the config type (for CONFIG objects only)"""
+
+    resourceType: "_tp.Optional[ResourceType]" = None
+
+    """Indicate the resource type (for RESOURCE objects only)"""
+
+
+@_dc.dataclass
+class ConfigDefinition:
+
+    """Definition of an individual config entry"""
+
+    configType: "ConfigType" = ConfigType.CONFIG_TYPE_NOT_SET
+
+    """The type of this config entry"""
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """The map of config properties (only available for PROPERTIES config entries)"""

tracdap/rt/metadata/object.py

@@ -13,6 +13,8 @@ from .job import *  # noqa
 from .file import *  # noqa
 from .custom import *  # noqa
 from .storage import *  # noqa
+from .config import *  # noqa
+from .resource import *  # noqa
 
 
 
@@ -72,4 +74,8 @@ class ObjectDefinition:
 
     result: "_tp.Optional[ResultDefinition]" = None
 
+    config: "_tp.Optional[ConfigDefinition]" = None
+
+    resource: "_tp.Optional[ResourceDefinition]" = None
+
     objectProps: "_tp.Dict[str, Value]" = _dc.field(default_factory=dict)

tracdap/rt/metadata/object_id.py

@@ -36,6 +36,10 @@ class ObjectType(_enum.Enum):
 
     RESULT = 9
 
+    CONFIG = 10
+
+    RESOURCE = 11
+
 
 @_dc.dataclass
 class TagHeader:

tracdap/rt/metadata/resource.py

@@ -5,7 +5,6 @@ import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
 from .object_id import *  # noqa
-from .object import *  # noqa
 
 
 class ResourceType(_enum.Enum):
@@ -21,3 +20,44 @@ class ResourceType(_enum.Enum):
     INTERNAL_STORAGE = 2
 
     """Storage location for data held internally by the TRAC platform"""
+
+    EXTERNAL_STORAGE = 3
+
+    """Storage location for upstream  downstream integration points"""
+
+
+@_dc.dataclass
+class ResourceDefinition:
+
+    """Definition of a platform resource"""
+
+    resourceType: "ResourceType" = ResourceType.RESOURCE_TYPE_NOT_SET
+
+    """The type of this resource"""
+
+    protocol: "str" = ""
+
+    """The protocol used to communicate with this resource (usually corresponds to a TRAC plugin)"""
+
+    subProtocol: "_tp.Optional[str]" = None
+
+    """Optional sub-protocol, to further distinguish the behavior of resources within a protocol"""
+
+    publicProperties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """Configuration properties for this resource, made visible to client applications"""
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """Configuration properties for this resource, not visible to client applications"""
+
+    secrets: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """
+    Configuration properties that are stored as secrets using TRAC's secret handling mechanism
+    
+    Only the secret alias is stored in metadata, handling of secret storage is delegated to the
+    configured secrets plugin. Secret values may be sent by clients when resource definitions are
+    created / updated, in which case they will be written to the configured secret store and
+    metadata values will be replaced with an alias.
+    """

{tracdap_runtime-0.8.0rc1.dist-info → tracdap_runtime-0.9.0rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: tracdap-runtime
-Version: 0.8.0rc1
+Version: 0.9.0rc1
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -21,6 +21,8 @@ Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: dulwich==0.22.7
 Requires-Dist: requests==2.32.3
+Requires-Dist: typing_extensions<4.13
+Requires-Dist: urllib3<2.4.0
 Provides-Extra: grpc
 Requires-Dist: grpcio==1.70.0; extra == "grpc"
 Requires-Dist: grpcio-status==1.70.0; extra == "grpc"
@@ -46,6 +48,7 @@ Requires-Dist: azure-core==1.30.1; extra == "azure"
 Requires-Dist: azure-identity==1.16.1; extra == "azure"
 Requires-Dist: azure-storage-blob==12.19.1; extra == "azure"
 Requires-Dist: adlfs==2024.4.1; extra == "azure"
+Dynamic: license-file
 
 # TRAC Model Runtime for Python