tracdap-runtime 0.9.0b1__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/api/model_api.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 import abc as _abc
+import dataclasses as _dc
 import typing as _tp
 import logging as _logging
 
@@ -35,6 +36,25 @@ if _tp.TYPE_CHECKING:
         pass
 
 
+@_dc.dataclass(frozen=True)
+class RuntimeMetadata:
+
+    """
+    The metadata associated with a TRAC object, made available for models at runtime
+
+    The metadata available for a particular object depends on the current job configuration, as
+    well as the type of object. For example, a model input supplied from a TRAC dataset will
+    have the metadata of that dataset available, but data passed into a model as an intermediate
+    dataset in a flow might not have an ID or any attributes.
+    """
+
+    objectId: _tp.Optional[TagHeader] = None
+    """TRAC object ID of the current object (if available)"""
+
+    attributes: _tp.Dict[str, _tp.Any] = _dc.field(default_factory=dict)
+    """TRAC metadata attributes of the current object (if available)"""
+
+
 class TracContext(metaclass=_abc.ABCMeta):
 
     """
@@ -202,6 +222,31 @@ class TracContext(metaclass=_abc.ABCMeta):
 
         pass
 
+    def get_metadata(self, item_name: str) -> _tp.Optional[RuntimeMetadata]:
+
+        """
+        Get the TRAC metadata associated with a model input.
+
+        Metadata is available for inputs supplied from real TRAC objects, including
+        both :py:attr:`DATA <tracdap.rt.metadata.ObjectType.DATA>` and
+        :py:attr:`FILE <tracdap.rt.metadata.ObjectType.DATA>` objects.
+
+        Calling :py:meth:`get_metadata()` for objects that are not inputs will return null,
+        since parameters have no metadata and output metadata does not exist until after a job completes.
+        :py:meth:`get_metadata()` will also return null for inputs supplied from other models
+        as intermediates in a flow, since these also have no persistent metadata.
+
+        Attempting to access metadata for objects that do not exist, including outputs that
+        have not been put yet, is an error.
+
+        :param item_name: The name of the file or dataset to get metadata for
+        :return: Runtime metadata for the named item, or None if no metadata is available
+        :type item_name: str
+        :rtype: :py:class:`RuntimeMetadata <tracdap.rt.api.RuntimeMetadata>`
+        """
+
+        pass
+
     def put_schema(self, dataset_name: str, schema: SchemaDefinition):
 
         """

tracdap/rt/config/__init__.py

@@ -1,30 +1,28 @@
 # Code generated by TRAC
 
-from .result import JobResultAttrs
-from .result import JobResult
+from .tenant import TenantConfigMap
+from .tenant import TenantConfig
 
 from .common import _ConfigFile
 from .common import PluginConfig
 from .common import PlatformInfo
-from .common import StorageConfig
 from .common import ServiceConfig
 
+from .runtime import RuntimeConfig
+
+from .result import JobResultAttrs
+from .result import JobResult
+
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
-from .platform import MetadataConfig
-from .platform import TenantConfig
 from .platform import GatewayConfig
 from .platform import GatewayRedirect
 from .platform import RouteConfig
 from .platform import RoutingMatch
 from .platform import RoutingTarget
 from .platform import DeploymentConfig
-from .platform import ClientConfig
 
-from .runtime import RuntimeConfig
-from .runtime import SparkSettings
+from .dynamic import DynamicConfig
 
 from .job import JobConfig
-
-from .dynamic import DynamicConfig

tracdap/rt/config/common.py

@@ -36,22 +36,6 @@ class PlatformInfo:
     deploymentInfo: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
 
-@_dc.dataclass
-class StorageConfig:
-
-    buckets: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    """TODO: Rename "buckets" as "internal" for 0.7"""
-
-    external: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    defaultBucket: "str" = ""
-
-    defaultFormat: "str" = ""
-
-    defaultLayout: "metadata.StorageLayout" = metadata.StorageLayout.OBJECT_ID_LAYOUT
-
-
 @_dc.dataclass
 class ServiceConfig:
 

tracdap/rt/config/job.py

@@ -30,3 +30,7 @@ class JobConfig:
     """Preallocated IDs for job outputs"""
 
     preallocatedIds: "_tp.List[metadata.TagHeader]" = _dc.field(default_factory=list)
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """Allow setting per-job configuration"""

tracdap/rt/config/platform.py

@@ -6,6 +6,7 @@ import enum as _enum  # noqa
 
 import tracdap.rt.metadata as metadata
 from .common import *  # noqa
+from .tenant import *  # noqa
 
 
 class RoutingProtocol(_enum.Enum):
@@ -20,6 +21,8 @@ class RoutingProtocol(_enum.Enum):
 
     REST = 4
 
+    INTERNAL = 5
+
 
 class DeploymentLayout(_enum.Enum):
 
@@ -39,43 +42,21 @@ class PlatformConfig:
 
     platformInfo: "PlatformInfo" = _dc.field(default_factory=lambda: PlatformInfo())
 
-    metadata: "MetadataConfig" = _dc.field(default_factory=lambda: MetadataConfig())
-
-    storage: "StorageConfig" = _dc.field(default_factory=lambda: StorageConfig())
-
-    repositories: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    executor: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
+    metadataStore: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
 
     jobCache: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
 
-    tenants: "_tp.Dict[str, TenantConfig]" = _dc.field(default_factory=dict)
-
-    gateway: "GatewayConfig" = _dc.field(default_factory=lambda: GatewayConfig())
+    executor: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
 
     services: "_tp.Dict[str, ServiceConfig]" = _dc.field(default_factory=dict)
 
     deployment: "DeploymentConfig" = _dc.field(default_factory=lambda: DeploymentConfig())
 
-    clientConfig: "_tp.Dict[str, ClientConfig]" = _dc.field(default_factory=dict)
+    gateway: "GatewayConfig" = _dc.field(default_factory=lambda: GatewayConfig())
 
     extensions: "_tp.Dict[str, protobuf.Any]" = _dc.field(default_factory=dict)
 
-
-@_dc.dataclass
-class MetadataConfig:
-
-    database: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
-
-    format: "metadata.MetadataFormat" = metadata.MetadataFormat.METADATA_FORMAT_NOT_SET
-
-
-@_dc.dataclass
-class TenantConfig:
-
-    defaultBucket: "_tp.Optional[str]" = None
-
-    defaultFormat: "_tp.Optional[str]" = None
+    setupTasks: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
 
 
 @_dc.dataclass
@@ -109,6 +90,8 @@ class RouteConfig:
 
     target: "RoutingTarget" = _dc.field(default_factory=lambda: RoutingTarget())
 
+    routeKey: "str" = ""
+
 
 @_dc.dataclass
 class RoutingMatch:
@@ -136,9 +119,3 @@ class RoutingTarget:
 class DeploymentConfig:
 
     layout: "DeploymentLayout" = DeploymentLayout.LAYOUT_NOT_SET
-
-
-@_dc.dataclass
-class ClientConfig:
-
-    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)

tracdap/rt/config/runtime.py

@@ -4,6 +4,7 @@ import typing as _tp  # noqa
 import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
+import tracdap.rt.metadata as metadata
 from .common import *  # noqa
 
 
@@ -13,18 +14,10 @@ class RuntimeConfig:
 
     config: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
-    storage: "StorageConfig" = _dc.field(default_factory=lambda: StorageConfig())
-
-    repositories: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    sparkSettings: "SparkSettings" = _dc.field(default_factory=lambda: SparkSettings())
-
-    runtimeApi: "ServiceConfig" = _dc.field(default_factory=lambda: ServiceConfig())
-
     properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
+    secrets: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
-@_dc.dataclass
-class SparkSettings:
+    resources: "_tp.Dict[str, metadata.ResourceDefinition]" = _dc.field(default_factory=dict)
 
-    sparkProps: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+    runtimeApi: "_tp.Optional[ServiceConfig]" = None

tracdap/rt/config/tenant.py

@@ -0,0 +1,28 @@
+# Code generated by TRAC
+
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+import tracdap.rt.metadata as metadata
+
+
+
+@_dc.dataclass
+class TenantConfigMap:
+
+    tenants: "_tp.Dict[str, TenantConfig]" = _dc.field(default_factory=dict)
+
+    autoActivate: "bool" = False
+
+
+@_dc.dataclass
+class TenantConfig:
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    secrets: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    config: "_tp.Dict[str, metadata.ConfigDefinition]" = _dc.field(default_factory=dict)
+
+    resources: "_tp.Dict[str, metadata.ResourceDefinition]" = _dc.field(default_factory=dict)

tracdap/rt/launch/cli.py

@@ -35,14 +35,6 @@ def _cli_args(programmatic_args = None):
         "--dev-mode", dest="dev_mode", default=False, action="store_true",
         help="Enable development mode config translation")
 
-    parser.add_argument(
-        "--job-result-dir", dest="job_result_dir", type=pathlib.Path, required=False,
-        help="Output the result metadata for a batch job to the given directory")
-
-    parser.add_argument(
-        "--job-result-format", dest="job_result_format", choices=["json", "yaml", "proto"], default="json",
-        help="Output format for the result metadata (only meaningful if --job-result-dir is set)")
-
     parser.add_argument(
         "--scratch-dir", dest="scratch_dir", type=pathlib.Path, required=False,
         help="Scratch directory for working files" +

tracdap/rt/launch/launch.py

@@ -36,7 +36,7 @@ def _resolve_config_file(
     if isinstance(config_path, str):
         scheme_sep = config_path.find(":")
         # Single letter scheme is a Windows file path (C:\...)
-        scheme = scheme = config_path[:scheme_sep] if scheme_sep > 1 else "file"
+        scheme = config_path[:scheme_sep] if scheme_sep > 1 else "file"
         if scheme != "file":
             return config_path
 
@@ -191,8 +191,6 @@ def launch_cli(programmatic_args: _tp.Optional[_tp.List[str]] = None):
     runtime_instance = _runtime.TracRuntime(
         _sys_config,
         dev_mode=launch_args.dev_mode,
-        job_result_dir=launch_args.job_result_dir,
-        job_result_format=launch_args.job_result_format,
         scratch_dir=launch_args.scratch_dir,
         scratch_dir_persist=launch_args.scratch_dir_persist,
         plugin_packages=launch_args.plugin_packages)

tracdap/rt/metadata/__init__.py

@@ -20,12 +20,20 @@ from .search import LogicalExpression
 from .search import SearchExpression
 from .search import SearchParameters
 
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
+
+from .custom import CustomDefinition
+
+from .common import MetadataFormat
+from .common import MetadataVersion
+from .common import TenantInfo
+
 from .data import SchemaType
 from .data import PartType
 from .data import FieldSchema
+from .data import EnumValues
 from .data import TableSchema
-from .data import StructField
-from .data import StructSchema
 from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
@@ -39,20 +47,12 @@ from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
 from .flow import FlowNodeType
 from .flow import FlowNode
 from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
-from .custom import CustomDefinition
-
-from .resource import ResourceType
-from .resource import ResourceDefinition
-
 from .job import JobType
 from .job import JobStatusCode
 from .job import JobGroupType
@@ -67,15 +67,6 @@ from .job import JobGroup
 from .job import SequentialJobGroup
 from .job import ParallelJobGroup
 
-from .common import MetadataFormat
-from .common import MetadataVersion
-from .common import TenantInfo
-
-from .config import ConfigType
-from .config import ConfigEntry
-from .config import ConfigDetails
-from .config import ConfigDefinition
-
 from .storage import CopyStatus
 from .storage import IncarnationStatus
 from .storage import StorageLayout
@@ -84,6 +75,14 @@ from .storage import StorageIncarnation
 from .storage import StorageItem
 from .storage import StorageDefinition
 
+from .resource import ResourceType
+from .resource import ResourceDefinition
+
+from .config import ConfigType
+from .config import ConfigEntry
+from .config import ConfigDetails
+from .config import ConfigDefinition
+
 from .object import ObjectDefinition
 
 from .tag import Tag

tracdap/rt/metadata/data.py

@@ -25,7 +25,9 @@ class SchemaType(_enum.Enum):
 
     """Tabular data"""
 
-    STRUCT = 2
+    TABLE_SCHEMA = 1
+
+    STRUCT_SCHEMA = 2
 
     """Structured objects"""
 
@@ -38,6 +40,8 @@ class PartType(_enum.Enum):
 
     """Dataset has a single partition called the root partition (this is the default)"""
 
+    NOT_PARTITIONED = 0
+
     PART_BY_RANGE = 1
 
     """Partition by range over an ordered variable (not available yet)"""
@@ -75,47 +79,27 @@ class FieldSchema:
 
     formatCode: "_tp.Optional[str]" = None
 
+    defaultValue: "_tp.Optional[Value]" = None
 
-@_dc.dataclass
-class TableSchema:
+    namedType: "_tp.Optional[str]" = None
 
-    """Schema for a tabular dataset"""
+    namedEnum: "_tp.Optional[str]" = None
 
-    fields: "_tp.List[FieldSchema]" = _dc.field(default_factory=list)
+    children: "_tp.List[FieldSchema]" = _dc.field(default_factory=list)
 
 
 @_dc.dataclass
-class StructField:
-
-    """Schema for an individual field in a structured object dataset"""
-
-    fieldType: "TypeDescriptor" = _dc.field(default_factory=lambda: TypeDescriptor())
-
-    label: "str" = ""
-
-    businessKey: "bool" = False
-
-    categorical: "bool" = False
+class EnumValues:
 
-    notNull: "_tp.Optional[bool]" = None
-
-    """This could become mandatory with the next metadata update"""
-
-    formatCode: "_tp.Optional[str]" = None
-
-    defaultValue: "Value" = _dc.field(default_factory=lambda: Value())
-
-    structType: "_tp.Optional[str]" = None
+    values: "_tp.List[str]" = _dc.field(default_factory=list)
 
 
 @_dc.dataclass
-class StructSchema:
-
-    """Schema for a structured object dataset"""
+class TableSchema:
 
-    fields: "_tp.Dict[str, StructField]" = _dc.field(default_factory=dict)
+    """Schema for a tabular dataset"""
 
-    namedTypes: "_tp.Dict[str, StructSchema]" = _dc.field(default_factory=dict)
+    fields: "_tp.List[FieldSchema]" = _dc.field(default_factory=list)
 
 
 @_dc.dataclass
@@ -141,7 +125,11 @@ class SchemaDefinition:
 
     table: "_tp.Optional[TableSchema]" = None
 
-    struct: "_tp.Optional[StructSchema]" = None
+    fields: "_tp.List[FieldSchema]" = _dc.field(default_factory=list)
+
+    namedTypes: "_tp.Dict[str, SchemaDefinition]" = _dc.field(default_factory=dict)
+
+    namedEnums: "_tp.Dict[str, EnumValues]" = _dc.field(default_factory=dict)
 
 
 @_dc.dataclass

tracdap/rt/metadata/job.py

@@ -131,7 +131,7 @@ class ResultDefinition:
 
     statusMessage: "str" = ""
 
-    logFileId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+    logFileId: "_tp.Optional[TagSelector]" = None
 
     outputs: "_tp.Dict[str, TagSelector]" = _dc.field(default_factory=dict)
 

tracdap/rt/metadata/type.py

@@ -74,16 +74,16 @@ class BasicType(_enum.Enum):
 
     """An key-value map with string keys, values may be primitive or composite values.
     
-    Maps may be uniform, in which case all the values are of the same type, or non-
-    uniform in which case values can be of any type. For uniform maps the type
-    descriptor will specify the type contained in the map. For non-uniform maps the
-    type descriptor can only specify that the map is non-uniform, values must be
-    examined at run time to determine their type.
+    All values in a map must have the same type (i.e. the same type descriptor).
     
     .. seealso::
         :py:obj:`TypeDescriptor <TypeDescriptor>`
     """
 
+    STRUCT = 10
+
+    """A nested structure with a named set of fields, which may be primitive or composite values."""
+
 
 @_dc.dataclass
 class TypeDescriptor:
@@ -116,6 +116,10 @@ class TypeDescriptor:
     be inspected individually to determine their type.
     """
 
+    structTypes: "_tp.Dict[str, TypeDescriptor]" = _dc.field(default_factory=dict)
+
+    typeName: "_tp.Optional[str]" = None
+
 
 @_dc.dataclass
 class DecimalValue:

{tracdap_runtime-0.9.0b1.dist-info → tracdap_runtime-0.9.0b2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tracdap-runtime
-Version: 0.9.0b1
+Version: 0.9.0b2
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -16,11 +16,11 @@ Classifier: Operating System :: OS Independent
 Requires-Python: <3.14,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: protobuf==5.29.3
+Requires-Dist: protobuf==5.29.5
 Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
 Requires-Dist: dulwich==0.22.7
-Requires-Dist: requests==2.32.3
+Requires-Dist: requests==2.32.4
 Requires-Dist: typing_extensions<4.13
 Requires-Dist: urllib3<2.4.0
 Provides-Extra: grpc