tracdap-runtime 0.7.0rc1__py3-none-any.whl → 0.8.0__py3-none-any.whl

tracdap/rt/ext/plugins.py

@@ -14,13 +14,13 @@
 #  limitations under the License.
 
 import typing as _tp
-import logging as _log
 import pkgutil as _pkg
 import importlib as _il
 
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
-import tracdap.rt._impl.guard_rails as _guard  # noqa
+import tracdap.rt._impl.core.guard_rails as _guard  # noqa
+import tracdap.rt._impl.core.logging as _logging  # noqa
 from tracdap.rt.exceptions import EStartup
 
 
@@ -28,7 +28,7 @@ class PluginManager:
 
     T_SERVICE = _tp.TypeVar("T_SERVICE")
 
-    __log = _log.getLogger(f"{__name__}.PluginManager")
+    __log = _logging.getLogger(f"{__name__}.PluginManager")
 
     __core_registered = False
     __3rd_party_registered = list()

tracdap/rt/launch/launch.py

@@ -20,9 +20,9 @@ import pathlib as _pathlib
 import typing as _tp
 
 import tracdap.rt.api as _api
-import tracdap.rt._impl.config_parser as _cparse  # noqa
-import tracdap.rt._impl.util as _util  # noqa
-import tracdap.rt._exec.runtime as _runtime  # noqa
+import tracdap.rt._impl.core.config_parser as _cparse  # noqa
+import tracdap.rt._impl.core.util as _util  # noqa
+import tracdap.rt._impl.runtime as _runtime  # noqa
 
 from .cli import _cli_args
 
@@ -112,9 +112,10 @@ def launch_model(
     runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=True, plugin_packages=plugin_packages)
     runtime_instance.pre_start()
 
-    job = runtime_instance.load_job_config(_job_config, model_class=model_class)
-
     with runtime_instance as rt:
+
+        job = rt.load_job_config(_job_config, model_class=model_class)
+
         rt.submit_job(job)
         rt.wait_for_job(job.jobId)
 
@@ -160,9 +161,10 @@ def launch_job(
     runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=dev_mode, plugin_packages=plugin_packages)
     runtime_instance.pre_start()
 
-    job = runtime_instance.load_job_config(_job_config)
-
     with runtime_instance as rt:
+
+        job = rt.load_job_config(_job_config)
+
         rt.submit_job(job)
         rt.wait_for_job(job.jobId)
 
@@ -184,6 +186,7 @@ def launch_cli(programmatic_args: _tp.Optional[_tp.List[str]] = None):
         launch_args = _cli_args()
 
     _sys_config = _resolve_config_file(launch_args.sys_config, None)
+    _job_config = _resolve_config_file(launch_args.job_config, None) if launch_args.job_config else None
 
     runtime_instance = _runtime.TracRuntime(
         _sys_config,
@@ -196,15 +199,10 @@ def launch_cli(programmatic_args: _tp.Optional[_tp.List[str]] = None):
 
     runtime_instance.pre_start()
 
-    if launch_args.job_config is not None:
-        _job_config = _resolve_config_file(launch_args.job_config, None)
-        job = runtime_instance.load_job_config(_job_config)
-    else:
-        job = None
-
     with runtime_instance as rt:
 
-        if job is not None:
+        if _job_config is not None:
+            job = rt.load_job_config(_job_config)
             rt.submit_job(job)
 
         if rt.is_oneshot():

tracdap/rt/metadata/__init__.py

@@ -13,27 +13,32 @@ from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
 
+from .search import SearchOperator
+from .search import LogicalOperator
+from .search import SearchTerm
+from .search import LogicalExpression
+from .search import SearchExpression
+from .search import SearchParameters
+
 from .data import SchemaType
 from .data import PartType
 from .data import FieldSchema
 from .data import TableSchema
+from .data import StructField
+from .data import StructSchema
 from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
 
+from .file import FileDefinition
+from .file import FileType
+
 from .model import ModelType
 from .model import ModelParameter
 from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .search import SearchOperator
-from .search import LogicalOperator
-from .search import SearchTerm
-from .search import LogicalExpression
-from .search import SearchExpression
-from .search import SearchParameters
-
 from .tag_update import TagOperation
 from .tag_update import TagUpdate
 
@@ -43,10 +48,16 @@ from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
+from .custom import CustomDefinition
+
+from .resource import ResourceType
+from .resource import ResourceDefinition
+
 from .job import JobType
 from .job import JobStatusCode
 from .job import JobGroupType
 from .job import JobDefinition
+from .job import ResultDefinition
 from .job import RunModelJob
 from .job import RunFlowJob
 from .job import ImportModelJob
@@ -56,23 +67,22 @@ from .job import JobGroup
 from .job import SequentialJobGroup
 from .job import ParallelJobGroup
 
-from .file import FileDefinition
+from .common import MetadataFormat
+from .common import MetadataVersion
+from .common import TenantInfo
 
-from .custom import CustomDefinition
+from .config import ConfigType
+from .config import ConfigEntry
+from .config import ConfigDetails
+from .config import ConfigDefinition
 
-from .stoarge import CopyStatus
-from .stoarge import IncarnationStatus
-from .stoarge import StorageCopy
-from .stoarge import StorageIncarnation
-from .stoarge import StorageItem
-from .stoarge import StorageDefinition
+from .storage import CopyStatus
+from .storage import IncarnationStatus
+from .storage import StorageCopy
+from .storage import StorageIncarnation
+from .storage import StorageItem
+from .storage import StorageDefinition
 
 from .object import ObjectDefinition
 
-from .resource import ResourceType
-
-from .common import MetadataFormat
-from .common import MetadataVersion
-from .common import TenantInfo
-
 from .tag import Tag

tracdap/rt/metadata/config.py

@@ -0,0 +1,95 @@
+# Code generated by TRAC
+
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+from .type import *  # noqa
+from .object_id import *  # noqa
+from .resource import *  # noqa
+
+
+class ConfigType(_enum.Enum):
+
+    """Enumeration of available config types"""
+
+    CONFIG_TYPE_NOT_SET = 0
+
+    PROPERTIES = 1
+
+    """A key-value properties map, follows the convention used for .properties files"""
+
+
+@_dc.dataclass
+class ConfigEntry:
+
+    """Represents the mapping of a config key to a config object."""
+
+    configClass: "str" = ""
+
+    """Config class for this config entry."""
+
+    configKey: "str" = ""
+
+    """Config key for this config entry."""
+
+    configVersion: "int" = 0
+
+    """Version of the config entry."""
+
+    configTimestamp: "DatetimeValue" = _dc.field(default_factory=lambda: DatetimeValue())
+
+    """Timestamp for when this version of the config entry was created."""
+
+    isLatestConfig: "bool" = False
+
+    """Flag indicating whether this is the latest version of the config entry."""
+
+    configDeleted: "bool" = False
+
+    """Flag indicating whether this config entry has been deleted."""
+
+    details: "_tp.Optional[ConfigDetails]" = None
+
+    """Describes the content of this config entry, can be used for list / summary views (optional)."""
+
+
+@_dc.dataclass
+class ConfigDetails:
+
+    """
+    Summary content of a config entry.
+    
+    .. seealso::
+        :py:obj:`ConfigEntry <ConfigEntry>`
+    """
+
+    objectSelector: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
+    """Selector for the metadata object holding the content of this config entry."""
+
+    objectType: "ObjectType" = ObjectType.OBJECT_TYPE_NOT_SET
+
+    """Type of the metadata object referred to by this config entry."""
+
+    configType: "_tp.Optional[ConfigType]" = None
+
+    """Indicate the config type (for CONFIG objects only)"""
+
+    resourceType: "_tp.Optional[ResourceType]" = None
+
+    """Indicate the resource type (for RESOURCE objects only)"""
+
+
+@_dc.dataclass
+class ConfigDefinition:
+
+    """Definition of an individual config entry"""
+
+    configType: "ConfigType" = ConfigType.CONFIG_TYPE_NOT_SET
+
+    """The type of this config entry"""
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """The map of config properties (only available for PROPERTIES config entries)"""

tracdap/rt/metadata/data.py

@@ -25,6 +25,10 @@ class SchemaType(_enum.Enum):
 
     """Tabular data"""
 
+    STRUCT = 2
+
+    """Structured objects"""
+
 
 class PartType(_enum.Enum):
 
@@ -80,6 +84,40 @@ class TableSchema:
     fields: "_tp.List[FieldSchema]" = _dc.field(default_factory=list)
 
 
+@_dc.dataclass
+class StructField:
+
+    """Schema for an individual field in a structured object dataset"""
+
+    fieldType: "TypeDescriptor" = _dc.field(default_factory=lambda: TypeDescriptor())
+
+    label: "str" = ""
+
+    businessKey: "bool" = False
+
+    categorical: "bool" = False
+
+    notNull: "_tp.Optional[bool]" = None
+
+    """This could become mandatory with the next metadata update"""
+
+    formatCode: "_tp.Optional[str]" = None
+
+    defaultValue: "Value" = _dc.field(default_factory=lambda: Value())
+
+    structType: "_tp.Optional[str]" = None
+
+
+@_dc.dataclass
+class StructSchema:
+
+    """Schema for a structured object dataset"""
+
+    fields: "_tp.Dict[str, StructField]" = _dc.field(default_factory=dict)
+
+    namedTypes: "_tp.Dict[str, StructSchema]" = _dc.field(default_factory=dict)
+
+
 @_dc.dataclass
 class SchemaDefinition:
 
@@ -103,6 +141,8 @@ class SchemaDefinition:
 
     table: "_tp.Optional[TableSchema]" = None
 
+    struct: "_tp.Optional[StructSchema]" = None
+
 
 @_dc.dataclass
 class PartKey:

tracdap/rt/metadata/file.py

@@ -24,3 +24,13 @@ class FileDefinition:
     storageId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
 
     dataItem: "str" = ""
+
+
+@_dc.dataclass
+class FileType:
+
+    """Specify a file type that is produced or consumed by a model, flow or job"""
+
+    extension: "str" = ""
+
+    mimeType: "str" = ""

tracdap/rt/metadata/job.py

@@ -117,6 +117,22 @@ class JobDefinition:
 
     jobGroup: "_tp.Optional[JobGroup]" = None
 
+    resultId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
+
+@_dc.dataclass
+class ResultDefinition:
+
+    """Define the result of a job after it has completed"""
+
+    jobId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
+    statusCode: "JobStatusCode" = JobStatusCode.JOB_STATUS_CODE_NOT_SET
+
+    statusMessage: "str" = ""
+
+    logFileId: "TagSelector" = _dc.field(default_factory=lambda: TagSelector())
+
 
 @_dc.dataclass
 class RunModelJob:

tracdap/rt/metadata/model.py

@@ -5,7 +5,9 @@ import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
 from .type import *  # noqa
+from .object_id import *  # noqa
 from .data import *  # noqa
+from .file import *  # noqa
 
 
 class ModelType(_enum.Enum):
@@ -54,7 +56,11 @@ class ModelInputSchema:
     other options may be required. These capabilities may be added in future releases.
     """
 
-    schema: "SchemaDefinition" = _dc.field(default_factory=lambda: SchemaDefinition())
+    objectType: "ObjectType" = ObjectType.OBJECT_TYPE_NOT_SET
+
+    schema: "_tp.Optional[SchemaDefinition]" = None
+
+    fileType: "_tp.Optional[FileType]" = None
 
     label: "_tp.Optional[str]" = None
 
@@ -81,7 +87,11 @@ class ModelOutputSchema:
     added in future releases.
     """
 
-    schema: "SchemaDefinition" = _dc.field(default_factory=lambda: SchemaDefinition())
+    objectType: "ObjectType" = ObjectType.OBJECT_TYPE_NOT_SET
+
+    schema: "_tp.Optional[SchemaDefinition]" = None
+
+    fileType: "_tp.Optional[FileType]" = None
 
     label: "_tp.Optional[str]" = None
 

tracdap/rt/metadata/object.py

@@ -12,7 +12,9 @@ from .flow import *  # noqa
 from .job import *  # noqa
 from .file import *  # noqa
 from .custom import *  # noqa
-from .stoarge import *  # noqa
+from .storage import *  # noqa
+from .config import *  # noqa
+from .resource import *  # noqa
 
 
 
@@ -70,4 +72,10 @@ class ObjectDefinition:
 
     schema: "_tp.Optional[SchemaDefinition]" = None
 
+    result: "_tp.Optional[ResultDefinition]" = None
+
+    config: "_tp.Optional[ConfigDefinition]" = None
+
+    resource: "_tp.Optional[ResourceDefinition]" = None
+
     objectProps: "_tp.Dict[str, Value]" = _dc.field(default_factory=dict)

tracdap/rt/metadata/object_id.py

@@ -34,6 +34,12 @@ class ObjectType(_enum.Enum):
 
     SCHEMA = 8
 
+    RESULT = 9
+
+    CONFIG = 10
+
+    RESOURCE = 11
+
 
 @_dc.dataclass
 class TagHeader:

tracdap/rt/metadata/resource.py

@@ -5,7 +5,6 @@ import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
 from .object_id import *  # noqa
-from .object import *  # noqa
 
 
 class ResourceType(_enum.Enum):
@@ -21,3 +20,44 @@ class ResourceType(_enum.Enum):
     INTERNAL_STORAGE = 2
 
     """Storage location for data held internally by the TRAC platform"""
+
+    EXTERNAL_STORAGE = 3
+
+    """Storage location for upstream  downstream integration points"""
+
+
+@_dc.dataclass
+class ResourceDefinition:
+
+    """Definition of a platform resource"""
+
+    resourceType: "ResourceType" = ResourceType.RESOURCE_TYPE_NOT_SET
+
+    """The type of this resource"""
+
+    protocol: "str" = ""
+
+    """The protocol used to communicate with this resource (usually corresponds to a TRAC plugin)"""
+
+    subProtocol: "_tp.Optional[str]" = None
+
+    """Optional sub-protocol, to further distinguish the behavior of resources within a protocol"""
+
+    publicProperties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """Configuration properties for this resource, made visible to client applications"""
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """Configuration properties for this resource, not visible to client applications"""
+
+    secrets: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    """
+    Configuration properties that are stored as secrets using TRAC's secret handling mechanism
+    
+    Only the secret alias is stored in metadata, handling of secret storage is delegated to the
+    configured secrets plugin. Secret values may be sent by clients when resource definitions are
+    created / updated, in which case they will be written to the configured secret store and
+    metadata values will be replaced with an alias.
+    """

{tracdap_runtime-0.7.0rc1.dist-info → tracdap_runtime-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: tracdap-runtime
-Version: 0.7.0rc1
+Version: 0.8.0
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -13,38 +13,42 @@ Platform: any
 Classifier: Programming Language :: Python :: 3
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
-Requires-Python: <3.13,>=3.8
+Requires-Python: <3.14,>=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: protobuf==5.28.2
-Requires-Dist: pyarrow==16.1.0
+Requires-Dist: protobuf==5.29.3
+Requires-Dist: pyarrow==18.1.0
 Requires-Dist: pyyaml==6.0.2
-Requires-Dist: dulwich==0.22.1
+Requires-Dist: dulwich==0.22.7
 Requires-Dist: requests==2.32.3
-Requires-Dist: pandas<2.3.0,>=1.2.0
-Requires-Dist: numpy<2.0.0
+Requires-Dist: typing_extensions<4.13
+Requires-Dist: urllib3<2.4.0
+Provides-Extra: grpc
+Requires-Dist: grpcio==1.70.0; extra == "grpc"
+Requires-Dist: grpcio-status==1.70.0; extra == "grpc"
+Provides-Extra: pandas
+Requires-Dist: pandas<2.3.0,>=1.2.0; extra == "pandas"
+Requires-Dist: numpy<2.3.0,>=1.20; extra == "pandas"
+Provides-Extra: polars
+Requires-Dist: polars<2.0.0,>=1.0.0; extra == "polars"
+Provides-Extra: pyspark
+Requires-Dist: pyspark<3.6.0,>=3.0.0; extra == "pyspark"
+Provides-Extra: sql
+Requires-Dist: sqlalchemy<2.1.0,>=2.0.0; extra == "sql"
 Provides-Extra: aws
 Requires-Dist: botocore==1.34.93; extra == "aws"
 Requires-Dist: boto3==1.34.93; extra == "aws"
-Provides-Extra: azure
-Requires-Dist: azure-core==1.30.1; extra == "azure"
-Requires-Dist: azure-identity==1.16.1; extra == "azure"
-Requires-Dist: azure-storage-blob==12.19.1; extra == "azure"
-Requires-Dist: adlfs==2024.4.1; extra == "azure"
 Provides-Extra: gcp
 Requires-Dist: google-auth==2.34.0; extra == "gcp"
 Requires-Dist: google-api-core==2.19.2; extra == "gcp"
 Requires-Dist: google-cloud-storage==2.18.2; extra == "gcp"
 Requires-Dist: gcsfs==2024.3.1; extra == "gcp"
-Provides-Extra: grpc
-Requires-Dist: grpcio==1.66.1; extra == "grpc"
-Requires-Dist: grpcio-status==1.66.1; extra == "grpc"
-Provides-Extra: polars
-Requires-Dist: polars<2.0.0,>=1.0.0; extra == "polars"
-Provides-Extra: spark
-Requires-Dist: pyspark<3.6.0,>=3.0.0; extra == "spark"
-Provides-Extra: sql
-Requires-Dist: sqlalchemy<2.1.0,>=2.0.0; extra == "sql"
+Provides-Extra: azure
+Requires-Dist: azure-core==1.30.1; extra == "azure"
+Requires-Dist: azure-identity==1.16.1; extra == "azure"
+Requires-Dist: azure-storage-blob==12.19.1; extra == "azure"
+Requires-Dist: adlfs==2024.4.1; extra == "azure"
+Dynamic: license-file
 
 # TRAC Model Runtime for Python
 
@@ -64,12 +68,14 @@ Documentation for the TRAC platform is available on our website at
 
 The TRAC runtime for Python has these requirements:
 
-* Python: 3.8 up to 3.12
-* Pandas: 1.2 up to 2.2
-* PySpark 3.0 up to 3.5 (optional)
+* Python: 3.9 up to 3.13
+* Pandas: 1.2 up to 2.2 (optional)
+* NumPy: 1.2 up to 2.2 (optional, required by Pandas)
+* Polars: 1.X (optional)
 
-3rd party libraries may impose additional constraints on supported versions of Python, Pandas or PySpark.
-As of February 2024, the Python libraries for GCP do not yet support Python 3.12.
+3rd party libraries may impose additional constraints on supported versions of key libraries.
+For example, Pandas 1.5 is not available for Python 3.12 or 3.13, while NumPy 2.0 is only
+compatible with Pandas 2.1 and later.
 
 ## Installing the runtime