tracdap-runtime 0.6.4__py3-none-any.whl → 0.6.6__py3-none-any.whl

tracdap/rt/api/experimental.py

@@ -0,0 +1,267 @@
+#  Copyright 2024 Accenture Global Solutions Limited
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import abc as _abc
+import dataclasses as _dc
+import datetime as _dt
+import enum as _enum
+import typing as _tp
+
+from tracdap.rt.api import *
+from .hook import _StaticApiHook
+
+
+_PROTOCOL = _tp.TypeVar('_PROTOCOL')
+
+@_dc.dataclass(frozen=True)
+class _Protocol(_tp.Generic[_PROTOCOL]):
+
+    protocol_name: str
+    api_type: _tp.Type[_PROTOCOL]
+
+    def __str__(self):
+        return self.protocol_name
+
+
+def __pandas_api_type() -> "_tp.Type[pandas.DataFrame]":
+    try:
+        import pandas
+        return pandas.DataFrame
+    except ModuleNotFoundError:
+        return None  # noqa
+
+def __polars_api_type() -> "_tp.Type[polars.DataFrame]":
+    try:
+        import polars
+        return polars.DataFrame
+    except ModuleNotFoundError:
+        return None  # noqa
+
+DATA_API = _tp.TypeVar('DATA_API', __pandas_api_type(), __polars_api_type())
+
+class DataFramework(_Protocol[DATA_API]):
+
+    def __init__(self, protocol_name: str, api_type: _tp.Type[DATA_API]):
+        super().__init__(protocol_name, api_type)
+
+    @classmethod
+    def pandas(cls) -> "DataFramework[pandas.DataFrame]":
+        return DataFramework("pandas", DATA_API.__constraints__[0])
+
+    @classmethod
+    def polars(cls) -> "DataFramework[polars.DataFrame]":
+        return DataFramework("polars", DATA_API.__constraints__[1])
+
+PANDAS = DataFramework.pandas()
+"""Data framework constant for the Pandas data library"""
+
+POLARS = DataFramework.polars()
+"""Data framework constant for the Polars data library"""
+
+
+class TracContext(TracContext):
+
+    @_abc.abstractmethod
+    def get_table(self, dataset_name: str, framework: DataFramework[DATA_API]) -> DATA_API:
+
+        pass
+
+    @_abc.abstractmethod
+    def put_table(self, dataset_name: str, dataset: DATA_API):
+
+        pass
+
+
+def init_static():
+    """Ensure TRAC's static model API is available to use (for static definitions at module or class scope)"""
+    import tracdap.rt._impl.static_api as _static_impl  # noqa
+    _static_impl.StaticApiImpl.register_impl()
+
+
+def infer_schema(dataset: DATA_API) -> SchemaDefinition:
+    """Infer the full TRAC schema of an existing dataset"""
+    sa = _StaticApiHook.get_instance()
+    return sa.infer_schema(dataset)
+
+
+def array_type(item_type: BasicType) -> TypeDescriptor:
+    """Build a type descriptor for an ARRAY type"""
+    sa = _StaticApiHook.get_instance()
+    return sa.array_type(item_type)
+
+
+def map_type(entry_type: BasicType) -> TypeDescriptor:
+    """Build a type descriptor for a MAP type"""
+    sa = _StaticApiHook.get_instance()
+    return sa.map_type(entry_type)
+
+
+class FileType(_enum.Enum):
+
+    FILE = 1
+    DIRECTORY = 2
+
+
+@_dc.dataclass
+class FileStat:
+
+    """
+    Dataclass to represent some basic  file stat info independent of the storage technology used
+    I.e. do not depend on Python stat_result class that refers to locally-mounted filesystems
+    Timestamps are held in UTC
+    """
+
+    file_name: str
+    file_type: FileType
+    storage_path: str
+    size: int
+
+    mtime: _tp.Optional[_dt.datetime] = None
+    atime: _tp.Optional[_dt.datetime] = None
+
+
+class TracFileStorage:
+
+    @_abc.abstractmethod
+    def get_storage_key(self) -> str:
+        pass
+
+    @_abc.abstractmethod
+    def exists(self, storage_path: str) -> bool:
+        """The exists method can be used for both files and directories"""
+        pass
+
+    @_abc.abstractmethod
+    def size(self, storage_path: str) -> int:
+        """The rm method only works on regular files, it cannot be used for directories"""
+        pass
+
+    @_abc.abstractmethod
+    def stat(self, storage_path: str) -> FileStat:
+        """The stat method can be used for both files and directories, so long as they exist"""
+        pass
+
+    @_abc.abstractmethod
+    def ls(self, storage_path: str, recursive: bool = False) -> _tp.List[FileStat]:
+        """The ls method only works on directories, it cannot be used for regular files"""
+        pass
+
+    @_abc.abstractmethod
+    def mkdir(self, storage_path: str, recursive: bool = False):
+        """The mkdir method will succeed silently if the directory already exists"""
+        pass
+
+    @_abc.abstractmethod
+    def rm(self, storage_path: str):
+        """The rm method only works on regular files, it cannot be used for directories and is not recursive"""
+        pass
+
+    @_abc.abstractmethod
+    def rmdir(self, storage_path: str):
+        """The rmdir method only works on directories and is always recursive"""
+        pass
+
+    @_abc.abstractmethod
+    def read_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The read_byte_stream method only works for existing files"""
+        pass
+
+    @_abc.abstractmethod
+    def write_byte_stream(self, storage_path: str) -> _tp.ContextManager[_tp.BinaryIO]:
+        """The write_byte_stream method will always overwrite an existing file if it exists"""
+        pass
+
+    def read_bytes(self, storage_path: str) -> bytes:
+        """The read_bytes method only works for existing files"""
+        with self.read_byte_stream(storage_path) as stream:
+            return stream.read()
+
+    def write_bytes(self, storage_path: str, data: bytes):
+        """The write_bytes method will always overwrite an existing file if it exists"""
+        with self.write_byte_stream(storage_path) as stream:
+            stream.write(data)
+
+
+class TracDataStorage(_tp.Generic[DATA_API]):
+
+    @_abc.abstractmethod
+    def has_table(self, table_name: str) -> bool:
+        pass
+
+    @_abc.abstractmethod
+    def list_tables(self) -> _tp.List[str]:
+        pass
+
+    @_abc.abstractmethod
+    def create_table(self, table_name: str, schema: SchemaDefinition):
+        pass
+
+    @_abc.abstractmethod
+    def read_table(self, table_name: str) -> DATA_API:
+        pass
+
+    @_abc.abstractmethod
+    def write_table(self, table_name: str, dataset: DATA_API):
+        pass
+
+    @_abc.abstractmethod
+    def native_read_query(self, query: str, **parameters) -> DATA_API:
+        pass
+
+
+class TracDataContext(TracContext):
+
+    @_abc.abstractmethod
+    def get_file_storage(self, storage_key: str) -> TracFileStorage:
+        pass
+
+    @_abc.abstractmethod
+    def get_data_storage(self, storage_key: str, framework: DataFramework[DATA_API], **framework_args) -> TracDataStorage[DATA_API]:
+        pass
+
+    @_abc.abstractmethod
+    def add_data_import(self, dataset_key: str):
+        pass
+
+    @_abc.abstractmethod
+    def set_source_metadata(self, dataset_key: str, storage_key: str, source_info: _tp.Union[FileStat, str]):
+        pass
+
+    @_abc.abstractmethod
+    def set_attribute(self, dataset_key: str, attribute_name: str, value: _tp.Any):
+        pass
+
+    @_abc.abstractmethod
+    def set_schema(self, dataset_key: str, schema: SchemaDefinition):
+        pass
+
+
+class TracDataImport(TracModel):
+
+    def define_inputs(self) -> _tp.Dict[str, ModelInputSchema]:
+        return dict()
+
+    @_abc.abstractmethod
+    def run_model(self, ctx: TracDataContext):
+        pass
+
+
+class TracDataExport(TracModel):
+
+    def define_outputs(self) -> _tp.Dict[str, ModelOutputSchema]:
+        return dict()
+
+    @_abc.abstractmethod
+    def run_model(self, ctx: TracDataContext):
+        pass

tracdap/rt/api/hook.py

@@ -61,6 +61,16 @@ class _StaticApiHook:
 
         return cls.__static_api_hook
 
+    @_abc.abstractmethod
+    def array_type(self, item_type: _meta.BasicType) -> _meta.TypeDescriptor:
+
+        pass
+
+    @_abc.abstractmethod
+    def map_type(self, entry_type: _meta.BasicType) -> _meta.TypeDescriptor:
+
+        pass
+
     @_abc.abstractmethod
     def define_attributes(
             self, *attrs: _tp.Union[_meta.TagUpdate, _tp.List[_meta.TagUpdate]]) \
@@ -118,6 +128,10 @@ class _StaticApiHook:
 
         pass
 
+    @_abc.abstractmethod
+    def infer_schema(self, dataset: _tp.Any) -> _meta.SchemaDefinition:
+        pass
+
     @_abc.abstractmethod
     def define_input_table(
             self, *fields: _tp.Union[_meta.FieldSchema, _tp.List[_meta.FieldSchema]],

tracdap/rt/api/model_api.py

@@ -12,8 +12,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import abc as _abc
 import typing as _tp
 import logging as _logging
@@ -22,8 +20,18 @@ import logging as _logging
 # This significantly improves type hinting, inline documentation and auto-complete in JetBrains IDEs
 from tracdap.rt.metadata import *  # DOCGEN_REMOVE
 
+
 if _tp.TYPE_CHECKING:
-    import pandas
+
+    try:
+        import pandas
+    except ModuleNotFoundError:
+        pass
+
+    try:
+        import polars
+    except ModuleNotFoundError:
+        pass
 
 
 class TracContext:
@@ -134,7 +142,8 @@ class TracContext:
         pass
 
     @_abc.abstractmethod
-    def get_pandas_table(self, dataset_name: str, use_temporal_objects: _tp.Optional[bool] = None) -> pandas.DataFrame:
+    def get_pandas_table(self, dataset_name: str, use_temporal_objects: _tp.Optional[bool] = None) \
+            -> "pandas.DataFrame":
 
         """
         Get the data for a model input or output as a Pandas dataframe
@@ -165,6 +174,22 @@ class TracContext:
         """
         pass
 
+    @_abc.abstractmethod
+    def get_polars_table(self, dataset_name: str) -> "polars.DataFrame":
+
+        """
+        Get the data for a model input or output as a Polars dataframe
+
+        This method has equivalent semantics to :py:meth:`get_pandas_table`, but returns
+        a Polars dataframe.
+
+        :param dataset_name: The name of the model input or output to get data for
+        :return: A polars dataframe containing the data for the named dataset
+        :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
+        """
+
+        pass
+
     @_abc.abstractmethod
     def put_schema(self, dataset_name: str, schema: SchemaDefinition):
 
@@ -196,7 +221,7 @@ class TracContext:
         pass
 
     @_abc.abstractmethod
-    def put_pandas_table(self, dataset_name: str, dataset: pandas.DataFrame):
+    def put_pandas_table(self, dataset_name: str, dataset: "pandas.DataFrame"):
 
         """
         Save the data for a model output as a Pandas dataframe
@@ -217,7 +242,24 @@ class TracContext:
         :param dataset_name: The name of the model output to save data for
         :param dataset: A pandas dataframe containing the data for the named dataset
         :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`,
-                 :py:class:`EDataValidation <tracdap.rt.exceptions.EDataValidation>`
+                 :py:class:`EDataConformance <tracdap.rt.exceptions.EDataConformance>`
+        """
+
+        pass
+
+    @_abc.abstractmethod
+    def put_polars_table(self, dataset_name: str, dataset: "polars.DataFrame"):
+
+        """
+        Save the data for a model output as a Polars dataframe
+
+        This method has equivalent semantics to :py:meth:`put_pandas_table`, but accepts
+        a Polars dataframe.
+
+        :param dataset_name: The name of the model output to save data for
+        :param dataset: A polars dataframe containing the data for the named dataset
+        :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`,
+                 :py:class:`EDataConformance <tracdap.rt.exceptions.EDataConformance>`
         """
 
         pass

tracdap/rt/config/__init__.py

@@ -10,6 +10,8 @@ from .common import ServiceConfig
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
 
+from .job import JobConfig
+
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
@@ -24,7 +26,5 @@ from .platform import RoutingMatch
 from .platform import RoutingTarget
 from .platform import DeploymentConfig
 
-from .job import JobConfig
-
 from .result import TagUpdateList
 from .result import JobResult

tracdap/rt/config/common.py

@@ -17,6 +17,8 @@ class PluginConfig:
 
     protocol: str = ""
 
+    publicProperties: _tp.Dict[str, str] = _dc.field(default_factory=dict)
+
     properties: _tp.Dict[str, str] = _dc.field(default_factory=dict)
 
     secrets: _tp.Dict[str, str] = _dc.field(default_factory=dict)
@@ -63,6 +65,10 @@ class StorageConfig:
 
     buckets: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
 
+    """TODO: Rename "buckets" as "internal" for 0.7"""
+
+    external: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
+
     defaultBucket: str = ""
 
     defaultFormat: str = ""

tracdap/rt/metadata/__init__.py

@@ -21,30 +21,12 @@ from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
 
-from .stoarge import CopyStatus
-from .stoarge import IncarnationStatus
-from .stoarge import StorageCopy
-from .stoarge import StorageIncarnation
-from .stoarge import StorageItem
-from .stoarge import StorageDefinition
-
+from .model import ModelType
 from .model import ModelParameter
 from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
-from .job import JobType
-from .job import JobStatusCode
-from .job import JobDefinition
-from .job import RunModelJob
-from .job import RunFlowJob
-from .job import ImportModelJob
-
-from .file import FileDefinition
-
 from .search import SearchOperator
 from .search import LogicalOperator
 from .search import SearchTerm
@@ -52,18 +34,45 @@ from .search import LogicalExpression
 from .search import SearchExpression
 from .search import SearchParameters
 
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
+
 from .flow import FlowNodeType
 from .flow import FlowNode
 from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
+from .job import JobType
+from .job import JobStatusCode
+from .job import JobGroupType
+from .job import JobDefinition
+from .job import RunModelJob
+from .job import RunFlowJob
+from .job import ImportModelJob
+from .job import ImportDataJob
+from .job import ExportDataJob
+from .job import JobGroup
+from .job import SequentialJobGroup
+from .job import ParallelJobGroup
+
+from .file import FileDefinition
+
 from .custom import CustomDefinition
 
+from .stoarge import CopyStatus
+from .stoarge import IncarnationStatus
+from .stoarge import StorageCopy
+from .stoarge import StorageIncarnation
+from .stoarge import StorageItem
+from .stoarge import StorageDefinition
+
 from .object import ObjectDefinition
 
-from .tag import Tag
+from .resource import ResourceType
 
 from .common import MetadataFormat
 from .common import MetadataVersion
 from .common import TenantInfo
+
+from .tag import Tag

tracdap/rt/metadata/job.py

@@ -32,6 +32,14 @@ class JobType(_enum.Enum):
 
     """Import data into the platform"""
 
+    EXPORT_DATA = 5
+
+    """Export data to external locations"""
+
+    JOB_GROUP = 6
+
+    """A job built from a collection of other jobs"""
+
 
 class JobStatusCode(_enum.Enum):
 
@@ -80,6 +88,17 @@ class JobStatusCode(_enum.Enum):
     """The job was cancelled by a user of the platform"""
 
 
+class JobGroupType(_enum.Enum):
+
+    """Specify the group type for a JOB_GROUP job"""
+
+    JOB_GROUP_TYPE_NOT_SET = 0
+
+    SEQUENTIAL_JOB_GROUP = 1
+
+    PARALLEL_JOB_GROUP = 2
+
+
 @_dc.dataclass
 class JobDefinition:
 
@@ -93,6 +112,12 @@ class JobDefinition:
 
     importModel: _tp.Optional[ImportModelJob] = None
 
+    importData: _tp.Optional[ImportDataJob] = None
+
+    exportData: _tp.Optional[ExportDataJob] = None
+
+    jobGroup: _tp.Optional[JobGroup] = None
+
 
 @_dc.dataclass
 class RunModelJob:
@@ -152,3 +177,77 @@ class ImportModelJob:
     path: str = ""
 
     modelAttrs: _tp.List[TagUpdate] = _dc.field(default_factory=list)
+
+
+@_dc.dataclass
+class ImportDataJob:
+
+    """Specification for an IMPORT_DATA job"""
+
+    model: TagSelector = _dc.field(default_factory=lambda: TagSelector())
+
+    parameters: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
+
+    inputs: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    outputs: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    priorOutputs: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    storageAccess: _tp.List[str] = _dc.field(default_factory=list)
+
+    imports: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    outputAttrs: _tp.List[TagUpdate] = _dc.field(default_factory=list)
+
+    importAttrs: _tp.List[TagUpdate] = _dc.field(default_factory=list)
+
+
+@_dc.dataclass
+class ExportDataJob:
+
+    """Specification for an EXPORT_DATA job"""
+
+    model: TagSelector = _dc.field(default_factory=lambda: TagSelector())
+
+    parameters: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
+
+    inputs: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    outputs: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    priorOutputs: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    storageAccess: _tp.List[str] = _dc.field(default_factory=list)
+
+    exports: _tp.Dict[str, TagSelector] = _dc.field(default_factory=dict)
+
+    outputAttrs: _tp.List[TagUpdate] = _dc.field(default_factory=list)
+
+
+@_dc.dataclass
+class JobGroup:
+
+    """Specification for a JOB_GROUP job, which runs a collection of other jobs"""
+
+    jobGroupType: JobGroupType = JobGroupType.JOB_GROUP_TYPE_NOT_SET
+
+    sequential: _tp.Optional[SequentialJobGroup] = None
+
+    parallel: _tp.Optional[ParallelJobGroup] = None
+
+
+@_dc.dataclass
+class SequentialJobGroup:
+
+    """A job group where each job runs in sequence"""
+
+    jobs: _tp.List[JobDefinition] = _dc.field(default_factory=list)
+
+
+@_dc.dataclass
+class ParallelJobGroup:
+
+    """A job group where all jobs runs in parallel"""
+
+    jobs: _tp.List[JobDefinition] = _dc.field(default_factory=list)

tracdap/rt/metadata/model.py

@@ -9,6 +9,22 @@ from .type import *  # noqa
 from .data import *  # noqa
 
 
+class ModelType(_enum.Enum):
+
+    """Identify specialized model types for specific tasks"""
+
+    STANDARD_MODEL = 0
+
+    """A regular model with parameters, inputs and outputs (this is the default)"""
+
+    DATA_IMPORT_MODEL = 1
+
+    """A model with read access to external storage for importing data"""
+
+    DATA_EXPORT_MODEL = 2
+
+    """A model with write access to external storage for exporting data"""
+
 
 @_dc.dataclass
 class ModelParameter:
@@ -105,3 +121,5 @@ class ModelDefinition:
     staticAttributes: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
 
     """Static attributes defined in model code"""
+
+    modelType: ModelType = ModelType.STANDARD_MODEL

tracdap/rt/metadata/resource.py

@@ -0,0 +1,24 @@
+# Code generated by TRAC
+
+from __future__ import annotations
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+from .object_id import *  # noqa
+from .object import *  # noqa
+
+
+class ResourceType(_enum.Enum):
+
+    """Enumeration of infrastructure resources that can be added to a tenant"""
+
+    RESOURCE_TYPE_NOT_SET = 0
+
+    MODEL_REPOSITORY = 1
+
+    """Model repository, which can be a source or binary repository"""
+
+    INTERNAL_STORAGE = 2
+
+    """Storage location for data held internally by the TRAC platform"""

{tracdap_runtime-0.6.4.dist-info → tracdap_runtime-0.6.6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tracdap-runtime
-Version: 0.6.4
+Version: 0.6.6
 Summary: Runtime package for building models on the TRAC Data & Analytics Platform
 Home-page: https://tracdap.finos.org/
 Author: Martin Traverse
@@ -39,8 +39,12 @@ Requires-Dist: gcsfs==2024.3.1; extra == "gcp"
 Provides-Extra: grpc
 Requires-Dist: grpcio==1.66.1; extra == "grpc"
 Requires-Dist: grpcio-status==1.66.1; extra == "grpc"
+Provides-Extra: polars
+Requires-Dist: polars<2.0.0,>=1.0.0; extra == "polars"
 Provides-Extra: spark
 Requires-Dist: pyspark<3.6.0,>=3.0.0; extra == "spark"
+Provides-Extra: sql
+Requires-Dist: sqlalchemy<2.1.0,>=2.0.0; extra == "sql"
 
 # TRAC Model Runtime for Python