tracdap-runtime 0.6.3__py3-none-any.whl → 0.6.5__py3-none-any.whl

tracdap/rt/api/model_api.py

@@ -12,8 +12,6 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-from __future__ import annotations
-
 import abc as _abc
 import typing as _tp
 import logging as _logging
@@ -22,8 +20,18 @@ import logging as _logging
 # This significantly improves type hinting, inline documentation and auto-complete in JetBrains IDEs
 from tracdap.rt.metadata import *  # DOCGEN_REMOVE
 
+
 if _tp.TYPE_CHECKING:
-    import pandas
+
+    try:
+        import pandas
+    except ModuleNotFoundError:
+        pass
+
+    try:
+        import polars
+    except ModuleNotFoundError:
+        pass
 
 
 class TracContext:
@@ -100,13 +108,26 @@ class TracContext:
         """
         Get the schema of a model input or output
 
-        The schema of an input or output can be retrieved and examined at runtime using this method.
-        Inputs must be defined in :py:meth:`TracModel.define_inputs`
-        and outputs in :py:meth:`TracModel.define_outputs`.
-        Input and output names are case-sensitive.
-
-        In the current version of the runtime all model inputs and outputs are defined statically,
-        :py:meth:`get_schema` will return the schema as it was defined.
+        Use this method to get the :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
+        for any input or output of the current model.
+        For datasets with static schemas, this will be the same schema that was defined in the
+        :py:class:`TracModel <tracdap.rt.api.TracModel>` methods
+        :py:meth:`define_inputs() <tracdap.rt.api.TracModel.define_inputs>` and
+        :py:meth:`define_outputs() <tracdap.rt.api.TracModel.define_outputs>`.
+
+        For inputs with dynamic schemas, the schema of the provided input dataset will be returned.
+        For outputs with dynamic schemas the schema must be set by calling
+        :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>`, after which this method
+        will return that schema. Calling :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>`
+        for a dynamic output before the schema is set will result in a runtime validation error.
+
+        For optional inputs, use :py:meth:`has_dataset() <tracdap.rt.api.TracContext.has_dataset>`
+        to check whether the input was provided. Calling :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>`
+        for an optional input that was not provided will always result in a validation error,
+        regardless of whether the input using a static or dynamic schema. For optional outputs
+        :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` can be called, with the
+        normal proviso that dynamic schemas must first be set by calling
+        :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>`.
 
         Attempting to retrieve the schema for a dataset that is not defined as a model input or output
         will result in a runtime validation error, even if that dataset exists in the job config and
@@ -121,7 +142,8 @@ class TracContext:
         pass
 
     @_abc.abstractmethod
-    def get_pandas_table(self, dataset_name: str, use_temporal_objects: _tp.Optional[bool] = None) -> pandas.DataFrame:
+    def get_pandas_table(self, dataset_name: str, use_temporal_objects: _tp.Optional[bool] = None) \
+            -> "pandas.DataFrame":
 
         """
         Get the data for a model input or output as a Pandas dataframe
@@ -153,7 +175,53 @@ class TracContext:
         pass
 
     @_abc.abstractmethod
-    def put_pandas_table(self, dataset_name: str, dataset: pandas.DataFrame):
+    def get_polars_table(self, dataset_name: str) -> "polars.DataFrame":
+
+        """
+        Get the data for a model input or output as a Polars dataframe
+
+        This method has equivalent semantics to :py:meth:`get_pandas_table`, but returns
+        a Polars dataframe.
+
+        :param dataset_name: The name of the model input or output to get data for
+        :return: A polars dataframe containing the data for the named dataset
+        :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
+        """
+
+        pass
+
+    @_abc.abstractmethod
+    def put_schema(self, dataset_name: str, schema: SchemaDefinition):
+
+        """
+        Set the schema of a dynamic model output
+
+        For outputs marked as dynamic, a :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
+        must be supplied before attempting to save the data. TRAC API functions are available to help with
+        building schemas, such as :py:func:`trac.F() <tracdap.rt.api.F>` to define fields or
+        :py:func:`load_schema() <tracdap.rt.api.load_schema>` to load predefined schemas.
+        Once a schema has been set, it can be retrieved by calling
+        :py:meth:`get_schema() <tracdap.rt.api.TracContext.get_schema>` as normal.
+        If :py:meth:`put_schema() <tracdap.rt.api.TracContext.put_schema>` is called for an optional
+        output the model must also supply data for that output, otherwise TRAC will report a runtime
+        validation error after the model completes.
+
+        Each schema can only be set once and the schema will be validated using the normal
+        validation rules. If validation fails this method will raise
+        :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`.
+        Attempting to set the schema for a dataset that is not defined as a dynamic model output
+        for the current model will result in a runtime validation error.
+
+        :param dataset_name: The name of the output to set the schema for
+        :param schema: A TRAC schema definition to use for the named output
+        :type schema: :py:class:`SchemaDefinition <tracdap.rt.metadata.SchemaDefinition>`
+        :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`
+        """
+
+        pass
+
+    @_abc.abstractmethod
+    def put_pandas_table(self, dataset_name: str, dataset: "pandas.DataFrame"):
 
         """
         Save the data for a model output as a Pandas dataframe
@@ -174,7 +242,24 @@ class TracContext:
         :param dataset_name: The name of the model output to save data for
         :param dataset: A pandas dataframe containing the data for the named dataset
         :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`,
-                 :py:class:`EDataValidation <tracdap.rt.exceptions.EDataValidation>`
+                 :py:class:`EDataConformance <tracdap.rt.exceptions.EDataConformance>`
+        """
+
+        pass
+
+    @_abc.abstractmethod
+    def put_polars_table(self, dataset_name: str, dataset: "polars.DataFrame"):
+
+        """
+        Save the data for a model output as a Polars dataframe
+
+        This method has equivalent semantics to :py:meth:`put_pandas_table`, but accepts
+        a Polars dataframe.
+
+        :param dataset_name: The name of the model output to save data for
+        :param dataset: A polars dataframe containing the data for the named dataset
+        :raises: :py:class:`ERuntimeValidation <tracdap.rt.exceptions.ERuntimeValidation>`,
+                 :py:class:`EDataConformance <tracdap.rt.exceptions.EDataConformance>`
         """
 
         pass

tracdap/rt/api/static_api.py

@@ -451,8 +451,7 @@ def load_schema(
 
 def define_input_table(
         *fields: _tp.Union[FieldSchema, _tp.List[FieldSchema]],
-        label: _tp.Optional[str] = None,
-        optional: bool = False,
+        label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
         input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
         -> ModelInputSchema:
 
@@ -469,6 +468,7 @@ def define_input_table(
     :param fields: A set of fields to make up a :py:class:`TableSchema <tracdap.rt.metadata.TableSchema>`
     :param label: An optional label (of type str) for a model input schema. Default value: None.
     :param optional: Mark this input as an optional model input
+    :param dynamic: Mark this input as a dynamic model input (the list of fields must be empty)
     :param input_props: Associate key-value properties with this input (not used by the TRAC engine)
     :return: A model input schema, suitable for returning from :py:meth:`TracModel.define_inputs`
 
@@ -476,12 +476,16 @@ def define_input_table(
                   List[:py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`]
     :type label: Optional[str]
     :type optional: bool
+    :type dynamic: bool
     :type input_props: Optional[Dict[str, Any]]
     :rtype: :py:class:`ModelInputSchema <tracdap.rt.metadata.ModelInputSchema>`
     """
 
     sa = _StaticApiHook.get_instance()
-    return sa.define_input_table(*fields, label=label, optional=optional, input_props=input_props)
+
+    return sa.define_input_table(
+        *fields, label=label, optional=optional, dynamic=dynamic,
+        input_props=input_props)
 
 
 def declare_input_table(
@@ -507,8 +511,7 @@ def declare_input_table(
 
 def define_output_table(
         *fields: _tp.Union[FieldSchema, _tp.List[FieldSchema]],
-        label: _tp.Optional[str] = None,
-        optional: bool = False,
+        label: _tp.Optional[str] = None, optional: bool = False, dynamic: bool = False,
         output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
         -> ModelOutputSchema:
 
@@ -525,6 +528,7 @@ def define_output_table(
     :param fields: A set of fields to make up a :py:class:`TableSchema <tracdap.rt.metadata.TableSchema>`
     :param label: An optional label (of type str) for a model output schema. Default value: None.
     :param optional: Mark this output as an optional model output
+    :param dynamic: Mark this output as a dynamic model output (the list of fields must be empty)
     :param output_props: Associate key-value properties with this output (not used by the TRAC engine)
     :return: A model output schema, suitable for returning from :py:meth:`TracModel.define_outputs`
 
@@ -532,12 +536,16 @@ def define_output_table(
                   List[:py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`]
     :type label: Optional[str]
     :type optional: bool
+    :type dynamic: bool
     :type output_props: Optional[Dict[str, Any]]
     :rtype: :py:class:`ModelOutputSchema <tracdap.rt.metadata.ModelOutputSchema>`
     """
 
     sa = _StaticApiHook.get_instance()
-    return sa.define_output_table(*fields, label=label, optional=optional, output_props=output_props)
+
+    return sa.define_output_table(
+        *fields, label=label, optional=optional, dynamic=dynamic,
+        output_props=output_props)
 
 
 def declare_output_table(

tracdap/rt/config/__init__.py

@@ -10,6 +10,8 @@ from .common import ServiceConfig
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
 
+from .job import JobConfig
+
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
@@ -24,7 +26,5 @@ from .platform import RoutingMatch
 from .platform import RoutingTarget
 from .platform import DeploymentConfig
 
-from .job import JobConfig
-
 from .result import TagUpdateList
 from .result import JobResult

tracdap/rt/config/common.py

@@ -15,7 +15,9 @@ class _ConfigFile:
 @_dc.dataclass
 class PluginConfig:
 
-    protocol: str = None
+    protocol: str = ""
+
+    publicProperties: _tp.Dict[str, str] = _dc.field(default_factory=dict)
 
     properties: _tp.Dict[str, str] = _dc.field(default_factory=dict)
 
@@ -25,9 +27,9 @@ class PluginConfig:
 @_dc.dataclass
 class PlatformInfo:
 
-    environment: str = None
+    environment: str = ""
 
-    production: bool = None
+    production: bool = False
 
     deploymentInfo: _tp.Dict[str, str] = _dc.field(default_factory=dict)
 
@@ -35,27 +37,27 @@ class PlatformInfo:
 @_dc.dataclass
 class AuthenticationConfig:
 
-    jwtIssuer: str = None
+    jwtIssuer: str = ""
 
-    jwtExpiry: int = None
+    jwtExpiry: int = 0
 
-    jwtLimit: int = None
+    jwtLimit: int = 0
 
-    jwtRefresh: int = None
+    jwtRefresh: int = 0
 
     provider: _tp.Optional[PluginConfig] = None
 
-    disableAuth: bool = None
+    disableAuth: bool = False
 
-    disableSigning: bool = None
+    disableSigning: bool = False
 
-    systemUserId: str = None
+    systemUserId: str = ""
 
-    systemUserName: str = None
+    systemUserName: str = ""
 
-    systemTicketDuration: int = None
+    systemTicketDuration: int = 0
 
-    systemTicketRefresh: int = None
+    systemTicketRefresh: int = 0
 
 
 @_dc.dataclass
@@ -63,9 +65,13 @@ class StorageConfig:
 
     buckets: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
 
-    defaultBucket: str = None
+    """TODO: Rename "buckets" as "internal" for 0.7"""
+
+    external: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
+
+    defaultBucket: str = ""
 
-    defaultFormat: str = None
+    defaultFormat: str = ""
 
 
 @_dc.dataclass
@@ -73,6 +79,6 @@ class ServiceConfig:
 
     enabled: _tp.Optional[bool] = None
 
-    alias: str = None
+    alias: str = ""
 
-    port: int = None
+    port: int = 0

tracdap/rt/config/job.py

@@ -12,9 +12,9 @@ import tracdap.rt.metadata as metadata
 @_dc.dataclass
 class JobConfig:
 
-    jobId: metadata.TagHeader = None
+    jobId: metadata.TagHeader = _dc.field(default_factory=lambda: metadata.TagHeader())
 
-    job: metadata.JobDefinition = None
+    job: metadata.JobDefinition = _dc.field(default_factory=lambda: metadata.JobDefinition())
 
     resources: _tp.Dict[str, metadata.ObjectDefinition] = _dc.field(default_factory=dict)
 

tracdap/rt/config/platform.py

@@ -38,19 +38,19 @@ class PlatformConfig:
 
     config: _tp.Dict[str, str] = _dc.field(default_factory=dict)
 
-    platformInfo: PlatformInfo = None
+    platformInfo: PlatformInfo = _dc.field(default_factory=lambda: PlatformInfo())
 
-    authentication: AuthenticationConfig = None
+    authentication: AuthenticationConfig = _dc.field(default_factory=lambda: AuthenticationConfig())
 
-    metadata: MetadataConfig = None
+    metadata: MetadataConfig = _dc.field(default_factory=lambda: MetadataConfig())
 
-    storage: StorageConfig = None
+    storage: StorageConfig = _dc.field(default_factory=lambda: StorageConfig())
 
     repositories: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
 
-    executor: PluginConfig = None
+    executor: PluginConfig = _dc.field(default_factory=lambda: PluginConfig())
 
-    jobCache: PluginConfig = None
+    jobCache: PluginConfig = _dc.field(default_factory=lambda: PluginConfig())
 
     tenants: _tp.Dict[str, TenantConfig] = _dc.field(default_factory=dict)
 
@@ -60,13 +60,13 @@ class PlatformConfig:
 
     services: _tp.Dict[str, ServiceConfig] = _dc.field(default_factory=dict)
 
-    deployment: DeploymentConfig = None
+    deployment: DeploymentConfig = _dc.field(default_factory=lambda: DeploymentConfig())
 
 
 @_dc.dataclass
 class MetadataConfig:
 
-    database: PluginConfig = None
+    database: PluginConfig = _dc.field(default_factory=lambda: PluginConfig())
 
     format: metadata.MetadataFormat = metadata.MetadataFormat.METADATA_FORMAT_NOT_SET
 
@@ -82,9 +82,9 @@ class TenantConfig:
 @_dc.dataclass
 class WebServerConfig:
 
-    enabled: bool = None
+    enabled: bool = False
 
-    contentRoot: PluginConfig = None
+    contentRoot: PluginConfig = _dc.field(default_factory=lambda: PluginConfig())
 
     rewriteRules: _tp.List[WebServerRewriteRule] = _dc.field(default_factory=list)
 
@@ -94,25 +94,25 @@ class WebServerConfig:
 @_dc.dataclass
 class WebServerRewriteRule:
 
-    source: str = None
+    source: str = ""
 
-    target: str = None
+    target: str = ""
 
 
 @_dc.dataclass
 class WebServerRedirect:
 
-    source: str = None
+    source: str = ""
 
-    target: str = None
+    target: str = ""
 
-    status: int = None
+    status: int = 0
 
 
 @_dc.dataclass
 class GatewayConfig:
 
-    idleTimeout: int = None
+    idleTimeout: int = 0
 
     routes: _tp.List[RouteConfig] = _dc.field(default_factory=list)
 
@@ -122,35 +122,35 @@ class GatewayConfig:
 @_dc.dataclass
 class RouteConfig:
 
-    routeName: str = None
+    routeName: str = ""
 
     routeType: RoutingProtocol = RoutingProtocol.PROTOCOL_NOT_SET
 
     protocols: _tp.List[RoutingProtocol] = _dc.field(default_factory=list)
 
-    match: RoutingMatch = None
+    match: RoutingMatch = _dc.field(default_factory=lambda: RoutingMatch())
 
-    target: RoutingTarget = None
+    target: RoutingTarget = _dc.field(default_factory=lambda: RoutingTarget())
 
 
 @_dc.dataclass
 class RoutingMatch:
 
-    host: str = None
+    host: str = ""
 
-    path: str = None
+    path: str = ""
 
 
 @_dc.dataclass
 class RoutingTarget:
 
-    scheme: str = None
+    scheme: str = ""
 
-    host: str = None
+    host: str = ""
 
-    port: int = None
+    port: int = 0
 
-    path: str = None
+    path: str = ""
 
 
 @_dc.dataclass

tracdap/rt/config/result.py

@@ -18,10 +18,10 @@ class TagUpdateList:
 @_dc.dataclass
 class JobResult:
 
-    jobId: metadata.TagHeader = None
+    jobId: metadata.TagHeader = _dc.field(default_factory=lambda: metadata.TagHeader())
 
     statusCode: metadata.JobStatusCode = metadata.JobStatusCode.JOB_STATUS_CODE_NOT_SET
 
-    statusMessage: str = None
+    statusMessage: str = ""
 
     results: _tp.Dict[str, metadata.ObjectDefinition] = _dc.field(default_factory=dict)

tracdap/rt/config/runtime.py

@@ -14,13 +14,13 @@ class RuntimeConfig:
 
     config: _tp.Dict[str, str] = _dc.field(default_factory=dict)
 
-    storage: StorageConfig = None
+    storage: StorageConfig = _dc.field(default_factory=lambda: StorageConfig())
 
     repositories: _tp.Dict[str, PluginConfig] = _dc.field(default_factory=dict)
 
-    sparkSettings: SparkSettings = None
+    sparkSettings: SparkSettings = _dc.field(default_factory=lambda: SparkSettings())
 
-    runtimeApi: ServiceConfig = None
+    runtimeApi: ServiceConfig = _dc.field(default_factory=lambda: ServiceConfig())
 
 
 @_dc.dataclass

tracdap/rt/launch/cli.py

@@ -16,18 +16,18 @@ import argparse
 import pathlib
 
 
-def _cli_args():
+def _cli_args(programmatic_args = None):
 
     parser = argparse.ArgumentParser(
         prog="python -m tracdap.rt.launch",
         description="TRAC D.A.P. Runtime for Python")
 
     parser.add_argument(
-        "--sys-config", dest="sys_config", type=pathlib.Path, required=True,
+        "--sys-config", dest="sys_config", type=str, required=True,
         help="Path to the system configuration file for the TRAC runtime")
 
     parser.add_argument(
-        "--job-config", dest="job_config", type=pathlib.Path, required=True,
+        "--job-config", dest="job_config", type=str, required=True,
         help="Path to the job configuration for the job to be executed")
 
     parser.add_argument(
@@ -55,4 +55,7 @@ def _cli_args():
         "--plugin-package", dest="plugin_packages", type=str, action="append",
         help="Do not clean up the scratch location on exit")
 
-    return parser.parse_args()
+    if programmatic_args:
+        return parser.parse_args(programmatic_args)
+    else:
+        return parser.parse_args()

tracdap/rt/launch/launch.py

@@ -29,7 +29,15 @@ from .cli import _cli_args
 def _resolve_config_file(
         config_path: _tp.Union[str, _pathlib.Path],
         model_dir: _tp.Optional[_pathlib.Path] = None) \
-        -> _pathlib.Path:
+        -> _tp.Union[_pathlib.Path, str]:
+
+    # If the config path is a URL, do not convert it into a path
+    if isinstance(config_path, str):
+        scheme_sep = config_path.find(":")
+        # Single letter scheme is a Windows file path (C:\...)
+        scheme = scheme = config_path[:scheme_sep] if scheme_sep > 1 else "file"
+        if scheme != "file":
+            return config_path
 
     if _pathlib.Path(config_path).is_absolute():
         return config_path
@@ -136,13 +144,21 @@ def launch_job(
         rt.wait_for_job(job.jobId)
 
 
-def launch_cli():
+def launch_cli(programmatic_args: _tp.Optional[_tp.List[str]] = None):
 
     """
     Launch the TRAC runtime using the command line interface
+
+    CLI arguments are read from the process command line by default. To pass CLI args
+    explicitly, provide the list of arguments using the programmatic_args parameter.
+
+    :param programmatic_args: Optional parameter to pass CLI args explicitly in code
     """
 
-    launch_args = _cli_args()
+    if programmatic_args:
+        launch_args = _cli_args(programmatic_args)
+    else:
+        launch_args = _cli_args()
 
     _sys_config = _resolve_config_file(launch_args.sys_config, None)
 

tracdap/rt/metadata/__init__.py

@@ -21,30 +21,12 @@ from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
 
-from .stoarge import CopyStatus
-from .stoarge import IncarnationStatus
-from .stoarge import StorageCopy
-from .stoarge import StorageIncarnation
-from .stoarge import StorageItem
-from .stoarge import StorageDefinition
-
+from .model import ModelType
 from .model import ModelParameter
 from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
-from .job import JobType
-from .job import JobStatusCode
-from .job import JobDefinition
-from .job import RunModelJob
-from .job import RunFlowJob
-from .job import ImportModelJob
-
-from .file import FileDefinition
-
 from .search import SearchOperator
 from .search import LogicalOperator
 from .search import SearchTerm
@@ -52,18 +34,41 @@ from .search import LogicalExpression
 from .search import SearchExpression
 from .search import SearchParameters
 
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
+
 from .flow import FlowNodeType
 from .flow import FlowNode
 from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
+from .job import JobType
+from .job import JobStatusCode
+from .job import JobDefinition
+from .job import RunModelJob
+from .job import RunFlowJob
+from .job import ImportModelJob
+from .job import ImportDataJob
+from .job import ExportDataJob
+
+from .file import FileDefinition
+
 from .custom import CustomDefinition
 
+from .stoarge import CopyStatus
+from .stoarge import IncarnationStatus
+from .stoarge import StorageCopy
+from .stoarge import StorageIncarnation
+from .stoarge import StorageItem
+from .stoarge import StorageDefinition
+
 from .object import ObjectDefinition
 
-from .tag import Tag
+from .resource import ResourceType
 
 from .common import MetadataFormat
 from .common import MetadataVersion
 from .common import TenantInfo
+
+from .tag import Tag

tracdap/rt/metadata/common.py

@@ -49,10 +49,10 @@ class TenantInfo:
 
     """Information about a tenant that is set up on the TRAC platform."""
 
-    tenantCode: str = None
+    tenantCode: str = ""
 
     """* Unique code used to identify the tenant, required by most API calls."""
 
-    description: str = None
+    description: str = ""
 
     """* A short description of the tenant, suitable for displaying to users in lists."""

tracdap/rt/metadata/custom.py

@@ -11,8 +11,8 @@ class CustomDefinition:
 
     """Define a custom object that can be stored and managed in the TRAC metadata store"""
 
-    customSchemaType: str = None
+    customSchemaType: str = ""
 
-    customSchemaVersion: int = None
+    customSchemaVersion: int = 0
 
-    customData: bytes = None
+    customData: bytes = b""