tracdap-runtime 0.6.2__py3-none-any.whl → 0.6.4__py3-none-any.whl

tracdap/rt/ext/plugins.py

@@ -19,7 +19,8 @@ import importlib as _il
 
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
-import tracdap.rt.ext._guard as _guard
+import tracdap.rt._impl.guard_rails as _guard  # noqa
+from tracdap.rt.exceptions import EStartup
 
 
 class PluginManager:
@@ -29,6 +30,8 @@ class PluginManager:
     __log = _log.getLogger(f"{__name__}.PluginManager")
 
     __core_registered = False
+    __3rd_party_registered = list()
+
     __plugins = {}
 
     @classmethod
@@ -40,17 +43,42 @@ class PluginManager:
             return
 
         cls.__log.info("Register core plugins...")
+        cls.__register_plugin_package("tracdap.rt._plugins")
+
+        cls.__core_registered = True
+
+    @classmethod
+    def register_plugin_package(cls, plugin_package_name: str):
+
+        _guard.run_model_guard()
 
-        plugins_package = _il.import_module("tracdap.rt._plugins")
+        if plugin_package_name in cls.__3rd_party_registered:
+            return
+
+        if plugin_package_name.startswith("tracdap."):
+            raise EStartup("3rd party plugins cannot be registered from the tracdap namespace")
+
+        cls.__log.info(f"Register plugins from package [{plugin_package_name}]...")
+        cls.__register_plugin_package(plugin_package_name)
+
+        cls.__3rd_party_registered.append(plugin_package_name)
+
+    @classmethod
+    def __register_plugin_package(cls, plugin_package_name: str):
+
+        _guard.run_model_guard()
+
+        plugins_package = _il.import_module(plugin_package_name)
 
         for module in _pkg.iter_modules(plugins_package.__path__):
+
+            module_name = f"{plugin_package_name}.{module.name}"
+
             try:
-                module_name = f"tracdap.rt._plugins.{module.name}"
                 _il.import_module(module_name)
             except ImportError:
-                pass  # Ignore plugins that fail to load
-
-        cls.__core_registered = True
+                # It is not a fatal error if some plugins fail to load
+                cls.__log.warning(f"Failed to load plugins from module [{module_name}]")
 
     @classmethod
     def register_plugin(
@@ -94,3 +122,16 @@ class PluginManager:
         plugin = plugin_class(config.properties)
 
         return plugin
+
+    @classmethod
+    def load_config_plugin(cls,
+            service_type: _tp.Type[T_SERVICE],
+            config: _cfg.PluginConfig) \
+            -> T_SERVICE:
+
+        # Currently config plugins are loaded the same way as regular plugins
+        # However, regular plugins can be modified to take ConfigManager as an init parameter
+        # This is useful for loading secondary config files needed in particularly plugins
+        # Config plugins can never do this, because the config manager is not yet initialized
+
+        return cls.load_plugin(service_type, config)

tracdap/rt/launch/cli.py

@@ -16,18 +16,18 @@ import argparse
 import pathlib
 
 
-def _cli_args():
+def _cli_args(programmatic_args = None):
 
     parser = argparse.ArgumentParser(
         prog="python -m tracdap.rt.launch",
         description="TRAC D.A.P. Runtime for Python")
 
     parser.add_argument(
-        "--sys-config", dest="sys_config", type=pathlib.Path, required=True,
+        "--sys-config", dest="sys_config", type=str, required=True,
         help="Path to the system configuration file for the TRAC runtime")
 
     parser.add_argument(
-        "--job-config", dest="job_config", type=pathlib.Path, required=True,
+        "--job-config", dest="job_config", type=str, required=True,
         help="Path to the job configuration for the job to be executed")
 
     parser.add_argument(
@@ -51,4 +51,11 @@ def _cli_args():
         "--scratch-dir-persist", dest="scratch_dir_persist", default=False, action="store_true",
         help="Do not clean up the scratch location on exit")
 
-    return parser.parse_args()
+    parser.add_argument(
+        "--plugin-package", dest="plugin_packages", type=str, action="append",
+        help="Do not clean up the scratch location on exit")
+
+    if programmatic_args:
+        return parser.parse_args(programmatic_args)
+    else:
+        return parser.parse_args()

tracdap/rt/launch/launch.py

@@ -29,7 +29,15 @@ from .cli import _cli_args
 def _resolve_config_file(
         config_path: _tp.Union[str, _pathlib.Path],
         model_dir: _tp.Optional[_pathlib.Path] = None) \
-        -> _pathlib.Path:
+        -> _tp.Union[_pathlib.Path, str]:
+
+    # If the config path is a URL, do not convert it into a path
+    if isinstance(config_path, str):
+        scheme_sep = config_path.find(":")
+        # Single letter scheme is a Windows file path (C:\...)
+        scheme = scheme = config_path[:scheme_sep] if scheme_sep > 1 else "file"
+        if scheme != "file":
+            return config_path
 
     if _pathlib.Path(config_path).is_absolute():
         return config_path
@@ -53,10 +61,16 @@ def _resolve_config_file(
         return _pathlib.Path(config_path)
 
 
+def _optional_arg(launch_args: _tp.Dict[str, _tp.Any], arg_name: str) -> _tp.Any:
+
+    return launch_args.get(arg_name, None)
+
+
 def launch_model(
         model_class: _api.TracModel.__class__,
         job_config: _tp.Union[str, _pathlib.Path],
-        sys_config: _tp.Union[str, _pathlib.Path]):
+        sys_config: _tp.Union[str, _pathlib.Path],
+        **launch_args):
 
     """
     Launch an individual model using its Python class
@@ -77,6 +91,7 @@ def launch_model(
     :param model_class: The model class that will be launched
     :param job_config: Path to the job configuration file
     :param sys_config: Path to the system configuration file
+    :param launch_args: Additional keyword args to control behavior of the TRAC runtime (not normally required)
     """
 
     model_file = _inspect.getfile(model_class)
@@ -85,7 +100,10 @@ def launch_model(
     _sys_config = _resolve_config_file(sys_config, model_dir)
     _job_config = _resolve_config_file(job_config, model_dir)
 
-    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=True)
+    plugin_package = _optional_arg(launch_args, 'plugin_package')
+    plugin_packages = [plugin_package] if plugin_package else None
+
+    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=True, plugin_packages=plugin_packages)
     runtime_instance.pre_start()
 
     job = runtime_instance.load_job_config(_job_config, model_class=model_class)
@@ -98,7 +116,8 @@ def launch_model(
 def launch_job(
         job_config: _tp.Union[str, _pathlib.Path],
         sys_config: _tp.Union[str, _pathlib.Path],
-        dev_mode: bool = False):
+        dev_mode: bool = False,
+        **launch_args):
 
     """
     Launch a TRAC job using external configuration files
@@ -106,12 +125,16 @@ def launch_job(
     :param job_config: Path to the job configuration file
     :param sys_config: Path to the system configuration file
     :param dev_mode: Whether to launch in dev mode (applies dev mode translation to the job inputs)
+    :param launch_args: Additional keyword args to control behavior of the TRAC runtime (not normally required)
     """
 
     _sys_config = _resolve_config_file(sys_config, None)
     _job_config = _resolve_config_file(job_config, None)
 
-    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=dev_mode)
+    plugin_package = _optional_arg(launch_args, 'plugin_package')
+    plugin_packages = [plugin_package] if plugin_package else None
+
+    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=dev_mode, plugin_packages=plugin_packages)
     runtime_instance.pre_start()
 
     job = runtime_instance.load_job_config(_job_config)
@@ -121,16 +144,23 @@ def launch_job(
         rt.wait_for_job(job.jobId)
 
 
-def launch_cli():
+def launch_cli(programmatic_args: _tp.Optional[_tp.List[str]] = None):
 
     """
     Launch the TRAC runtime using the command line interface
+
+    CLI arguments are read from the process command line by default. To pass CLI args
+    explicitly, provide the list of arguments using the programmatic_args parameter.
+
+    :param programmatic_args: Optional parameter to pass CLI args explicitly in code
     """
 
-    launch_args = _cli_args()
+    if programmatic_args:
+        launch_args = _cli_args(programmatic_args)
+    else:
+        launch_args = _cli_args()
 
     _sys_config = _resolve_config_file(launch_args.sys_config, None)
-    _job_config = _resolve_config_file(launch_args.job_config, None)
 
     runtime_instance = _runtime.TracRuntime(
         _sys_config,
@@ -138,12 +168,23 @@ def launch_cli():
         job_result_dir=launch_args.job_result_dir,
         job_result_format=launch_args.job_result_format,
         scratch_dir=launch_args.scratch_dir,
-        scratch_dir_persist=launch_args.scratch_dir_persist)
+        scratch_dir_persist=launch_args.scratch_dir_persist,
+        plugin_packages=launch_args.plugin_packages)
 
     runtime_instance.pre_start()
 
-    job = runtime_instance.load_job_config(_job_config)
+    if launch_args.job_config is not None:
+        _job_config = _resolve_config_file(launch_args.job_config, None)
+        job = runtime_instance.load_job_config(_job_config)
+    else:
+        job = None
 
     with runtime_instance as rt:
-        rt.submit_job(job)
-        rt.wait_for_job(job.jobId)
+
+        if job is not None:
+            rt.submit_job(job)
+
+        if rt.is_oneshot():
+            rt.wait_for_job(job.jobId)
+        else:
+            rt.run_until_done()

tracdap/rt/metadata/__init__.py

@@ -13,10 +13,6 @@ from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
 
-from .file import FileDefinition
-
-from .custom import CustomDefinition
-
 from .data import SchemaType
 from .data import PartType
 from .data import FieldSchema
@@ -25,6 +21,13 @@ from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
 
+from .stoarge import CopyStatus
+from .stoarge import IncarnationStatus
+from .stoarge import StorageCopy
+from .stoarge import StorageIncarnation
+from .stoarge import StorageItem
+from .stoarge import StorageDefinition
+
 from .model import ModelParameter
 from .model import ModelInputSchema
 from .model import ModelOutputSchema
@@ -33,6 +36,15 @@ from .model import ModelDefinition
 from .tag_update import TagOperation
 from .tag_update import TagUpdate
 
+from .job import JobType
+from .job import JobStatusCode
+from .job import JobDefinition
+from .job import RunModelJob
+from .job import RunFlowJob
+from .job import ImportModelJob
+
+from .file import FileDefinition
+
 from .search import SearchOperator
 from .search import LogicalOperator
 from .search import SearchTerm
@@ -46,19 +58,7 @@ from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
-from .job import JobType
-from .job import JobStatusCode
-from .job import JobDefinition
-from .job import RunModelJob
-from .job import RunFlowJob
-from .job import ImportModelJob
-
-from .stoarge import CopyStatus
-from .stoarge import IncarnationStatus
-from .stoarge import StorageCopy
-from .stoarge import StorageIncarnation
-from .stoarge import StorageItem
-from .stoarge import StorageDefinition
+from .custom import CustomDefinition
 
 from .object import ObjectDefinition
 

tracdap/rt/metadata/common.py

@@ -49,10 +49,10 @@ class TenantInfo:
 
     """Information about a tenant that is set up on the TRAC platform."""
 
-    tenantCode: str = None
+    tenantCode: str = ""
 
     """* Unique code used to identify the tenant, required by most API calls."""
 
-    description: str = None
+    description: str = ""
 
     """* A short description of the tenant, suitable for displaying to users in lists."""

tracdap/rt/metadata/custom.py

@@ -11,8 +11,8 @@ class CustomDefinition:
 
     """Define a custom object that can be stored and managed in the TRAC metadata store"""
 
-    customSchemaType: str = None
+    customSchemaType: str = ""
 
-    customSchemaVersion: int = None
+    customSchemaVersion: int = 0
 
-    customData: bytes = None
+    customData: bytes = b""

tracdap/rt/metadata/data.py

@@ -54,17 +54,17 @@ class FieldSchema:
         :class:`TableSchema <TableSchema>`
     """
 
-    fieldName: str = None
+    fieldName: str = ""
 
-    fieldOrder: int = None
+    fieldOrder: int = 0
 
     fieldType: BasicType = BasicType.BASIC_TYPE_NOT_SET
 
-    label: str = None
+    label: str = ""
 
-    businessKey: bool = None
+    businessKey: bool = False
 
-    categorical: bool = None
+    categorical: bool = False
 
     notNull: _tp.Optional[bool] = None
 
@@ -110,7 +110,7 @@ class PartKey:
 
     """Partition key for tabular datasets"""
 
-    opaqueKey: str = None
+    opaqueKey: str = ""
 
     partType: PartType = PartType.PART_ROOT
 
@@ -129,23 +129,23 @@ class DataDefinition:
     @_dc.dataclass
     class Delta:
 
-        deltaIndex: int = None
+        deltaIndex: int = 0
 
-        dataItem: str = None
+        dataItem: str = ""
 
     @_dc.dataclass
     class Snap:
 
-        snapIndex: int = None
+        snapIndex: int = 0
 
         deltas: _tp.List[DataDefinition.Delta] = _dc.field(default_factory=list)
 
     @_dc.dataclass
     class Part:
 
-        partKey: PartKey = None
+        partKey: PartKey = _dc.field(default_factory=lambda: PartKey())
 
-        snap: DataDefinition.Snap = None
+        snap: DataDefinition.Snap = _dc.field(default_factory=lambda: DataDefinition.Snap())
 
     schemaId: _tp.Optional[TagSelector] = None
 
@@ -153,4 +153,4 @@ class DataDefinition:
 
     parts: _tp.Dict[str, DataDefinition.Part] = _dc.field(default_factory=dict)
 
-    storageId: TagSelector = None
+    storageId: TagSelector = _dc.field(default_factory=lambda: TagSelector())

tracdap/rt/metadata/file.py

@@ -14,14 +14,14 @@ class FileDefinition:
 
     """Describes a file object stored in the TRAC platform"""
 
-    name: str = None
+    name: str = ""
 
-    extension: str = None
+    extension: str = ""
 
-    mimeType: str = None
+    mimeType: str = ""
 
-    size: int = None
+    size: int = 0
 
-    storageId: TagSelector = None
+    storageId: TagSelector = _dc.field(default_factory=lambda: TagSelector())
 
-    dataItem: str = None
+    dataItem: str = ""

tracdap/rt/metadata/flow.py

@@ -59,13 +59,13 @@ class FlowNode:
 
     outputs: _tp.List[str] = _dc.field(default_factory=list)
 
-    nodeSearch: SearchExpression = None
+    nodeSearch: SearchExpression = _dc.field(default_factory=lambda: SearchExpression())
 
     nodeAttrs: _tp.List[TagUpdate] = _dc.field(default_factory=list)
 
     nodeProps: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
 
-    label: str = None
+    label: str = ""
 
 
 @_dc.dataclass
@@ -85,9 +85,9 @@ class FlowSocket:
         :class:`FlowEdge <FlowEdge>`
     """
 
-    node: str = None
+    node: str = ""
 
-    socket: str = None
+    socket: str = ""
 
 
 @_dc.dataclass
@@ -100,9 +100,9 @@ class FlowEdge:
         :class:`FlowSocket <FlowSocket>`
     """
 
-    source: FlowSocket = None
+    source: FlowSocket = _dc.field(default_factory=lambda: FlowSocket())
 
-    target: FlowSocket = None
+    target: FlowSocket = _dc.field(default_factory=lambda: FlowSocket())
 
 
 @_dc.dataclass

tracdap/rt/metadata/job.py

@@ -99,7 +99,7 @@ class RunModelJob:
 
     """Specification for a RuN_MODEL job"""
 
-    model: TagSelector = None
+    model: TagSelector = _dc.field(default_factory=lambda: TagSelector())
 
     parameters: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
 
@@ -117,7 +117,7 @@ class RunFlowJob:
 
     """Specification for a RUN_FLOW job"""
 
-    flow: TagSelector = None
+    flow: TagSelector = _dc.field(default_factory=lambda: TagSelector())
 
     parameters: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
 
@@ -137,18 +137,18 @@ class ImportModelJob:
 
     """Specification for an IMPORT_MODEL job"""
 
-    language: str = None
+    language: str = ""
 
-    repository: str = None
+    repository: str = ""
 
     packageGroup: _tp.Optional[str] = None
 
-    package: str = None
+    package: str = ""
 
-    version: str = None
+    version: str = ""
 
-    entryPoint: str = None
+    entryPoint: str = ""
 
-    path: str = None
+    path: str = ""
 
     modelAttrs: _tp.List[TagUpdate] = _dc.field(default_factory=list)

tracdap/rt/metadata/model.py

@@ -15,12 +15,14 @@ class ModelParameter:
 
     """Describes an individual parameter of a model"""
 
-    paramType: TypeDescriptor = None
+    paramType: TypeDescriptor = _dc.field(default_factory=lambda: TypeDescriptor())
 
-    label: str = None
+    label: str = ""
 
     defaultValue: _tp.Optional[Value] = None
 
+    paramProps: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
+
 
 @_dc.dataclass
 class ModelInputSchema:
@@ -37,11 +39,15 @@ class ModelInputSchema:
     other options may be required. These capabilities may be added in future releases.
     """
 
-    schema: SchemaDefinition = None
+    schema: SchemaDefinition = _dc.field(default_factory=lambda: SchemaDefinition())
 
     label: _tp.Optional[str] = None
 
-    optional: bool = None
+    optional: bool = False
+
+    dynamic: bool = False
+
+    inputProps: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
 
 
 @_dc.dataclass
@@ -60,11 +66,15 @@ class ModelOutputSchema:
     added in future releases.
     """
 
-    schema: SchemaDefinition = None
+    schema: SchemaDefinition = _dc.field(default_factory=lambda: SchemaDefinition())
 
     label: _tp.Optional[str] = None
 
-    optional: bool = None
+    optional: bool = False
+
+    dynamic: bool = False
+
+    outputProps: _tp.Dict[str, Value] = _dc.field(default_factory=dict)
 
 
 @_dc.dataclass
@@ -72,17 +82,17 @@ class ModelDefinition:
 
     """Define a model for execution on the TRAC platform"""
 
-    language: str = None
+    language: str = ""
 
-    repository: str = None
+    repository: str = ""
 
     packageGroup: _tp.Optional[str] = None
 
-    package: str = None
+    package: str = ""
 
-    version: str = None
+    version: str = ""
 
-    entryPoint: str = None
+    entryPoint: str = ""
 
     path: _tp.Optional[str] = None
 

tracdap/rt/metadata/object.py

@@ -6,6 +6,7 @@ import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
 from .object_id import *  # noqa
+from .type import *  # noqa
 from .data import *  # noqa
 from .model import *  # noqa
 from .flow import *  # noqa
@@ -69,3 +70,5 @@ class ObjectDefinition:
     storage: _tp.Optional[StorageDefinition] = None
 
     schema: _tp.Optional[SchemaDefinition] = None
+
+    objectProps: _tp.Dict[str, Value] = _dc.field(default_factory=dict)

tracdap/rt/metadata/object_id.py

@@ -56,7 +56,7 @@ class TagHeader:
         :class:`ObjectType <ObjectType>`
     """
 
-    objectId: str = None
+    objectId: str = ""
 
     """
     Object ID of the object this tag is associated with.
@@ -64,27 +64,27 @@ class TagHeader:
     Object IDs are UUIDs (RFC4122, https://www.ietf.org/rfc/rfc4122.txt)
     """
 
-    objectVersion: int = None
+    objectVersion: int = 0
 
     """Version of the object this tag is associated with."""
 
-    objectTimestamp: DatetimeValue = None
+    objectTimestamp: DatetimeValue = _dc.field(default_factory=lambda: DatetimeValue())
 
     """Timestamp for when this version of the object was created."""
 
-    tagVersion: int = None
+    tagVersion: int = 0
 
     """Version of this tag."""
 
-    tagTimestamp: DatetimeValue = None
+    tagTimestamp: DatetimeValue = _dc.field(default_factory=lambda: DatetimeValue())
 
     """Timestamp for when this version of the tag was created."""
 
-    isLatestObject: bool = None
+    isLatestObject: bool = False
 
     """isLatest flag for the object the tag is associated with."""
 
-    isLatestTag: bool = None
+    isLatestTag: bool = False
 
     """isLatest flag for the tag."""
 
@@ -132,7 +132,7 @@ class TagSelector:
         :class:`ObjectType <ObjectType>`
     """
 
-    objectId: str = None
+    objectId: str = ""
 
     """
     Object ID of the tag being selected.