tracdap-runtime 0.6.1.dev3__py3-none-any.whl → 0.6.3__py3-none-any.whl

tracdap/rt/api/static_api.py

@@ -14,6 +14,7 @@
 
 from __future__ import annotations
 
+import sys
 import typing as _tp
 import types as _ts
 
@@ -111,7 +112,8 @@ def A(  # noqa
 
 def define_parameter(
         param_name: str, param_type: _tp.Union[TypeDescriptor, BasicType],
-        label: str, default_value: _tp.Optional[_tp.Any] = None) \
+        label: str, default_value: _tp.Optional[_tp.Any] = None,
+        *, param_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
         -> _Named[ModelParameter]:
 
     """
@@ -128,6 +130,10 @@ def define_parameter(
     the model. TRAC will apply type coercion where possible to ensure the default value matches the parameter type,
     if the default value cannot be coerced to match the parameter type then model validation will fail.
 
+    You can use param_props to associate arbitrary key-value properties with this model parameter.
+    These properties are not used by the TRAC engine, but are stored in the model metadata for
+    the parameter and can be used as needed in 3rd-party applications.
+
     Once defined model parameters can be passed to :py:func:`define_parameters`,
     either as a list or as individual arguments, to create the set of parameters for a model.
 
@@ -135,6 +141,7 @@ def define_parameter(
     :param param_type: The parameter type, expressed in the TRAC type system
     :param label: A descriptive label for the parameter (required)
     :param default_value: A default value to use if no explicit value is supplied (optional)
+    :param param_props: Associate key-value properties with this parameter (not used by the TRAC engine)
     :return: A named model parameter, suitable for passing to :py:func:`define_parameters`
 
     :type param_name: str
@@ -142,11 +149,12 @@ def define_parameter(
                       :py:class:`BasicType <tracdap.rt.metadata.BasicType>`
     :type label: str
     :type default_value: Optional[Any]
+    :type param_props: Optional[Dict[str, Any]]
     :rtype: _Named[:py:class:`ModelParameter <tracdap.rt.metadata.ModelParameter>`]
     """
 
     sa = _StaticApiHook.get_instance()
-    return sa.define_parameter(param_name, param_type, label, default_value)
+    return sa.define_parameter(param_name, param_type, label, default_value, param_props=param_props)
 
 
 def declare_parameter(
@@ -160,6 +168,9 @@ def declare_parameter(
     .. deprecated:: 0.4.4
        Use :py:func:`define_parameter` or :py:func:`P` instead.
 
+    This function is deprecated and will be removed in a future version.
+    Please use :py:func:`define_parameter() <tracdap.rt.api.define_parameter>` instead.
+
     :type param_name: str
     :type param_type: :py:class:`TypeDescriptor <tracdap.rt.metadata.TypeDescriptor>` |
                       :py:class:`BasicType <tracdap.rt.metadata.BasicType>`
@@ -168,6 +179,8 @@ def declare_parameter(
     :rtype: _Named[:py:class:`ModelParameter <tracdap.rt.metadata.ModelParameter>`]
     """
 
+    print("TRAC Warning: declare_parameter() is deprecated, please use define_parameter()", file=sys.stderr)
+
     return define_parameter(param_name, param_type, label, default_value)
 
 
@@ -175,7 +188,8 @@ def P(  # noqa
         param_name: str,
         param_type: _tp.Union[TypeDescriptor, BasicType],
         label: str,
-        default_value: _tp.Optional[_tp.Any] = None) \
+        default_value: _tp.Optional[_tp.Any] = None,
+        *, param_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
         -> _Named[ModelParameter]:
 
     """
@@ -186,12 +200,11 @@ def P(  # noqa
                       :py:class:`BasicType <tracdap.rt.metadata.BasicType>`
     :type label: str
     :type default_value: Optional[Any]
+    :type param_props: Optional[Dict[str, Any]]
     :rtype: _Named[:py:class:`ModelParameter <tracdap.rt.metadata.ModelParameter>`]
     """
 
-    return declare_parameter(
-        param_name, param_type, label,
-        default_value)
+    return define_parameter(param_name, param_type, label, default_value, param_props=param_props)
 
 
 def define_parameters(
@@ -226,11 +239,16 @@ def declare_parameters(
     .. deprecated:: 0.4.4
        Use :py:func:`define_parameters` instead.
 
+    This function is deprecated and will be removed in a future version.
+    Please use :py:func:`define_parameters() <tracdap.rt.api.define_parameters>` instead.
+
     :type params: _Named[:py:class:`ModelParameter <tracdap.rt.metadata.ModelParameter>`] |
                   List[_Named[:py:class:`ModelParameter <tracdap.rt.metadata.ModelParameter>`]]
     :rtype: Dict[str, :py:class:`ModelParameter <tracdap.rt.metadata.ModelParameter>`]
     """
 
+    print("TRAC Warning: declare_parameters() is deprecated, please use define_parameters()", file=sys.stderr)
+
     return define_parameters(*params)
 
 
@@ -278,9 +296,9 @@ def define_field(
     :type label: str
     :type business_key: bool
     :type categorical: bool
-    :type not_null: _tp.Optional[bool]
-    :type format_code: _tp.Optional[str]
-    :type field_order: _tp.Optional[int]
+    :type not_null: Optional[bool]
+    :type format_code: Optional[str]
+    :type field_order: Optional[int]
     :rtype: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`
     """
 
@@ -307,17 +325,22 @@ def declare_field(
     .. deprecated:: 0.4.4
        Use :py:func:`define_field` or :py:func:`F` instead.
 
+    This function is deprecated and will be removed in a future version.
+    Please use :py:func:`define_field() <tracdap.rt.api.define_field>` instead.
+
     :type field_name: str
     :type field_type: :py:class:`BasicType <tracdap.rt.metadata.BasicType>`
     :type label: str
     :type business_key: bool
     :type categorical: bool
-    :type not_null: _tp.Optional[bool]
-    :type format_code: _tp.Optional[str]
-    :type field_order: _tp.Optional[int]
+    :type not_null: Optional[bool]
+    :type format_code: Optional[str]
+    :type field_order: Optional[int]
     :rtype: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`
     """
 
+    print("TRAC Warning: declare_field() is deprecated, please use define_field()", file=sys.stderr)
+
     return define_field(
         field_name, field_type, label,
         business_key, categorical, not_null,
@@ -343,9 +366,9 @@ def F(  # noqa
     :type label: str
     :type business_key: bool
     :type categorical: bool
-    :type not_null: _tp.Optional[bool]
-    :type format_code: _tp.Optional[str]
-    :type field_order: _tp.Optional[int]
+    :type not_null: Optional[bool]
+    :type format_code: Optional[str]
+    :type field_order: Optional[int]
     :rtype: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`
     """
 
@@ -427,7 +450,10 @@ def load_schema(
 
 
 def define_input_table(
-        *fields: _tp.Union[FieldSchema, _tp.List[FieldSchema]], label: _tp.Optional[str] = None) \
+        *fields: _tp.Union[FieldSchema, _tp.List[FieldSchema]],
+        label: _tp.Optional[str] = None,
+        optional: bool = False,
+        input_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
         -> ModelInputSchema:
 
     """
@@ -436,17 +462,26 @@ def define_input_table(
     Fields can be supplied either as individual arguments to this function or as a list.
     Individual fields should be defined using :py:func:`define_field` or the shorthand alias :py:func:`F`.
 
+    You can use input_props to associate arbitrary key-value properties with this model input.
+    These properties are not used by the TRAC engine, but are stored in the model metadata for
+    the input and can be used as needed in 3rd-party applications.
+
     :param fields: A set of fields to make up a :py:class:`TableSchema <tracdap.rt.metadata.TableSchema>`
     :param label: An optional label (of type str) for a model input schema. Default value: None.
+    :param optional: Mark this input as an optional model input
+    :param input_props: Associate key-value properties with this input (not used by the TRAC engine)
     :return: A model input schema, suitable for returning from :py:meth:`TracModel.define_inputs`
 
     :type fields: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>` |
                   List[:py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`]
+    :type label: Optional[str]
+    :type optional: bool
+    :type input_props: Optional[Dict[str, Any]]
     :rtype: :py:class:`ModelInputSchema <tracdap.rt.metadata.ModelInputSchema>`
     """
 
     sa = _StaticApiHook.get_instance()
-    return sa.define_input_table(*fields, label=label)
+    return sa.define_input_table(*fields, label=label, optional=optional, input_props=input_props)
 
 
 def declare_input_table(
@@ -457,16 +492,24 @@ def declare_input_table(
     .. deprecated:: 0.4.4
        Use :py:func:`define_input_table` instead.
 
+    This function is deprecated and will be removed in a future version.
+    Please use :py:func:`define_input_table() <tracdap.rt.api.define_input_table>` instead.
+
     :type fields: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>` |
                   List[:py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`]
     :rtype: :py:class:`ModelInputSchema <tracdap.rt.metadata.ModelInputSchema>`
     """
 
+    print("TRAC Warning: declare_input_table() is deprecated, please use define_input_table()", file=sys.stderr)
+
     return define_input_table(*fields)
 
 
 def define_output_table(
-        *fields: _tp.Union[FieldSchema, _tp.List[FieldSchema]], label: _tp.Optional[str] = None) \
+        *fields: _tp.Union[FieldSchema, _tp.List[FieldSchema]],
+        label: _tp.Optional[str] = None,
+        optional: bool = False,
+        output_props: _tp.Optional[_tp.Dict[str, _tp.Any]] = None) \
         -> ModelOutputSchema:
 
     """
@@ -475,17 +518,26 @@ def define_output_table(
     Fields can be supplied either as individual arguments to this function or as a list.
     Individual fields should be defined using :py:func:`define_field` or the shorthand alias :py:func:`F`.
 
+    You can use output_props to associate arbitrary key-value properties with this model output.
+    These properties are not used by the TRAC engine, but are stored in the model metadata for
+    the output and can be used as needed in 3rd-party applications.
+
     :param fields: A set of fields to make up a :py:class:`TableSchema <tracdap.rt.metadata.TableSchema>`
     :param label: An optional label (of type str) for a model output schema. Default value: None.
+    :param optional: Mark this output as an optional model output
+    :param output_props: Associate key-value properties with this output (not used by the TRAC engine)
     :return: A model output schema, suitable for returning from :py:meth:`TracModel.define_outputs`
 
     :type fields: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>` |
                   List[:py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`]
+    :type label: Optional[str]
+    :type optional: bool
+    :type output_props: Optional[Dict[str, Any]]
     :rtype: :py:class:`ModelOutputSchema <tracdap.rt.metadata.ModelOutputSchema>`
     """
 
     sa = _StaticApiHook.get_instance()
-    return sa.define_output_table(*fields, label=label)
+    return sa.define_output_table(*fields, label=label, optional=optional, output_props=output_props)
 
 
 def declare_output_table(
@@ -496,9 +548,17 @@ def declare_output_table(
     .. deprecated:: 0.4.4
        Use :py:func:`define_output_table` instead.
 
+    This function is deprecated and will be removed in a future version.
+    Please use :py:func:`define_output_table() <tracdap.rt.api.define_output_table>` instead.
+
+    This function is deprecated and will be removed in a future version.
+    Please use define_output_table() instead.
+
     :type fields: :py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>` |
                   List[:py:class:`FieldSchema <tracdap.rt.metadata.FieldSchema>`]
     :rtype: :py:class:`ModelOutputSchema <tracdap.rt.metadata.ModelOutputSchema>`
     """
 
+    print("TRAC Warning: declare_output_table() is deprecated, please use define_output_table()", file==sys.stderr)
+
     return define_output_table(*fields)

tracdap/rt/config/__init__.py

@@ -5,6 +5,7 @@ from .common import PluginConfig
 from .common import PlatformInfo
 from .common import AuthenticationConfig
 from .common import StorageConfig
+from .common import ServiceConfig
 
 from .runtime import RuntimeConfig
 from .runtime import SparkSettings
@@ -21,10 +22,9 @@ from .platform import GatewayConfig
 from .platform import RouteConfig
 from .platform import RoutingMatch
 from .platform import RoutingTarget
-from .platform import ServiceConfig
 from .platform import DeploymentConfig
 
+from .job import JobConfig
+
 from .result import TagUpdateList
 from .result import JobResult
-
-from .job import JobConfig

tracdap/rt/config/common.py

@@ -66,3 +66,13 @@ class StorageConfig:
     defaultBucket: str = None
 
     defaultFormat: str = None
+
+
+@_dc.dataclass
+class ServiceConfig:
+
+    enabled: _tp.Optional[bool] = None
+
+    alias: str = None
+
+    port: int = None

tracdap/rt/config/platform.py

@@ -11,26 +11,26 @@ from .common import *  # noqa
 
 class RoutingProtocol(_enum.Enum):
 
-    PROTOCOL_NOT_SET = 0, 
+    PROTOCOL_NOT_SET = 0
 
-    HTTP = 1, 
+    HTTP = 1
 
-    GRPC = 2, 
+    GRPC = 2
 
-    GRPC_WEB = 3, 
+    GRPC_WEB = 3
 
-    REST = 4, 
+    REST = 4
 
 
 class DeploymentLayout(_enum.Enum):
 
-    LAYOUT_NOT_SET = 0, 
+    LAYOUT_NOT_SET = 0
 
-    SANDBOX = 1, 
+    SANDBOX = 1
 
-    HOSTED = 2, 
+    HOSTED = 2
 
-    CUSTOM = 3, 
+    CUSTOM = 3
 
 
 @_dc.dataclass
@@ -153,16 +153,6 @@ class RoutingTarget:
     path: str = None
 
 
-@_dc.dataclass
-class ServiceConfig:
-
-    enabled: _tp.Optional[bool] = None
-
-    alias: str = None
-
-    port: int = None
-
-
 @_dc.dataclass
 class DeploymentConfig:
 

tracdap/rt/config/runtime.py

@@ -20,6 +20,8 @@ class RuntimeConfig:
 
     sparkSettings: SparkSettings = None
 
+    runtimeApi: ServiceConfig = None
+
 
 @_dc.dataclass
 class SparkSettings:

tracdap/rt/ext/config.py

@@ -0,0 +1,34 @@
+#  Copyright 2024 Accenture Global Solutions Limited
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+# 
+#      http://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import abc as _abc
+
+
+class IConfigLoader:
+
+    @_abc.abstractmethod
+    def has_config_file(self, config_url: str) -> bool:
+        pass
+
+    @_abc.abstractmethod
+    def load_config_file(self, config_url: str) -> bytes:
+        pass
+
+    @_abc.abstractmethod
+    def has_config_dict(self, config_url: str) -> bool:
+        pass
+
+    @_abc.abstractmethod
+    def load_config_dict(self, config_url: str) -> dict:
+        pass

tracdap/rt/ext/embed.py

@@ -13,9 +13,7 @@
 #  limitations under the License.
 
 import tracdap.rt.config as _cfg
-import tracdap.rt.ext._guard as _guard
-
-# Dependency back into the main runtime implementation
+import tracdap.rt._impl.guard_rails as _guard  # noqa
 import tracdap.rt._exec.runtime as _rt  # noqa
 
 

tracdap/rt/ext/plugins.py

@@ -19,7 +19,8 @@ import importlib as _il
 
 import tracdap.rt.config as _cfg
 import tracdap.rt.exceptions as _ex
-import tracdap.rt.ext._guard as _guard
+import tracdap.rt._impl.guard_rails as _guard  # noqa
+from tracdap.rt.exceptions import EStartup
 
 
 class PluginManager:
@@ -29,6 +30,8 @@ class PluginManager:
     __log = _log.getLogger(f"{__name__}.PluginManager")
 
     __core_registered = False
+    __3rd_party_registered = list()
+
     __plugins = {}
 
     @classmethod
@@ -40,17 +43,42 @@ class PluginManager:
             return
 
         cls.__log.info("Register core plugins...")
+        cls.__register_plugin_package("tracdap.rt._plugins")
+
+        cls.__core_registered = True
+
+    @classmethod
+    def register_plugin_package(cls, plugin_package_name: str):
+
+        _guard.run_model_guard()
 
-        plugins_package = _il.import_module("tracdap.rt._plugins")
+        if plugin_package_name in cls.__3rd_party_registered:
+            return
+
+        if plugin_package_name.startswith("tracdap."):
+            raise EStartup("3rd party plugins cannot be registered from the tracdap namespace")
+
+        cls.__log.info(f"Register plugins from package [{plugin_package_name}]...")
+        cls.__register_plugin_package(plugin_package_name)
+
+        cls.__3rd_party_registered.append(plugin_package_name)
+
+    @classmethod
+    def __register_plugin_package(cls, plugin_package_name: str):
+
+        _guard.run_model_guard()
+
+        plugins_package = _il.import_module(plugin_package_name)
 
         for module in _pkg.iter_modules(plugins_package.__path__):
+
+            module_name = f"{plugin_package_name}.{module.name}"
+
             try:
-                module_name = f"tracdap.rt._plugins.{module.name}"
                 _il.import_module(module_name)
             except ImportError:
-                pass  # Ignore plugins that fail to load
-
-        cls.__core_registered = True
+                # It is not a fatal error if some plugins fail to load
+                cls.__log.warning(f"Failed to load plugins from module [{module_name}]")
 
     @classmethod
     def register_plugin(
@@ -94,3 +122,16 @@ class PluginManager:
         plugin = plugin_class(config.properties)
 
         return plugin
+
+    @classmethod
+    def load_config_plugin(cls,
+            service_type: _tp.Type[T_SERVICE],
+            config: _cfg.PluginConfig) \
+            -> T_SERVICE:
+
+        # Currently config plugins are loaded the same way as regular plugins
+        # However, regular plugins can be modified to take ConfigManager as an init parameter
+        # This is useful for loading secondary config files needed in particularly plugins
+        # Config plugins can never do this, because the config manager is not yet initialized
+
+        return cls.load_plugin(service_type, config)

tracdap/rt/launch/cli.py

@@ -16,7 +16,7 @@ import argparse
 import pathlib
 
 
-def cli_args():
+def _cli_args():
 
     parser = argparse.ArgumentParser(
         prog="python -m tracdap.rt.launch",
@@ -45,12 +45,14 @@ def cli_args():
     parser.add_argument(
         "--scratch-dir", dest="scratch_dir", type=pathlib.Path, required=False,
         help="Scratch directory for working files" +
-             " (if not supplied the system's temp location will be used)"
-    )
+             " (if not supplied the system's temp location will be used)")
 
     parser.add_argument(
         "--scratch-dir-persist", dest="scratch_dir_persist", default=False, action="store_true",
-        help="Do not clean up the scratch location on exit"
-    )
+        help="Do not clean up the scratch location on exit")
+
+    parser.add_argument(
+        "--plugin-package", dest="plugin_packages", type=str, action="append",
+        help="Do not clean up the scratch location on exit")
 
     return parser.parse_args()

tracdap/rt/launch/launch.py

@@ -23,7 +23,7 @@ import tracdap.rt._impl.config_parser as _cparse  # noqa
 import tracdap.rt._impl.util as _util  # noqa
 import tracdap.rt._exec.runtime as _runtime  # noqa
 
-from .cli import cli_args
+from .cli import _cli_args
 
 
 def _resolve_config_file(
@@ -53,13 +53,19 @@ def _resolve_config_file(
         return _pathlib.Path(config_path)
 
 
+def _optional_arg(launch_args: _tp.Dict[str, _tp.Any], arg_name: str) -> _tp.Any:
+
+    return launch_args.get(arg_name, None)
+
+
 def launch_model(
         model_class: _api.TracModel.__class__,
         job_config: _tp.Union[str, _pathlib.Path],
-        sys_config: _tp.Union[str, _pathlib.Path]):
+        sys_config: _tp.Union[str, _pathlib.Path],
+        **launch_args):
 
     """
-    Launch an individual model component by class (embedded launch)
+    Launch an individual model using its Python class
 
     This launch method launches the supplied model class directly, it must be called
     from the Python codebase containing the model class. The TRAC runtime will launch
@@ -77,6 +83,7 @@ def launch_model(
     :param model_class: The model class that will be launched
     :param job_config: Path to the job configuration file
     :param sys_config: Path to the system configuration file
+    :param launch_args: Additional keyword args to control behavior of the TRAC runtime (not normally required)
     """
 
     model_file = _inspect.getfile(model_class)
@@ -85,7 +92,10 @@ def launch_model(
     _sys_config = _resolve_config_file(sys_config, model_dir)
     _job_config = _resolve_config_file(job_config, model_dir)
 
-    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=True)
+    plugin_package = _optional_arg(launch_args, 'plugin_package')
+    plugin_packages = [plugin_package] if plugin_package else None
+
+    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=True, plugin_packages=plugin_packages)
     runtime_instance.pre_start()
 
     job = runtime_instance.load_job_config(_job_config, model_class=model_class)
@@ -98,12 +108,25 @@ def launch_model(
 def launch_job(
         job_config: _tp.Union[str, _pathlib.Path],
         sys_config: _tp.Union[str, _pathlib.Path],
-        dev_mode: bool = False):
+        dev_mode: bool = False,
+        **launch_args):
+
+    """
+    Launch a TRAC job using external configuration files
+
+    :param job_config: Path to the job configuration file
+    :param sys_config: Path to the system configuration file
+    :param dev_mode: Whether to launch in dev mode (applies dev mode translation to the job inputs)
+    :param launch_args: Additional keyword args to control behavior of the TRAC runtime (not normally required)
+    """
 
     _sys_config = _resolve_config_file(sys_config, None)
     _job_config = _resolve_config_file(job_config, None)
 
-    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=dev_mode)
+    plugin_package = _optional_arg(launch_args, 'plugin_package')
+    plugin_packages = [plugin_package] if plugin_package else None
+
+    runtime_instance = _runtime.TracRuntime(_sys_config, dev_mode=dev_mode, plugin_packages=plugin_packages)
     runtime_instance.pre_start()
 
     job = runtime_instance.load_job_config(_job_config)
@@ -115,10 +138,13 @@ def launch_job(
 
 def launch_cli():
 
-    launch_args = cli_args()
+    """
+    Launch the TRAC runtime using the command line interface
+    """
+
+    launch_args = _cli_args()
 
     _sys_config = _resolve_config_file(launch_args.sys_config, None)
-    _job_config = _resolve_config_file(launch_args.job_config, None)
 
     runtime_instance = _runtime.TracRuntime(
         _sys_config,
@@ -126,12 +152,23 @@ def launch_cli():
         job_result_dir=launch_args.job_result_dir,
         job_result_format=launch_args.job_result_format,
         scratch_dir=launch_args.scratch_dir,
-        scratch_dir_persist=launch_args.scratch_dir_persist)
+        scratch_dir_persist=launch_args.scratch_dir_persist,
+        plugin_packages=launch_args.plugin_packages)
 
     runtime_instance.pre_start()
 
-    job = runtime_instance.load_job_config(_job_config)
+    if launch_args.job_config is not None:
+        _job_config = _resolve_config_file(launch_args.job_config, None)
+        job = runtime_instance.load_job_config(_job_config)
+    else:
+        job = None
 
     with runtime_instance as rt:
-        rt.submit_job(job)
-        rt.wait_for_job(job.jobId)
+
+        if job is not None:
+            rt.submit_job(job)
+
+        if rt.is_oneshot():
+            rt.wait_for_job(job.jobId)
+        else:
+            rt.run_until_done()