tracdap-runtime 0.8.0rc2__py3-none-any.whl → 0.9.0b2__py3-none-any.whl

tracdap/rt/_plugins/storage_gcp.py

@@ -28,20 +28,27 @@ from pyarrow import fs as pa_fs
 from . import _helpers
 
 
-try:
-    # These dependencies are provided by the optional [gcp] feature
-    # For local development, pip install -r requirements_plugins.txt
-    import google.cloud.storage as gcs  # noqa
-    import gcsfs  # noqa
-    __gcp_available = True
-except ImportError:
-    gcs = None
-    gcsfs = None
-    __gcp_available = False
+def _gcp_arrow_available():
+    try:
+        # Shipped as part of PyArrow, but may not be available on all platforms
+        return pa_fs.GcsFileSystem is not None
+    except ImportError:
+        return False
+
+def _gcp_fsspec_available():
+    try:
+        # These dependencies are provided by the optional [gcp] feature
+        # For local development, pip install -r requirements_plugins.txt
+        import google.cloud.storage as gcs  # noqa
+        import gcsfs  # noqa
+        return True
+    except ImportError:
+        return False
 
 
 class GcpStorageProvider(IStorageProvider):
 
+    PROJECT_PROPERTY = "project"
     BUCKET_PROPERTY = "bucket"
     PREFIX_PROPERTY = "prefix"
     REGION_PROPERTY = "region"
@@ -62,20 +69,17 @@ class GcpStorageProvider(IStorageProvider):
     RUNTIME_FS_DEFAULT = RUNTIME_FS_AUTO
 
     ARROW_CLIENT_ARGS = {
+        PROJECT_PROPERTY: "project_id",
         REGION_PROPERTY: "default_bucket_location",
-        ENDPOINT_PROPERTY: "endpoint_override"
+        ENDPOINT_PROPERTY: "endpoint_override",
     }
 
     FSSPEC_CLIENT_ARGS = {
+        PROJECT_PROPERTY: "project",
         REGION_PROPERTY: "default_location",
         ENDPOINT_PROPERTY: "endpoint_url"
     }
 
-    try:
-        __arrow_available = pa_fs.GcsFileSystem is not None
-    except ImportError:
-        __arrow_available = False
-
     def __init__(self, properties: tp.Dict[str, str]):
 
         self._log = _helpers.logger_for_object(self)
@@ -91,7 +95,7 @@ class GcpStorageProvider(IStorageProvider):
     def get_arrow_native(self) -> pa_fs.SubTreeFileSystem:
 
         if self._runtime_fs == self.RUNTIME_FS_AUTO:
-            gcs_fs = self.create_arrow() if self.__arrow_available else self.create_fsspec()
+            gcs_fs = self.create_arrow() if _gcp_arrow_available() else self.create_fsspec()
         elif self._runtime_fs == self.RUNTIME_FS_ARROW:
             gcs_fs = self.create_arrow()
         elif self._runtime_fs == self.RUNTIME_FS_FSSPEC:
@@ -115,12 +119,20 @@ class GcpStorageProvider(IStorageProvider):
 
     def create_arrow(self) -> pa_fs.FileSystem:
 
+        if not _gcp_arrow_available():
+            raise ex.EStorage(f"GCS storage setup failed: Plugin for [{self.RUNTIME_FS_ARROW}] is not available")
+
         gcs_arrow_args = self.setup_client_args(self.ARROW_CLIENT_ARGS)
 
         return pa_fs.GcsFileSystem(**gcs_arrow_args)
 
     def create_fsspec(self) -> pa_fs.FileSystem:
 
+        if not _gcp_fsspec_available():
+            raise ex.EStorage(f"GCS storage setup failed: Plugin for [{self.RUNTIME_FS_FSSPEC}] is not available")
+
+        import gcsfs  # noqa
+
         gcs_fsspec_args = self.setup_client_args(self.FSSPEC_CLIENT_ARGS)
         gcs_fsspec = gcsfs.GCSFileSystem(**gcs_fsspec_args)
 
@@ -130,9 +142,14 @@ class GcpStorageProvider(IStorageProvider):
 
         client_args = dict()
 
+        project = _helpers.get_plugin_property(self._properties, self.PROJECT_PROPERTY)
         region = _helpers.get_plugin_property(self._properties, self.REGION_PROPERTY)
         endpoint = _helpers.get_plugin_property(self._properties, self.ENDPOINT_PROPERTY)
 
+        if project is not None:
+            project_key = arg_mapping[self.PROJECT_PROPERTY]
+            client_args[project_key] = project
+
         if region is not None:
             region_key = arg_mapping[self.REGION_PROPERTY]
             client_args[region_key] = region
@@ -180,5 +197,5 @@ class GcpStorageProvider(IStorageProvider):
         raise ex.EStartup(message)
 
 
-if __gcp_available:
+if _gcp_arrow_available() or _gcp_fsspec_available():
     plugins.PluginManager.register_plugin(IStorageProvider, GcpStorageProvider, ["GCS"])

tracdap/rt/_version.py

@@ -13,4 +13,4 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 
-__version__ = "0.8.0rc2"
+__version__ = "0.9.0b2"

tracdap/rt/api/model_api.py

@@ -14,6 +14,7 @@
 #  limitations under the License.
 
 import abc as _abc
+import dataclasses as _dc
 import typing as _tp
 import logging as _logging
 
@@ -35,6 +36,25 @@ if _tp.TYPE_CHECKING:
         pass
 
 
+@_dc.dataclass(frozen=True)
+class RuntimeMetadata:
+
+    """
+    The metadata associated with a TRAC object, made available for models at runtime
+
+    The metadata available for a particular object depends on the current job configuration, as
+    well as the type of object. For example, a model input supplied from a TRAC dataset will
+    have the metadata of that dataset available, but data passed into a model as an intermediate
+    dataset in a flow might not have an ID or any attributes.
+    """
+
+    objectId: _tp.Optional[TagHeader] = None
+    """TRAC object ID of the current object (if available)"""
+
+    attributes: _tp.Dict[str, _tp.Any] = _dc.field(default_factory=dict)
+    """TRAC metadata attributes of the current object (if available)"""
+
+
 class TracContext(metaclass=_abc.ABCMeta):
 
     """
@@ -202,6 +222,31 @@ class TracContext(metaclass=_abc.ABCMeta):
 
         pass
 
+    def get_metadata(self, item_name: str) -> _tp.Optional[RuntimeMetadata]:
+
+        """
+        Get the TRAC metadata associated with a model input.
+
+        Metadata is available for inputs supplied from real TRAC objects, including
+        both :py:attr:`DATA <tracdap.rt.metadata.ObjectType.DATA>` and
+        :py:attr:`FILE <tracdap.rt.metadata.ObjectType.DATA>` objects.
+
+        Calling :py:meth:`get_metadata()` for objects that are not inputs will return null,
+        since parameters have no metadata and output metadata does not exist until after a job completes.
+        :py:meth:`get_metadata()` will also return null for inputs supplied from other models
+        as intermediates in a flow, since these also have no persistent metadata.
+
+        Attempting to access metadata for objects that do not exist, including outputs that
+        have not been put yet, is an error.
+
+        :param item_name: The name of the file or dataset to get metadata for
+        :return: Runtime metadata for the named item, or None if no metadata is available
+        :type item_name: str
+        :rtype: :py:class:`RuntimeMetadata <tracdap.rt.api.RuntimeMetadata>`
+        """
+
+        pass
+
     def put_schema(self, dataset_name: str, schema: SchemaDefinition):
 
         """

tracdap/rt/config/__init__.py

@@ -1,30 +1,28 @@
 # Code generated by TRAC
 
-from .job import JobConfig
+from .tenant import TenantConfigMap
+from .tenant import TenantConfig
 
 from .common import _ConfigFile
 from .common import PluginConfig
 from .common import PlatformInfo
-from .common import StorageConfig
 from .common import ServiceConfig
 
-from .dynamic import DynamicConfig
+from .runtime import RuntimeConfig
 
-from .result import TagUpdateList
+from .result import JobResultAttrs
 from .result import JobResult
 
 from .platform import RoutingProtocol
 from .platform import DeploymentLayout
 from .platform import PlatformConfig
-from .platform import MetadataConfig
-from .platform import TenantConfig
 from .platform import GatewayConfig
 from .platform import GatewayRedirect
 from .platform import RouteConfig
 from .platform import RoutingMatch
 from .platform import RoutingTarget
 from .platform import DeploymentConfig
-from .platform import ClientConfig
 
-from .runtime import RuntimeConfig
-from .runtime import SparkSettings
+from .dynamic import DynamicConfig
+
+from .job import JobConfig

tracdap/rt/config/common.py

@@ -4,6 +4,9 @@ import typing as _tp  # noqa
 import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
+import tracdap.rt.metadata as metadata
+
+
 
 @_dc.dataclass
 class _ConfigFile:
@@ -33,20 +36,6 @@ class PlatformInfo:
     deploymentInfo: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
 
-@_dc.dataclass
-class StorageConfig:
-
-    buckets: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    """TODO: Rename "buckets" as "internal" for 0.7"""
-
-    external: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    defaultBucket: "str" = ""
-
-    defaultFormat: "str" = ""
-
-
 @_dc.dataclass
 class ServiceConfig:
 

tracdap/rt/config/job.py

@@ -13,10 +13,24 @@ class JobConfig:
 
     jobId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
+    """ID and definition of the job being submitted"""
+
     job: "metadata.JobDefinition" = _dc.field(default_factory=lambda: metadata.JobDefinition())
 
-    resources: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+    objectMapping: "_tp.Dict[str, metadata.TagHeader]" = _dc.field(default_factory=dict)
+
+    """Metadata needed to execute the job (objects referred to in the job definition)"""
+
+    objects: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+
+    tags: "_tp.Dict[str, metadata.Tag]" = _dc.field(default_factory=dict)
+
+    resultId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
+
+    """Preallocated IDs for job outputs"""
+
+    preallocatedIds: "_tp.List[metadata.TagHeader]" = _dc.field(default_factory=list)
 
-    resourceMapping: "_tp.Dict[str, metadata.TagHeader]" = _dc.field(default_factory=dict)
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
-    resultMapping: "_tp.Dict[str, metadata.TagHeader]" = _dc.field(default_factory=dict)
+    """Allow setting per-job configuration"""

tracdap/rt/config/platform.py

@@ -6,6 +6,7 @@ import enum as _enum  # noqa
 
 import tracdap.rt.metadata as metadata
 from .common import *  # noqa
+from .tenant import *  # noqa
 
 
 class RoutingProtocol(_enum.Enum):
@@ -20,6 +21,8 @@ class RoutingProtocol(_enum.Enum):
 
     REST = 4
 
+    INTERNAL = 5
+
 
 class DeploymentLayout(_enum.Enum):
 
@@ -39,43 +42,21 @@ class PlatformConfig:
 
     platformInfo: "PlatformInfo" = _dc.field(default_factory=lambda: PlatformInfo())
 
-    metadata: "MetadataConfig" = _dc.field(default_factory=lambda: MetadataConfig())
-
-    storage: "StorageConfig" = _dc.field(default_factory=lambda: StorageConfig())
-
-    repositories: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
-
-    executor: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
+    metadataStore: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
 
     jobCache: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
 
-    tenants: "_tp.Dict[str, TenantConfig]" = _dc.field(default_factory=dict)
-
-    gateway: "GatewayConfig" = _dc.field(default_factory=lambda: GatewayConfig())
+    executor: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
 
     services: "_tp.Dict[str, ServiceConfig]" = _dc.field(default_factory=dict)
 
     deployment: "DeploymentConfig" = _dc.field(default_factory=lambda: DeploymentConfig())
 
-    clientConfig: "_tp.Dict[str, ClientConfig]" = _dc.field(default_factory=dict)
+    gateway: "GatewayConfig" = _dc.field(default_factory=lambda: GatewayConfig())
 
     extensions: "_tp.Dict[str, protobuf.Any]" = _dc.field(default_factory=dict)
 
-
-@_dc.dataclass
-class MetadataConfig:
-
-    database: "PluginConfig" = _dc.field(default_factory=lambda: PluginConfig())
-
-    format: "metadata.MetadataFormat" = metadata.MetadataFormat.METADATA_FORMAT_NOT_SET
-
-
-@_dc.dataclass
-class TenantConfig:
-
-    defaultBucket: "_tp.Optional[str]" = None
-
-    defaultFormat: "_tp.Optional[str]" = None
+    setupTasks: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
 
 
 @_dc.dataclass
@@ -109,6 +90,8 @@ class RouteConfig:
 
     target: "RoutingTarget" = _dc.field(default_factory=lambda: RoutingTarget())
 
+    routeKey: "str" = ""
+
 
 @_dc.dataclass
 class RoutingMatch:
@@ -136,9 +119,3 @@ class RoutingTarget:
 class DeploymentConfig:
 
     layout: "DeploymentLayout" = DeploymentLayout.LAYOUT_NOT_SET
-
-
-@_dc.dataclass
-class ClientConfig:
-
-    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)

tracdap/rt/config/result.py

@@ -9,7 +9,7 @@ import tracdap.rt.metadata as metadata
 
 
 @_dc.dataclass
-class TagUpdateList:
+class JobResultAttrs:
 
     attrs: "_tp.List[metadata.TagUpdate]" = _dc.field(default_factory=list)
 
@@ -19,8 +19,12 @@ class JobResult:
 
     jobId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
-    statusCode: "metadata.JobStatusCode" = metadata.JobStatusCode.JOB_STATUS_CODE_NOT_SET
+    resultId: "metadata.TagHeader" = _dc.field(default_factory=lambda: metadata.TagHeader())
 
-    statusMessage: "str" = ""
+    result: "metadata.ResultDefinition" = _dc.field(default_factory=lambda: metadata.ResultDefinition())
 
-    results: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+    objectIds: "_tp.List[metadata.TagHeader]" = _dc.field(default_factory=list)
+
+    objects: "_tp.Dict[str, metadata.ObjectDefinition]" = _dc.field(default_factory=dict)
+
+    attrs: "_tp.Dict[str, JobResultAttrs]" = _dc.field(default_factory=dict)

tracdap/rt/config/runtime.py

@@ -4,6 +4,7 @@ import typing as _tp  # noqa
 import dataclasses as _dc  # noqa
 import enum as _enum  # noqa
 
+import tracdap.rt.metadata as metadata
 from .common import *  # noqa
 
 
@@ -13,16 +14,10 @@ class RuntimeConfig:
 
     config: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
-    storage: "StorageConfig" = _dc.field(default_factory=lambda: StorageConfig())
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
-    repositories: "_tp.Dict[str, PluginConfig]" = _dc.field(default_factory=dict)
+    secrets: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
 
-    sparkSettings: "SparkSettings" = _dc.field(default_factory=lambda: SparkSettings())
+    resources: "_tp.Dict[str, metadata.ResourceDefinition]" = _dc.field(default_factory=dict)
 
-    runtimeApi: "ServiceConfig" = _dc.field(default_factory=lambda: ServiceConfig())
-
-
-@_dc.dataclass
-class SparkSettings:
-
-    sparkProps: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+    runtimeApi: "_tp.Optional[ServiceConfig]" = None

tracdap/rt/config/tenant.py

@@ -0,0 +1,28 @@
+# Code generated by TRAC
+
+import typing as _tp  # noqa
+import dataclasses as _dc  # noqa
+import enum as _enum  # noqa
+
+import tracdap.rt.metadata as metadata
+
+
+
+@_dc.dataclass
+class TenantConfigMap:
+
+    tenants: "_tp.Dict[str, TenantConfig]" = _dc.field(default_factory=dict)
+
+    autoActivate: "bool" = False
+
+
+@_dc.dataclass
+class TenantConfig:
+
+    properties: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    secrets: "_tp.Dict[str, str]" = _dc.field(default_factory=dict)
+
+    config: "_tp.Dict[str, metadata.ConfigDefinition]" = _dc.field(default_factory=dict)
+
+    resources: "_tp.Dict[str, metadata.ResourceDefinition]" = _dc.field(default_factory=dict)

tracdap/rt/launch/cli.py

@@ -35,14 +35,6 @@ def _cli_args(programmatic_args = None):
         "--dev-mode", dest="dev_mode", default=False, action="store_true",
         help="Enable development mode config translation")
 
-    parser.add_argument(
-        "--job-result-dir", dest="job_result_dir", type=pathlib.Path, required=False,
-        help="Output the result metadata for a batch job to the given directory")
-
-    parser.add_argument(
-        "--job-result-format", dest="job_result_format", choices=["json", "yaml", "proto"], default="json",
-        help="Output format for the result metadata (only meaningful if --job-result-dir is set)")
-
     parser.add_argument(
         "--scratch-dir", dest="scratch_dir", type=pathlib.Path, required=False,
         help="Scratch directory for working files" +

tracdap/rt/launch/launch.py

@@ -36,7 +36,7 @@ def _resolve_config_file(
     if isinstance(config_path, str):
         scheme_sep = config_path.find(":")
         # Single letter scheme is a Windows file path (C:\...)
-        scheme = scheme = config_path[:scheme_sep] if scheme_sep > 1 else "file"
+        scheme = config_path[:scheme_sep] if scheme_sep > 1 else "file"
         if scheme != "file":
             return config_path
 
@@ -191,8 +191,6 @@ def launch_cli(programmatic_args: _tp.Optional[_tp.List[str]] = None):
     runtime_instance = _runtime.TracRuntime(
         _sys_config,
         dev_mode=launch_args.dev_mode,
-        job_result_dir=launch_args.job_result_dir,
-        job_result_format=launch_args.job_result_format,
         scratch_dir=launch_args.scratch_dir,
         scratch_dir_persist=launch_args.scratch_dir_persist,
         plugin_packages=launch_args.plugin_packages)

tracdap/rt/metadata/__init__.py

@@ -9,47 +9,31 @@ from .type import Value
 from .type import ArrayValue
 from .type import MapValue
 
-from .tag_update import TagOperation
-from .tag_update import TagUpdate
-
 from .object_id import ObjectType
 from .object_id import TagHeader
 from .object_id import TagSelector
 
-from .job import JobType
-from .job import JobStatusCode
-from .job import JobGroupType
-from .job import JobDefinition
-from .job import ResultDefinition
-from .job import RunModelJob
-from .job import RunFlowJob
-from .job import ImportModelJob
-from .job import ImportDataJob
-from .job import ExportDataJob
-from .job import JobGroup
-from .job import SequentialJobGroup
-from .job import ParallelJobGroup
+from .search import SearchOperator
+from .search import LogicalOperator
+from .search import SearchTerm
+from .search import LogicalExpression
+from .search import SearchExpression
+from .search import SearchParameters
+
+from .tag_update import TagOperation
+from .tag_update import TagUpdate
+
+from .custom import CustomDefinition
 
 from .common import MetadataFormat
 from .common import MetadataVersion
 from .common import TenantInfo
 
-from .resource import ResourceType
-from .resource import ResourceDefinition
-
-from .config import ConfigType
-from .config import ConfigEntry
-from .config import ConfigDetails
-from .config import ConfigDefinition
-
-from .custom import CustomDefinition
-
 from .data import SchemaType
 from .data import PartType
 from .data import FieldSchema
+from .data import EnumValues
 from .data import TableSchema
-from .data import StructField
-from .data import StructSchema
 from .data import SchemaDefinition
 from .data import PartKey
 from .data import DataDefinition
@@ -63,26 +47,42 @@ from .model import ModelInputSchema
 from .model import ModelOutputSchema
 from .model import ModelDefinition
 
-from .search import SearchOperator
-from .search import LogicalOperator
-from .search import SearchTerm
-from .search import LogicalExpression
-from .search import SearchExpression
-from .search import SearchParameters
-
 from .flow import FlowNodeType
 from .flow import FlowNode
 from .flow import FlowSocket
 from .flow import FlowEdge
 from .flow import FlowDefinition
 
+from .job import JobType
+from .job import JobStatusCode
+from .job import JobGroupType
+from .job import JobDefinition
+from .job import ResultDefinition
+from .job import RunModelJob
+from .job import RunFlowJob
+from .job import ImportModelJob
+from .job import ImportDataJob
+from .job import ExportDataJob
+from .job import JobGroup
+from .job import SequentialJobGroup
+from .job import ParallelJobGroup
+
 from .storage import CopyStatus
 from .storage import IncarnationStatus
+from .storage import StorageLayout
 from .storage import StorageCopy
 from .storage import StorageIncarnation
 from .storage import StorageItem
 from .storage import StorageDefinition
 
+from .resource import ResourceType
+from .resource import ResourceDefinition
+
+from .config import ConfigType
+from .config import ConfigEntry
+from .config import ConfigDetails
+from .config import ConfigDefinition
+
 from .object import ObjectDefinition
 
 from .tag import Tag