deriva-ml 1.10.0__py3-none-any.whl → 1.11.0__py3-none-any.whl

deriva_ml/execution_configuration.py

@@ -1,14 +1,11 @@
 from __future__ import annotations
 
 import json
-from typing import Optional
+from typing import Optional, Any
 
-from pydantic import (
-    BaseModel,
-    conlist,
-    ConfigDict,
-)
+from pydantic import BaseModel, conlist, ConfigDict, field_validator, Field
 from pathlib import Path
+import sys
 
 
 from .dataset_aux_classes import DatasetSpec
@@ -43,6 +40,7 @@ class ExecutionConfiguration(BaseModel):
         datasets: List of dataset specifications which specify the dataset RID, version and if the dataset
             should be materialized.
         assets: List of assets to be downloaded prior to execution.  The values must be RIDs in an asset table
+        parameters: Either a dictionary or a path to a JSON file that contains configuration parameters for the execution.
         workflow: A RID for a workflow instance.  Must have a name, URI to the workflow instance, and a type.
         description: A description of the execution.  Can use Markdown format.
     """
@@ -50,10 +48,22 @@ class ExecutionConfiguration(BaseModel):
     datasets: conlist(DatasetSpec) = []
     assets: list[RID | str] = []  # List of RIDs to model files.
     workflow: RID | Workflow
+    parameters: dict[str, Any] = {}
     description: str = ""
+    argv: conlist(str) = Field(default_factory=lambda: sys.argv)
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
+    @field_validator("parameters", mode="before")
+    @classmethod
+    def validate_parameters(cls, value: Any) -> Any:
+        """If parameter is a file, assume that it has JSON contents for configuration parameters"""
+        if isinstance(value, str) or isinstance(value, Path):
+            with open(value, "r") as f:
+                return json.load(f)
+        else:
+            return value
+
     @staticmethod
     def load_configuration(path: Path) -> ExecutionConfiguration:
         """Create a ExecutionConfiguration from a JSON configuration file.

deriva_ml/feature.py

@@ -25,12 +25,11 @@ class FeatureRecord(BaseModel):
     """
 
     # model_dump of this feature should be compatible with feature table columns.
-    Execution: str
+    Execution: Optional[str] = None
     Feature_Name: str
     feature: ClassVar[Optional["Feature"]] = None
 
     class Config:
-
         arbitrary_types_allowed = True
 
     @classmethod

deriva_ml/schema_setup/create_schema.py

@@ -1,6 +1,6 @@
 import argparse
 import sys
-from typing import Optional
+from typing import Optional, Any
 
 from deriva.core import DerivaServer, get_credential
 from deriva.core.ermrest_model import Model
@@ -19,28 +19,48 @@ from deriva_ml.schema_setup.annotations import generate_annotation
 from deriva_ml.deriva_model import DerivaModel
 
 
-def define_table_workflow(workflow_annotation: dict):
-    return Table.define(
-        "Workflow",
-        column_defs=[
-            Column.define("Name", builtin_types.text),
-            Column.define("Description", builtin_types.markdown),
-            Column.define("URL", builtin_types.ermrest_uri),
-            Column.define("Checksum", builtin_types.text),
-            Column.define("Version", builtin_types.text),
-        ],
-        annotations=workflow_annotation,
+def create_dataset_table(
+    schema: Schema,
+    execution_table: Table,
+    project_name: str,
+    dataset_annotation: Optional[dict] = None,
+):
+    dataset_table = schema.create_table(
+        Table.define(
+            tname="Dataset",
+            column_defs=[
+                Column.define("Description", builtin_types.markdown),
+                Column.define("Deleted", builtin_types.boolean),
+            ],
+            annotations=dataset_annotation if dataset_annotation is not None else {},
+        )
     )
 
+    dataset_type = schema.create_table(
+        Table.define_vocabulary(MLVocab.dataset_type, f"{project_name}:{{RID}}")
+    )
+    schema.create_table(
+        Table.define_association(
+            associates=[
+                ("Dataset", dataset_table),
+                (MLVocab.dataset_type, dataset_type),
+            ]
+        )
+    )
 
-def define_table_dataset(dataset_annotation: Optional[dict] = None):
-    return Table.define(
-        tname="Dataset",
-        column_defs=[
-            Column.define("Description", builtin_types.markdown),
-            Column.define("Deleted", builtin_types.boolean),
-        ],
-        annotations=dataset_annotation if dataset_annotation is not None else {},
+    dataset_version = schema.create_table(define_table_dataset_version(schema.name))
+    dataset_table.create_reference(("Version", True, dataset_version))
+
+    # Nested datasets.
+    schema.create_table(
+        Table.define_association(
+            associates=[("Dataset", dataset_table), ("Nested_Dataset", dataset_table)]
+        )
+    )
+    schema.create_table(
+        Table.define_association(
+            associates=[("Dataset", dataset_table), ("Execution", execution_table)]
+        )
     )
 
 
@@ -66,92 +86,120 @@ def define_table_dataset_version(sname: str):
     )
 
 
-def define_table_execution(sname: str, execution_annotation: dict):
-    table_def = Table.define(
-        "Execution",
-        column_defs=[
-            Column.define("Workflow", builtin_types.text),
-            Column.define("Description", builtin_types.markdown),
-            Column.define("Duration", builtin_types.text),
-            Column.define("Status", builtin_types.text),
-            Column.define("Status_Detail", builtin_types.text),
-        ],
-        fkey_defs=[ForeignKey.define(["Workflow"], sname, "Workflow", ["RID"])],
-        annotations=execution_annotation,
+def create_execution_table(schema, annotation: Optional[dict] = None):
+    annotation = annotation if annotation is not None else {}
+    execution = schema.create_table(
+        Table.define(
+            "Execution",
+            column_defs=[
+                Column.define("Workflow", builtin_types.text),
+                Column.define("Description", builtin_types.markdown),
+                Column.define("Duration", builtin_types.text),
+                Column.define("Status", builtin_types.text),
+                Column.define("Status_Detail", builtin_types.text),
+            ],
+            fkey_defs=[
+                ForeignKey.define(["Workflow"], schema.name, "Workflow", ["RID"])
+            ],
+            annotations=annotation,
+        )
     )
-    return table_def
+    return execution
 
 
-def define_asset_execution_metadata(sname: str, execution_metadata_annotation: dict):
-    return Table.define_asset(
-        sname=sname,
-        tname="Execution_Metadata",
-        hatrac_template="/hatrac/metadata/{{MD5}}.{{Filename}}",
-        annotations=execution_metadata_annotation,
+def create_asset_table(
+    schema,
+    asset_name: str,
+    execution_table,
+    asset_type_table,
+    asset_role_table,
+    annotation: Optional[dict] = None,
+):
+    annotation = annotation if annotation is not None else {}
+    asset_table = schema.create_table(
+        Table.define_asset(
+            sname=schema.name,
+            tname=asset_name,
+            hatrac_template="/hatrac/metadata/{{MD5}}.{{Filename}}",
+            annotations=annotation,
+        )
+    )
+    atable = schema.create_table(
+        Table.define_association(
+            [
+                (asset_name, asset_table),
+                ("Asset_Type", asset_type_table),
+            ],
+        )
     )
 
-
-def define_asset_execution_asset(sname: str, execution_asset_annotation: dict):
-    table_def = Table.define_asset(
-        sname=sname,
-        tname="Execution_Asset",
-        hatrac_template="/hatrac/execution_asset/{{MD5}}.{{Filename}}",
-        annotations=execution_asset_annotation,
+    atable = schema.create_table(
+        Table.define_association(
+            [
+                (asset_name, asset_table),
+                ("Execution", execution_table),
+            ],
+        )
     )
-    return table_def
+    atable.create_reference(asset_role_table)
+    return asset_table
 
 
-def define_table_file(sname):
+def create_file_table(
+    schema: Schema,
+    execution_table: Table,
+    project_name: str,
+    annotation: Optional[dict] = None,
+):
     """Define files table structure"""
-    return Table.define_asset(
-        sname=sname,
-        tname="File",
+    annotation = annotation or {}
+    file_table = schema.create_table(
+        Table.define_asset(sname=schema.name, tname="File")
     )
 
+    file_type = schema.create_table(
+        Table.define_vocabulary(MLVocab.file_type, f"{project_name}:{{RID}}")
+    )
 
-def create_www_schema(model: Model):
-    """
-    Set up a new schema and tables to hold web-page like content.  The tables include a page table, and an asset
-    table that can have images that are referred to by the web page.  Pages are written using markdown.
-    :return:
-    """
-    if model.schemas.get("www"):
-        model.schemas["www"].drop(cascade=True)
-    www_schema = model.create_schema(
-        Schema.define(
-            "www", comment="Schema for tables that will be displayed as web content"
+    schema.create_table(
+        Table.define_association(
+            associates=[
+                ("File", file_table),
+                (MLVocab.file_type, file_type),
+            ]
         )
     )
-    www_schema.create_table(
+    schema.create_table(
+        Table.define_association(
+            [
+                ("File", file_table),
+                ("Execution", execution_table),
+            ]
+        )
+    )
+
+
+def create_workflow_table(schema: Schema, annotations: Optional[dict[str, Any]] = None):
+    annotations = annotations or {}
+    workflow_table = schema.create_table(
         Table.define(
-            "Page",
+            "Workflow",
             column_defs=[
-                Column.define(
-                    "Title",
-                    builtin_types.text,
-                    nullok=False,
-                    comment="Unique title for the page",
-                ),
-                Column.define(
-                    "Content",
-                    builtin_types.markdown,
-                    comment="Content of the page in markdown",
-                ),
+                Column.define("Name", builtin_types.text),
+                Column.define("Description", builtin_types.markdown),
+                Column.define("URL", builtin_types.ermrest_uri),
+                Column.define("Checksum", builtin_types.text),
+                Column.define("Version", builtin_types.text),
             ],
-            key_defs=[Key.define(["Title"])],
-            annotations={
-                chaise_tags.table_display: {
-                    "detailed": {
-                        "hide_column_headers": True,
-                        "collapse_toc_panel": True,
-                    }
-                },
-                chaise_tags.visible_foreign_keys: {"detailed": {}},
-                chaise_tags.visible_columns: {"detailed": ["Content"]},
-            },
+            annotations=annotations,
         )
     )
-    return www_schema
+    workflow_table.create_reference(
+        schema.create_table(
+            Table.define_vocabulary(MLVocab.workflow_type, f"{schema.name}:{{RID}}")
+        )
+    )
+    return workflow_table
 
 
 def create_ml_schema(
@@ -182,124 +230,100 @@ def create_ml_schema(
         Schema.define(schema_name, annotations=annotations["schema_annotation"])
     )
     project_name = project_name or schema_name
-    # Workflow
-    schema.create_table(
-        Table.define_vocabulary("Feature_Name", f"{project_name}:{{RID}}")
-    )
-
-    workflow_table = schema.create_table(
-        define_table_workflow(annotations["workflow_annotation"])
-    )
-    workflow_table.create_reference(
-        schema.create_table(
-            Table.define_vocabulary(MLVocab.workflow_type, f"{schema_name}:{{RID}}")
-        )
-    )
 
-    execution_table = schema.create_table(
-        define_table_execution(schema_name, annotations["execution_annotation"])
-    )
+    # Create workflow and execution table.
 
-    dataset_table = schema.create_table(
-        define_table_dataset(annotations["dataset_annotation"])
-    )
-    dataset_type = schema.create_table(
-        Table.define_vocabulary(MLVocab.dataset_type, f"{project_name}:{{RID}}")
-    )
     schema.create_table(
-        Table.define_association(
-            associates=[
-                ("Dataset", dataset_table),
-                (MLVocab.dataset_type, dataset_type),
-            ]
-        )
+        Table.define_vocabulary("Feature_Name", f"{project_name}:{{RID}}")
     )
-    schema.create_table(
-        Table.define_association(
-            associates=[("Dataset", dataset_table), ("Execution", execution_table)]
-        )
+    asset_type_table = schema.create_table(
+        Table.define_vocabulary("Asset_Type", f"{project_name}:{{RID}}")
     )
-
-    dataset_version = schema.create_table(define_table_dataset_version(schema_name))
-    dataset_table.create_reference(("Version", True, dataset_version))
-
-    # Nested datasets.
-    schema.create_table(
-        Table.define_association(
-            associates=[("Dataset", dataset_table), ("Nested_Dataset", dataset_table)]
-        )
+    asset_role_table = schema.create_table(
+        Table.define_vocabulary("Asset_Role", f"{project_name}:{{RID}}")
     )
 
-    # Execution Metadata
-    execution_metadata_table = schema.create_table(
-        define_asset_execution_metadata(
-            schema.name, annotations["execution_metadata_annotation"]
-        )
+    create_workflow_table(schema, annotations["workflow_annotation"])
+    execution_table = create_execution_table(
+        schema, annotations["execution_annotation"]
     )
-    execution_metadata_table.create_reference(
-        schema.create_table(
-            Table.define_vocabulary(
-                "Execution_Metadata_Type", f"{project_name}:{{RID}}"
-            )
-        )
-    )
-    schema.create_table(
-        Table.define_association(
-            [
-                ("Execution_Metadata", execution_metadata_table),
-                ("Execution", execution_table),
-            ]
-        )
+    create_dataset_table(
+        schema, execution_table, project_name, annotations["dataset_annotation"]
     )
 
-    # Execution Asset
-    execution_asset_table = schema.create_table(
-        define_asset_execution_asset(
-            schema.name, annotations["execution_asset_annotation"]
-        )
-    )
-    execution_asset_table.create_reference(
-        schema.create_table(
-            Table.define_vocabulary("Execution_Asset_Type", f"{project_name}:{{RID}}")
-        )
+    create_asset_table(
+        schema,
+        "Execution_Metadata",
+        execution_table,
+        asset_type_table,
+        asset_role_table,
+        annotations["execution_metadata_annotation"],
     )
-    schema.create_table(
-        Table.define_association(
-            [("Execution_Asset", execution_asset_table), ("Execution", execution_table)]
-        )
+    create_asset_table(
+        schema,
+        "Execution_Asset",
+        execution_table,
+        asset_type_table,
+        asset_role_table,
+        annotations["execution_asset_annotation"],
     )
 
     # File table
-    file_table = schema.create_table(define_table_file(schema_name))
-    file_type = schema.create_table(
-        Table.define_vocabulary(MLVocab.file_type, f"{project_name}:{{RID}}")
-    )
-    schema.create_table(
-        Table.define_association(
-            associates=[
-                ("File", file_table),
-                (MLVocab.file_type, file_type),
-            ]
+    create_file_table(schema, execution_table, project_name)
+
+    create_www_schema(model)
+    initialize_ml_schema(model, schema_name)
+
+
+def create_www_schema(model: Model):
+    """
+    Set up a new schema and tables to hold web-page like content.  The tables include a page table, and an asset
+    table that can have images that are referred to by the web page.  Pages are written using markdown.
+    :return:
+    """
+    if model.schemas.get("www"):
+        model.schemas["www"].drop(cascade=True)
+    www_schema = model.create_schema(
+        Schema.define(
+            "www", comment="Schema for tables that will be displayed as web content"
         )
     )
-    schema.create_table(
-        Table.define_association(
-            [
-                ("File", file_table),
-                ("Execution", execution_table),
-            ]
+    www_schema.create_table(
+        Table.define(
+            "Page",
+            column_defs=[
+                Column.define(
+                    "Title",
+                    builtin_types.text,
+                    nullok=False,
+                    comment="Unique title for the page",
+                ),
+                Column.define(
+                    "Content",
+                    builtin_types.markdown,
+                    comment="Content of the page in markdown",
+                ),
+            ],
+            key_defs=[Key.define(["Title"])],
+            annotations={
+                chaise_tags.table_display: {
+                    "detailed": {
+                        "hide_column_headers": True,
+                        "collapse_toc_panel": True,
+                    }
+                },
+                chaise_tags.visible_foreign_keys: {"detailed": {}},
+                chaise_tags.visible_columns: {"detailed": ["Content"]},
+            },
         )
     )
-    create_www_schema(model)
-    initialize_ml_schema(model, schema_name)
+    return www_schema
 
 
 def initialize_ml_schema(model: Model, schema_name: str = "deriva-ml"):
     catalog = model.catalog
-    execution_metadata_type = (
-        catalog.getPathBuilder().schemas[schema_name].tables["Execution_Metadata_Type"]
-    )
-    execution_metadata_type.insert(
+    asset_type = catalog.getPathBuilder().schemas[schema_name].tables["Asset_Type"]
+    asset_type.insert(
         [
             {
                 "Name": "Execution_Config",
@@ -309,6 +333,22 @@ def initialize_ml_schema(model: Model, schema_name: str = "deriva-ml"):
                 "Name": "Runtime_Env",
                 "Description": "Information about the execution environment",
             },
+            {
+                "Name": "Execution_Metadata",
+                "Description": "Information about the execution environment",
+            },
+            {
+                "Name": "Execution_Asset",
+                "Description": "A file generated by an execution",
+            },
+        ],
+        defaults={"ID", "URI"},
+    )
+    asset_role = catalog.getPathBuilder().schemas[schema_name].tables["Asset_Role"]
+    asset_role.insert(
+        [
+            {"Name": "Input", "Description": "Asset used for input of an execution."},
+            {"Name": "Output", "Description": "Asset used for output of an execution."},
         ],
         defaults={"ID", "URI"},
     )