deriva-ml 1.13.2__py3-none-any.whl → 1.13.3__py3-none-any.whl

deriva_ml/schema_setup/create_schema.py

@@ -2,7 +2,7 @@ import argparse
 import sys
 from typing import Optional, Any
 
-from deriva.core import DerivaServer, get_credential
+from deriva.core import DerivaServer, get_credential, ErmrestCatalog
 from deriva.core.ermrest_model import Model
 from deriva.core.ermrest_model import (
     builtin_types,
@@ -12,11 +12,9 @@ from deriva.core.ermrest_model import (
     ForeignKey,
     Key,
 )
-from deriva.core.utils.core_utils import tag as chaise_tags
 
 from deriva_ml import MLVocab
-from deriva_ml.schema_setup.annotations import generate_annotation
-from deriva_ml.deriva_model import DerivaModel
+from deriva_ml.schema_setup.annotations import generate_annotation, asset_annotation
 
 
 def create_dataset_table(
@@ -24,6 +22,7 @@ def create_dataset_table(
     execution_table: Table,
     project_name: str,
     dataset_annotation: Optional[dict] = None,
+    version_annotation: Optional[dict] = None,
 ):
     dataset_table = schema.create_table(
         Table.define(
@@ -48,7 +47,9 @@ def create_dataset_table(
         )
     )
 
-    dataset_version = schema.create_table(define_table_dataset_version(schema.name))
+    dataset_version = schema.create_table(
+        define_table_dataset_version(schema.name, version_annotation)
+    )
     dataset_table.create_reference(("Version", True, dataset_version))
 
     # Nested datasets.
@@ -64,7 +65,7 @@ def create_dataset_table(
     )
 
 
-def define_table_dataset_version(sname: str):
+def define_table_dataset_version(sname: str, annotation: Optional[dict] = None):
     return Table.define(
         tname="Dataset_Version",
         column_defs=[
@@ -80,9 +81,18 @@ def define_table_dataset_version(sname: str):
             Column.define(
                 "Minid", builtin_types.text, comment="URL to MINID for dataset"
             ),
+            Column.define(
+                "Snapshot",
+                builtin_types.text,
+                comment="Catalog Snapshot ID for dataset",
+            ),
         ],
+        annotations=annotation,
         key_defs=[Key.define(["Dataset", "Version"])],
-        fkey_defs=[ForeignKey.define(["Dataset"], sname, "Dataset", ["RID"])],
+        fkey_defs=[
+            ForeignKey.define(["Dataset"], sname, "Dataset", ["RID"]),
+            ForeignKey.define(["Execution"], sname, "Execution", ["RID"]),
+        ],
     )
 
 
@@ -113,18 +123,15 @@ def create_asset_table(
     execution_table,
     asset_type_table,
     asset_role_table,
-    annotation: Optional[dict] = None,
 ):
-    annotation = annotation if annotation is not None else {}
     asset_table = schema.create_table(
         Table.define_asset(
             sname=schema.name,
             tname=asset_name,
             hatrac_template="/hatrac/metadata/{{MD5}}.{{Filename}}",
-            annotations=annotation,
         )
     )
-    atable = schema.create_table(
+    schema.create_table(
         Table.define_association(
             [
                 (asset_name, asset_table),
@@ -142,6 +149,7 @@ def create_asset_table(
         )
     )
     atable.create_reference(asset_role_table)
+    asset_annotation(asset_table)
     return asset_table
 
 
@@ -203,15 +211,19 @@ def create_workflow_table(schema: Schema, annotations: Optional[dict[str, Any]]
 
 
 def create_ml_schema(
-    model: Model, schema_name: str = "deriva-ml", project_name: Optional[str] = None
+    catalog: ErmrestCatalog,
+    schema_name: str = "deriva-ml",
+    project_name: Optional[str] = None,
 ):
+    project_name = project_name or schema_name
+
+    model = catalog.getCatalogModel()
     if model.schemas.get(schema_name):
         model.schemas[schema_name].drop(cascade=True)
+
     # get annotations
-    deriva_model = DerivaModel(model)
-    annotations = generate_annotation(deriva_model)
+    annotations = generate_annotation(model, schema_name)
 
-    model.annotations.update(annotations["catalog_annotation"])
     client_annotation = {
         "tag:misd.isi.edu,2015:display": {"name": "Users"},
         "tag:isrd.isi.edu,2016:table-display": {
@@ -229,7 +241,6 @@ def create_ml_schema(
     schema = model.create_schema(
         Schema.define(schema_name, annotations=annotations["schema_annotation"])
     )
-    project_name = project_name or schema_name
 
     # Create workflow and execution table.
 
@@ -248,7 +259,11 @@ def create_ml_schema(
         schema, annotations["execution_annotation"]
     )
     create_dataset_table(
-        schema, execution_table, project_name, annotations["dataset_annotation"]
+        schema,
+        execution_table,
+        project_name,
+        annotations["dataset_annotation"],
+        annotations["dataset_version_annotation"],
     )
 
     create_asset_table(
@@ -257,69 +272,22 @@ def create_ml_schema(
         execution_table,
         asset_type_table,
         asset_role_table,
-        annotations["execution_metadata_annotation"],
     )
+
     create_asset_table(
         schema,
         "Execution_Asset",
         execution_table,
         asset_type_table,
         asset_role_table,
-        annotations["execution_asset_annotation"],
     )
 
     # File table
     create_file_table(schema, execution_table, project_name)
 
-    create_www_schema(model)
     initialize_ml_schema(model, schema_name)
 
 
-def create_www_schema(model: Model):
-    """
-    Set up a new schema and tables to hold web-page like content.  The tables include a page table, and an asset
-    table that can have images that are referred to by the web page.  Pages are written using markdown.
-    :return:
-    """
-    if model.schemas.get("www"):
-        model.schemas["www"].drop(cascade=True)
-    www_schema = model.create_schema(
-        Schema.define(
-            "www", comment="Schema for tables that will be displayed as web content"
-        )
-    )
-    www_schema.create_table(
-        Table.define(
-            "Page",
-            column_defs=[
-                Column.define(
-                    "Title",
-                    builtin_types.text,
-                    nullok=False,
-                    comment="Unique title for the page",
-                ),
-                Column.define(
-                    "Content",
-                    builtin_types.markdown,
-                    comment="Content of the page in markdown",
-                ),
-            ],
-            key_defs=[Key.define(["Title"])],
-            annotations={
-                chaise_tags.table_display: {
-                    "detailed": {
-                        "hide_column_headers": True,
-                        "collapse_toc_panel": True,
-                    }
-                },
-                chaise_tags.visible_foreign_keys: {"detailed": {}},
-                chaise_tags.visible_columns: {"detailed": ["Content"]},
-            },
-        )
-    )
-    return www_schema
-
-
 def initialize_ml_schema(model: Model, schema_name: str = "deriva-ml"):
     catalog = model.catalog
     asset_type = catalog.getPathBuilder().schemas[schema_name].tables["Asset_Type"]

deriva_ml/schema_setup/policy.json

@@ -3,16 +3,20 @@
     "isrd-systems": ["https://auth.globus.org/3938e0d0-ed35-11e5-8641-22000ab4b42b"],
     "isrd-staff": ["https://auth.globus.org/176baec4-ed26-11e5-8e88-22000ab4b42b"],
     "isrd-testers": ["https://auth.globus.org/9d596ac6-22b9-11e6-b519-22000aef184d"],
+    "eye-ai": ["https://auth.globus.org/d38e6f4d-ac96-11ed-8ab4-7b5a369c0a53"],
 
+    "local-admin": ["admin"],
     "project-admins":   ["isrd-systems"],
-    "project-curators": ["isrd-systems"],
-    "project-writers":  ["isrd-systems"],
-    "project-users":    ["isrd-systems"],
+    "project-writers":  ["isrd-systems", "eye-ai"],
+
+    "project-curators": ["isrd-systems", "local-admin", "eye-ai" ],
+    "project-users":    ["isrd-systems", "local-admin", "eye-ai"],
 
     "empty": [],
     "public": ["*"],
     "admins": [
       "project-admins",
+      "local-admin",
       "isrd-systems"
     ],
     "curators": ["project-curators"],

deriva_ml/upload.py

@@ -98,10 +98,10 @@ def is_feature_dir(path: Path) -> Optional[re.Match]:
 
 def normalize_asset_dir(path: str) -> Optional[tuple[str, str]]:
     """Parse a path to an asset file and return the asset table name and file name"""
-
-    if not (m := re.match(asset_path_regex, path)):
+    path = Path(path)
+    if not (m := re.match(asset_path_regex, path.as_posix())):
         return None
-    return f"{m['schema']}/{m['asset_table']}", Path(path).name
+    return f"{m['schema']}/{m['asset_table']}", path.name
 
 
 def upload_root(prefix: Path | str) -> Path:

{deriva_ml-1.13.2.dist-info → deriva_ml-1.13.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.13.2
+Version: 1.13.3
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
@@ -13,7 +13,7 @@ Requires-Dist: regex~=2024.7.24
 Requires-Dist: pydantic>=2.10.6
 Requires-Dist: semver>3.0.0
 Requires-Dist: setuptools>=64
-Requires-Dist: setuptools-scm<=6.0
+Requires-Dist: setuptools-scm>=8.0
 Requires-Dist: nbstripout
 Requires-Dist: papermill
 Dynamic: license-file

deriva_ml-1.13.3.dist-info/RECORD

@@ -0,0 +1,31 @@
+deriva_ml/__init__.py,sha256=GfneBq7xDphMqUQY96sW9ixRj74M3UTUCmD4KMIRSaM,1101
+deriva_ml/database_model.py,sha256=D5vY0HqsCzWzbQVuK5jsImeEfjgg9wMe6fjDz8ucZqc,14695
+deriva_ml/dataset.py,sha256=UmVX3CQxjvi3pVudnbdpAddE4MJz230QnepICAb5Rag,60136
+deriva_ml/dataset_aux_classes.py,sha256=IyPAbZUoVWjnm4hcpLet2mkJvKHRGN__IEwcAIrH-u4,6597
+deriva_ml/dataset_bag.py,sha256=yS8oYVshfFtRDyhGPRqtbvxjyd3ZFF29lrB783OP4vM,11849
+deriva_ml/demo_catalog.py,sha256=NBBsnF955jGpsGTgYFjeBn2580MjRuzC1MqqdrkaHAU,12081
+deriva_ml/deriva_definitions.py,sha256=3rkl0TsOoVR3QdZdAHvR--Zgt1FvW1et3BEL-_bjLVM,10768
+deriva_ml/deriva_ml_base.py,sha256=FQirlTQvOUs25G0WGxnlVj3uF3PQbMX96nNj9s9arTI,38565
+deriva_ml/deriva_model.py,sha256=yMuJ3C8-anl1qf7lFBkiswVMWcheI_TUV5G_GvlzVCk,13565
+deriva_ml/execution.py,sha256=qcxqkF0gxmPfwLQb-ooyagFawgO9sK4CQfzJOfBmMp4,38330
+deriva_ml/execution_configuration.py,sha256=7fiIbtzz9nmkxA9-GTiN6Ln2twfaOLivwJwGZb8gAL0,14163
+deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
+deriva_ml/feature.py,sha256=07g0uSrhumdopJluWuWSRMrzagaikAOihqB09bzXBP4,5475
+deriva_ml/history.py,sha256=XRZ_mmxOnspUJAoC_XEH_DhSgfLLW8Kiub3CxpQPftw,2830
+deriva_ml/run_notebook.py,sha256=vhmij4P1Va52MIj8hOc-WmjLRp3sTmK6p7LXCWrzejc,6308
+deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
+deriva_ml/test_notebook.ipynb,sha256=_5D6rkSGbmENPJZbDgfZ6-yt94BNEwxytVUDmG3RE3w,10166
+deriva_ml/upload.py,sha256=VgCb0fUfm_siOW1p3g1MwenFIL8dpHzZHrVhzZGbHb4,16113
+deriva_ml/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deriva_ml/schema_setup/annotations.py,sha256=0lGFa7PLwlFXXps26OYQm1vuWdsfCtbL2v2FqjyehC8,17739
+deriva_ml/schema_setup/create_schema.py,sha256=PTbUL00A-IwuMqADFEHvGTeRgU9UyBmQlw94EGseHhE,10305
+deriva_ml/schema_setup/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVEanfY8,1826
+deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
+deriva_ml/test-files/execution-parameters.json,sha256=1vBqXlaMa0cysonE20TweVDfTGRdSi9CUuAkW1xiYNo,36
+deriva_ml/test-files/notebook-parameters.json,sha256=7uEE2sLQSrSc9cEGQ_RKE7t5dwkEYv0qLo5mRbzo8Og,108
+deriva_ml-1.13.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.13.3.dist-info/METADATA,sha256=D4ar2arJ8tkT9-YTAWl-r_CsJJA42rb2RKqcD8tOkF8,999
+deriva_ml-1.13.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.13.3.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
+deriva_ml-1.13.3.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.13.3.dist-info/RECORD,,

{deriva_ml-1.13.2.dist-info → deriva_ml-1.13.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (79.0.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

deriva_ml-1.13.2.dist-info/RECORD

@@ -1,31 +0,0 @@
-deriva_ml/__init__.py,sha256=GfneBq7xDphMqUQY96sW9ixRj74M3UTUCmD4KMIRSaM,1101
-deriva_ml/database_model.py,sha256=lMbAEqn4n0m7h_JstMX_LX9gbvBIEydG3sRilPn3eLU,14885
-deriva_ml/dataset.py,sha256=W1TSHgkdXNw2v5hC0UBrivCKadMK1LaFd6YIjHE9jZA,60786
-deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
-deriva_ml/dataset_bag.py,sha256=yS8oYVshfFtRDyhGPRqtbvxjyd3ZFF29lrB783OP4vM,11849
-deriva_ml/demo_catalog.py,sha256=9Qo3JD4bUIwnL3ngPctc2QBeWApvMR_5UyaK9ockTrY,11536
-deriva_ml/deriva_definitions.py,sha256=avdOgxtB60yb8XsWm-AYtCdvg2QkQbyfkZuA9xx9t2U,9221
-deriva_ml/deriva_ml_base.py,sha256=FYSTQl4mNePC8IxC70rS5D0VmLNPccfFkkiVneDxJpY,38678
-deriva_ml/deriva_model.py,sha256=wytGCAHutiUaRfnRKr80Ks_P6ci0_wXRU3vq3lthfYU,13260
-deriva_ml/execution.py,sha256=otMkdjF15SEWg99mvWrTpnKz7-BWp9b8XbFf6iwfmtg,37697
-deriva_ml/execution_configuration.py,sha256=7fiIbtzz9nmkxA9-GTiN6Ln2twfaOLivwJwGZb8gAL0,14163
-deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
-deriva_ml/feature.py,sha256=07g0uSrhumdopJluWuWSRMrzagaikAOihqB09bzXBP4,5475
-deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
-deriva_ml/run_notebook.py,sha256=vhmij4P1Va52MIj8hOc-WmjLRp3sTmK6p7LXCWrzejc,6308
-deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
-deriva_ml/test_notebook.ipynb,sha256=_5D6rkSGbmENPJZbDgfZ6-yt94BNEwxytVUDmG3RE3w,10166
-deriva_ml/upload.py,sha256=gHTGXAVlf56EwNzmw5zY0gbBf8h08eU2q2GBbb2FdVc,16087
-deriva_ml/schema_setup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-deriva_ml/schema_setup/annotations.py,sha256=v0gTpmWYxRqsQ-bcnQzsr8WowGv2pi9pZUsO3WWnu1U,9528
-deriva_ml/schema_setup/create_schema.py,sha256=hNMc-v5tferd0UjfdB6nBw7Rc_o-Mg6NkPqQGie9YOw,11700
-deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjtkk538,1633
-deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
-deriva_ml/test-files/execution-parameters.json,sha256=1vBqXlaMa0cysonE20TweVDfTGRdSi9CUuAkW1xiYNo,36
-deriva_ml/test-files/notebook-parameters.json,sha256=7uEE2sLQSrSc9cEGQ_RKE7t5dwkEYv0qLo5mRbzo8Og,108
-deriva_ml-1.13.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.13.2.dist-info/METADATA,sha256=uuvCztFgxOwWM34egjr65pW8-2pYGCtV_xofT5TmcLg,999
-deriva_ml-1.13.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
-deriva_ml-1.13.2.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
-deriva_ml-1.13.2.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.13.2.dist-info/RECORD,,