deriva-ml 1.12.1__tar.gz → 1.12.3__tar.gz

{deriva_ml-1.12.1/src/deriva_ml.egg-info → deriva_ml-1.12.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.12.1
+Version: 1.12.3
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

deriva_ml-1.12.3/docs/user-guide/file-assets.md

@@ -0,0 +1,3 @@
+#  File Assets
+
+

{deriva_ml-1.12.1 → deriva_ml-1.12.3}/src/deriva_ml/dataset.py

@@ -41,7 +41,14 @@ from tempfile import TemporaryDirectory, NamedTemporaryFile
 from typing import Any, Callable, Optional, Iterable, Iterator, TYPE_CHECKING
 
 from deriva_ml import DatasetBag
-from .deriva_definitions import ML_SCHEMA, DerivaMLException, MLVocab, Status, RID
+from .deriva_definitions import (
+    ML_SCHEMA,
+    DerivaMLException,
+    MLVocab,
+    Status,
+    RID,
+    DRY_RUN_RID,
+)
 from .history import iso_to_snap
 from .deriva_model import DerivaModel
 from .database_model import DatabaseModel
@@ -957,7 +964,7 @@ class Dataset:
             for the dataset.
         """
         if (
-            execution_rid
+            execution_rid != DRY_RUN_RID
             and self._model.catalog.resolve_rid(execution_rid).table.name != "Execution"
         ):
             raise DerivaMLException(f"RID {execution_rid} is not an execution")

{deriva_ml-1.12.1 → deriva_ml-1.12.3}/src/deriva_ml/deriva_definitions.py

@@ -21,6 +21,7 @@ from pydantic import (
 from socket import gethostname
 
 ML_SCHEMA = "deriva-ml"
+DRY_RUN_RID = "0000"
 
 # We are going to use schema as a field name and this collides with method in pydantic base class
 warnings.filterwarnings(

{deriva_ml-1.12.1 → deriva_ml-1.12.3}/src/deriva_ml/execution.py

@@ -5,21 +5,30 @@ This module defined the Execution class which is used to interact with the state
 from __future__ import annotations
 
 from collections import defaultdict
+from datetime import datetime
 import json
 import logging
 import os
-import shutil
-from datetime import datetime
 from pathlib import Path
-from typing import Iterable, Any, Optional
 
-from deriva.core import format_exception
 from pydantic import validate_call, ConfigDict
+import regex as re
 import sys
-from deriva.core.hatrac_store import HatracStore
+import shutil
+from typing import Iterable, Any, Optional
 
+from deriva.core import format_exception
+from deriva.core.datapath import DataPathException
+from deriva.core.hatrac_store import HatracStore
 from .deriva_definitions import ExecMetadataVocab
-from .deriva_definitions import RID, Status, FileUploadState, DerivaMLException, MLVocab
+from .deriva_definitions import (
+    RID,
+    Status,
+    FileUploadState,
+    DerivaMLException,
+    MLVocab,
+    DRY_RUN_RID,
+)
 from .deriva_ml_base import DerivaML, FeatureRecord
 from .dataset_aux_classes import DatasetSpec, DatasetVersion, VersionPart
 from .dataset_bag import DatasetBag
@@ -167,7 +176,7 @@ class Execution:
             self.workflow_rid = (
                 self._ml_object.add_workflow(self.configuration.workflow)
                 if not self._dry_run
-                else "0000"
+                else DRY_RUN_RID
             )
         else:
             self.workflow_rid = self.configuration.workflow
@@ -195,10 +204,10 @@ class Execution:
         schema_path = self._ml_object.pathBuilder.schemas[self._ml_object.ml_schema]
         if reload:
             self.execution_rid = reload
-            if self.execution_rid == "0000":
+            if self.execution_rid == DRY_RUN_RID:
                 self._dry_run = True
         elif self._dry_run:
-            self.execution_rid = "0000"
+            self.execution_rid = DRY_RUN_RID
         else:
             self.execution_rid = schema_path.Execution.insert(
                 [
@@ -625,9 +634,20 @@ class Execution:
         with open(feature_file, "r") as feature_values:
             entities = [json.loads(line.strip()) for line in feature_values]
         # Update the asset columns in the feature and add to the catalog.
-        self._ml_object.domain_path.tables[feature_table].insert(
-            [map_path(e) for e in entities]
-        )
+        try:
+            self._ml_object.domain_path.tables[feature_table].insert(
+                [map_path(e) for e in entities]
+            )
+        except DataPathException as e:
+            if re.match(
+                rf'DETAIL: +Key +\("Execution", +"{target_table}", +"Feature_Name"\)=\(.*\) already exists',
+                e.message,
+            ):
+                self._logger.info(
+                    f"Skipping reload of feature values for {feature_table}"
+                )
+            else:
+                raise e
 
     def _update_asset_execution_table(
         self,
@@ -652,16 +672,27 @@ class Execution:
             asset_exe = self._model.find_association(asset_table_name, "Execution")
             asset_exe_path = pb.schemas[asset_exe.schema.name].tables[asset_exe.name]
 
-            asset_exe_path.insert(
-                [
-                    {
-                        asset_table_name: asset_path.asset_rid,
-                        "Execution": self.execution_rid,
-                        "Asset_Role": asset_role,
-                    }
-                    for asset_path in asset_list
-                ]
-            )
+            try:
+                asset_exe_path.insert(
+                    [
+                        {
+                            asset_table_name: asset_path.asset_rid,
+                            "Execution": self.execution_rid,
+                            "Asset_Role": asset_role,
+                        }
+                        for asset_path in asset_list
+                    ]
+                )
+            except DataPathException as e:
+                if re.match(
+                    rf'DETAIL: +Key +\("{asset_table_name}", +"Execution"\)=\(.*\) already exists',
+                    e.message,
+                ):
+                    self._logger.info(
+                        f"Skipping reload of execution assocations for {asset_table_name}"
+                    )
+                else:
+                    raise e
 
             # Now add in the type names via the asset_asset_type association table.
             # Get the list of types for each file in the asset.
@@ -687,13 +718,24 @@ class Execution:
             type_path = pb.schemas[asset_asset_type.schema.name].tables[
                 asset_asset_type.name
             ]
-            type_path.insert(
-                [
-                    {asset_table_name: asset.asset_rid, "Asset_Type": t}
-                    for asset in asset_list
-                    for t in asset_type_map[asset.file_name]
-                ]
-            )
+            try:
+                type_path.insert(
+                    [
+                        {asset_table_name: asset.asset_rid, "Asset_Type": t}
+                        for asset in asset_list
+                        for t in asset_type_map[asset.file_name]
+                    ]
+                )
+            except DataPathException as e:
+                if re.match(
+                    rf'DETAIL: +Key +\("{asset_table_name}", +"Asset_Type"\)=\(.*\) already exists',
+                    e.message,
+                ):
+                    self._logger.info(
+                        f"Skipping reload of execution asset types for {asset_table_name}"
+                    )
+                else:
+                    raise e
 
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def asset_file_path(

{deriva_ml-1.12.1 → deriva_ml-1.12.3}/src/deriva_ml/upload.py

@@ -74,17 +74,16 @@ feature_table_dir_regex = (
     + r"/(?P<schema>[-\w]+)/(?P<target_table>[-\w]+)/(?P<feature_name>[-\w]+)"
 )
 feature_value_regex = (
-    feature_table_dir_regex + r"/(?P=feature_name)[.](?P<file_ext>[(csv|json)]*)$"
+    feature_table_dir_regex + r"/(?P=feature_name)[.](?P<ext>[(csv|json)]*)$"
 )
 feature_asset_dir_regex = feature_table_dir_regex + r"/asset/(?P<asset_table>[-\w]+)"
 feature_asset_regex = (
-    feature_asset_dir_regex
-    + r"/(?P<file_name>[A-Za-z0-9_-]+)[.](?P<file_ext>[a-z0-9]*)$"
+    feature_asset_dir_regex + r"/(?P<file>[A-Za-z0-9_-]+)[.](?P<ext>[a-z0-9]*)$"
 )
 
 asset_path_regex = exec_dir_regex + r"/asset/(?P<schema>[-\w]+)/(?P<asset_table>[-\w]*)"
 
-asset_file_regex = r"(?P<file_name>[-\w]+)[.](?P<file_ext>[a-z0-9]*)$"
+asset_file_regex = r"(?P<file>[-\w]+)[.](?P<ext>[a-z0-9]*)$"
 
 table_regex = (
     exec_dir_regex
@@ -211,16 +210,16 @@ def asset_table_upload_spec(model: DerivaModel, asset_table: str | Table):
             "Filename": "{file_name}",
         }
         | {c: f"{{{c}}}" for c in metadata_columns},
-        "file_pattern": asset_path,  # Sets schema, asset_table, file_name, file_ext
+        "file_pattern": asset_path,  # Sets schema, asset_table, file
         "asset_type": "file",
         "target_table": [schema, asset_table.name],
         "checksum_types": ["sha256", "md5"],
         "hatrac_options": {"versioned_urls": True},
         "hatrac_templates": {
-            "hatrac_uri": f"/hatrac/{asset_table.name}/{{md5}}.{{file_name}}.{{file_ext}}",
-            "content-disposition": "filename*=UTF-8''{file_name}.{file_ext}",
+            "hatrac_uri": f"/hatrac/{asset_table.name}/{{md5}}.{{file_name}}",
+            "content-disposition": "filename*=UTF-8''{file_name}",
         },
-        "record_query_template": "/entity/{target_table}/MD5={{md5}}&Filename={file_name}.{file_ext}",
+        "record_query_template": "/entity/{target_table}/MD5={md5}&Filename={file_name}",
     }
 
 
@@ -249,14 +248,14 @@ def bulk_upload_configuration(model: DerivaModel) -> dict[str, Any]:
                 "target_table": ["{schema}", "{asset_table}"],
                 "file_pattern": asset_path_regex
                 + "/"
-                + asset_file_regex,  # Sets schema, asset_table, file_name, file_ext
+                + asset_file_regex,  # Sets schema, asset_table, name, ext
                 "checksum_types": ["sha256", "md5"],
                 "hatrac_options": {"versioned_urls": True},
                 "hatrac_templates": {
-                    "hatrac_uri": "/hatrac/{asset_table}/{md5}.{file_name}.{file_ext}",
-                    "content-disposition": "filename*=UTF-8''{file_name}.{file_ext}",
+                    "hatrac_uri": "/hatrac/{asset_table}/{md5}.{file_name}",
+                    "content-disposition": "filename*=UTF-8''{file_name}",
                 },
-                "record_query_template": "/entity/{target_table}/MD5={md5}&Filename={file_name}.{file_ext}",
+                "record_query_template": "/entity/{target_table}/MD5={md5}&Filename={file_name}",
             },
             # {
             #  Upload the records into a  table

{deriva_ml-1.12.1 → deriva_ml-1.12.3/src/deriva_ml.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.12.1
+Version: 1.12.3
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.12.1 → deriva_ml-1.12.3}/src/deriva_ml.egg-info/SOURCES.txt

@@ -31,6 +31,7 @@ docs/code-docs/feature.md
 docs/code-docs/upload.md
 docs/user-guide/datasets.md
 docs/user-guide/execution-configuration.md
+docs/user-guide/file-assets.md
 docs/user-guide/identifiers.md
 docs/user-guide/install.md
 docs/user-guide/ml_workflow_instruction.md