deriva-ml 1.13.1__py3-none-any.whl → 1.13.2__py3-none-any.whl

deriva_ml/dataset.py

@@ -964,7 +964,8 @@ class Dataset:
             for the dataset.
         """
         if (
-            execution_rid != DRY_RUN_RID
+            execution_rid
+            and execution_rid != DRY_RUN_RID
             and self._model.catalog.resolve_rid(execution_rid).table.name != "Execution"
         ):
             raise DerivaMLException(f"RID {execution_rid} is not an execution")
@@ -1120,17 +1121,18 @@ class Dataset:
 
         def update_status(status: Status, msg: str) -> None:
             """Update the current status for this execution in the catalog"""
-            self._model.catalog.getPathBuilder().schemas[
-                self._ml_schema
-            ].Execution.update(
-                [
-                    {
-                        "RID": execution_rid,
-                        "Status": status.value,
-                        "Status_Detail": msg,
-                    }
-                ]
-            )
+            if execution_rid and execution_rid != DRY_RUN_RID:
+                self._model.catalog.getPathBuilder().schemas[
+                    self._ml_schema
+                ].Execution.update(
+                    [
+                        {
+                            "RID": execution_rid,
+                            "Status": status.value,
+                            "Status_Detail": msg,
+                        }
+                    ]
+                )
             self._logger.info(msg)
 
         def fetch_progress_callback(current, total):

deriva_ml/deriva_ml_base.py

@@ -974,7 +974,7 @@ class DerivaML(Dataset):
     ) -> Workflow:
         """Identify current executing program and return a workflow RID for it
 
-        Determine the notebook or script that is currently being executed. Assume that  this is
+        Determine the notebook or script that is currently being executed. Assume that this is
         being executed from a cloned GitHub repository.  Determine the remote repository name for
         this object.  Then either retrieve an existing workflow for this executable or create
         a new one.
@@ -983,6 +983,9 @@ class DerivaML(Dataset):
             name: The name of the workflow.
             workflow_type: The type of the workflow.
             description: The description of the workflow.
+
+        Returns:
+            A workflow object.
         """
         # Make sure type is correct.
         self.lookup_term(MLVocab.workflow_type, workflow_type)
@@ -1001,6 +1004,9 @@ class DerivaML(Dataset):
         1. The datasets specified in the configuration are downloaded and placed in the cache-dir. If a version is
         not specified in the configuration, then a new minor version number is created for the dataset and downloaded.
 
+        2. If any execution assets are provided in the configuration, they are downloaded and placed in the working directory.
+
+
         Args:
             configuration: ExecutionConfiguration:
             dry_run: Do not create an execution record or upload results.

deriva_ml/execution.py

@@ -12,13 +12,11 @@ import os
 from pathlib import Path
 
 from pydantic import validate_call, ConfigDict
-import regex as re
 import sys
 import shutil
 from typing import Iterable, Any, Optional
 
 from deriva.core import format_exception
-from deriva.core.datapath import DataPathException
 from deriva.core.hatrac_store import HatracStore
 from .deriva_definitions import (
     RID,
@@ -66,29 +64,43 @@ except ImportError:
         return s
 
 
-class AssetFilePath(type(Path())):
-    """Derived class of Path that also includes information about a downloaded.
+# Platform-specific base class
+if sys.version_info >= (3, 12):
 
-    An AssetFilePath has all  the methods associated with a pathlib.Path object. In addition, it defines additional
-    attributes associated with a DerviaML asset.
+    class AssetFilePath(Path):
+        """
+        Create a new Path object that has additional information related to the use of this path as an asset.
 
-    Attributes:
-        asset_types: A list of the types associated with this asset.  From the Asset_Type controlled vocabulary.
-        asset_metadata: A dictionary of names and values of any additional columns  associated with this asset.
-        asset_name: The name of the asset table
-        file_name: The name of the file in the local file system that has the asset contents
-        asset_rid: The RID of the asset if it has been uploaded into an asset table
-    """
+        Args:
+            asset_path: Local path to the location of the asset.
+            asset_name:  The name of the asset in the catalog (e.g. the asset table name).
+            file_name:  Name of the local file that contains the contents of the asset.
+            asset_metadata: Any additional columns associated with this asset beyond the URL, Length, and checksum.
+            asset_types:  A list of terms from the Asset_Type controlled vocabulary.
+            asset_rid:  The RID of the asset if it has been uploaded into an asset table
+        """
 
-    def __new__(
-        cls,
-        asset_path,
-        asset_name: str,
-        file_name: str,
-        asset_metadata: dict[str, Any],
-        asset_types: list[str] | str,
-        asset_rid: Optional[RID] = None,
-    ):
+        def __init__(
+            self,
+            asset_path: str | Path,
+            asset_name: str,
+            file_name: str,
+            asset_metadata: dict[str, Any],
+            asset_types: list[str] | str,
+            asset_rid: Optional["RID"] = None,
+        ):
+            super().__init__(asset_path)
+            # These assignments happen after __new__ returns the instance
+            self.asset_name = asset_name
+            self.file_name = file_name
+            self.asset_metadata = asset_metadata
+            self.asset_types = (
+                asset_types if isinstance(asset_types, list) else [asset_types]
+            )
+            self.asset_rid = asset_rid
+else:
+
+    class AssetFilePath(type(Path())):
         """
         Create a new Path object that has additional information related to the use of this path as an asset.
 
@@ -100,15 +112,26 @@ class AssetFilePath(type(Path())):
             asset_types:  A list of terms from the Asset_Type controlled vocabulary.
             asset_rid:  The RID of the asset if it has been uploaded into an asset table
         """
-        obj = super().__new__(cls, asset_path)
-        obj.asset_types = (
-            asset_types if isinstance(asset_types, list) else [asset_types]
-        )
-        obj.asset_metadata = asset_metadata
-        obj.asset_name = asset_name
-        obj.file_name = file_name
-        obj.asset_rid = asset_rid
-        return obj
+
+        def __new__(
+            cls,
+            asset_path: str | Path,
+            asset_name: str,
+            file_name: str,
+            asset_metadata: dict[str, Any],
+            asset_types: list[str] | str,
+            asset_rid: Optional["RID"] = None,
+        ):
+            # Only pass the path to the base Path class
+            obj = super().__new__(cls, asset_path)
+            obj.asset_name = asset_name
+            obj.file_name = file_name
+            obj.asset_metadata = asset_metadata
+            obj.asset_types = (
+                asset_types if isinstance(asset_types, list) else [asset_types]
+            )
+            obj.asset_rid = asset_rid
+            return obj
 
 
 class Execution:
@@ -155,7 +178,7 @@ class Execution:
         Args:
             configuration: Execution configuration object that describes the execution.
             ml_object: The DerivaML instance that created the execution.
-            reload: RID of previously initialized execution object.
+            reload: RID of a previously initialized execution object.
         """
         self.asset_paths: list[AssetFilePath] = []
         self.configuration = configuration
@@ -476,7 +499,7 @@ class Execution:
         """Download an asset from a URL and place it in a local directory.
 
         Args:
-            asset_rid: URL of the asset.
+            asset_rid: RID of the asset.
             dest_dir: Destination directory for the asset.
             update_catalog: Whether to update the catalog execution information after downloading.
 
@@ -656,20 +679,9 @@ class Execution:
         with open(feature_file, "r") as feature_values:
             entities = [json.loads(line.strip()) for line in feature_values]
         # Update the asset columns in the feature and add to the catalog.
-        try:
-            self._ml_object.domain_path.tables[feature_table].insert(
-                [map_path(e) for e in entities]
-            )
-        except DataPathException as e:
-            if re.match(
-                rf'DETAIL: +Key +\("Execution", +"{target_table}", +"Feature_Name"\)=\(.*\) already exists',
-                e.message,
-            ):
-                self._logger.info(
-                    f"Skipping reload of feature values for {feature_table}"
-                )
-            else:
-                raise e
+        self._ml_object.domain_path.tables[feature_table].insert(
+            [map_path(e) for e in entities], on_conflict_skip=True
+        )
 
     def _update_asset_execution_table(
         self,
@@ -694,27 +706,17 @@ class Execution:
             asset_exe = self._model.find_association(asset_table_name, "Execution")
             asset_exe_path = pb.schemas[asset_exe.schema.name].tables[asset_exe.name]
 
-            try:
-                asset_exe_path.insert(
-                    [
-                        {
-                            asset_table_name: asset_path.asset_rid,
-                            "Execution": self.execution_rid,
-                            "Asset_Role": asset_role,
-                        }
-                        for asset_path in asset_list
-                    ]
-                )
-            except DataPathException as e:
-                if re.match(
-                    rf'DETAIL: +Key +\("{asset_table_name}", +"Execution"\)=\(.*\) already exists',
-                    e.message,
-                ):
-                    self._logger.info(
-                        f"Skipping reload of execution assocations for {asset_table_name}"
-                    )
-                else:
-                    raise e
+            asset_exe_path.insert(
+                [
+                    {
+                        asset_table_name: asset_path.asset_rid,
+                        "Execution": self.execution_rid,
+                        "Asset_Role": asset_role,
+                    }
+                    for asset_path in asset_list
+                ],
+                on_conflict_skip=True,
+            )
 
             # Now add in the type names via the asset_asset_type association table.
             # Get the list of types for each file in the asset.
@@ -740,24 +742,15 @@ class Execution:
             type_path = pb.schemas[asset_asset_type.schema.name].tables[
                 asset_asset_type.name
             ]
-            try:
-                type_path.insert(
-                    [
-                        {asset_table_name: asset.asset_rid, "Asset_Type": t}
-                        for asset in asset_list
-                        for t in asset_type_map[asset.file_name]
-                    ]
-                )
-            except DataPathException as e:
-                if re.match(
-                    rf'DETAIL: +Key +\("{asset_table_name}", +"Asset_Type"\)=\(.*\) already exists',
-                    e.message,
-                ):
-                    self._logger.info(
-                        f"Skipping reload of execution asset types for {asset_table_name}"
-                    )
-                else:
-                    raise e
+
+            type_path.insert(
+                [
+                    {asset_table_name: asset.asset_rid, "Asset_Type": t}
+                    for asset in asset_list
+                    for t in asset_type_map[asset.file_name]
+                ],
+                on_conflict_skip=True,
+            )
 
     @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
     def asset_file_path(

deriva_ml/execution_configuration.py

@@ -325,7 +325,8 @@ class ExecutionConfiguration(BaseModel):
             should be materialized.
         assets: List of assets to be downloaded prior to execution.  The values must be RIDs in an asset table
         parameters: Either a dictionary or a path to a JSON file that contains configuration parameters for the execution.
-        workflow: A RID for a workflow instance.  Must have a name, URI to the workflow instance, and a type.
+        workflow: Either a Workflow object, or a RID for a workflow instance.
+        parameters: Either a dictionary or a path to a JSON file that contains configuration parameters for the execution.
         description: A description of the execution.  Can use Markdown format.
     """
 

{deriva_ml-1.13.1.dist-info → deriva_ml-1.13.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.13.1
+Version: 1.13.2
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.13.1.dist-info → deriva_ml-1.13.2.dist-info}/RECORD

@@ -1,14 +1,14 @@
 deriva_ml/__init__.py,sha256=GfneBq7xDphMqUQY96sW9ixRj74M3UTUCmD4KMIRSaM,1101
 deriva_ml/database_model.py,sha256=lMbAEqn4n0m7h_JstMX_LX9gbvBIEydG3sRilPn3eLU,14885
-deriva_ml/dataset.py,sha256=OyWUKWnYeP0ctimSBQ4em-uJrzCNOohx4GPT2uIl6R4,60649
+deriva_ml/dataset.py,sha256=W1TSHgkdXNw2v5hC0UBrivCKadMK1LaFd6YIjHE9jZA,60786
 deriva_ml/dataset_aux_classes.py,sha256=YxjQnu2kS9kK_f8bGqhmgE6ty9GNeitCxfvReT9vaM0,6537
 deriva_ml/dataset_bag.py,sha256=yS8oYVshfFtRDyhGPRqtbvxjyd3ZFF29lrB783OP4vM,11849
 deriva_ml/demo_catalog.py,sha256=9Qo3JD4bUIwnL3ngPctc2QBeWApvMR_5UyaK9ockTrY,11536
 deriva_ml/deriva_definitions.py,sha256=avdOgxtB60yb8XsWm-AYtCdvg2QkQbyfkZuA9xx9t2U,9221
-deriva_ml/deriva_ml_base.py,sha256=JYTG_a8SURhrPQBTz6OaGMk0D0sSPWpXqCnoVnSNViI,38501
+deriva_ml/deriva_ml_base.py,sha256=FYSTQl4mNePC8IxC70rS5D0VmLNPccfFkkiVneDxJpY,38678
 deriva_ml/deriva_model.py,sha256=wytGCAHutiUaRfnRKr80Ks_P6ci0_wXRU3vq3lthfYU,13260
-deriva_ml/execution.py,sha256=Oyja5wonSBUDUIVSC01w3AojGEkWyw_8_kBMv3MTZBM,38126
-deriva_ml/execution_configuration.py,sha256=KKg2HhvOiOmYc3jJ9iJeeHYyRu05Bb8JpojmPn1gYW0,14072
+deriva_ml/execution.py,sha256=otMkdjF15SEWg99mvWrTpnKz7-BWp9b8XbFf6iwfmtg,37697
+deriva_ml/execution_configuration.py,sha256=7fiIbtzz9nmkxA9-GTiN6Ln2twfaOLivwJwGZb8gAL0,14163
 deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
 deriva_ml/feature.py,sha256=07g0uSrhumdopJluWuWSRMrzagaikAOihqB09bzXBP4,5475
 deriva_ml/history.py,sha256=qTDLDs8Ow_6r7mDO0gZm0Fg81SWKOAgtCU5pzZoDRgM,2828
@@ -23,9 +23,9 @@ deriva_ml/schema_setup/policy.json,sha256=77sf0Imy6CAQV0_VwwbA56_KROJ05WXsvT-Wjt
 deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
 deriva_ml/test-files/execution-parameters.json,sha256=1vBqXlaMa0cysonE20TweVDfTGRdSi9CUuAkW1xiYNo,36
 deriva_ml/test-files/notebook-parameters.json,sha256=7uEE2sLQSrSc9cEGQ_RKE7t5dwkEYv0qLo5mRbzo8Og,108
-deriva_ml-1.13.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.13.1.dist-info/METADATA,sha256=OKuCDvSR63ii7fO1W6tw-7-6RtYaKMHR59AbiURo_tI,999
-deriva_ml-1.13.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-deriva_ml-1.13.1.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
-deriva_ml-1.13.1.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.13.1.dist-info/RECORD,,
+deriva_ml-1.13.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.13.2.dist-info/METADATA,sha256=uuvCztFgxOwWM34egjr65pW8-2pYGCtV_xofT5TmcLg,999
+deriva_ml-1.13.2.dist-info/WHEEL,sha256=pxyMxgL8-pra_rKaQ4drOZAegBVuX-G_4nRHjjgWbmo,91
+deriva_ml-1.13.2.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
+deriva_ml-1.13.2.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.13.2.dist-info/RECORD,,

{deriva_ml-1.13.1.dist-info → deriva_ml-1.13.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (79.0.0)
 Root-Is-Purelib: true
 Tag: py3-none-any