deriva-ml 1.13.3__tar.gz → 1.14.0__tar.gz

{deriva_ml-1.13.3/src/deriva_ml.egg-info → deriva_ml-1.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.13.3
+Version: 1.14.0
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

deriva_ml-1.14.0/docs/code-docs/deriva_definitions.md

@@ -0,0 +1,2 @@
+::: deriva_ml.deriva_definitions
+    handler: python

{deriva_ml-1.13.3 → deriva_ml-1.14.0}/mkdocs.yml

@@ -57,6 +57,7 @@ nav:
       - Datasets: code-docs/dataset.md
       - DatasetBag Auxiliary Classes: code-docs/dataset_aux_classes.md
       - DatasetBag: code-docs/dataset_bag.md
+      - Helper Classes: code-docs/deriva_definitions.md
       - DerivaModel: code-docs/deriva_model.md
       - Execution: code-docs/execution.md
       - ExecutionConfiguration: code-docs/execution_configuration.md

{deriva_ml-1.13.3 → deriva_ml-1.14.0}/src/deriva_ml/dataset.py

@@ -21,7 +21,7 @@ import requests
 from tempfile import TemporaryDirectory
 from typing import Any, Callable, Optional, Iterable, Iterator, TYPE_CHECKING
 
-
+from .history import iso_to_snap
 from deriva.core.ermrest_model import Table
 from deriva.core.utils.core_utils import tag as deriva_tags, format_exception
 import deriva.core.utils.hash_utils as hash_utils
@@ -168,6 +168,21 @@ class Dataset:
             ]
         )
 
+    def _set_version_snapshot(self):
+        dataset_version_path = (
+            self._model.catalog.getPathBuilder()
+            .schemas[self._ml_schema]
+            .tables["Dataset_Version"]
+        )
+        versions = dataset_version_path.entities().fetch()
+        dataset_version_path.update(
+            [
+                {"RID": h["RID"], "Snapshot": iso_to_snap(h["RCT"])}
+                for h in versions
+                if not h["Snapshot"]
+            ]
+        )
+
     def dataset_history(self, dataset_rid: RID) -> list[DatasetHistory]:
         """Return a list of DatasetHistory objects representing the dataset
 

{deriva_ml-1.13.3 → deriva_ml-1.14.0}/src/deriva_ml/deriva_definitions.py

@@ -8,10 +8,11 @@ import warnings
 from datetime import date
 from enum import Enum
 from pathlib import Path
-from typing import Any, Iterable, Optional, Annotated, Generator
+from typing import Any, Iterable, Optional, Annotated, Generator, Iterator
 
 import deriva.core.ermrest_model as em
 import deriva.core.utils.hash_utils as hash_utils
+import json
 from urllib.parse import urlparse
 from deriva.core.ermrest_model import builtin_types
 from pydantic import (
@@ -192,6 +193,24 @@ class FileSpec(BaseModel):
 
         return (create_spec(file, description) for file in list_all_files(path))
 
+    @staticmethod
+    def read_filespec(path: Path | str) -> Iterator[FileSpec]:
+        """Get FileSpecs from a JSON lines file.
+
+        Arguments:
+         path: Path to the .jsonl file (string or Path).
+
+        Yields:
+             A FileSpec object.
+        """
+        path = Path(path)
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                yield FileSpec(**json.loads(line))
+
 
 class VocabularyTerm(BaseModel):
     """An entry in a vocabulary table.

{deriva_ml-1.13.3 → deriva_ml-1.14.0}/src/deriva_ml/history.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from dateutil.parser import isoparse
 from deriva.core import urlquote
 
 
@@ -57,7 +58,7 @@ def datetime_epoch_us(dt):
 
 
 def iso_to_snap(iso_datetime):
-    rmt = datetime.fromisoformat(iso_datetime)
+    rmt = isoparse(iso_datetime)  # datetime.fromisoformat(iso_datetime)
     return urlb32_encode(datetime_epoch_us(rmt))
 
 

{deriva_ml-1.13.3 → deriva_ml-1.14.0/src/deriva_ml.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.13.3
+Version: 1.14.0
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.13.3 → deriva_ml-1.14.0}/src/deriva_ml.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ docs/assets/sharing-at-home.pdf
 docs/code-docs/dataset.md
 docs/code-docs/dataset_aux_classes.md
 docs/code-docs/dataset_bag.md
+docs/code-docs/deriva_definitions.md
 docs/code-docs/deriva_ml_base.md
 docs/code-docs/deriva_model.md
 docs/code-docs/execution.md