deriva-ml 1.13.3__py3-none-any.whl → 1.14.0__py3-none-any.whl

deriva_ml/dataset.py

@@ -21,7 +21,7 @@ import requests
 from tempfile import TemporaryDirectory
 from typing import Any, Callable, Optional, Iterable, Iterator, TYPE_CHECKING
 
-
+from .history import iso_to_snap
 from deriva.core.ermrest_model import Table
 from deriva.core.utils.core_utils import tag as deriva_tags, format_exception
 import deriva.core.utils.hash_utils as hash_utils
@@ -168,6 +168,21 @@ class Dataset:
             ]
         )
 
+    def _set_version_snapshot(self):
+        dataset_version_path = (
+            self._model.catalog.getPathBuilder()
+            .schemas[self._ml_schema]
+            .tables["Dataset_Version"]
+        )
+        versions = dataset_version_path.entities().fetch()
+        dataset_version_path.update(
+            [
+                {"RID": h["RID"], "Snapshot": iso_to_snap(h["RCT"])}
+                for h in versions
+                if not h["Snapshot"]
+            ]
+        )
+
     def dataset_history(self, dataset_rid: RID) -> list[DatasetHistory]:
         """Return a list of DatasetHistory objects representing the dataset
 

deriva_ml/deriva_definitions.py

@@ -8,10 +8,11 @@ import warnings
 from datetime import date
 from enum import Enum
 from pathlib import Path
-from typing import Any, Iterable, Optional, Annotated, Generator
+from typing import Any, Iterable, Optional, Annotated, Generator, Iterator
 
 import deriva.core.ermrest_model as em
 import deriva.core.utils.hash_utils as hash_utils
+import json
 from urllib.parse import urlparse
 from deriva.core.ermrest_model import builtin_types
 from pydantic import (
@@ -192,6 +193,24 @@ class FileSpec(BaseModel):
 
         return (create_spec(file, description) for file in list_all_files(path))
 
+    @staticmethod
+    def read_filespec(path: Path | str) -> Iterator[FileSpec]:
+        """Get FileSpecs from a JSON lines file.
+
+        Arguments:
+         path: Path to the .jsonl file (string or Path).
+
+        Yields:
+             A FileSpec object.
+        """
+        path = Path(path)
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                yield FileSpec(**json.loads(line))
+
 
 class VocabularyTerm(BaseModel):
     """An entry in a vocabulary table.

deriva_ml/history.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from dateutil.parser import isoparse
 from deriva.core import urlquote
 
 
@@ -57,7 +58,7 @@ def datetime_epoch_us(dt):
 
 
 def iso_to_snap(iso_datetime):
-    rmt = datetime.fromisoformat(iso_datetime)
+    rmt = isoparse(iso_datetime)  # datetime.fromisoformat(iso_datetime)
     return urlb32_encode(datetime_epoch_us(rmt))
 
 

{deriva_ml-1.13.3.dist-info → deriva_ml-1.14.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.13.3
+Version: 1.14.0
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10

{deriva_ml-1.13.3.dist-info → deriva_ml-1.14.0.dist-info}/RECORD

@@ -1,17 +1,17 @@
 deriva_ml/__init__.py,sha256=GfneBq7xDphMqUQY96sW9ixRj74M3UTUCmD4KMIRSaM,1101
 deriva_ml/database_model.py,sha256=D5vY0HqsCzWzbQVuK5jsImeEfjgg9wMe6fjDz8ucZqc,14695
-deriva_ml/dataset.py,sha256=UmVX3CQxjvi3pVudnbdpAddE4MJz230QnepICAb5Rag,60136
+deriva_ml/dataset.py,sha256=NVyIRzy33QHnYvi5fZoxEIuUf-F2bHDCdIb6Zzchdpk,60649
 deriva_ml/dataset_aux_classes.py,sha256=IyPAbZUoVWjnm4hcpLet2mkJvKHRGN__IEwcAIrH-u4,6597
 deriva_ml/dataset_bag.py,sha256=yS8oYVshfFtRDyhGPRqtbvxjyd3ZFF29lrB783OP4vM,11849
 deriva_ml/demo_catalog.py,sha256=NBBsnF955jGpsGTgYFjeBn2580MjRuzC1MqqdrkaHAU,12081
-deriva_ml/deriva_definitions.py,sha256=3rkl0TsOoVR3QdZdAHvR--Zgt1FvW1et3BEL-_bjLVM,10768
+deriva_ml/deriva_definitions.py,sha256=LhreMYY_1a_MgpmmtG_ABVzK6psbfbiUcYf47KxNIGc,11309
 deriva_ml/deriva_ml_base.py,sha256=FQirlTQvOUs25G0WGxnlVj3uF3PQbMX96nNj9s9arTI,38565
 deriva_ml/deriva_model.py,sha256=yMuJ3C8-anl1qf7lFBkiswVMWcheI_TUV5G_GvlzVCk,13565
 deriva_ml/execution.py,sha256=qcxqkF0gxmPfwLQb-ooyagFawgO9sK4CQfzJOfBmMp4,38330
 deriva_ml/execution_configuration.py,sha256=7fiIbtzz9nmkxA9-GTiN6Ln2twfaOLivwJwGZb8gAL0,14163
 deriva_ml/execution_environment.py,sha256=bCRKrCELDbGQDo7_FKfw7e8iMzVjSRZK3baKkqH5-_0,3264
 deriva_ml/feature.py,sha256=07g0uSrhumdopJluWuWSRMrzagaikAOihqB09bzXBP4,5475
-deriva_ml/history.py,sha256=XRZ_mmxOnspUJAoC_XEH_DhSgfLLW8Kiub3CxpQPftw,2830
+deriva_ml/history.py,sha256=qP1fSAYUrMHdssT_0u2hzfgwb3X8-ZhI89q6xnVxTvI,2893
 deriva_ml/run_notebook.py,sha256=vhmij4P1Va52MIj8hOc-WmjLRp3sTmK6p7LXCWrzejc,6308
 deriva_ml/test_functions.py,sha256=-eqLHjjCQCLBNAr1ofbZekNiCOfMISSACRxT_YHER8I,4396
 deriva_ml/test_notebook.ipynb,sha256=_5D6rkSGbmENPJZbDgfZ6-yt94BNEwxytVUDmG3RE3w,10166
@@ -23,9 +23,9 @@ deriva_ml/schema_setup/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVE
 deriva_ml/schema_setup/table_comments_utils.py,sha256=-2_ubEpoH7ViLVb-ZfW9wZbQ26DTKNgjkCABMzGu4i4,2140
 deriva_ml/test-files/execution-parameters.json,sha256=1vBqXlaMa0cysonE20TweVDfTGRdSi9CUuAkW1xiYNo,36
 deriva_ml/test-files/notebook-parameters.json,sha256=7uEE2sLQSrSc9cEGQ_RKE7t5dwkEYv0qLo5mRbzo8Og,108
-deriva_ml-1.13.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-deriva_ml-1.13.3.dist-info/METADATA,sha256=D4ar2arJ8tkT9-YTAWl-r_CsJJA42rb2RKqcD8tOkF8,999
-deriva_ml-1.13.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-deriva_ml-1.13.3.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
-deriva_ml-1.13.3.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
-deriva_ml-1.13.3.dist-info/RECORD,,
+deriva_ml-1.14.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.14.0.dist-info/METADATA,sha256=DAIaEfvbsypLwhnNYZoYFnKi3jNln1je4n5KR4fiwvY,999
+deriva_ml-1.14.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.14.0.dist-info/entry_points.txt,sha256=cJnALMa6pjdk6RQCt4HFbKHqALpVa0k6wPeQDPedLJI,295
+deriva_ml-1.14.0.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.14.0.dist-info/RECORD,,