deriva-ml 1.14.43__tar.gz → 1.14.45__tar.gz

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.43
+Version: 1.14.45
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
@@ -20,6 +20,7 @@ Requires-Dist: setuptools-scm>=8.0
 Requires-Dist: nbstripout
 Requires-Dist: papermill
 Requires-Dist: pandas-stubs==2.2.3.250527
+Requires-Dist: pyyaml
 Dynamic: license-file
 
 # DerivaML

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "nbstripout",
     "papermill",
     "pandas-stubs==2.2.3.250527",
+    "pyyaml"
 ]
 
 [project.scripts]
@@ -43,8 +44,8 @@ build-backend = "setuptools.build_meta"
 [tool.uv]
 
 [tool.uv.sources]
-bdbag = {git = "https://github.com/fair-research/bdbag", branch = "master" }
-deriva = {git = "https://github.com/informatics-isi-edu/deriva-py", branch = "master" }
+#bdbag = {git = "https://github.com/fair-research/bdbag", branch = "master" }
+#deriva = {git = "https://github.com/informatics-isi-edu/deriva-py", branch = "master" }
 
 
 [tool.setuptools.package-data]

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml/core/base.py

@@ -120,6 +120,7 @@ class DerivaML(Dataset):
         logging_level=logging.WARNING,
         credential=None,
         use_minid: bool = True,
+        check_auth: bool = True,
     ):
         """Initializes a DerivaML instance.
 
@@ -139,6 +140,7 @@ class DerivaML(Dataset):
             working_dir: Directory path for storing data used by or generated by any computations. If no value is
                 provided, will default to  ${HOME}/deriva_ml
             use_minid: Use the MINID service when downloading dataset bags.
+            check_auth: Check if the user has access to the catalog.
         """
         # Get or use provided credentials for server access
         self.credential = credential or get_credential(hostname)
@@ -150,6 +152,16 @@ class DerivaML(Dataset):
             credentials=self.credential,
             session_config=self._get_session_config(),
         )
+
+        try:
+            if check_auth and server.get_authn_session():
+                pass
+        except Exception:
+            raise DerivaMLException(
+                "You are not authorized to access this catalog. "
+                "Please check your credentials and make sure you have logged in."
+            )
+
         self.catalog = server.connect_ermrest(catalog_id)
         self.model = DerivaModel(self.catalog.getCatalogModel(), domain_schema=domain_schema)
 

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml/dataset/dataset.py

@@ -1289,8 +1289,8 @@ class Dataset:
                 {
                     "processor": "fetch",
                     "processor_params": {
-                        "query_path": f"/attribute/{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5",
-                        "output_path": f"asset/{table.name}",
+                        "query_path": f"/attribute/{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5,asset_rid:=RID",
+                        "output_path": "asset/{asset_rid}/" + table.name,
                     },
                 }
             )
@@ -1341,9 +1341,9 @@ class Dataset:
                     "source": {
                         "skip_root_path": False,
                         "api": "attribute",
-                        "path": f"{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5",
+                        "path": f"{spath}/!(URL::null::)/url:=URL,length:=Length,filename:=Filename,md5:=MD5, asset_rid:=RID",
                     },
-                    "destination": {"name": f"asset/{table.name}", "type": "fetch"},
+                    "destination": {"name": "asset/{asset_rid}/" + table.name, "type": "fetch"},
                 }
             )
         return exports

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml/dataset/upload.py

@@ -296,7 +296,6 @@ def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, Fil
 
         with spec_file.open("w+") as cfile:
             json.dump(bulk_upload_configuration(model), cfile)
-        print("Creating GenericUploader...")
         uploader = GenericUploader(
             server={
                 "host": model.hostname,
@@ -318,7 +317,6 @@ def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, Fil
             }
         finally:
             uploader.cleanup()
-            print(f"Cleanup called...{uploader}")
         return results
 
 

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml/run_notebook.py

@@ -9,8 +9,9 @@ from pathlib import Path
 import nbformat
 import papermill as pm
 import regex as re
+import yaml
 from deriva.core import BaseCLI
-from nbconvert import HTMLExporter
+from nbconvert import MarkdownExporter
 
 from deriva_ml import DerivaML, ExecAssetType, Execution, ExecutionConfiguration, MLAsset, Workflow
 
@@ -28,7 +29,7 @@ class DerivaMLRunNotebookCLI(BaseCLI):
             "-f",
             type=Path,
             default=None,
-            help="JSON file with parameter values to inject into the notebook.",
+            help="JSON or YAML file with parameter values to inject into the notebook.",
         )
 
         self.parser.add_argument(
@@ -86,11 +87,14 @@ class DerivaMLRunNotebookCLI(BaseCLI):
         parameters = {key: self._coerce_number(val) for key, val in args.parameter}
 
         if parameter_file:
-            if not (parameter_file.is_file() and parameter_file.suffix == ".json"):
-                print("Parameter file must be an json file.")
-                exit(1)
-            with Path(parameter_file).open("r") as f:
-                parameters |= json.load(f)
+            with parameter_file.open("r") as f:
+                if parameter_file.suffix == ".json":
+                    parameters |= json.load(f)
+                elif parameter_file.suffix == ".yaml":
+                    parameters |= yaml.safe_load(f)
+                else:
+                    print("Parameter file must be an json or YAML file.")
+                    exit(1)
 
         if not (notebook_file.is_file() and notebook_file.suffix == ".ipynb"):
             print(f"Notebook file must be an ipynb file: {notebook_file.name}.")
@@ -157,11 +161,16 @@ class DerivaMLRunNotebookCLI(BaseCLI):
             )
 
             # Generate an HTML version of the output notebook.
+            notebook_output_md = notebook_output.with_suffix(".md")
+            with notebook_output.open() as f:
+                nb = nbformat.read(f, as_version=4)
+            # Convert to Markdown
+            exporter = MarkdownExporter()
+            (body, resources) = exporter.from_notebook_node(nb)
+
+            with notebook_output_md.open("w") as f:
+                f.write(body)
             nb = nbformat.read(notebook_output, as_version=4)
-            html_exporter = HTMLExporter(template_name="classic")
-            body, resources = html_exporter.from_notebook_node(nb)
-            notebook_output_html = notebook_output.with_suffix(".html")
-            notebook_output_html.write_text(body, encoding="utf-8")
 
             execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
@@ -171,12 +180,12 @@ class DerivaMLRunNotebookCLI(BaseCLI):
 
             execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
-                file_name=notebook_output_html,
+                file_name=notebook_output_md,
                 asset_types=ExecAssetType.notebook_output,
             )
             execution.asset_file_path(
                 asset_name=MLAsset.execution_asset,
-                file_name=notebook_output_html,
+                file_name=notebook_output_md,
                 asset_types=ExecAssetType.notebook_output,
             )
             print("parameter....")

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.14.43
+Version: 1.14.45
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
@@ -20,6 +20,7 @@ Requires-Dist: setuptools-scm>=8.0
 Requires-Dist: nbstripout
 Requires-Dist: papermill
 Requires-Dist: pandas-stubs==2.2.3.250527
+Requires-Dist: pyyaml
 Dynamic: license-file
 
 # DerivaML

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml.egg-info/SOURCES.txt

@@ -93,7 +93,6 @@ tests/dataset/__init__.py
 tests/dataset/demo-catalog-schema.json
 tests/dataset/deriva-ml-reference.json
 tests/dataset/eye-ai-catalog-schema.json
-tests/dataset/test_dataset_export.py
 tests/dataset/test_dataset_version.py
 tests/dataset/test_datasets.py
 tests/dataset/test_download.py

{deriva_ml-1.14.43 → deriva_ml-1.14.45}/src/deriva_ml.egg-info/requires.txt

@@ -12,3 +12,4 @@ setuptools-scm>=8.0
 nbstripout
 papermill
 pandas-stubs==2.2.3.250527
+pyyaml