deriva-ml 1.14.0__py3-none-any.whl → 1.14.27__py3-none-any.whl

deriva_ml/{upload.py → dataset/upload.py}

@@ -36,6 +36,7 @@ Here is the directory layout we support:
 """
 
 import json
+import os
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from typing import Any, Optional
@@ -46,49 +47,41 @@ from deriva.core.ermrest_model import Table
 from deriva.core.hatrac_store import HatracStore
 from deriva.core.utils import hash_utils, mime_utils
 from deriva.transfer.upload.deriva_upload import GenericUploader
-from pydantic import validate_call, ConfigDict
+from pydantic import ConfigDict, validate_call
 
-from deriva_ml.deriva_definitions import (
+from deriva_ml.core.definitions import (
     RID,
-    DerivaMLException,
+    DerivaSystemColumns,
     FileUploadState,
     UploadState,
-    DerivaSystemColumns,
 )
-from deriva_ml.deriva_model import DerivaModel
-
+from deriva_ml.core.exceptions import DerivaMLException
+from deriva_ml.model.catalog import DerivaModel
 
 try:
     from icecream import ic
 except ImportError:  # Graceful fallback if IceCream isn't installed.
     ic = lambda *a: None if not a else (a[0] if len(a) == 1 else a)  # noqa
 
+# Use os.path.sep for OS-agnostic paths in regex patterns
+SEP = re.escape(os.path.sep)
+upload_root_regex = f"(?i)^.*{SEP}deriva-ml"
 
-upload_root_regex = r"(?i)^.*/deriva-ml"
+exec_dir_regex = upload_root_regex + f"{SEP}execution{SEP}(?P<execution_rid>[-\\w]+)"
 
-exec_dir_regex = upload_root_regex + r"/execution/(?P<execution_rid>[-\w]+)"
-
-feature_dir_regex = exec_dir_regex + r"/feature"
+feature_dir_regex = exec_dir_regex + f"{SEP}feature"
 feature_table_dir_regex = (
-    feature_dir_regex
-    + r"/(?P<schema>[-\w]+)/(?P<target_table>[-\w]+)/(?P<feature_name>[-\w]+)"
-)
-feature_value_regex = (
-    feature_table_dir_regex + r"/(?P=feature_name)[.](?P<ext>[(csv|json)]*)$"
-)
-feature_asset_dir_regex = feature_table_dir_regex + r"/asset/(?P<asset_table>[-\w]+)"
-feature_asset_regex = (
-    feature_asset_dir_regex + r"/(?P<file>[A-Za-z0-9_-]+)[.](?P<ext>[a-z0-9]*)$"
+    feature_dir_regex + f"{SEP}(?P<schema>[-\\w]+){SEP}(?P<target_table>[-\\w]+){SEP}(?P<feature_name>[-\\w]+)"
 )
+feature_value_regex = feature_table_dir_regex + f"{SEP}(?P=feature_name)[.](?P<ext>[(csv|json)]*)$"
+feature_asset_dir_regex = feature_table_dir_regex + f"{SEP}asset{SEP}(?P<asset_table>[-\\w]+)"
+feature_asset_regex = feature_asset_dir_regex + f"{SEP}(?P<file>[A-Za-z0-9_-]+)[.](?P<ext>[a-z0-9]*)$"
 
-asset_path_regex = exec_dir_regex + r"/asset/(?P<schema>[-\w]+)/(?P<asset_table>[-\w]*)"
+asset_path_regex = exec_dir_regex + f"{SEP}asset{SEP}(?P<schema>[-\\w]+){SEP}(?P<asset_table>[-\\w]*)"
 
 asset_file_regex = r"(?P<file>[-\w]+)[.](?P<ext>[a-z0-9]*)$"
 
-table_regex = (
-    exec_dir_regex
-    + r"/table/(?P<schema>[-\w]+)/(?P<table>[-\w]+)/(?P=table)[.](csv|json)$"
-)
+table_regex = exec_dir_regex + f"{SEP}table{SEP}(?P<schema>[-\\w]+){SEP}(?P<table>[-\\w]+){SEP}(?P=table)[.](csv|json)$"
 
 
 def is_feature_dir(path: Path) -> Optional[re.Match]:
@@ -97,9 +90,16 @@ def is_feature_dir(path: Path) -> Optional[re.Match]:
 
 
 def normalize_asset_dir(path: str) -> Optional[tuple[str, str]]:
-    """Parse a path to an asset file and return the asset table name and file name"""
+    """Parse a path to an asset file and return the asset table name and file name.
+
+    Args:
+        path: Path to the asset file
+
+    Returns:
+        Tuple of (schema/table, filename) or None if path doesn't match pattern
+    """
     path = Path(path)
-    if not (m := re.match(asset_path_regex, path.as_posix())):
+    if not (m := re.match(asset_path_regex, str(path))):
         return None
     return f"{m['schema']}/{m['asset_table']}", path.name
 
@@ -138,18 +138,14 @@ def asset_root(prefix: Path | str, exec_rid: str) -> Path:
     return path
 
 
-def feature_dir(
-    prefix: Path | str, exec_rid: str, schema: str, target_table: str, feature_name: str
-) -> Path:
+def feature_dir(prefix: Path | str, exec_rid: str, schema: str, target_table: str, feature_name: str) -> Path:
     """Return the path to eht directory in which a named feature for an execution should be placed."""
     path = feature_root(prefix, exec_rid) / schema / target_table / feature_name
     path.mkdir(parents=True, exist_ok=True)
     return path
 
 
-def feature_value_path(
-    prefix: Path | str, exec_rid: str, schema: str, target_table: str, feature_name: str
-) -> Path:
+def feature_value_path(prefix: Path | str, exec_rid: str, schema: str, target_table: str, feature_name: str) -> Path:
     """Return the path to a CSV file in which to place feature values that are to be uploaded.
 
     Args:
@@ -162,10 +158,7 @@ def feature_value_path(
     Returns:
         Path to CSV file in which to place feature values
     """
-    return (
-        feature_dir(prefix, exec_rid, schema, target_table, feature_name)
-        / f"{feature_name}.jsonl"
-    )
+    return feature_dir(prefix, exec_rid, schema, target_table, feature_name) / f"{feature_name}.jsonl"
 
 
 def table_path(prefix: Path | str, schema: str, table: str) -> Path:
@@ -185,14 +178,14 @@ def table_path(prefix: Path | str, schema: str, table: str) -> Path:
 
 
 def asset_table_upload_spec(model: DerivaModel, asset_table: str | Table):
-    """Generate a pattern to an asset table that may include additional metadata columns.
+    """Generate upload specification for an asset table.
 
     Args:
-        model:
-        asset_table:
+        model: The DerivaModel instance.
+        asset_table: The asset table name or Table object.
 
     Returns:
-
+        A dictionary containing the upload specification for the asset table.
     """
     metadata_columns = model.asset_metadata(asset_table)
     asset_table = model.name_to_table(asset_table)
@@ -201,7 +194,9 @@ def asset_table_upload_spec(model: DerivaModel, asset_table: str | Table):
     asset_path = f"{exec_dir_regex}/asset/{schema}/{asset_table.name}/{metadata_path}/{asset_file_regex}"
     asset_table = model.name_to_table(asset_table)
     schema = model.name_to_table(asset_table).schema.name
-    return {
+
+    # Create upload specification
+    spec = {
         # Upload assets into an asset table of an asset table.
         "column_map": {
             "MD5": "{md5}",
@@ -221,6 +216,7 @@ def asset_table_upload_spec(model: DerivaModel, asset_table: str | Table):
         },
         "record_query_template": "/entity/{target_table}/MD5={md5}&Filename={file_name}",
     }
+    return spec
 
 
 def bulk_upload_configuration(model: DerivaModel) -> dict[str, Any]:
@@ -229,9 +225,7 @@ def bulk_upload_configuration(model: DerivaModel) -> dict[str, Any]:
         model: Model from which to generate the upload configuration
     """
     asset_tables_with_metadata = [
-        asset_table_upload_spec(model=model, asset_table=t)
-        for t in model.find_assets()
-        if model.asset_metadata(t)
+        asset_table_upload_spec(model=model, asset_table=t) for t in model.find_assets() if model.asset_metadata(t)
     ]
     return {
         "asset_mappings": asset_tables_with_metadata
@@ -246,9 +240,7 @@ def bulk_upload_configuration(model: DerivaModel) -> dict[str, Any]:
                 },
                 "asset_type": "file",
                 "target_table": ["{schema}", "{asset_table}"],
-                "file_pattern": asset_path_regex
-                + "/"
-                + asset_file_regex,  # Sets schema, asset_table, name, ext
+                "file_pattern": asset_path_regex + "/" + asset_file_regex,  # Sets schema, asset_table, name, ext
                 "checksum_types": ["sha256", "md5"],
                 "hatrac_options": {"versioned_urls": True},
                 "hatrac_templates": {
@@ -280,9 +272,7 @@ def bulk_upload_configuration(model: DerivaModel) -> dict[str, Any]:
 
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-def upload_directory(
-    model: DerivaModel, directory: Path | str
-) -> dict[Any, FileUploadState] | None:
+def upload_directory(model: DerivaModel, directory: Path | str) -> dict[Any, FileUploadState] | None:
     """Upload assets from a directory. This routine assumes that the current upload specification includes a
     configuration for the specified directory.  Every asset in the specified directory is uploaded
 
@@ -294,7 +284,7 @@ def upload_directory(
         Results of the upload operation.
 
     Raises:
-        DerivaMLException: If there is an issue uploading the assets.
+        DerivaMLException: If there is an issue with uploading the assets.
     """
     directory = Path(directory)
     if not directory.is_dir():
@@ -302,9 +292,9 @@ def upload_directory(
 
     # Now upload the files by creating an upload spec and then calling the uploader.
     with TemporaryDirectory() as temp_dir:
-        spec_file = f"{temp_dir}/config.json"
+        spec_file = Path(temp_dir) / "config.json"
 
-        with open(spec_file, "w+") as cfile:
+        with spec_file.open("w+") as cfile:
             json.dump(bulk_upload_configuration(model), cfile)
         uploader = GenericUploader(
             server={
@@ -331,9 +321,7 @@ def upload_directory(
 
 
 @validate_call(config=ConfigDict(arbitrary_types_allowed=True))
-def upload_asset(
-    model: DerivaModel, file: Path | str, table: Table, **kwargs: Any
-) -> dict:
+def upload_asset(model: DerivaModel, file: Path | str, table: Table, **kwargs: Any) -> dict:
     """Upload the specified file into Hatrac and update the associated asset table.
 
     Args:
@@ -359,9 +347,7 @@ def upload_asset(
         credentials=model.catalog.deriva_server.credentials,
     )
     md5_hashes = hash_utils.compute_file_hashes(file, ["md5"])["md5"]
-    sanitized_filename = urlquote(
-        re.sub("[^a-zA-Z0-9_.-]", "_", md5_hashes[0] + "." + file_name)
-    )
+    sanitized_filename = urlquote(re.sub("[^a-zA-Z0-9_.-]", "_", md5_hashes[0] + "." + file_name))
     hatrac_path = f"{hatrac_path}{sanitized_filename}"
 
     try:
@@ -377,9 +363,7 @@ def upload_asset(
         raise e
     try:
         # Now update the asset table.
-        ipath = (
-            model.catalog.getPathBuilder().schemas[table.schema.name].tables[table.name]
-        )
+        ipath = model.catalog.getPathBuilder().schemas[table.schema.name].tables[table.name]
         return list(
             ipath.insert(
                 [
@@ -429,9 +413,7 @@ def asset_file_path(
     }.union(set(DerivaSystemColumns))
     asset_metadata = {c.name for c in asset_table.columns} - asset_columns
     if not (asset_metadata >= set(metadata.keys())):
-        raise DerivaMLException(
-            f"Metadata {metadata} does not match asset metadata {asset_metadata}"
-        )
+        raise DerivaMLException(f"Metadata {metadata} does not match asset metadata {asset_metadata}")
 
     for m in asset_metadata:
         path = path / metadata.get(m, "None")
@@ -450,10 +432,6 @@ def asset_type_path(prefix: Path | str, exec_rid: RID, asset_table: Table) -> Pa
     Returns:
         Path to the file in which to place asset_type values for the named asset.
     """
-    path = (
-        execution_root(prefix, exec_rid=exec_rid)
-        / "asset-type"
-        / asset_table.schema.name
-    )
+    path = execution_root(prefix, exec_rid=exec_rid) / "asset-type" / asset_table.schema.name
     path.mkdir(parents=True, exist_ok=True)
     return path / f"{asset_table.name}.jsonl"