deriva-ml 1.13.3__py3-none-any.whl → 1.14.26__py3-none-any.whl

deriva_ml/schema/table_comments_utils.py

@@ -0,0 +1,57 @@
+import argparse
+import sys
+from pathlib import Path
+
+from deriva.core import ErmrestCatalog, get_credential
+
+
+def update_table_comments(model, schema_name: str, table_name: str, comments_dir: str) -> None:
+    table = model.schemas[schema_name].tables[table_name]
+    table_comments_dir = Path(comments_dir)/Path(f"{schema_name}/{table_name}")
+    for file in table_comments_dir.iterdir():
+        file_path = table_comments_dir / file.name
+        with file_path.open("r") as f:
+            comment_str = f.read()
+            if file.name.split(".")[0] == table_name:
+                table.comment = comment_str
+            else:
+                table.columns[file.name.split(".")[0]].comment = comment_str
+
+
+def update_schema_comments(model, schema_name: str, comments_dir: str) -> None:
+    schema_comments_dir = Path(comments_dir)/Path(schema_name)
+    for table in schema_comments_dir.iterdir():
+        if not table.name.endswith(".DS_Store"):
+            update_table_comments(model, schema_name, table.name, comments_dir)
+
+
+def main():
+    """Main entry point for the table comments utility CLI.
+    
+    Parses command line arguments and updates table comments.
+    
+    Returns:
+        None. Executes the CLI.
+    """
+    parser = argparse.ArgumentParser(description="Update table comments from files")
+    parser.add_argument("host", help="Hostname")
+    parser.add_argument("catalog_id", help="Catalog ID")
+    parser.add_argument("comments_dir", help="Directory containing comment files")
+    
+    args = parser.parse_args()
+    
+    catalog = ErmrestCatalog("https", args.host, args.catalog_id, credentials=get_credential(args.host))
+    model = catalog.getCatalogModel()
+    
+    # Update comments for all schemas
+    for schema_name in model.schemas:
+        if schema_name not in ["public", "deriva-ml"]:
+            update_schema_comments(model, schema_name, args.comments_dir)
+
+
+if __name__ == '__main__':
+    sys.exit(main())
+
+
+
+# docs/<schema-name>/<table-name>/[table|<column-name>.Md

{deriva_ml-1.13.3.dist-info → deriva_ml-1.14.26.dist-info}/METADATA

@@ -1,21 +1,22 @@
 Metadata-Version: 2.4
 Name: deriva-ml
-Version: 1.13.3
+Version: 1.14.26
 Summary: Utilities to simplify use of Dervia and Pandas to create reproducable ML pipelines
 Author-email: ISRD <isrd-dev@isi.edu>
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: bump-my-version
-Requires-Dist: deriva~=1.7.7
+Requires-Dist: deriva~=1.7.10
+Requires-Dist: deepdiff
 Requires-Dist: pandas
 Requires-Dist: regex~=2024.7.24
-Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pydantic>=2.11
 Requires-Dist: semver>3.0.0
 Requires-Dist: setuptools>=64
 Requires-Dist: setuptools-scm>=8.0
 Requires-Dist: nbstripout
 Requires-Dist: papermill
+Requires-Dist: pandas-stubs==2.2.3.250527
 Dynamic: license-file
 
 # DerivaML

deriva_ml-1.14.26.dist-info/RECORD

@@ -0,0 +1,40 @@
+deriva_ml/__init__.py,sha256=_aMdxGG4mRTcXodLZLNpXqH8v5uqMbqFUryE9KqNSB8,1158
+deriva_ml/demo_catalog.py,sha256=JjPAIac_hKPh5krEhGJydjXquRnivi7kQoR8W4Khp-s,14928
+deriva_ml/feature.py,sha256=L1XUXLWGnUGCjkxX5KsGu0I8SaUTJG7eDs__yUCWuCY,8445
+deriva_ml/run_notebook.py,sha256=SNjqRMDjy7zYnf6TwC7w0Y7nvqsgoYzW2NMxWD9BzUc,6479
+deriva_ml/core/__init__.py,sha256=V_i90pc5PB1F4UdOO6DZWzpEFaZDTaPRU-EzKXQ19eI,787
+deriva_ml/core/base.py,sha256=Rsa6fExlEJwQkU1PHf5ZK051ZB9oBTHrHx-_Omij__Y,60823
+deriva_ml/core/constants.py,sha256=6wBJ8qMxe-dbCjRGrjUIX-RK0mTWrLDTeUpaVbLFoM8,888
+deriva_ml/core/definitions.py,sha256=uq_8uYFBVBVHS691Ri2kdQsN37z0GNYTaZskJIb_ocM,1385
+deriva_ml/core/enums.py,sha256=sSN4B4OynbB-AXwxRszoFr-KWIWIAfhVa06EzAEHwVc,7194
+deriva_ml/core/ermrest.py,sha256=N0IJ3TE87jElaBChEIo5AFDTr0SIrb6F90yiimRfPr4,10182
+deriva_ml/core/exceptions.py,sha256=4MZNPOyN-UMaGeY9sqJDVwh_iOmz1ntp4usSyCNqVMg,934
+deriva_ml/core/filespec.py,sha256=BQAAcRXfXq1lDcsKlokLOOXCBtEZpPgXxrFOIZYAgLg,4229
+deriva_ml/dataset/__init__.py,sha256=ukl2laJqa9J2AVqb4zlpIYc-3RaAlfRR33NMIQaoNrQ,104
+deriva_ml/dataset/aux_classes.py,sha256=9mZAln7_rrzaRbKhKA6dJOp3xeD6dHOC9NXOtJKROo4,6933
+deriva_ml/dataset/dataset.py,sha256=FnQG98eEr7mnUmo1ySgVRxRoyhbP5u_ZxuexdAcYxJY,64305
+deriva_ml/dataset/dataset_bag.py,sha256=mPIZRX5aTbVRcJbCFtdkmlnexquF8NE-onbVK_8IxVk,14224
+deriva_ml/dataset/history.py,sha256=FK5AYYz11p4E4FWMVg4r7UPWOD4eobrq3b3xMjWF59g,3197
+deriva_ml/dataset/upload.py,sha256=Ad5JDfGvkIvefE-plP8SN9pNAxHzYrBoid5isz_bnNs,16411
+deriva_ml/execution/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deriva_ml/execution/environment.py,sha256=B7nywqxFTRUWgyu8n7rFoKcVC9on422kjeFG2FPQfvg,9302
+deriva_ml/execution/execution.py,sha256=tXWkFLDoSre836x6MMkcmhtmr3zP5_VoSioQ72-XmvE,44298
+deriva_ml/execution/execution_configuration.py,sha256=Rw4VWkBCZN9yatvSKdTqEWTfu470lpcVKfHFR0uN0jI,6248
+deriva_ml/execution/workflow.py,sha256=fd8dwCyDSpW_BOD_d0-zFs9yTNrbddwqD-xG3rKdvyU,13215
+deriva_ml/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deriva_ml/model/catalog.py,sha256=dzTBcRlqgEVkPY32AUax_iu75RgFiT4Pu5au7rmrv8k,14068
+deriva_ml/model/database.py,sha256=cTe9rJHMCDo89_dmAT5aJGhzzLiK2DkXvCvZE6fczcM,14805
+deriva_ml/model/sql_mapper.py,sha256=_0QsJEVSgSPtxrWKSgjfPZCQ1aMVcjR_Tk2OxLhWEvY,1696
+deriva_ml/schema/__init__.py,sha256=yV-MfzCF3FA4OOz7mZwMM2q6-x1vgOJ057kUvikFF6E,130
+deriva_ml/schema/annotations.py,sha256=TuQ3vWFnK0160fRmtvsCkHx9qAcRa63MSyERB4x5a98,18197
+deriva_ml/schema/check_schema.py,sha256=6dadLYHPqRex6AYVClmsESI8WhC7-rb-XnGf2G298xw,3609
+deriva_ml/schema/create_schema.py,sha256=0ydJSZEg3C3-m8hWPN6k2MoUvm-RWxAlKFzVChxcx3I,12791
+deriva_ml/schema/deriva-ml-reference.json,sha256=AEOMIgwKO3dNMMWHb0lxaXyamvfAEbUPh8qw0aAtsUQ,242460
+deriva_ml/schema/policy.json,sha256=5ykB8nnZFl-oCHzlAwppCFKJHWJFIkYognUMVEanfY8,1826
+deriva_ml/schema/table_comments_utils.py,sha256=4flCqnZAaqg_uSZ9I18pNUWAZoLfmMCXbmI5uERY5vM,2007
+deriva_ml-1.14.26.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deriva_ml-1.14.26.dist-info/METADATA,sha256=nhgVSz2mmn24u8xsLlAQRu6wYa37-z4DJZNy3Fi8odM,1034
+deriva_ml-1.14.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+deriva_ml-1.14.26.dist-info/entry_points.txt,sha256=dkf_z7E4V6_3_5Xjsm0hcixNg6ASHDw6NfYQuBvF1Wc,363
+deriva_ml-1.14.26.dist-info/top_level.txt,sha256=I1Q1dkH96cRghdsFRVqwpa2M7IqJpR2QPUNNc5-Bnpw,10
+deriva_ml-1.14.26.dist-info/RECORD,,

{deriva_ml-1.13.3.dist-info → deriva_ml-1.14.26.dist-info}/entry_points.txt

@@ -1,5 +1,6 @@
 [console_scripts]
 deriva-ml-alter-annotation = deriva_ml.schema_setup.alter_annotation:main
+deriva-ml-check-catalog-schema = deriva_ml.schema.check_schema:main
 deriva-ml-create-schema = deriva_ml.schema_setup.create_schema:main
 deriva-ml-run-notebook = deriva_ml.run_notebook:main
 deriva-ml-table-comments-utils = deriva_ml.schema_setup.table_comments_utils:main

deriva_ml/deriva_definitions.py

@@ -1,372 +0,0 @@
-"""
-Shared definitions that are used in different DerivaML modules.
-"""
-
-from __future__ import annotations
-
-import warnings
-from datetime import date
-from enum import Enum
-from pathlib import Path
-from typing import Any, Iterable, Optional, Annotated, Generator
-
-import deriva.core.ermrest_model as em
-import deriva.core.utils.hash_utils as hash_utils
-from urllib.parse import urlparse
-from deriva.core.ermrest_model import builtin_types
-from pydantic import (
-    BaseModel,
-    model_serializer,
-    Field,
-    computed_field,
-    field_validator,
-    ValidationError,
-)
-from socket import gethostname
-
-ML_SCHEMA = "deriva-ml"
-DRY_RUN_RID = "0000"
-
-# We are going to use schema as a field name and this collides with method in pydantic base class
-warnings.filterwarnings(
-    "ignore", message='Field name "schema"', category=Warning, module="pydantic"
-)
-
-warnings.filterwarnings(
-    "ignore",
-    message="fields may not start with an underscore",
-    category=Warning,
-    module="pydantic",
-)
-
-rid_part = r"(?P<rid>(?:[A-Z\d]{1,4}|[A-Z\d]{1,4}(?:-[A-Z\d]{4})+))"
-snapshot_part = r"(?:@(?P<snapshot>(?:[A-Z\d]{1,4}|[A-Z\d]{1,4}(?:-[A-Z\d]{4})+)))?"
-rid_regex = f"^{rid_part}{snapshot_part}$"
-RID = Annotated[str, Field(pattern=rid_regex)]
-
-DerivaSystemColumns = ["RID", "RCT", "RMT", "RCB", "RMB"]
-
-
-# For some reason, deriva-py doesn't use the proper enum class!!
-class UploadState(Enum):
-    """State of file upload"""
-
-    success = 0
-    failed = 1
-    pending = 2
-    running = 3
-    paused = 4
-    aborted = 5
-    cancelled = 6
-    timeout = 7
-
-
-class StrEnum(str, Enum):
-    pass
-
-
-class FileUploadState(BaseModel):
-    state: UploadState
-    status: str
-    result: Any
-
-    @computed_field
-    @property
-    def rid(self) -> Optional[RID]:
-        return self.result and self.result["RID"]
-
-
-class Status(StrEnum):
-    """Enumeration class defining execution status.
-
-    Attributes:
-        running: Execution is currently running.
-        pending: Execution is pending.
-        completed: Execution has been completed successfully.
-        failed: Execution has failed.
-
-    """
-
-    initializing = "Initializing"
-    created = "Created"
-    pending = "Pending"
-    running = "Running"
-    aborted = "Aborted"
-    completed = "Completed"
-    failed = "Failed"
-
-
-class BuiltinTypes(Enum):
-    text = builtin_types.text.typename
-    int2 = builtin_types.int2.typename
-    jsonb = builtin_types.json.typename
-    float8 = builtin_types.float8.typename
-    timestamp = builtin_types.timestamp.typename
-    int8 = builtin_types.int8.typename
-    boolean = builtin_types.boolean.typename
-    json = builtin_types.json.typename
-    float4 = builtin_types.float4.typename
-    int4 = builtin_types.int4.typename
-    timestamptz = builtin_types.timestamptz.typename
-    date = builtin_types.date.typename
-    ermrest_rid = builtin_types.ermrest_rid.typename
-    ermrest_rcb = builtin_types.ermrest_rcb.typename
-    ermrest_rmb = builtin_types.ermrest_rmb.typename
-    ermrest_rct = builtin_types.ermrest_rct.typename
-    ermrest_rmt = builtin_types.ermrest_rmt.typename
-    markdown = builtin_types.markdown.typename
-    longtext = builtin_types.longtext.typename
-    ermrest_curie = builtin_types.ermrest_curie.typename
-    ermrest_uri = builtin_types.ermrest_uri.typename
-    color_rgb_hex = builtin_types.color_rgb_hex.typename
-    serial2 = builtin_types.serial2.typename
-    serial4 = builtin_types.serial4.typename
-    serial8 = builtin_types.serial8.typename
-
-
-class FileSpec(BaseModel):
-    """An entry into the File table
-
-    Attributes:
-        url: The File url to the url.
-        description: The description of the file.
-    """
-
-    url: str
-    description: Optional[str] = ""
-    md5: str
-    length: int
-
-    @field_validator("url")
-    @classmethod
-    def validate_file_url(cls, v):
-        """Examine the provided URL.  If it's a local path, convert it into a tag URL."""
-        url_parts = urlparse(v)
-        if url_parts.scheme == "tag":
-            # Already a tag URL, so just return it.
-            return v
-        elif (not url_parts.scheme) or url_parts.scheme == "file":
-            # There is no scheme part tof the URL, or it is a file URL, so it is a local file path, so convert to a tag URL.
-            return f"tag://{gethostname()},{date.today()}:file://{url_parts.path}"
-        else:
-            raise ValidationError("url is not a file URL")
-
-    @model_serializer()
-    def serialize_filespec(self):
-        return {
-            "URL": self.url,
-            "Description": self.description,
-            "MD5": self.md5,
-            "Length": self.length,
-        }
-
-    @staticmethod
-    def create_filespecs(
-        path: Path | str, description: str
-    ) -> Generator["FileSpec", None, None]:
-        """Given a file or directory, generate the sequence of corresponding FileSpecs sutable to create a File table
-
-        Arguments:
-            path: Path to the file or directory.
-            description: The description of the file(s)
-
-        Returns:
-            An iterable of FileSpecs for each file in the directory.
-        """
-        path = Path(path)
-
-        def list_all_files(p) -> list[Path]:
-            return (
-                (f for f in Path(p).rglob("*") if f.is_file()) if path.is_dir() else [p]
-            )
-
-        def create_spec(p: Path, description: str) -> FileSpec:
-            hashes = hash_utils.compute_file_hashes(p, hashes=["md5", "sha256"])
-            md5 = hashes["md5"][0]
-            return FileSpec(
-                length=path.stat().st_size,
-                md5=md5,
-                description=description,
-                url=p.as_posix(),
-            )
-
-        return (create_spec(file, description) for file in list_all_files(path))
-
-
-class VocabularyTerm(BaseModel):
-    """An entry in a vocabulary table.
-
-    Attributes:
-       name: Name of vocabulary term
-       synonyms: List of alternative names for the term
-       id: CURI identifier for the term
-       uri: Unique URI for the term.
-       description: A description of the term meaning
-       rid: Resource identifier assigned to the term
-
-    Args:
-
-    Returns:
-
-    """
-
-    name: str = Field(alias="Name")
-    synonyms: Optional[list[str]] = Field(alias="Synonyms")
-    id: str = Field(alias="ID")
-    uri: str = Field(alias="URI")
-    description: str = Field(alias="Description")
-    rid: str = Field(alias="RID")
-
-    class Config:
-        extra = "ignore"
-
-
-class MLVocab(StrEnum):
-    """Names of controlled vocabulary for various types within DerivaML."""
-
-    dataset_type = "Dataset_Type"
-    workflow_type = "Workflow_Type"
-    file_type = "File_Type"
-    asset_type = "Asset_Type"
-    asset_role = "Asset_Role"
-
-
-class MLAsset(StrEnum):
-    execution_metadata = "Execution_Metadata"
-    execution_asset = "Execution_Asset"
-
-
-class ExecMetadataType(StrEnum):
-    """
-    Predefined execution metadata types.
-    """
-
-    execution_config = "Execution_Config"
-    runtime_env = "Runtime_Env"
-
-
-class ExecAssetType(StrEnum):
-    """
-    Predefined execution metadata types.
-    """
-
-    input_file = "Input_File"
-    output_file = "Output_File"
-    notebook_output = "Notebook_Output"
-
-
-class ColumnDefinition(BaseModel):
-    """Pydantic model for deriva_py Column.define"""
-
-    name: str
-    type: BuiltinTypes
-    nullok: bool = True
-    default: Any = None
-    comment: Optional[str] = None
-    acls: dict = Field(default_factory=dict)
-    acl_bindings: dict = Field(default_factory=dict)
-    annotations: dict = Field(default_factory=dict)
-
-    @field_validator("type", mode="before")
-    @classmethod
-    def extract_type_name(cls, value: Any) -> Any:
-        if isinstance(value, dict):
-            return BuiltinTypes(value["typename"])
-        else:
-            return value
-
-    @model_serializer()
-    def serialize_column_definition(self):
-        return em.Column.define(
-            self.name,
-            builtin_types[self.type.value],
-            nullok=self.nullok,
-            default=self.default,
-            comment=self.comment,
-            acls=self.acls,
-            acl_bindings=self.acl_bindings,
-            annotations=self.annotations,
-        )
-
-
-class KeyDefinition(BaseModel):
-    colnames: Iterable[str]
-    constraint_names: Iterable[str]
-    comment: Optional[str] = None
-    annotations: dict = Field(default_factory=dict)
-
-    @model_serializer()
-    def serialize_key_definition(self):
-        return em.Key.define(
-            colnames=self.colnames,
-            constraint_names=self.constraint_names,
-            comment=self.comment,
-            annotations=self.annotations,
-        )
-
-
-class ForeignKeyDefinition(BaseModel):
-    """Pydantic model for deriva_py ForeignKey.define"""
-
-    colnames: Iterable[str]
-    pk_sname: str
-    pk_tname: str
-    pk_colnames: Iterable[str]
-    constraint_names: Iterable[str] = Field(default_factory=list)
-    on_update: str = "NO ACTION"
-    on_delete: str = "NO ACTION"
-    comment: str = None
-    acls: dict[str, Any] = Field(default_factory=dict)
-    acl_bindings: dict[str, Any] = Field(default_factory=dict)
-    annotations: dict[str, Any] = Field(default_factory=dict)
-
-    @model_serializer()
-    def serialize_fk_definition(self):
-        return em.ForeignKey.define(
-            fk_colnames=self.colnames,
-            pk_sname=self.pk_sname,
-            pk_tname=self.pk_tname,
-            pk_colnames=self.pk_colnames,
-            on_update=self.on_update,
-            on_delete=self.on_delete,
-            comment=self.comment,
-            acls=self.acls,
-            acl_bindings=self.acl_bindings,
-            annotations=self.annotations,
-        )
-
-
-class TableDefinition(BaseModel):
-    name: str
-    column_defs: Iterable[ColumnDefinition]
-    key_defs: Iterable[KeyDefinition] = Field(default_factory=list)
-    fkey_defs: Iterable[ForeignKeyDefinition] = Field(default_factory=list)
-    comment: str = None
-    acls: dict = Field(default_factory=dict)
-    acl_bindings: dict = Field(default_factory=dict)
-    annotations: dict = Field(default_factory=dict)
-
-    @model_serializer()
-    def serialize_table_definition(self):
-        return em.Table.define(
-            tname=self.name,
-            column_defs=[c.model_dump() for c in self.column_defs],
-            key_defs=[k.model_dump() for k in self.key_defs],
-            fkey_defs=[fk.model_dump() for fk in self.fkey_defs],
-            comment=self.comment,
-            acls=self.acls,
-            acl_bindings=self.acl_bindings,
-            annotations=self.annotations,
-        )
-
-
-class DerivaMLException(Exception):
-    """Exception class specific to DerivaML module.
-
-    Args:
-        msg (str): Optional message for the exception.
-    """
-
-    def __init__(self, msg=""):
-        super().__init__(msg)
-        self._msg = msg