nmdc-runtime 1.0.8__py3-none-any.whl → 1.0.10__py3-none-any.whl

nmdc_runtime/site/changesheets/base.py

@@ -4,14 +4,17 @@ base.py: Provides data classes for creating changesheets for NMDC database objec
 """
 
 import logging
+import os
 import time
 from dataclasses import dataclass, field
+from dotenv import load_dotenv
 from pathlib import Path
 import requests
 from typing import Any, ClassVar, Dict, TypeAlias, Optional
 
-from nmdc_runtime.site.resources import RuntimeApiUserClient
+from nmdc_runtime.site.resources import GoldApiClient, RuntimeApiUserClient
 
+load_dotenv()
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s %(levelname)s %(" "message)s"
 )
@@ -83,3 +86,25 @@ class Changesheet:
             f.write(self.header + "\n")
             for line_item in self.line_items:
                 f.write(line_item.line + "\n")
+
+
+def get_runtime_client(use_dev_api):
+    if use_dev_api:
+        base_url = os.getenv("API_HOST_DEV")
+        logging.info("using Dev API...")
+    else:
+        base_url = os.getenv("API_HOST")
+        logging.info("using prod API...")
+    return RuntimeApiUserClient(
+        base_url=base_url,
+        username=os.getenv("API_QUERY_USER"),
+        password=os.getenv("API_QUERY_PASS"),
+    )
+
+
+def get_gold_client():
+    return GoldApiClient(
+        base_url=os.getenv("GOLD_API_BASE_URL"),
+        username=os.getenv("GOLD_API_USERNAME"),
+        password=os.getenv("GOLD_API_PASSWORD"),
+    )

nmdc_runtime/site/resources.py

@@ -17,7 +17,7 @@ from dagster import (
 from fastjsonschema import JsonSchemaValueException
 from frozendict import frozendict
 from linkml_runtime.dumpers import json_dumper
-from pydantic import BaseModel
+from pydantic import BaseModel, AnyUrl
 from pymongo import MongoClient, ReplaceOne, InsertOne
 from terminusdb_client import WOQLClient
 from toolz import get_in
@@ -27,6 +27,7 @@ from nmdc_runtime.api.core.util import expiry_dt_from_now, has_passed
 from nmdc_runtime.api.models.object import DrsObject, AccessURL, DrsObjectIn
 from nmdc_runtime.api.models.operation import ListOperationsResponse
 from nmdc_runtime.api.models.util import ListRequest
+from nmdc_runtime.site.normalization.gold import normalize_gold_id
 from nmdc_runtime.util import unfreeze, nmdc_jsonschema_validator_noidpatterns
 from nmdc_schema import nmdc
 
@@ -95,7 +96,8 @@ class RuntimeApiUserClient(RuntimeApiClient):
         return self.request("GET", f"/runs/{run_id}")
 
     def get_biosamples_by_gold_biosample_id(self, gold_biosample_id: str):
-        return self.request(
+        gold_biosample_id = normalize_gold_id(gold_biosample_id)
+        response = self.request(
             "POST",
             f"/queries:run",
             {
@@ -107,9 +109,40 @@ class RuntimeApiUserClient(RuntimeApiClient):
                 },
             },
         )
+        response.raise_for_status()
+        return response.json()["cursor"]["firstBatch"]
+
+    def get_omics_processing_records_by_gold_project_id(self, gold_project_id: str):
+        gold_project_id = normalize_gold_id(gold_project_id)
+        response = self.request(
+            "POST",
+            f"/queries:run",
+            {
+                "find": "omics_processing_set",
+                "filter": {
+                    "gold_sequencing_project_identifiers": {
+                        "$elemMatch": {"$eq": gold_project_id}
+                    }
+                },
+            },
+        )
+        response.raise_for_status()
+        return response.json()["cursor"]["firstBatch"]
+
+    def get_biosamples_for_study(self, study_id: str):
+        response = self.request(
+            "POST",
+            f"/queries:run",
+            {
+                "find": "biosample_set",
+                "filter": {"part_of": {"$elemMatch": {"$eq": study_id}}},
+            },
+        )
+        response.raise_for_status()
+        return response.json()["cursor"]["firstBatch"]
 
     def get_omics_processing_by_name(self, name: str):
-        return self.request(
+        response = self.request(
             "POST",
             f"/queries:run",
             {
@@ -117,6 +150,8 @@ class RuntimeApiUserClient(RuntimeApiClient):
                 "filter": {"name": {"$regex": name, "$options": "i"}},
             },
         )
+        response.raise_for_status()
+        return response.json()["cursor"]["firstBatch"]
 
 
 class RuntimeApiSiteClient(RuntimeApiClient):
@@ -194,15 +229,17 @@ class RuntimeApiSiteClient(RuntimeApiClient):
             access = AccessURL(
                 **self.get_object_access(object_id, method.access_id).json()
             )
-            if access.url.startswith(
+            if str(access.url).startswith(
                 os.getenv("API_HOST_EXTERNAL")
             ) and self.base_url == os.getenv("API_HOST"):
-                access.url = access.url.replace(
-                    os.getenv("API_HOST_EXTERNAL"), os.getenv("API_HOST")
+                access.url = AnyUrl(
+                    str(access.url).replace(
+                        os.getenv("API_HOST_EXTERNAL"), os.getenv("API_HOST")
+                    )
                 )
         else:
             access = AccessURL(url=method.access_url.url)
-        return requests.get(access.url)
+        return requests.get(str(access.url))
 
     def list_jobs(self, list_request=None):
         if list_request is None:

nmdc_runtime/util.py

@@ -10,6 +10,7 @@ from functools import lru_cache
 from io import BytesIO
 from pathlib import Path
 from uuid import uuid4
+from typing import List, Optional, Set, Dict
 
 import fastjsonschema
 import requests
@@ -27,13 +28,67 @@ from nmdc_runtime.api.models.object import DrsObjectIn
 from typing_extensions import Annotated
 
 
+def get_class_names_from_collection_spec(
+    spec: dict, prefix: Optional[str] = None
+) -> List[str]:
+    """
+    Returns the list of classes referenced by the `$ref` values in a JSON Schema snippet describing a collection,
+    applying an optional prefix to each class name.
+
+    >>> get_class_names_from_collection_spec({"items": {"foo": "#/$defs/A"}})
+    []
+    >>> get_class_names_from_collection_spec({"items": {"$ref": "#/$defs/A"}})
+    ['A']
+    >>> get_class_names_from_collection_spec({"items": {"$ref": "#/$defs/A"}}, "p:")
+    ['p:A']
+    >>> get_class_names_from_collection_spec({"items": {"anyOf": "not-a-list"}})
+    []
+    >>> get_class_names_from_collection_spec({"items": {"anyOf": []}})
+    []
+    >>> get_class_names_from_collection_spec({"items": {"anyOf": [{"$ref": "#/$defs/A"}]}})
+    ['A']
+    >>> get_class_names_from_collection_spec({"items": {"anyOf": [{"$ref": "#/$defs/A"}, {"$ref": "#/$defs/B"}]}})
+    ['A', 'B']
+    >>> get_class_names_from_collection_spec({"items": {"anyOf": [{"$ref": "#/$defs/A"}, {"$ref": "#/$defs/B"}]}}, "p:")
+    ['p:A', 'p:B']
+    """
+
+    class_names = []
+    if "items" in spec:
+        # If the `items` dictionary has a key named `$ref`, get the single class name from it.
+        if "$ref" in spec["items"]:
+            ref_dict = spec["items"]["$ref"]
+            class_name = ref_dict.split("/")[-1]  # e.g. `#/$defs/Foo` --> `Foo`
+            class_names.append(class_name)
+
+        # Else, if it has a key named `anyOf` whose value is a list, get the class name from each ref in the list.
+        elif "anyOf" in spec["items"] and isinstance(spec["items"]["anyOf"], list):
+            for element in spec["items"]["anyOf"]:
+                ref_dict = element["$ref"]
+                class_name = ref_dict.split("/")[-1]  # e.g. `#/$defs/Foo` --> `Foo`
+                class_names.append(class_name)
+
+    # Apply the specified prefix, if any, to each class name.
+    if isinstance(prefix, str):
+        class_names = list(map(lambda name: f"{prefix}{name}", class_names))
+
+    return class_names
+
+
 @lru_cache
-def get_type_collections():
-    return {
-        f'nmdc:{spec["items"]["$ref"].split("/")[-1]}': collection_name
-        for collection_name, spec in nmdc_jsonschema["properties"].items()
-        if collection_name.endswith("_set")
-    }
+def get_type_collections() -> dict:
+    """Returns a dictionary mapping class names to Mongo collection names."""
+
+    mappings = {}
+
+    # Process the `items` dictionary of each collection whose name ends with `_set`.
+    for collection_name, spec in nmdc_jsonschema["properties"].items():
+        if collection_name.endswith("_set"):
+            class_names = get_class_names_from_collection_spec(spec, "nmdc:")
+            for class_name in class_names:
+                mappings[class_name] = collection_name
+
+    return mappings
 
 
 def without_id_patterns(nmdc_jsonschema):
@@ -82,7 +137,7 @@ def put_object(filepath, url, mime_type=None):
         return requests.put(url, data=f, headers={"Content-Type": mime_type})
 
 
-def drs_metadata_for(filepath, base=None):
+def drs_metadata_for(filepath, base=None, timestamp=None):
     """given file path, get drs metadata
 
     required: size, created_time, and at least one checksum.
@@ -96,7 +151,7 @@ def drs_metadata_for(filepath, base=None):
         )
     if "checksums" not in base:
         base["checksums"] = [
-            {"type": "sha256", "checksum": sha256hash_from_file(filepath)}
+            {"type": "sha256", "checksum": sha256hash_from_file(filepath, timestamp)}
         ]
     if "mime_type" not in base:
         base["mime_type"] = mimetypes.guess_type(filepath)[0]
@@ -312,22 +367,30 @@ def specialize_activity_set_docs(docs):
     return docs, validation_errors
 
 
-collection_name_to_class_name = {
-    db_prop: db_prop_spec["items"]["$ref"].split("/")[-1]
-    for db_prop, db_prop_spec in get_nmdc_jsonschema_dict()["$defs"]["Database"][
+# Define a mapping from collection name to a list of class names allowable for that collection's documents.
+collection_name_to_class_names: Dict[str, List[str]] = {
+    collection_name: get_class_names_from_collection_spec(spec)
+    for collection_name, spec in nmdc_jsonschema["$defs"]["Database"][
         "properties"
     ].items()
-    if "items" in db_prop_spec and "$ref" in db_prop_spec["items"]
 }
 
 
 @lru_cache
-def schema_collection_names_with_id_field():
-    return {
-        coll_name
-        for coll_name, class_name in collection_name_to_class_name.items()
-        if "id" in get_nmdc_jsonschema_dict()["$defs"][class_name].get("properties", {})
-    }
+def schema_collection_names_with_id_field() -> Set[str]:
+    """
+    Returns the set of collection names with which _any_ of the associated classes contains an `id` field.
+    """
+
+    target_collection_names = set()
+
+    for collection_name, class_names in collection_name_to_class_names.items():
+        for class_name in class_names:
+            if "id" in nmdc_jsonschema["$defs"][class_name].get("properties", {}):
+                target_collection_names.add(collection_name)
+                break
+
+    return target_collection_names
 
 
 def ensure_unique_id_indexes(mdb: MongoDatabase):

{nmdc_runtime-1.0.8.dist-info → nmdc_runtime-1.0.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nmdc-runtime
-Version: 1.0.8
+Version: 1.0.10
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston

{nmdc_runtime-1.0.8.dist-info → nmdc_runtime-1.0.10.dist-info}/RECORD

@@ -1,7 +1,7 @@
 nmdc_runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/containers.py,sha256=WBzPue0PRoyKXFxgLR-aQcuHetTa8yC5JjI0dGBblYA,419
 nmdc_runtime/main.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nmdc_runtime/util.py,sha256=Ai9mmRPiiyK0-Y5QnI71aWWzR08fEt42LSdkpGOJeAU,17088
+nmdc_runtime/util.py,sha256=o74ZKOmSD79brPFAcQFsYpA6wh9287m0hDhDlIpn9VM,19872
 nmdc_runtime/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/core/db/Database.py,sha256=WamgBUbq85A7-fr3p5B9Tk92U__yPdr9pBb4zyQok-4,377
@@ -38,14 +38,14 @@ nmdc_runtime/site/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 nmdc_runtime/site/graphs.py,sha256=AiCQzwrITZvpfGwvm2PAr2pL96juZduBw2e4M_2N1LM,5965
 nmdc_runtime/site/ops.py,sha256=i3Oy3_yEaFxsYpDRpnUmWMJSyL2W_2wL1XEHPxGojyA,28779
 nmdc_runtime/site/repository.py,sha256=DFvF0PqMK0TCuZJfjtpVQqCUl9F5rMcEo5_M6Dw0wYk,23304
-nmdc_runtime/site/resources.py,sha256=kqAnVPVrEHLQBsGvNVsKDznJDmwVGEm96-drkQToLag,16401
+nmdc_runtime/site/resources.py,sha256=pQSwg1dRpL_D91gYLzzaOIDZ3qa69rPqSlsq5dS9i_M,17783
 nmdc_runtime/site/util.py,sha256=6hyVPpb6ZkWEG8Nm7uQxnZ-QmuPOG9hgWvl0mUBr5JU,1303
 nmdc_runtime/site/backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/site/backup/nmdcdb_mongodump.py,sha256=H5uosmEiXwLwklJrYJWrNhb_Nuf_ew8dBpZLl6_dYhs,2699
 nmdc_runtime/site/backup/nmdcdb_mongoexport.py,sha256=XIFI_AI3zl0dFr-ELOEmwvT41MyRKBGFaAT3RcamTNE,4166
 nmdc_runtime/site/backup/nmdcdb_mongoimport.py,sha256=k6w5yscMNYoMBVkaAA9soWS0Dj2CB0FRBSFlifRO3Ro,1739
 nmdc_runtime/site/changesheets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nmdc_runtime/site/changesheets/base.py,sha256=ODUNd0Ivpv0Jl_qPeb_-XBSvWOaTF6j1v0YdY6qG9ek,2501
+nmdc_runtime/site/changesheets/base.py,sha256=lZT6WCsEBl-FsTr7Ki8_ploT93uMiVyIWWKM36aOrRk,3171
 nmdc_runtime/site/drsobjects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/site/drsobjects/ingest.py,sha256=pcMP69WSzFHFqHB9JIL55ePFhilnCLRc2XHCQ97w1Ik,3107
 nmdc_runtime/site/drsobjects/registration.py,sha256=D1T3QUuxEOxqKZIvB5rkb_6ZxFZiA-U9SMPajyeWC2Y,3572
@@ -71,9 +71,9 @@ nmdc_runtime/site/validation/emsl.py,sha256=TgckqKkFquHDLso77sn-jZRu5ZaBevGCt5p8
 nmdc_runtime/site/validation/gold.py,sha256=kJ1L081SZb-8qKpF731r5aQOueM206SUfUYMTTNTFMc,802
 nmdc_runtime/site/validation/jgi.py,sha256=lBo-FCtEYedT74CpW-Kdj512Ib963ik-4YIYmY5puDo,1298
 nmdc_runtime/site/validation/util.py,sha256=GGbMDSwR090sr_E_fHffCN418gpYESaiot6XghS7OYk,3349
-nmdc_runtime-1.0.8.dist-info/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
-nmdc_runtime-1.0.8.dist-info/METADATA,sha256=UsGWNojU5Bfzi3-awRbUPOq7leHL-yuoJzGu-HAwfxo,6806
-nmdc_runtime-1.0.8.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-nmdc_runtime-1.0.8.dist-info/entry_points.txt,sha256=nfH6-K9tDKv7va8ENfShsBnxVQoYJdEe7HHdwtkbh1Y,289
-nmdc_runtime-1.0.8.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
-nmdc_runtime-1.0.8.dist-info/RECORD,,
+nmdc_runtime-1.0.10.dist-info/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
+nmdc_runtime-1.0.10.dist-info/METADATA,sha256=SrqcOdwVvwblF8WwVYjpZapxCwvcijms5vNaOuK5eng,6807
+nmdc_runtime-1.0.10.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
+nmdc_runtime-1.0.10.dist-info/entry_points.txt,sha256=nfH6-K9tDKv7va8ENfShsBnxVQoYJdEe7HHdwtkbh1Y,289
+nmdc_runtime-1.0.10.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
+nmdc_runtime-1.0.10.dist-info/RECORD,,