nmdc-runtime 2.6.0__py3-none-any.whl → 2.12.0__py3-none-any.whl

nmdc_runtime/site/resources.py

@@ -14,7 +14,6 @@ from dagster import (
     StringSource,
     InitResourceContext,
 )
-from fastjsonschema import JsonSchemaValueException
 from frozendict import frozendict
 from linkml_runtime.dumpers import json_dumper
 from pydantic import BaseModel, AnyUrl
@@ -27,7 +26,7 @@ from nmdc_runtime.api.models.object import DrsObject, AccessURL, DrsObjectIn
 from nmdc_runtime.api.models.operation import ListOperationsResponse
 from nmdc_runtime.api.models.util import ListRequest
 from nmdc_runtime.site.normalization.gold import normalize_gold_id
-from nmdc_runtime.util import unfreeze, nmdc_jsonschema_validator_noidpatterns
+from nmdc_runtime.util import unfreeze, get_nmdc_schema_validator
 from nmdc_schema import nmdc
 
 
@@ -109,7 +108,7 @@ class RuntimeApiUserClient(RuntimeApiClient):
             },
         )
         response.raise_for_status()
-        return response.json()["cursor"]["firstBatch"]
+        return response.json()["cursor"]["batch"]
 
     def get_omics_processing_records_by_gold_project_id(self, gold_project_id: str):
         gold_project_id = normalize_gold_id(gold_project_id)
@@ -126,7 +125,7 @@ class RuntimeApiUserClient(RuntimeApiClient):
             },
         )
         response.raise_for_status()
-        return response.json()["cursor"]["firstBatch"]
+        return response.json()["cursor"]["batch"]
 
     def get_biosamples_for_study(self, study_id: str):
         # TODO: 10000 is an arbitrarily large number that has been chosen for the max_page_size param.
@@ -147,6 +146,19 @@ class RuntimeApiUserClient(RuntimeApiClient):
         response.raise_for_status()
         return response.json()["resources"]
 
+    def get_data_generation_records_for_study(self, study_id: str):
+        # TODO: same as above, we are using a large max_page_size to avoid pagination.
+        response = self.request(
+            "GET",
+            f"/nmdcschema/data_generation_set",
+            {
+                "filter": json.dumps({"associated_studies": study_id}),
+                "max_page_size": 10000,
+            },
+        )
+        response.raise_for_status()
+        return response.json()["resources"]
+
     def get_omics_processing_by_name(self, name: str):
         response = self.request(
             "POST",
@@ -157,7 +169,7 @@ class RuntimeApiUserClient(RuntimeApiClient):
             },
         )
         response.raise_for_status()
-        return response.json()["cursor"]["firstBatch"]
+        return response.json()["cursor"]["batch"]
 
     def get_study(self, study_id: str):
         response = self.request(
@@ -169,7 +181,7 @@ class RuntimeApiUserClient(RuntimeApiClient):
             },
         )
         response.raise_for_status()
-        return response.json()["cursor"]["firstBatch"]
+        return response.json()["cursor"]["batch"]
 
 
 class RuntimeApiSiteClient(RuntimeApiClient):
@@ -453,6 +465,17 @@ class NmdcPortalApiClient:
         response.raise_for_status()
         return response.json()
 
+    def make_submission_images_public(
+        self, submission_id: str, *, study_id: str
+    ) -> Dict[str, Any]:
+        response = self._request(
+            "POST",
+            f"/api/metadata_submission/{submission_id}/image/make_public",
+            json={"study_id": study_id},
+        )
+        response.raise_for_status()
+        return response.json()
+
 
 @resource(
     config_schema={
@@ -507,36 +530,49 @@ class MongoDB:
         self.db = self.client[dbname]
 
     def add_docs(self, docs, validate=True, replace=True):
-        try:
-            if validate:
-                nmdc_jsonschema_validator_noidpatterns(docs)
-            rv = {}
-            for collection_name, docs in docs.items():
-                rv[collection_name] = self.db[collection_name].bulk_write(
-                    [
-                        (
-                            ReplaceOne({"id": d["id"]}, d, upsert=True)
-                            if replace
-                            else InsertOne(d)
-                        )
-                        for d in docs
-                    ]
-                )
-                now = datetime.now(timezone.utc)
-                self.db.txn_log.insert_many(
-                    [
-                        {
-                            "tgt": {"id": d.get("id"), "c": collection_name},
-                            "type": "upsert",
-                            "ts": now,
-                            # "dtl": {},
-                        }
-                        for d in docs
-                    ]
-                )
-            return rv
-        except JsonSchemaValueException as e:
-            raise ValueError(e.message)
+        """
+        TODO: Document this function.
+        """
+        if validate:
+            validator = get_nmdc_schema_validator()
+            # Fail fast on first validation error.
+            for result in validator.iter_results(docs, target_class="Database"):
+                raise ValueError(result.message)
+        rv = {}
+        for collection_name, collection_docs in docs.items():
+            # If `collection_docs` is empty, abort this iteration.
+            #
+            # Note: We do this because the `bulk_write` method called below will raise
+            #       an `InvalidOperation` exception if it is passed 0 operations.
+            #
+            # Reference: https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.bulk_write
+            #
+            if len(collection_docs) == 0:
+                continue
+
+            rv[collection_name] = self.db[collection_name].bulk_write(
+                [
+                    (
+                        ReplaceOne({"id": d["id"]}, d, upsert=True)
+                        if replace
+                        else InsertOne(d)
+                    )
+                    for d in collection_docs
+                ]
+            )
+            now = datetime.now(timezone.utc)
+            self.db.txn_log.insert_many(
+                [
+                    {
+                        "tgt": {"id": d.get("id"), "c": collection_name},
+                        "type": "upsert",
+                        "ts": now,
+                        # "dtl": {},
+                    }
+                    for d in collection_docs
+                ]
+            )
+        return rv
 
 
 @resource(

nmdc_runtime/site/translation/gold_translator.py

@@ -1,5 +1,4 @@
 import collections
-import csv
 import re
 from typing import List, Tuple, Union
 from nmdc_schema import nmdc
@@ -45,6 +44,7 @@ class GoldStudyTranslator(Translator):
         analysis_projects: List[JSON_OBJECT] = [],
         gold_nmdc_instrument_map_df: pd.DataFrame = pd.DataFrame(),
         include_field_site_info: bool = False,
+        enable_biosample_filtering: bool = True,
         *args,
         **kwargs,
     ) -> None:
@@ -53,15 +53,20 @@ class GoldStudyTranslator(Translator):
         self.study = study
         self.study_type = nmdc.StudyCategoryEnum(study_type)
         self.include_field_site_info = include_field_site_info
+        self.enable_biosample_filtering = enable_biosample_filtering
         # Filter biosamples to only those with `sequencingStrategy` of
-        # "Metagenome" or "Metatranscriptome"
-        self.biosamples = [
-            biosample
-            for biosample in biosamples
-            if any(
-                _is_valid_project(project) for project in biosample.get("projects", [])
-            )
-        ]
+        # "Metagenome" or "Metatranscriptome" if filtering is enabled
+        if enable_biosample_filtering:
+            self.biosamples = [
+                biosample
+                for biosample in biosamples
+                if any(
+                    _is_valid_project(project)
+                    for project in biosample.get("projects", [])
+                )
+            ]
+        else:
+            self.biosamples = biosamples
         # Fetch the valid projectGoldIds that are associated with filtered
         # biosamples on their `projects` field
         valid_project_ids = {
@@ -116,6 +121,9 @@ class GoldStudyTranslator(Translator):
         :param gold_entity: GOLD entity object
         :return: PersonValue corresponding to the first PI in the `contacts` field
         """
+        if "contacts" not in gold_entity:
+            return None
+
         pi_dict = next(
             (
                 contact
@@ -169,7 +177,7 @@ class GoldStudyTranslator(Translator):
                 project["ncbiBioSampleAccession"], default_prefix="biosample"
             )
             for project in biosample_projects
-            if project["ncbiBioSampleAccession"]
+            if project.get("ncbiBioSampleAccession")
         ]
 
     def _get_samp_taxon_id(
@@ -333,12 +341,7 @@ class GoldStudyTranslator(Translator):
         if field_value is None:
             return None
 
-        return nmdc.QuantityValue(
-            has_raw_value=field_value,
-            has_numeric_value=nmdc.Double(field_value),
-            has_unit=unit,
-            type="nmdc:QuantityValue",
-        )
+        return self._parse_quantity_value(str(field_value), unit)
 
     def _get_text_value(
         self, gold_entity: JSON_OBJECT, gold_field: str
@@ -564,13 +567,11 @@ class GoldStudyTranslator(Translator):
         gold_biosample_id = gold_biosample["biosampleGoldId"]
         return nmdc.Biosample(
             add_date=gold_biosample.get("addDate"),
-            alt=self._get_quantity_value(
-                gold_biosample, "altitudeInMeters", unit="meters"
-            ),
+            alt=self._get_quantity_value(gold_biosample, "altitudeInMeters", unit="m"),
             collected_from=nmdc_field_site_id,
             collection_date=self._get_collection_date(gold_biosample),
             depth=self._get_quantity_value(
-                gold_biosample, ("depthInMeters", "depthInMeters2"), unit="meters"
+                gold_biosample, ("depthInMeters", "depthInMeters2"), unit="m"
             ),
             description=gold_biosample.get("description"),
             diss_oxygen=self._get_quantity_value(gold_biosample, "oxygenConcentration"),
@@ -609,7 +610,7 @@ class GoldStudyTranslator(Translator):
             ),
             specific_ecosystem=gold_biosample.get("specificEcosystem"),
             subsurface_depth=self._get_quantity_value(
-                gold_biosample, "subsurfaceDepthInMeters", unit="meters"
+                gold_biosample, "subsurfaceDepthInMeters", unit="m"
             ),
             temp=self._get_quantity_value(
                 gold_biosample, "sampleCollectionTemperature"

nmdc_runtime/site/translation/neon_benthic_translator.py

@@ -11,7 +11,6 @@ from nmdc_runtime.site.util import get_basename
 from nmdc_runtime.site.translation.neon_utils import (
     _get_value_or_none,
     _create_controlled_identified_term_value,
-    _create_controlled_term_value,
     _create_geolocation_value,
     _create_quantity_value,
     _create_timestamp_value,
@@ -349,6 +348,7 @@ class NeonBenthicDataTranslator(Translator):
             description=f"sequencing results for {basename}",
             type="nmdc:DataObject",
             data_object_type=do_type,
+            data_category=nmdc.DataCategoryEnum.instrument_data.text,
             in_manifest=manifest_id,
         )
 

nmdc_runtime/site/translation/neon_soil_translator.py

@@ -10,7 +10,6 @@ from nmdc_runtime.site.util import get_basename
 from nmdc_runtime.site.translation.neon_utils import (
     _get_value_or_none,
     _create_controlled_identified_term_value,
-    _create_controlled_term_value,
     _create_geolocation_value,
     _create_quantity_value,
     _create_timestamp_value,
@@ -153,7 +152,7 @@ class NeonSoilDataTranslator(Translator):
             collection_date=_create_timestamp_value(
                 biosample_row["collectDate"].values[0]
             ),
-            temp=_create_quantity_value(biosample_row["soilTemp"].values[0], "Celsius"),
+            temp=_create_quantity_value(biosample_row["soilTemp"].values[0], "Cel"),
             depth=nmdc.QuantityValue(
                 has_minimum_numeric_value=_get_value_or_none(
                     biosample_row, "sampleTopDepth"
@@ -169,13 +168,13 @@ class NeonSoilDataTranslator(Translator):
             analysis_type=_get_value_or_none(biosample_row, "sequenceAnalysisType"),
             env_package=_create_text_value(biosample_row["sampleType"].values[0]),
             nitro=_create_quantity_value(
-                biosample_row["nitrogenPercent"].values[0], "percent"
+                biosample_row["nitrogenPercent"].values[0], "%"
             ),
             org_carb=_create_quantity_value(
-                biosample_row["organicCPercent"].values[0], "percent"
+                biosample_row["organicCPercent"].values[0], "%"
             ),
             carb_nitro_ratio=_create_quantity_value(
-                biosample_row["CNratio"].values[0], None
+                biosample_row["CNratio"].values[0], "ratio"
             ),
             ph=_create_double_value(biosample_row["soilInWaterpH"].values[0]),
             water_content=(
@@ -264,6 +263,7 @@ class NeonSoilDataTranslator(Translator):
             description=f"sequencing results for {basename}",
             type="nmdc:DataObject",
             md5_checksum=checksum,
+            data_category=nmdc.DataCategoryEnum.instrument_data.text,
             data_object_type=do_type,
         )
 

nmdc_runtime/site/translation/neon_surface_water_translator.py

@@ -3,7 +3,6 @@ import sqlite3
 from typing import Dict, Optional, Union
 
 import pandas as pd
-import requests
 import requests_cache
 
 from nmdc_schema import nmdc
@@ -12,7 +11,6 @@ from nmdc_runtime.site.util import get_basename
 from nmdc_runtime.site.translation.neon_utils import (
     _get_value_or_none,
     _create_controlled_identified_term_value,
-    _create_controlled_term_value,
     _create_geolocation_value,
     _create_quantity_value,
     _create_timestamp_value,
@@ -397,6 +395,7 @@ class NeonSurfaceWaterDataTranslator(Translator):
             description=f"sequencing results for {basename}",
             type="nmdc:DataObject",
             data_object_type=do_type,
+            data_category=nmdc.DataCategoryEnum.instrument_data.text,
             in_manifest=manifest_id,
         )