nmdc-runtime 2.4.0__py3-none-any.whl → 2.6.0__py3-none-any.whl

nmdc_runtime/site/export/ncbi_xml.py

@@ -7,7 +7,6 @@ import xml.dom.minidom
 from typing import Any, List, Union
 from urllib.parse import urlparse
 from nmdc_runtime.site.export.ncbi_xml_utils import (
-    get_instruments,
     handle_controlled_identified_term_value,
     handle_controlled_term_value,
     handle_geolocation_value,

nmdc_runtime/site/export/ncbi_xml_utils.py

@@ -24,31 +24,6 @@ def get_classname_from_typecode(doc_id):
     return class_map.get(typecode)
 
 
-def get_instruments(instrument_set_collection):
-    # dictionary to capture a list of all instruments
-    # Structure of dict:
-    # {"instrument_id": {"vendor": "vendor_name", "model": "model_name"}}
-    all_instruments = {}
-
-    try:
-        query = {"type": "nmdc:Instrument"}
-        cursor = instrument_set_collection.find(query)
-
-        for document in cursor:
-            instrument_id = document.get("id")
-            vendor = document.get("vendor")
-            model = document.get("model")
-
-            if not instrument_id or not vendor or not model:
-                continue
-
-            all_instruments[instrument_id] = {"vendor": vendor, "model": model}
-
-        return all_instruments
-    except Exception as e:
-        raise RuntimeError(f"An error occurred while fetching instrument data: {e}")
-
-
 def fetch_data_objects_from_biosamples(
     all_docs_collection: Collection,
     data_object_set: Collection,

nmdc_runtime/site/graphs.py

@@ -61,6 +61,8 @@ from nmdc_runtime.site.ops import (
     get_database_updater_inputs,
     post_submission_portal_biosample_ingest_record_stitching_filename,
     generate_data_generation_set_post_biosample_ingest,
+    get_instrument_ids_by_model,
+    log_database_ids,
 )
 from nmdc_runtime.site.export.study_metadata import get_biosamples_by_study_id
 
@@ -181,6 +183,7 @@ def translate_metadata_submission_to_nmdc_schema_database():
     biosample_extras_slot_mapping = get_csv_rows_from_url(
         biosample_extras_slot_mapping_file_url
     )
+    instrument_mapping = get_instrument_ids_by_model()
 
     database = translate_portal_submission_to_nmdc_schema_database(
         metadata_submission,
@@ -188,10 +191,13 @@ def translate_metadata_submission_to_nmdc_schema_database():
         data_object_mapping=data_object_mapping,
         biosample_extras=biosample_extras,
         biosample_extras_slot_mapping=biosample_extras_slot_mapping,
+        instrument_mapping=instrument_mapping,
     )
 
     validate_metadata(database)
 
+    log_database_ids(database)
+
     database_dict = nmdc_schema_object_to_dict(database)
     filename = nmdc_schema_database_export_filename(metadata_submission)
     outputs = export_json_to_drs(database_dict, filename)
@@ -217,6 +223,7 @@ def ingest_metadata_submission():
     biosample_extras_slot_mapping = get_csv_rows_from_url(
         biosample_extras_slot_mapping_file_url
     )
+    instrument_mapping = get_instrument_ids_by_model()
 
     database = translate_portal_submission_to_nmdc_schema_database(
         metadata_submission,
@@ -224,7 +231,11 @@ def ingest_metadata_submission():
         data_object_mapping=data_object_mapping,
         biosample_extras=biosample_extras,
         biosample_extras_slot_mapping=biosample_extras_slot_mapping,
+        instrument_mapping=instrument_mapping,
     )
+
+    log_database_ids(database)
+
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)
 

nmdc_runtime/site/ops.py

@@ -7,6 +7,7 @@ import tempfile
 from collections import defaultdict
 from datetime import datetime, timezone
 from io import BytesIO, StringIO
+from pprint import pformat
 from toolz.dicttoolz import keyfilter
 from typing import Tuple
 from zipfile import ZipFile
@@ -38,7 +39,7 @@ from dagster import (
     Bool,
 )
 from gridfs import GridFS
-from linkml_runtime.dumpers import json_dumper
+from linkml_runtime.utils.dictutils import as_simple_dict
 from linkml_runtime.utils.yamlutils import YAMLRoot
 from nmdc_runtime.api.db.mongo import get_mongo_db
 from nmdc_runtime.api.core.idgen import generate_one_id
@@ -69,7 +70,6 @@ from nmdc_runtime.site.export.ncbi_xml_utils import (
     fetch_data_objects_from_biosamples,
     fetch_nucleotide_sequencing_from_biosamples,
     fetch_library_preparation_from_biosamples,
-    get_instruments,
 )
 from nmdc_runtime.site.drsobjects.ingest import mongo_add_docs_result_as_dict
 from nmdc_runtime.site.resources import (
@@ -96,6 +96,7 @@ from nmdc_runtime.site.util import (
     run_and_log,
     schema_collection_has_index_on_id,
     nmdc_study_id_to_filename,
+    get_instruments_by_id,
 )
 from nmdc_runtime.util import (
     drs_object_in_for,
@@ -720,9 +721,8 @@ def translate_portal_submission_to_nmdc_schema_database(
     metadata_submission: Dict[str, Any],
     nucleotide_sequencing_mapping: List,
     data_object_mapping: List,
+    instrument_mapping: Dict[str, str],
     study_category: Optional[str],
-    study_doi_category: Optional[str],
-    study_doi_provider: Optional[str],
     study_pi_image_url: Optional[str],
     biosample_extras: Optional[list[dict]],
     biosample_extras_slot_mapping: Optional[list[dict]],
@@ -739,11 +739,10 @@ def translate_portal_submission_to_nmdc_schema_database(
         data_object_mapping=data_object_mapping,
         id_minter=id_minter,
         study_category=study_category,
-        study_doi_category=study_doi_category,
-        study_doi_provider=study_doi_provider,
         study_pi_image_url=study_pi_image_url,
         biosample_extras=biosample_extras,
         biosample_extras_slot_mapping=biosample_extras_slot_mapping,
+        illumina_instrument_mapping=instrument_mapping,
     )
     database = translator.get_database()
     return database
@@ -761,7 +760,7 @@ def nmdc_schema_database_export_filename(study: Dict[str, Any]) -> str:
 
 @op
 def nmdc_schema_object_to_dict(object: YAMLRoot) -> Dict[str, Any]:
-    return json_dumper.to_dict(object)
+    return as_simple_dict(object)
 
 
 @op(required_resource_keys={"mongo"}, config_schema={"username": str})
@@ -1100,7 +1099,12 @@ def materialize_alldocs(context) -> int:
         write_operations = []
         documents_processed_counter = 0
         for doc in mdb[coll_name].find():
-            doc_type = doc["type"][5:]  # lop off "nmdc:" prefix
+            try:
+                doc_type = doc["type"][5:]  # lop off "nmdc:" prefix
+            except KeyError:
+                raise Exception(
+                    f"doc {doc['id']} in collection {coll_name} has no 'type'!"
+                )
             slots_to_include = ["id", "type"] + document_reference_ranged_slots[
                 doc_type
             ]
@@ -1222,11 +1226,26 @@ def get_library_preparation_from_biosamples(
 
 
 @op(required_resource_keys={"mongo"})
-def get_all_instruments(context: OpExecutionContext):
+def get_all_instruments(context: OpExecutionContext) -> dict[str, dict]:
+    mdb = context.resources.mongo.db
+    return get_instruments_by_id(mdb)
+
+
+@op(required_resource_keys={"mongo"})
+def get_instrument_ids_by_model(context: OpExecutionContext) -> dict[str, str]:
     mdb = context.resources.mongo.db
-    instrument_set_collection = mdb["instrument_set"]
-    all_instruments = get_instruments(instrument_set_collection)
-    return all_instruments
+    instruments_by_id = get_instruments_by_id(mdb)
+    instruments_by_model: dict[str, str] = {}
+    for inst_id, instrument in instruments_by_id.items():
+        model = instrument.get("model")
+        if model is None:
+            context.log.warning(f"Instrument {inst_id} has no model.")
+            continue
+        if model in instruments_by_model:
+            context.log.warning(f"Instrument model {model} is not unique.")
+        instruments_by_model[model] = inst_id
+    context.log.info("Instrument models: %s", pformat(instruments_by_model))
+    return instruments_by_model
 
 
 @op
@@ -1340,3 +1359,26 @@ def generate_biosample_set_for_nmdc_study_from_gold(
     database = database_updater.generate_biosample_set_from_gold_api_for_study()
 
     return database
+
+
+@op
+def log_database_ids(
+    context: OpExecutionContext,
+    database: nmdc.Database,
+) -> None:
+    """Log the IDs of the database."""
+    database_dict = as_simple_dict(database)
+    message = ""
+    for collection_name, collection in database_dict.items():
+        if not isinstance(collection, list):
+            continue
+        message += f"{collection_name} ({len(collection)}):\n"
+        if len(collection) < 10:
+            message += "\n".join(f"  {doc['id']}" for doc in collection)
+        else:
+            message += "\n".join(f"  {doc['id']}" for doc in collection[:4])
+            message += f"\n  ... {len(collection) - 8} more\n"
+            message += "\n".join(f"  {doc['id']}" for doc in collection[-4:])
+        message += "\n"
+    if message:
+        context.log.info(message)

nmdc_runtime/site/repair/database_updater.py

@@ -199,8 +199,20 @@ class DatabaseUpdater:
             if gbs.get("biosampleGoldId") not in nmdc_gold_ids
         ]
 
+        # use the GOLD study id to fetch all sequencing project records associated with the study
+        gold_sequencing_projects_for_study = (
+            self.gold_api_client.fetch_projects_by_study(gold_study_id)
+        )
+
+        # use the GOLD study id to fetch all analysis project records associated with the study
+        gold_analysis_projects_for_study = (
+            self.gold_api_client.fetch_analysis_projects_by_study(gold_study_id)
+        )
+
         gold_study_translator = GoldStudyTranslator(
             biosamples=missing_gold_biosamples,
+            projects=gold_sequencing_projects_for_study,
+            analysis_projects=gold_analysis_projects_for_study,
             gold_nmdc_instrument_map_df=self.gold_nmdc_instrument_map_df,
         )
 

nmdc_runtime/site/repository.py

@@ -553,8 +553,6 @@ def biosample_submission_ingest():
                     "translate_portal_submission_to_nmdc_schema_database": {
                         "inputs": {
                             "study_category": "research_study",
-                            "study_doi_category": None,
-                            "study_doi_provider": None,
                             "study_pi_image_url": None,
                         }
                     },
@@ -591,8 +589,6 @@ def biosample_submission_ingest():
                     "translate_portal_submission_to_nmdc_schema_database": {
                         "inputs": {
                             "study_category": None,
-                            "study_doi_category": None,
-                            "study_doi_provider": None,
                             "study_pi_image_url": None,
                         }
                     },
@@ -744,7 +740,7 @@ def biosample_submission_ingest():
                         "config": {
                             "benthic_data_product": {
                                 "product_id": "DP1.20279.001",
-                                "product_tables": "mms_benthicMetagenomeSequencing, mms_benthicMetagenomeDnaExtraction, mms_benthicRawDataFiles, amb_fieldParent",
+                                "product_tables": "mms_benthicMetagenomeSequencing, mms_benthicMetagenomeDnaExtraction, mms_benthicRawDataFiles, amb_fieldParent, mms_mms_benthicRawDataFiles",
                             }
                         }
                     },
@@ -771,7 +767,7 @@ def biosample_submission_ingest():
                         "config": {
                             "benthic_data_product": {
                                 "product_id": "DP1.20279.001",
-                                "product_tables": "mms_benthicMetagenomeSequencing, mms_benthicMetagenomeDnaExtraction, mms_benthicRawDataFiles, amb_fieldParent",
+                                "product_tables": "mms_benthicMetagenomeSequencing, mms_benthicMetagenomeDnaExtraction, mms_benthicRawDataFiles, amb_fieldParent, mms_mms_benthicRawDataFiles",
                             }
                         }
                     },

nmdc_runtime/site/translation/gold_translator.py

@@ -639,6 +639,16 @@ class GoldStudyTranslator(Translator):
         :return: nmdc:NucleotideSequencing object
         """
         gold_project_id = gold_project["projectGoldId"]
+        ncbi_bioproject_identifier = gold_project.get("ncbiBioProjectAccession")
+        insdc_bioproject_identifiers = []
+        if ncbi_bioproject_identifier:
+            insdc_bioproject_identifiers.append(
+                self._ensure_curie(
+                    ncbi_bioproject_identifier,
+                    default_prefix="bioproject",
+                )
+            )
+
         return nmdc.NucleotideSequencing(
             id=nmdc_nucleotide_sequencing_id,
             name=gold_project.get("projectName"),
@@ -650,6 +660,7 @@ class GoldStudyTranslator(Translator):
             has_input=nmdc_biosample_id,
             add_date=gold_project.get("addDate"),
             mod_date=self._get_mod_date(gold_project),
+            insdc_bioproject_identifiers=insdc_bioproject_identifiers,
             principal_investigator=self._get_pi(gold_project),
             processing_institution=self._get_processing_institution(gold_project),
             instrument_used=self._get_instrument(gold_project),