nmdc-runtime 2.6.0__py3-none-any.whl → 2.12.0__py3-none-any.whl

nmdc_runtime/site/export/ncbi_xml_utils.py

@@ -1,5 +1,5 @@
 from io import BytesIO, StringIO
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List
 
 from nmdc_runtime.api.endpoints.util import strip_oid
 from nmdc_runtime.minter.config import typecodes
@@ -99,31 +99,36 @@ def fetch_nucleotide_sequencing_from_biosamples(
     for biosample in biosamples_list:
         current_ids = [biosample["id"]]
         collected_ntseq_objects = []
+        processed_ids = set()  # Track already processed nucleotide sequencing IDs
 
         while current_ids:
             new_current_ids = []
             for current_id in current_ids:
-                query = {"has_input": current_id}
-                document = all_docs_collection.find_one(query)
-
-                if not document:
-                    continue
-
-                has_output = document.get("has_output")
-                if not has_output:
-                    continue
-
-                for output_id in has_output:
-                    if get_classname_from_typecode(output_id) == "DataObject":
-                        nucleotide_sequencing_doc = data_generation_set.find_one(
-                            {"id": document["id"]}
-                        )
-                        if nucleotide_sequencing_doc:
-                            collected_ntseq_objects.append(
-                                strip_oid(nucleotide_sequencing_doc)
-                            )
-                    else:
-                        new_current_ids.append(output_id)
+                # Find all documents with current_id as input instead of just one
+                for document in all_docs_collection.find({"has_input": current_id}):
+                    has_output = document.get("has_output")
+                    if not has_output:
+                        continue
+
+                    for output_id in has_output:
+                        if get_classname_from_typecode(output_id) == "DataObject":
+                            # Only process if we haven't seen this document ID before
+                            if document["id"] not in processed_ids:
+                                nucleotide_sequencing_doc = (
+                                    data_generation_set.find_one(
+                                        {
+                                            "id": document["id"],
+                                            "type": "nmdc:NucleotideSequencing",
+                                        }
+                                    )
+                                )
+                                if nucleotide_sequencing_doc:
+                                    collected_ntseq_objects.append(
+                                        strip_oid(nucleotide_sequencing_doc)
+                                    )
+                                    processed_ids.add(document["id"])
+                        else:
+                            new_current_ids.append(output_id)
 
             current_ids = new_current_ids
 
@@ -187,10 +192,7 @@ def handle_quantity_value(slot_value):
         and "has_minimum_numeric_value" in slot_value
         and "has_unit" in slot_value
     ):
-        range_value = (
-            slot_value["has_maximum_numeric_value"]
-            - slot_value["has_minimum_numeric_value"]
-        )
+        range_value = f"{slot_value['has_minimum_numeric_value']} - {slot_value['has_maximum_numeric_value']}"
         return f"{range_value} {slot_value['has_unit']}"
     elif "has_raw_value" in slot_value:
         return slot_value["has_raw_value"]
@@ -273,6 +275,120 @@ def load_mappings(url):
     return attribute_mappings, slot_range_mappings
 
 
+def check_pooling_for_biosamples(
+    material_processing_set: Collection, biosamples_list: List[Dict[str, Any]]
+) -> Dict[str, Dict[str, Any]]:
+    """Check which biosamples are part of pooling processes and return pooling information.
+
+    The way in which we check if a biosample is part of a Pooling process is by checking if
+    the biosample id has been asserted on the `has_input` slot/key of an `nmdc:Pooling` process
+    instance.
+
+    :param material_processing_set: reference to the material_processing_set collection
+    :param biosamples_list: list of all biosamples to check
+    :return: dictionary mapping biosample_id to pooling information (empty dict if not pooled)
+    """
+    result = {}
+    # get list of all biosample IDs that are part of a given study
+    biosample_lookup = {bs["id"]: bs for bs in biosamples_list}
+
+    # get list of all pooling processes
+    pooling_processes = list(material_processing_set.find({"type": "nmdc:Pooling"}))
+
+    # initialize all biosamples as not pooled
+    for biosample in biosamples_list:
+        result[biosample["id"]] = {}
+
+    # process each pooling process
+    for pooling_process in pooling_processes:
+        pooled_biosample_ids = pooling_process.get("has_input", [])
+
+        # get the processed sample output from the pooling process
+        has_output = pooling_process.get("has_output", [])
+        processed_sample_id = None
+
+        for output_id in has_output:
+            if get_classname_from_typecode(output_id) == "ProcessedSample":
+                processed_sample_id = output_id
+                break
+
+        # aggregate the values on `collection_date` and `depth` slots
+        # here, we are collecting the `collection_date` and `depth` values
+        # asserted on each of the biosamples that are part of a given pooling
+        # process in the following way:
+        # example of aggregated `collection_date`: 2017-06-05T16:50Z/2017-06-05T17:47Z
+        # example of aggregated `depth`: 0-10 m
+        collection_dates = []
+        depths = []
+
+        for bs_id in pooled_biosample_ids:
+            biosample = biosample_lookup.get(bs_id)
+            if not biosample:
+                continue
+
+            if "collection_date" in biosample:
+                collection_date = biosample["collection_date"]
+                if (
+                    isinstance(collection_date, dict)
+                    and "has_raw_value" in collection_date
+                ):
+                    collection_dates.append(collection_date["has_raw_value"])
+                elif isinstance(collection_date, str):
+                    collection_dates.append(collection_date)
+
+            if "depth" in biosample:
+                depth = biosample["depth"]
+                if isinstance(depth, dict):
+                    if "has_numeric_value" in depth:
+                        depths.append(depth["has_numeric_value"])
+                    elif (
+                        "has_minimum_numeric_value" in depth
+                        and "has_maximum_numeric_value" in depth
+                    ):
+                        depths.extend(
+                            [
+                                depth["has_minimum_numeric_value"],
+                                depth["has_maximum_numeric_value"],
+                            ]
+                        )
+                elif isinstance(depth, (int, float)):
+                    depths.append(depth)
+
+        # create aggregated (forward slash separated) value for `collection_date`
+        aggregated_collection_date = None
+        if collection_dates:
+            sorted_dates = sorted(collection_dates)
+            if len(sorted_dates) > 1:
+                aggregated_collection_date = f"{sorted_dates[0]}/{sorted_dates[-1]}"
+            else:
+                aggregated_collection_date = sorted_dates[0]
+
+        # create aggregated (hyphen separated) value for `depth`
+        aggregated_depth = None
+        if depths:
+            min_depth = min(depths)
+            max_depth = max(depths)
+            if min_depth != max_depth:
+                aggregated_depth = f"{min_depth}-{max_depth} m"
+            else:
+                aggregated_depth = f"{min_depth} m"
+
+        # update all biosamples that are part of this pooling process
+        pooling_info = {
+            "processed_sample_id": processed_sample_id,
+            "pooling_process_id": pooling_process.get("id"),
+            "pooled_biosample_ids": pooled_biosample_ids,
+            "aggregated_collection_date": aggregated_collection_date,
+            "aggregated_depth": aggregated_depth,
+        }
+
+        for bs_id in pooled_biosample_ids:
+            if bs_id in result:
+                result[bs_id] = pooling_info
+
+    return result
+
+
 def validate_xml(xml, xsd_url):
     response = requests.get(xsd_url)
     response.raise_for_status()

nmdc_runtime/site/graphs.py

@@ -1,7 +1,6 @@
 from dagster import graph
 
 from nmdc_runtime.site.ops import (
-    build_merged_db,
     generate_biosample_set_for_nmdc_study_from_gold,
     nmdc_schema_database_export_filename,
     nmdc_schema_database_from_gold_study,
@@ -12,8 +11,6 @@ from nmdc_runtime.site.ops import (
     gold_projects_by_study,
     gold_study,
     poll_for_run_completion,
-    run_etl,
-    local_file_to_api_object,
     get_operation,
     produce_curated_db,
     delete_operations,
@@ -22,6 +19,7 @@ from nmdc_runtime.site.ops import (
     filter_ops_done_object_puts,
     hello,
     mongo_stats,
+    run_script_to_update_insdc_biosample_identifiers,
     submit_metadata_to_db,
     filter_ops_undone_expired,
     construct_jobs,
@@ -50,41 +48,27 @@ from nmdc_runtime.site.ops import (
     get_df_from_url,
     site_code_mapping,
     materialize_alldocs,
+    load_ontology,
     get_ncbi_export_pipeline_study,
     get_data_objects_from_biosamples,
     get_nucleotide_sequencing_from_biosamples,
     get_library_preparation_from_biosamples,
+    get_aggregated_pooled_biosamples,
     get_all_instruments,
     get_ncbi_export_pipeline_inputs,
     ncbi_submission_xml_from_nmdc_study,
     ncbi_submission_xml_asset,
+    render_text,
     get_database_updater_inputs,
     post_submission_portal_biosample_ingest_record_stitching_filename,
     generate_data_generation_set_post_biosample_ingest,
     get_instrument_ids_by_model,
     log_database_ids,
+    add_public_image_urls,
 )
 from nmdc_runtime.site.export.study_metadata import get_biosamples_by_study_id
 
 
-@graph
-def gold_translation():
-    """
-    Translating an export of the JGI GOLD [1] SQL database to the NMDC database JSON schema.
-
-    [1] Genomes OnLine Database (GOLD) <https://gold.jgi.doe.gov/>.
-    """
-    local_file_to_api_object(run_etl(build_merged_db()))
-
-
-@graph()
-def gold_translation_curation():
-    # TODO
-    #   - have produce_curated_db do actual curation (see notebook), persisting to db.
-    #   - more steps in pipeline? Or handoff via run_status_sensor on DagsterRunStatus.SUCCESS.
-    produce_curated_db(get_operation())
-
-
 @graph()
 def create_objects_from_site_object_puts():
     delete_operations(
@@ -112,6 +96,16 @@ def ensure_alldocs():
     materialize_alldocs()
 
 
+@graph
+def run_ontology_load():
+    """
+    A graph for loading ontologies.
+    The source_ontology parameter is provided by the job configuration
+    and passed to the load_ontology op.
+    """
+    load_ontology()
+
+
 @graph
 def ensure_jobs():
     jobs = construct_jobs()
@@ -120,17 +114,24 @@ def ensure_jobs():
 
 @graph
 def apply_changesheet():
+    # Note: We use `_` as a "placeholder" variable.
+    #       It's a variable to whose value we assign no significance. In this case, we use it to
+    #       tell Dagster that one op depends upon the output of the other (so Dagster runs them
+    #       in that order), without implying to maintainers that its value is significant to us.
+    #       Reference (this strategy): https://docs.dagster.io/api/dagster/types#dagster.Nothing
+    #       Reference (`_` variables): https://stackoverflow.com/a/47599668
     sheet_in = get_changesheet_in()
     outputs = perform_changesheet_updates(sheet_in)
-    add_output_run_event(outputs)
-    materialize_alldocs()
+    _ = add_output_run_event(outputs)
+    materialize_alldocs(waits_for=_)
 
 
 @graph
 def apply_metadata_in():
+    # Note: We use `_` as a "placeholder" variable.
     outputs = perform_mongo_updates(get_json_in())
-    add_output_run_event(outputs)
-    materialize_alldocs()
+    _ = add_output_run_event(outputs)
+    materialize_alldocs(waits_for=_)
 
 
 @graph
@@ -140,6 +141,7 @@ def gold_study_to_database():
         study_type,
         gold_nmdc_instrument_mapping_file_url,
         include_field_site_info,
+        enable_biosample_filtering,
     ) = get_gold_study_pipeline_inputs()
 
     projects = gold_projects_by_study(study_id)
@@ -156,6 +158,7 @@ def gold_study_to_database():
         analysis_projects,
         gold_nmdc_instrument_map_df,
         include_field_site_info,
+        enable_biosample_filtering,
     )
     database_dict = nmdc_schema_object_to_dict(database)
     filename = nmdc_schema_database_export_filename(study)
@@ -172,6 +175,7 @@ def translate_metadata_submission_to_nmdc_schema_database():
         data_object_mapping_file_url,
         biosample_extras_file_url,
         biosample_extras_slot_mapping_file_url,
+        study_id,
     ) = get_submission_portal_pipeline_inputs()
 
     metadata_submission = fetch_nmdc_portal_submission_by_id(submission_id)
@@ -192,6 +196,7 @@ def translate_metadata_submission_to_nmdc_schema_database():
         biosample_extras=biosample_extras,
         biosample_extras_slot_mapping=biosample_extras_slot_mapping,
         instrument_mapping=instrument_mapping,
+        study_id=study_id,
     )
 
     validate_metadata(database)
@@ -212,6 +217,7 @@ def ingest_metadata_submission():
         data_object_mapping_file_url,
         biosample_extras_file_url,
         biosample_extras_slot_mapping_file_url,
+        study_id,
     ) = get_submission_portal_pipeline_inputs()
 
     metadata_submission = fetch_nmdc_portal_submission_by_id(submission_id)
@@ -232,7 +238,9 @@ def ingest_metadata_submission():
         biosample_extras=biosample_extras,
         biosample_extras_slot_mapping=biosample_extras_slot_mapping,
         instrument_mapping=instrument_mapping,
+        study_id=study_id,
     )
+    database = add_public_image_urls(database, submission_id)
 
     log_database_ids(database)
 
@@ -471,6 +479,7 @@ def nmdc_study_to_ncbi_submission_export():
     )
     data_object_records = get_data_objects_from_biosamples(biosamples)
     library_preparation_records = get_library_preparation_from_biosamples(biosamples)
+    pooled_biosamples_data = get_aggregated_pooled_biosamples(biosamples)
     all_instruments = get_all_instruments()
     xml_data = ncbi_submission_xml_from_nmdc_study(
         nmdc_study,
@@ -480,17 +489,26 @@ def nmdc_study_to_ncbi_submission_export():
         data_object_records,
         library_preparation_records,
         all_instruments,
+        pooled_biosamples_data,
     )
     ncbi_submission_xml_asset(xml_data)
 
 
 @graph
 def generate_data_generation_set_for_biosamples_in_nmdc_study():
-    (study_id, gold_nmdc_instrument_mapping_file_url) = get_database_updater_inputs()
+    (
+        study_id,
+        gold_nmdc_instrument_mapping_file_url,
+        include_field_site_info,
+        enable_biosample_filtering,
+    ) = get_database_updater_inputs()
     gold_nmdc_instrument_map_df = get_df_from_url(gold_nmdc_instrument_mapping_file_url)
 
     database = generate_data_generation_set_post_biosample_ingest(
-        study_id, gold_nmdc_instrument_map_df
+        study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
 
     database_dict = nmdc_schema_object_to_dict(database)
@@ -503,11 +521,19 @@ def generate_data_generation_set_for_biosamples_in_nmdc_study():
 
 @graph
 def generate_biosample_set_from_samples_in_gold():
-    (study_id, gold_nmdc_instrument_mapping_file_url) = get_database_updater_inputs()
+    (
+        study_id,
+        gold_nmdc_instrument_mapping_file_url,
+        include_field_site_info,
+        enable_biosample_filtering,
+    ) = get_database_updater_inputs()
     gold_nmdc_instrument_map_df = get_df_from_url(gold_nmdc_instrument_mapping_file_url)
 
     database = generate_biosample_set_for_nmdc_study_from_gold(
-        study_id, gold_nmdc_instrument_map_df
+        study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
     database_dict = nmdc_schema_object_to_dict(database)
     filename = post_submission_portal_biosample_ingest_record_stitching_filename(
@@ -515,3 +541,28 @@ def generate_biosample_set_from_samples_in_gold():
     )
     outputs = export_json_to_drs(database_dict, filename)
     add_output_run_event(outputs)
+
+
+@graph
+def generate_update_script_for_insdc_biosample_identifiers():
+    """Generate a MongoDB update script to add INSDC biosample identifiers to biosamples based on GOLD data.
+
+    This graph fetches the necessary inputs, then calls the run_script_to_update_insdc_biosample_identifiers op
+    to generate a script for updating biosample records with INSDC identifiers obtained from GOLD.
+    The script is returned as a dictionary that can be executed against MongoDB.
+    """
+    (
+        study_id,
+        gold_nmdc_instrument_mapping_file_url,
+        include_field_site_info,
+        enable_biosample_filtering,
+    ) = get_database_updater_inputs()
+    gold_nmdc_instrument_map_df = get_df_from_url(gold_nmdc_instrument_mapping_file_url)
+
+    update_script = run_script_to_update_insdc_biosample_identifiers(
+        study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
+    )
+    render_text(update_script)