nmdc-runtime 1.6.0__py3-none-any.whl → 1.8.0__py3-none-any.whl

nmdc_runtime/site/export/ncbi_xml_utils.py

@@ -0,0 +1,206 @@
+from io import BytesIO, StringIO
+from nmdc_runtime.minter.config import typecodes
+from lxml import etree
+
+import csv
+import requests
+
+
+def _build_class_map(class_map_data):
+    return {
+        entry["name"]: entry["schema_class"].split(":")[1] for entry in class_map_data
+    }
+
+
+def get_classname_from_typecode(doc_id):
+    class_map_data = typecodes()
+    class_map = _build_class_map(class_map_data)
+
+    typecode = doc_id.split(":")[1].split("-")[0]
+    return class_map.get(typecode)
+
+
+def fetch_data_objects_from_biosamples(all_docs_collection, biosamples_list):
+    biosample_data_objects = []
+
+    for biosample in biosamples_list:
+        current_ids = [biosample["id"]]
+        collected_data_objects = []
+
+        while current_ids:
+            new_current_ids = []
+            for current_id in current_ids:
+                query = {"has_input": current_id}
+                document = all_docs_collection.find_one(query)
+
+                if not document:
+                    continue
+
+                has_output = document.get("has_output")
+                if not has_output:
+                    continue
+
+                for output_id in has_output:
+                    if get_classname_from_typecode(output_id) == "DataObject":
+                        data_object_doc = all_docs_collection.find_one(
+                            {"id": output_id}
+                        )
+                        if data_object_doc:
+                            collected_data_objects.append(data_object_doc)
+                    else:
+                        new_current_ids.append(output_id)
+
+            current_ids = new_current_ids
+
+        if collected_data_objects:
+            biosample_data_objects.append({biosample["id"]: collected_data_objects})
+
+    return biosample_data_objects
+
+
+def fetch_omics_processing_from_biosamples(all_docs_collection, biosamples_list):
+    biosample_data_objects = []
+
+    for biosample in biosamples_list:
+        current_ids = [biosample["id"]]
+        collected_data_objects = []
+
+        while current_ids:
+            new_current_ids = []
+            for current_id in current_ids:
+                query = {"has_input": current_id}
+                document = all_docs_collection.find_one(query)
+
+                if not document:
+                    continue
+
+                has_output = document.get("has_output")
+                if not has_output:
+                    continue
+
+                for output_id in has_output:
+                    if get_classname_from_typecode(output_id) == "DataObject":
+                        omics_processing_doc = all_docs_collection.find_one(
+                            {"id": document["id"]}
+                        )
+                        if omics_processing_doc:
+                            collected_data_objects.append(omics_processing_doc)
+                    else:
+                        new_current_ids.append(output_id)
+
+            current_ids = new_current_ids
+
+        if collected_data_objects:
+            biosample_data_objects.append({biosample["id"]: collected_data_objects})
+
+    return biosample_data_objects
+
+
+def handle_quantity_value(slot_value):
+    if "has_numeric_value" in slot_value and "has_unit" in slot_value:
+        return f"{slot_value['has_numeric_value']} {slot_value['has_unit']}"
+    elif (
+        "has_maximum_numeric_value" in slot_value
+        and "has_minimum_numeric_value" in slot_value
+        and "has_unit" in slot_value
+    ):
+        range_value = (
+            slot_value["has_maximum_numeric_value"]
+            - slot_value["has_minimum_numeric_value"]
+        )
+        return f"{range_value} {slot_value['has_unit']}"
+    elif "has_raw_value" in slot_value:
+        return slot_value["has_raw_value"]
+    return "Unknown format"
+
+
+def handle_text_value(slot_value):
+    return slot_value.get("has_raw_value", "Unknown format")
+
+
+def handle_timestamp_value(slot_value):
+    return slot_value.get("has_raw_value", "Unknown format")
+
+
+def handle_controlled_term_value(slot_value):
+    if "term" in slot_value:
+        term = slot_value["term"]
+        if "name" in term and "id" in term:
+            return f"{term['name']} [{term['id']}]"
+        elif "id" in term:
+            return term["id"]
+        elif "name" in term:
+            return term["name"]
+    elif "has_raw_value" in slot_value:
+        return slot_value["has_raw_value"]
+    return "Unknown format"
+
+
+def handle_controlled_identified_term_value(slot_value):
+    if "term" in slot_value:
+        term = slot_value["term"]
+        if "name" in term and "id" in term:
+            return f"{term['name']} [{term['id']}]"
+        elif "id" in term:
+            return term["id"]
+    elif "has_raw_value" in slot_value:
+        return slot_value["has_raw_value"]
+    return "Unknown format"
+
+
+def handle_geolocation_value(slot_value):
+    if "latitude" in slot_value and "longitude" in slot_value:
+        return f"{slot_value['latitude']} {slot_value['longitude']}"
+    elif "has_raw_value" in slot_value:
+        return slot_value["has_raw_value"]
+    return "Unknown format"
+
+
+def handle_float_value(slot_value):
+    return f"{slot_value:.2f}"
+
+
+def handle_string_value(slot_value):
+    return f"{slot_value}"
+
+
+def load_mappings(url):
+    response = requests.get(url)
+    response.raise_for_status()
+    file_content = response.text
+
+    attribute_mappings = {}
+    slot_range_mappings = {}
+    reader = csv.DictReader(StringIO(file_content), delimiter="\t")
+    for row in reader:
+        if row["ignore"].strip():
+            continue
+
+        json_key = row["nmdc_schema_slot"]
+        # attribute mappings
+        xml_attribute_name = row["ncbi_biosample_attribute_name"]
+        attribute_mappings[json_key] = (
+            xml_attribute_name if xml_attribute_name else json_key
+        )
+
+        # slot range mappings
+        data_type = row["nmdc_schema_slot_range"]
+        slot_range_mappings[json_key] = data_type if data_type else "default"
+
+    return attribute_mappings, slot_range_mappings
+
+
+def validate_xml(xml, xsd_url):
+    response = requests.get(xsd_url)
+    response.raise_for_status()
+    xsd_content = response.text
+
+    xml_schema_doc = etree.parse(BytesIO(xsd_content.encode("utf-8")))
+    xml_schema = etree.XMLSchema(xml_schema_doc)
+
+    xml_doc = etree.parse(BytesIO(xml.encode("utf-8")))
+
+    if not xml_schema.validate(xml_doc):
+        raise ValueError(f"There were errors while validating against: {xsd_url}")
+
+    return True

nmdc_runtime/site/export/study_metadata.py

@@ -5,7 +5,6 @@ Get NMDC study-associated metadata from search api
 import csv
 from io import StringIO
 
-import requests
 from dagster import (
     op,
     get_dagster_logger,
@@ -26,13 +25,27 @@ def get_all_docs(client, collection, filter_):
     per_page = 200
     url_base = f"/{collection}?filter={filter_}&per_page={per_page}"
     results = []
-    rv = client.request("GET", url_base).json()
+    response = client.request("GET", url_base)
+    if response.status_code != 200:
+        raise Exception(
+            f"Runtime API request failed with status {response.status_code}."
+            f" Check URL: {url_base}"
+        )
+    rv = response.json()
     results.extend(rv.get("results", []))
     page, count = rv["meta"]["page"], rv["meta"]["count"]
     assert count <= 10_000
     while page * per_page < count:
-        rv = requests.get(url_base + f"&page={page + 1}").json()
-        results.extend(rv["results"])
+        page += 1
+        url = f"{url_base}&page={page}"
+        response = client.request("GET", url)
+        if response.status_code != 200:
+            raise Exception(
+                f"Runtime API request failed with status {response.status_code}."
+                f" Check URL: {url}"
+            )
+        rv = response.json()
+        results.extend(rv.get("results", []))
     return results
 
 
@@ -115,3 +128,10 @@ def export_study_biosamples_as_csv(context: OpExecutionContext, study_export_inf
 def export_study_biosamples_metadata():
     outputs = export_study_biosamples_as_csv(get_study_biosamples_metadata())
     add_output_run_event(outputs)
+
+
+@op(required_resource_keys={"runtime_api_site_client"})
+def get_biosamples_by_study_id(context: OpExecutionContext, nmdc_study: dict):
+    client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
+    biosamples = get_all_docs(client, "biosamples", f"part_of:{nmdc_study['id']}")
+    return biosamples

nmdc_runtime/site/graphs.py

@@ -22,7 +22,6 @@ from nmdc_runtime.site.ops import (
     hello,
     mongo_stats,
     submit_metadata_to_db,
-    update_schema,
     filter_ops_undone_expired,
     construct_jobs,
     maybe_post_jobs,
@@ -49,7 +48,15 @@ from nmdc_runtime.site.ops import (
     get_neon_pipeline_inputs,
     get_df_from_url,
     site_code_mapping,
+    materialize_alldocs,
+    get_ncbi_export_pipeline_study,
+    get_data_objects_from_biosamples,
+    get_omics_processing_from_biosamples,
+    get_ncbi_export_pipeline_inputs,
+    ncbi_submission_xml_from_nmdc_study,
+    ncbi_submission_xml_asset,
 )
+from nmdc_runtime.site.export.study_metadata import get_biosamples_by_study_id
 
 
 @graph
@@ -88,19 +95,13 @@ def hello_mongo():
 
 
 @graph
-def update_terminus():
-    """
-    A pipeline definition. This example pipeline has a single solid.
-
-    For more hints on writing Dagster pipelines, see our documentation overview on Pipelines:
-    https://docs.dagster.io/overview/solids-pipelines/pipelines
-    """
-    update_schema()
+def housekeeping():
+    delete_operations(list_operations(filter_ops_undone_expired()))
 
 
 @graph
-def housekeeping():
-    delete_operations(list_operations(filter_ops_undone_expired()))
+def ensure_alldocs():
+    materialize_alldocs()
 
 
 @graph
@@ -381,3 +382,20 @@ def ingest_neon_surface_water_metadata():
     )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)
+
+
+@graph
+def nmdc_study_to_ncbi_submission_export():
+    nmdc_study = get_ncbi_export_pipeline_study()
+    ncbi_submission_metadata = get_ncbi_export_pipeline_inputs()
+    biosamples = get_biosamples_by_study_id(nmdc_study)
+    omics_processing_records = get_omics_processing_from_biosamples(biosamples)
+    data_objects = get_data_objects_from_biosamples(biosamples)
+    xml_data = ncbi_submission_xml_from_nmdc_study(
+        nmdc_study,
+        ncbi_submission_metadata,
+        biosamples,
+        omics_processing_records,
+        data_objects,
+    )
+    ncbi_submission_xml_asset(xml_data)

nmdc_runtime/site/ops.py

@@ -9,9 +9,11 @@ from datetime import datetime, timezone
 from io import BytesIO, StringIO
 from typing import Tuple
 from zipfile import ZipFile
+
 import pandas as pd
 import requests
 
+
 from bson import ObjectId, json_util
 from dagster import (
     Any,
@@ -29,10 +31,14 @@ from dagster import (
     String,
     op,
     Optional,
+    Field,
+    Permissive,
+    Bool,
 )
 from gridfs import GridFS
 from linkml_runtime.dumpers import json_dumper
 from linkml_runtime.utils.yamlutils import YAMLRoot
+from nmdc_runtime.api.db.mongo import get_mongo_db
 from nmdc_runtime.api.core.idgen import generate_one_id
 from nmdc_runtime.api.core.metadata import (
     _validate_changesheet,
@@ -42,6 +48,7 @@ from nmdc_runtime.api.core.metadata import (
 )
 from nmdc_runtime.api.core.util import dotted_path_for, hash_from_str, json_clean, now
 from nmdc_runtime.api.endpoints.util import persist_content_and_get_drs_object
+from nmdc_runtime.api.endpoints.find import find_study_by_id
 from nmdc_runtime.api.models.job import Job, JobOperationMetadata
 from nmdc_runtime.api.models.metadata import ChangesheetIn
 from nmdc_runtime.api.models.operation import (
@@ -55,6 +62,11 @@ from nmdc_runtime.api.models.run import (
     _add_run_complete_event,
 )
 from nmdc_runtime.api.models.util import ResultT
+from nmdc_runtime.site.export.ncbi_xml import NCBISubmissionXML
+from nmdc_runtime.site.export.ncbi_xml_utils import (
+    fetch_data_objects_from_biosamples,
+    fetch_omics_processing_from_biosamples,
+)
 from nmdc_runtime.site.drsobjects.ingest import mongo_add_docs_result_as_dict
 from nmdc_runtime.site.resources import (
     NmdcPortalApiClient,
@@ -81,12 +93,15 @@ from nmdc_runtime.util import (
     put_object,
     validate_json,
     specialize_activity_set_docs,
+    collection_name_to_class_names,
+    class_hierarchy_as_list,
+    populated_schema_collection_names_with_id_field,
 )
 from nmdc_schema import nmdc
+from nmdc_schema.nmdc import Database as NMDCDatabase
 from pydantic import BaseModel
 from pymongo.database import Database as MongoDatabase
 from starlette import status
-from terminusdb_client.woqlquery import WOQLQuery as WQ
 from toolz import assoc, dissoc, get_in, valfilter, identity
 
 
@@ -111,14 +126,6 @@ def log_env(context):
     context.log.info("\n".join(out))
 
 
-@op(required_resource_keys={"terminus"})
-def list_databases(context) -> List[String]:
-    client = context.resources.terminus.client
-    list_ = client.list_databases()
-    context.log.info(f"databases: {list_}")
-    return list_
-
-
 @op(required_resource_keys={"mongo"})
 def mongo_stats(context) -> List[str]:
     db = context.resources.mongo.db
@@ -127,41 +134,6 @@ def mongo_stats(context) -> List[str]:
     return collection_names
 
 
-@op(required_resource_keys={"terminus"})
-def update_schema(context):
-    with tempfile.TemporaryDirectory() as tmpdirname:
-        try:
-            context.log.info("shallow-cloning nmdc-schema repo")
-            subprocess.check_output(
-                "git clone https://github.com/microbiomedata/nmdc-schema.git"
-                f" --branch main --single-branch {tmpdirname}/nmdc-schema",
-                shell=True,
-            )
-            context.log.info("generating TerminusDB JSON-LD from NMDC LinkML")
-            subprocess.check_output(
-                f"gen-terminusdb {tmpdirname}/nmdc-schema/src/schema/nmdc.yaml"
-                f" > {tmpdirname}/nmdc.terminus.json",
-                shell=True,
-            )
-        except subprocess.CalledProcessError as e:
-            if e.stdout:
-                context.log.debug(e.stdout.decode())
-            if e.stderr:
-                context.log.error(e.stderr.decode())
-            context.log.debug(str(e.returncode))
-            raise e
-
-        with open(f"{tmpdirname}/nmdc.terminus.json") as f:
-            woql_dict = json.load(f)
-
-    context.log.info("Updating terminus schema via WOQLQuery")
-    rv = WQ(query=woql_dict).execute(
-        context.resources.terminus.client, "update schema via WOQL"
-    )
-    context.log.info(str(rv))
-    return rv
-
-
 @op(
     required_resource_keys={"mongo", "runtime_api_site_client"},
     retry_policy=RetryPolicy(max_retries=2),
@@ -768,6 +740,33 @@ def export_json_to_drs(
     return ["/objects/" + drs_object["id"]]
 
 
+@op(
+    description="NCBI Submission XML file rendered in a Dagster Asset",
+    out=Out(description="XML content rendered through Dagit UI"),
+)
+def ncbi_submission_xml_asset(context: OpExecutionContext, data: str):
+    filename = "ncbi_submission.xml"
+    file_path = os.path.join(context.instance.storage_directory(), filename)
+
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+    with open(file_path, "w") as f:
+        f.write(data)
+
+    context.log_event(
+        AssetMaterialization(
+            asset_key="ncbi_submission_xml",
+            description="NCBI Submission XML Data",
+            metadata={
+                "file_path": MetadataValue.path(file_path),
+                "xml": MetadataValue.text(data),
+            },
+        )
+    )
+
+    return Output(data)
+
+
 def unique_field_values(docs: List[Dict[str, Any]], field: str):
     return {doc[field] for doc in docs if field in doc}
 
@@ -977,3 +976,140 @@ def site_code_mapping() -> dict:
         raise Exception(
             f"Failed to fetch site data from {endpoint}. Status code: {response.status_code}, Content: {response.content}"
         )
+
+
+@op(required_resource_keys={"mongo"})
+def materialize_alldocs(context) -> int:
+    mdb = context.resources.mongo.db
+    collection_names = populated_schema_collection_names_with_id_field(mdb)
+
+    for name in collection_names:
+        assert (
+            len(collection_name_to_class_names[name]) == 1
+        ), f"{name} collection has class name of {collection_name_to_class_names[name]} and len {len(collection_name_to_class_names[name])}"
+
+    context.log.info(f"{collection_names=}")
+
+    # Drop any existing `alldocs` collection (e.g. from previous use of this op).
+    mdb.alldocs.drop()
+
+    # Build alldocs
+    context.log.info("constructing `alldocs` collection")
+
+    for collection in collection_names:
+        # Calculate class_hierarchy_as_list once per collection, using the first document in list
+        try:
+            nmdcdb = NMDCDatabase(
+                **{collection: [dissoc(mdb[collection].find_one(), "_id")]}
+            )
+            exemplar = getattr(nmdcdb, collection)[0]
+            newdoc_type: list[str] = class_hierarchy_as_list(exemplar)
+        except ValueError as e:
+            context.log.info(f"Collection {collection} does not exist.")
+            raise e
+
+        context.log.info(
+            f"Found {mdb[collection].estimated_document_count()} estimated documents for {collection=}."
+        )
+        # For each document in this collection, replace the value of the `type` field with
+        # a _list_ of the document's own class and ancestor classes, remove the `_id` field,
+        # and insert the resulting document into the `alldocs` collection.
+
+        inserted_many_result = mdb.alldocs.insert_many(
+            [
+                assoc(dissoc(doc, "type", "_id"), "type", newdoc_type)
+                for doc in mdb[collection].find()
+            ]
+        )
+        context.log.info(
+            f"Inserted {len(inserted_many_result.inserted_ids)} documents for {collection=}."
+        )
+
+    # Re-idx for `alldocs` collection
+    mdb.alldocs.create_index("id", unique=True)
+    context.log.info(
+        f"refreshed {mdb.alldocs} collection with {mdb.alldocs.estimated_document_count()} docs."
+    )
+    return mdb.alldocs.estimated_document_count()
+
+
+@op(config_schema={"nmdc_study_id": str}, required_resource_keys={"mongo"})
+def get_ncbi_export_pipeline_study(context: OpExecutionContext) -> Any:
+    nmdc_study = find_study_by_id(
+        context.op_config["nmdc_study_id"], context.resources.mongo.db
+    )
+    return nmdc_study
+
+
+@op(
+    config_schema={
+        "nmdc_ncbi_attribute_mapping_file_url": str,
+        "ncbi_submission_metadata": Field(
+            Permissive(
+                {
+                    "organization": String,
+                }
+            ),
+            is_required=True,
+            description="General metadata about the NCBI submission.",
+        ),
+        "ncbi_biosample_metadata": Field(
+            Permissive(
+                {
+                    "organism_name": String,
+                }
+            ),
+            is_required=True,
+            description="Metadata for one or many NCBI BioSample in the Submission.",
+        ),
+    },
+    out=Out(Dict),
+)
+def get_ncbi_export_pipeline_inputs(context: OpExecutionContext) -> str:
+    nmdc_ncbi_attribute_mapping_file_url = context.op_config[
+        "nmdc_ncbi_attribute_mapping_file_url"
+    ]
+    ncbi_submission_metadata = context.op_config.get("ncbi_submission_metadata", {})
+    ncbi_biosample_metadata = context.op_config.get("ncbi_biosample_metadata", {})
+
+    return {
+        "nmdc_ncbi_attribute_mapping_file_url": nmdc_ncbi_attribute_mapping_file_url,
+        "ncbi_submission_metadata": ncbi_submission_metadata,
+        "ncbi_biosample_metadata": ncbi_biosample_metadata,
+    }
+
+
+@op(required_resource_keys={"mongo"})
+def get_data_objects_from_biosamples(context: OpExecutionContext, biosamples: list):
+    mdb = context.resources.mongo.db
+    alldocs_collection = mdb["alldocs"]
+    biosample_data_objects = fetch_data_objects_from_biosamples(
+        alldocs_collection, biosamples
+    )
+    return biosample_data_objects
+
+
+@op(required_resource_keys={"mongo"})
+def get_omics_processing_from_biosamples(context: OpExecutionContext, biosamples: list):
+    mdb = context.resources.mongo.db
+    alldocs_collection = mdb["alldocs"]
+    biosample_omics_processing = fetch_omics_processing_from_biosamples(
+        alldocs_collection, biosamples
+    )
+    return biosample_omics_processing
+
+
+@op
+def ncbi_submission_xml_from_nmdc_study(
+    context: OpExecutionContext,
+    nmdc_study: Any,
+    ncbi_exporter_metadata: dict,
+    biosamples: list,
+    omics_processing_records: list,
+    data_objects: list,
+) -> str:
+    ncbi_exporter = NCBISubmissionXML(nmdc_study, ncbi_exporter_metadata)
+    ncbi_xml = ncbi_exporter.get_submission_xml(
+        biosamples, omics_processing_records, data_objects
+    )
+    return ncbi_xml