nmdc-runtime 1.5.0__py3-none-any.whl → 1.7.0__py3-none-any.whl

nmdc_runtime/site/graphs.py

@@ -22,7 +22,6 @@ from nmdc_runtime.site.ops import (
     hello,
     mongo_stats,
     submit_metadata_to_db,
-    update_schema,
     filter_ops_undone_expired,
     construct_jobs,
     maybe_post_jobs,
@@ -38,16 +37,25 @@ from nmdc_runtime.site.ops import (
     neon_data_by_product,
     nmdc_schema_database_from_neon_soil_data,
     nmdc_schema_database_from_neon_benthic_data,
+    nmdc_schema_database_from_neon_surface_water_data,
     nmdc_schema_database_export_filename_neon,
     get_neon_pipeline_mms_data_product,
     get_neon_pipeline_sls_data_product,
+    get_neon_pipeline_surface_water_data_product,
     get_submission_portal_pipeline_inputs,
     get_csv_rows_from_url,
     get_neon_pipeline_benthic_data_product,
     get_neon_pipeline_inputs,
     get_df_from_url,
     site_code_mapping,
+    get_ncbi_export_pipeline_study,
+    get_data_objects_from_biosamples,
+    get_omics_processing_from_biosamples,
+    get_ncbi_export_pipeline_inputs,
+    ncbi_submission_xml_from_nmdc_study,
+    ncbi_submission_xml_asset,
 )
+from nmdc_runtime.site.export.study_metadata import get_biosamples_by_study_id
 
 
 @graph
@@ -85,17 +93,6 @@ def hello_mongo():
     mongo_stats()
 
 
-@graph
-def update_terminus():
-    """
-    A pipeline definition. This example pipeline has a single solid.
-
-    For more hints on writing Dagster pipelines, see our documentation overview on Pipelines:
-    https://docs.dagster.io/overview/solids-pipelines/pipelines
-    """
-    update_schema()
-
-
 @graph
 def housekeeping():
     delete_operations(list_operations(filter_ops_undone_expired()))
@@ -317,3 +314,82 @@ def ingest_neon_benthic_metadata():
     )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)
+
+
+@graph
+def translate_neon_api_surface_water_metadata_to_nmdc_schema_database():
+    mms_surface_water_data_product = get_neon_pipeline_surface_water_data_product()
+
+    mms_surface_water = neon_data_by_product(mms_surface_water_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_surface_water_data(
+        mms_surface_water,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+
+    database_dict = nmdc_schema_object_to_dict(database)
+    filename = nmdc_schema_database_export_filename_neon()
+
+    outputs = export_json_to_drs(database_dict, filename)
+    add_output_run_event(outputs)
+
+
+@graph
+def ingest_neon_surface_water_metadata():
+    mms_surface_water_data_product = get_neon_pipeline_surface_water_data_product()
+
+    mms_surface_water = neon_data_by_product(mms_surface_water_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_surface_water,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+    run_id = submit_metadata_to_db(database)
+    poll_for_run_completion(run_id)
+
+
+@graph
+def nmdc_study_to_ncbi_submission_export():
+    nmdc_study = get_ncbi_export_pipeline_study()
+    ncbi_submission_metadata = get_ncbi_export_pipeline_inputs()
+    biosamples = get_biosamples_by_study_id(nmdc_study)
+    omics_processing_records = get_omics_processing_from_biosamples(biosamples)
+    data_objects = get_data_objects_from_biosamples(biosamples)
+    xml_data = ncbi_submission_xml_from_nmdc_study(
+        nmdc_study,
+        ncbi_submission_metadata,
+        biosamples,
+        omics_processing_records,
+        data_objects,
+    )
+    ncbi_submission_xml_asset(xml_data)

nmdc_runtime/site/ops.py

@@ -9,6 +9,7 @@ from datetime import datetime, timezone
 from io import BytesIO, StringIO
 from typing import Tuple
 from zipfile import ZipFile
+
 import pandas as pd
 import requests
 
@@ -29,10 +30,14 @@ from dagster import (
     String,
     op,
     Optional,
+    Field,
+    Permissive,
+    Bool,
 )
 from gridfs import GridFS
 from linkml_runtime.dumpers import json_dumper
 from linkml_runtime.utils.yamlutils import YAMLRoot
+from nmdc_runtime.api.db.mongo import get_mongo_db
 from nmdc_runtime.api.core.idgen import generate_one_id
 from nmdc_runtime.api.core.metadata import (
     _validate_changesheet,
@@ -42,6 +47,7 @@ from nmdc_runtime.api.core.metadata import (
 )
 from nmdc_runtime.api.core.util import dotted_path_for, hash_from_str, json_clean, now
 from nmdc_runtime.api.endpoints.util import persist_content_and_get_drs_object
+from nmdc_runtime.api.endpoints.find import find_study_by_id
 from nmdc_runtime.api.models.job import Job, JobOperationMetadata
 from nmdc_runtime.api.models.metadata import ChangesheetIn
 from nmdc_runtime.api.models.operation import (
@@ -55,6 +61,11 @@ from nmdc_runtime.api.models.run import (
     _add_run_complete_event,
 )
 from nmdc_runtime.api.models.util import ResultT
+from nmdc_runtime.site.export.ncbi_xml import NCBISubmissionXML
+from nmdc_runtime.site.export.ncbi_xml_utils import (
+    fetch_data_objects_from_biosamples,
+    fetch_omics_processing_from_biosamples,
+)
 from nmdc_runtime.site.drsobjects.ingest import mongo_add_docs_result_as_dict
 from nmdc_runtime.site.resources import (
     NmdcPortalApiClient,
@@ -68,6 +79,9 @@ from nmdc_runtime.site.translation.neon_soil_translator import NeonSoilDataTrans
 from nmdc_runtime.site.translation.neon_benthic_translator import (
     NeonBenthicDataTranslator,
 )
+from nmdc_runtime.site.translation.neon_surface_water_translator import (
+    NeonSurfaceWaterDataTranslator,
+)
 from nmdc_runtime.site.translation.submission_portal_translator import (
     SubmissionPortalTranslator,
 )
@@ -83,7 +97,6 @@ from nmdc_schema import nmdc
 from pydantic import BaseModel
 from pymongo.database import Database as MongoDatabase
 from starlette import status
-from terminusdb_client.woqlquery import WOQLQuery as WQ
 from toolz import assoc, dissoc, get_in, valfilter, identity
 
 
@@ -108,14 +121,6 @@ def log_env(context):
     context.log.info("\n".join(out))
 
 
-@op(required_resource_keys={"terminus"})
-def list_databases(context) -> List[String]:
-    client = context.resources.terminus.client
-    list_ = client.list_databases()
-    context.log.info(f"databases: {list_}")
-    return list_
-
-
 @op(required_resource_keys={"mongo"})
 def mongo_stats(context) -> List[str]:
     db = context.resources.mongo.db
@@ -124,41 +129,6 @@ def mongo_stats(context) -> List[str]:
     return collection_names
 
 
-@op(required_resource_keys={"terminus"})
-def update_schema(context):
-    with tempfile.TemporaryDirectory() as tmpdirname:
-        try:
-            context.log.info("shallow-cloning nmdc-schema repo")
-            subprocess.check_output(
-                "git clone https://github.com/microbiomedata/nmdc-schema.git"
-                f" --branch main --single-branch {tmpdirname}/nmdc-schema",
-                shell=True,
-            )
-            context.log.info("generating TerminusDB JSON-LD from NMDC LinkML")
-            subprocess.check_output(
-                f"gen-terminusdb {tmpdirname}/nmdc-schema/src/schema/nmdc.yaml"
-                f" > {tmpdirname}/nmdc.terminus.json",
-                shell=True,
-            )
-        except subprocess.CalledProcessError as e:
-            if e.stdout:
-                context.log.debug(e.stdout.decode())
-            if e.stderr:
-                context.log.error(e.stderr.decode())
-            context.log.debug(str(e.returncode))
-            raise e
-
-        with open(f"{tmpdirname}/nmdc.terminus.json") as f:
-            woql_dict = json.load(f)
-
-    context.log.info("Updating terminus schema via WOQLQuery")
-    rv = WQ(query=woql_dict).execute(
-        context.resources.terminus.client, "update schema via WOQL"
-    )
-    context.log.info(str(rv))
-    return rv
-
-
 @op(
     required_resource_keys={"mongo", "runtime_api_site_client"},
     retry_policy=RetryPolicy(max_retries=2),
@@ -765,6 +735,33 @@ def export_json_to_drs(
     return ["/objects/" + drs_object["id"]]
 
 
+@op(
+    description="NCBI Submission XML file rendered in a Dagster Asset",
+    out=Out(description="XML content rendered through Dagit UI"),
+)
+def ncbi_submission_xml_asset(context: OpExecutionContext, data: str):
+    filename = "ncbi_submission.xml"
+    file_path = os.path.join(context.instance.storage_directory(), filename)
+
+    os.makedirs(os.path.dirname(file_path), exist_ok=True)
+
+    with open(file_path, "w") as f:
+        f.write(data)
+
+    context.log_event(
+        AssetMaterialization(
+            asset_key="ncbi_submission_xml",
+            description="NCBI Submission XML Data",
+            metadata={
+                "file_path": MetadataValue.path(file_path),
+                "xml": MetadataValue.text(data),
+            },
+        )
+    )
+
+    return Output(data)
+
+
 def unique_field_values(docs: List[Dict[str, Any]], field: str):
     return {doc[field] for doc in docs if field in doc}
 
@@ -784,6 +781,11 @@ def get_neon_pipeline_benthic_data_product(context: OpExecutionContext) -> dict:
     return context.op_config["benthic_data_product"]
 
 
+@op(config_schema={"surface_water_data_product": dict})
+def get_neon_pipeline_surface_water_data_product(context: OpExecutionContext) -> dict:
+    return context.op_config["surface_water_data_product"]
+
+
 @op(required_resource_keys={"neon_api_client"})
 def neon_data_by_product(
     context: OpExecutionContext, data_product: dict
@@ -862,6 +864,32 @@ def nmdc_schema_database_from_neon_benthic_data(
     return database
 
 
+@op(required_resource_keys={"runtime_api_site_client"})
+def nmdc_schema_database_from_neon_surface_water_data(
+    context: OpExecutionContext,
+    surface_water_data: Dict[str, pd.DataFrame],
+    site_code_mapping: Dict[str, str],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
+) -> nmdc.Database:
+    client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
+
+    def id_minter(*args, **kwargs):
+        response = client.mint_id(*args, **kwargs)
+        return response.json()
+
+    translator = NeonSurfaceWaterDataTranslator(
+        surface_water_data,
+        site_code_mapping,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
+
+    database = translator.get_database()
+    return database
+
+
 @op(
     out={
         "neon_envo_mappings_file_url": Out(),
@@ -943,3 +971,85 @@ def site_code_mapping() -> dict:
         raise Exception(
             f"Failed to fetch site data from {endpoint}. Status code: {response.status_code}, Content: {response.content}"
         )
+
+
+@op(config_schema={"nmdc_study_id": str}, required_resource_keys={"mongo"})
+def get_ncbi_export_pipeline_study(context: OpExecutionContext) -> Any:
+    nmdc_study = find_study_by_id(
+        context.op_config["nmdc_study_id"], context.resources.mongo.db
+    )
+    return nmdc_study
+
+
+@op(
+    config_schema={
+        "nmdc_ncbi_attribute_mapping_file_url": str,
+        "ncbi_submission_metadata": Field(
+            Permissive(
+                {
+                    "organization": String,
+                }
+            ),
+            is_required=True,
+            description="General metadata about the NCBI submission.",
+        ),
+        "ncbi_biosample_metadata": Field(
+            Permissive(
+                {
+                    "organism_name": String,
+                }
+            ),
+            is_required=True,
+            description="Metadata for one or many NCBI BioSample in the Submission.",
+        ),
+    },
+    out=Out(Dict),
+)
+def get_ncbi_export_pipeline_inputs(context: OpExecutionContext) -> str:
+    nmdc_ncbi_attribute_mapping_file_url = context.op_config[
+        "nmdc_ncbi_attribute_mapping_file_url"
+    ]
+    ncbi_submission_metadata = context.op_config.get("ncbi_submission_metadata", {})
+    ncbi_biosample_metadata = context.op_config.get("ncbi_biosample_metadata", {})
+
+    return {
+        "nmdc_ncbi_attribute_mapping_file_url": nmdc_ncbi_attribute_mapping_file_url,
+        "ncbi_submission_metadata": ncbi_submission_metadata,
+        "ncbi_biosample_metadata": ncbi_biosample_metadata,
+    }
+
+
+@op(required_resource_keys={"mongo"})
+def get_data_objects_from_biosamples(context: OpExecutionContext, biosamples: list):
+    mdb = context.resources.mongo.db
+    alldocs_collection = mdb["alldocs"]
+    biosample_data_objects = fetch_data_objects_from_biosamples(
+        alldocs_collection, biosamples
+    )
+    return biosample_data_objects
+
+
+@op(required_resource_keys={"mongo"})
+def get_omics_processing_from_biosamples(context: OpExecutionContext, biosamples: list):
+    mdb = context.resources.mongo.db
+    alldocs_collection = mdb["alldocs"]
+    biosample_omics_processing = fetch_omics_processing_from_biosamples(
+        alldocs_collection, biosamples
+    )
+    return biosample_omics_processing
+
+
+@op
+def ncbi_submission_xml_from_nmdc_study(
+    context: OpExecutionContext,
+    nmdc_study: Any,
+    ncbi_exporter_metadata: dict,
+    biosamples: list,
+    omics_processing_records: list,
+    data_objects: list,
+) -> str:
+    ncbi_exporter = NCBISubmissionXML(nmdc_study, ncbi_exporter_metadata)
+    ncbi_xml = ncbi_exporter.get_submission_xml(
+        biosamples, omics_processing_records, data_objects
+    )
+    return ncbi_xml

nmdc_runtime/site/repository.py

@@ -38,8 +38,11 @@ from nmdc_runtime.site.graphs import (
     hello_graph,
     translate_neon_api_soil_metadata_to_nmdc_schema_database,
     translate_neon_api_benthic_metadata_to_nmdc_schema_database,
+    translate_neon_api_surface_water_metadata_to_nmdc_schema_database,
     ingest_neon_soil_metadata,
     ingest_neon_benthic_metadata,
+    ingest_neon_surface_water_metadata,
+    nmdc_study_to_ncbi_submission_export,
 )
 from nmdc_runtime.site.resources import (
     get_mongo,
@@ -48,7 +51,6 @@ from nmdc_runtime.site.resources import (
     nmdc_portal_api_client_resource,
     gold_api_client_resource,
     neon_api_client_resource,
-    terminus_resource,
     mongo_resource,
 )
 from nmdc_runtime.site.resources import (
@@ -66,7 +68,6 @@ resource_defs = {
     "nmdc_portal_api_client": nmdc_portal_api_client_resource,
     "gold_api_client": gold_api_client_resource,
     "neon_api_client": neon_api_client_resource,
-    "terminus": terminus_resource,
     "mongo": mongo_resource,
 }
 
@@ -513,8 +514,8 @@ def biosample_submission_ingest():
                         "nmdc_portal_api_client": {
                             "config": {
                                 "base_url": {"env": "NMDC_PORTAL_API_BASE_URL"},
-                                "session_cookie": {
-                                    "env": "NMDC_PORTAL_API_SESSION_COOKIE"
+                                "refresh_token": {
+                                    "env": "NMDC_PORTAL_API_REFRESH_TOKEN"
                                 },
                             }
                         }
@@ -553,8 +554,8 @@ def biosample_submission_ingest():
                         "nmdc_portal_api_client": {
                             "config": {
                                 "base_url": {"env": "NMDC_PORTAL_API_BASE_URL"},
-                                "session_cookie": {
-                                    "env": "NMDC_PORTAL_API_SESSION_COOKIE"
+                                "refresh_token": {
+                                    "env": "NMDC_PORTAL_API_REFRESH_TOKEN"
                                 },
                             }
                         }
@@ -764,6 +765,140 @@ def biosample_submission_ingest():
                 },
             },
         ),
+        translate_neon_api_surface_water_metadata_to_nmdc_schema_database.to_job(
+            description="This job fetches the metadata associated with a given NEON data product code and translates it into an equivalent nmdc:Database object. The object is serialized to JSON and stored in DRS. This can be considered a dry-run for the `ingest_neon_metadata` job.",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
+                    },
+                ),
+                "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                    "get_neon_pipeline_surface_water_data_product": {
+                        "config": {
+                            "surface_water_data_product": {
+                                "product_id": "DP1.20281.001",
+                                "product_tables": "mms_swMetagenomeSequencing, mms_swMetagenomeDnaExtraction, amc_fieldGenetic, amc_fieldSuperParent",
+                            }
+                        }
+                    },
+                },
+            },
+        ),
+        ingest_neon_surface_water_metadata.to_job(
+            description="",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        }
+                    },
+                ),
+                "ops": {
+                    "get_neon_pipeline_surface_water_data_product": {
+                        "config": {
+                            "surface_water_data_product": {
+                                "product_id": "DP1.20281.001",
+                                "product_tables": "mms_swMetagenomeSequencing, mms_swMetagenomeDnaExtraction, amc_fieldGenetic, amc_fieldSuperParent",
+                            }
+                        }
+                    },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                },
+            },
+        ),
+    ]
+
+
+@repository
+def biosample_export():
+    normal_resources = run_config_frozen__normal_env["resources"]
+    return [
+        nmdc_study_to_ncbi_submission_export.to_job(
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "mongo": {
+                            "config": {
+                                "host": {"env": "MONGO_HOST"},
+                                "username": {"env": "MONGO_USERNAME"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "dbname": {"env": "MONGO_DBNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
+                    },
+                ),
+                "ops": {
+                    "get_ncbi_export_pipeline_study": {
+                        "config": {
+                            "nmdc_study_id": "",
+                        }
+                    },
+                    "get_ncbi_export_pipeline_inputs": {
+                        "config": {
+                            "nmdc_ncbi_attribute_mapping_file_url": "",
+                            "ncbi_submission_metadata": {
+                                "organization": "",
+                            },
+                            "ncbi_biosample_metadata": {
+                                "organism_name": "",
+                            },
+                        }
+                    },
+                },
+            },
+        ),
     ]