nmdc-runtime 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

nmdc_runtime/containers.py

@@ -9,7 +9,6 @@ from nmdc_runtime.infrastructure.database.impl.mongo.models.user import (
 
 
 class Container(containers.DeclarativeContainer):
-
     user_queries = providers.Singleton(UserQueries)
 
     user_service = providers.Factory(UserService, user_queries=user_queries)

nmdc_runtime/lib/nmdc_etl_class.py

@@ -196,7 +196,6 @@ class NMDC_ETL:
         print_df=False,
         print_dict=False,
     ) -> list:
-
         ## used for testing
         if test_rows != 0:
             nmdc_df = nmdc_df.head(test_rows)

nmdc_runtime/lib/transform_nmdc_data.py

@@ -995,7 +995,6 @@ def make_quantity_value(nmdc_objs: list, tx_attributes: list, **kwargs) -> list:
     for attribute in tx_attributes:
         for obj in nmdc_objs:
             if has_raw_value(obj, attribute):
-
                 val = getattr(obj, attribute)
 
                 ## split raw value after first space

nmdc_runtime/site/graphs.py

@@ -36,12 +36,17 @@ from nmdc_runtime.site.ops import (
     translate_portal_submission_to_nmdc_schema_database,
     validate_metadata,
     neon_data_by_product,
-    nmdc_schema_database_from_neon_data,
+    nmdc_schema_database_from_neon_soil_data,
+    nmdc_schema_database_from_neon_benthic_data,
     nmdc_schema_database_export_filename_neon,
     get_neon_pipeline_mms_data_product,
     get_neon_pipeline_sls_data_product,
     get_submission_portal_pipeline_inputs,
     get_csv_rows_from_url,
+    get_neon_pipeline_benthic_data_product,
+    get_neon_pipeline_inputs,
+    get_df_from_url,
+    site_code_mapping,
 )
 
 
@@ -198,14 +203,27 @@ def ingest_metadata_submission():
 
 
 @graph
-def translate_neon_api_metadata_to_nmdc_schema_database():
+def translate_neon_api_soil_metadata_to_nmdc_schema_database():
     mms_data_product = get_neon_pipeline_mms_data_product()
     sls_data_product = get_neon_pipeline_sls_data_product()
 
     mms_data = neon_data_by_product(mms_data_product)
     sls_data = neon_data_by_product(sls_data_product)
 
-    database = nmdc_schema_database_from_neon_data(mms_data, sls_data)
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_soil_data(
+        mms_data, sls_data, neon_envo_mappings_file, neon_raw_data_file_mappings_file
+    )
 
     database_dict = nmdc_schema_object_to_dict(database)
     filename = nmdc_schema_database_export_filename_neon()
@@ -215,13 +233,87 @@ def translate_neon_api_metadata_to_nmdc_schema_database():
 
 
 @graph
-def ingest_neon_metadata():
+def ingest_neon_soil_metadata():
     mms_data_product = get_neon_pipeline_mms_data_product()
     sls_data_product = get_neon_pipeline_sls_data_product()
 
     mms_data = neon_data_by_product(mms_data_product)
     sls_data = neon_data_by_product(sls_data_product)
 
-    database = nmdc_schema_database_from_neon_data(mms_data, sls_data)
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_soil_data(
+        mms_data, sls_data, neon_envo_mappings_file, neon_raw_data_file_mappings_file
+    )
+    run_id = submit_metadata_to_db(database)
+    poll_for_run_completion(run_id)
+
+
+@graph
+def translate_neon_api_benthic_metadata_to_nmdc_schema_database():
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    mms_benthic_data_product = get_neon_pipeline_benthic_data_product()
+    mms_benthic = neon_data_by_product(mms_benthic_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_benthic,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+
+    database_dict = nmdc_schema_object_to_dict(database)
+    filename = nmdc_schema_database_export_filename_neon()
+
+    outputs = export_json_to_drs(database_dict, filename)
+    add_output_run_event(outputs)
+
+
+@graph
+def ingest_neon_benthic_metadata():
+    mms_benthic_data_product = get_neon_pipeline_benthic_data_product()
+
+    mms_benthic = neon_data_by_product(mms_benthic_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_benthic,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)

nmdc_runtime/site/ops.py

@@ -6,7 +6,7 @@ import subprocess
 import tempfile
 from collections import defaultdict
 from datetime import datetime, timezone
-from io import BytesIO
+from io import BytesIO, StringIO
 from typing import Tuple
 from zipfile import ZipFile
 import pandas as pd
@@ -64,7 +64,10 @@ from nmdc_runtime.site.resources import (
     NeonApiClient,
 )
 from nmdc_runtime.site.translation.gold_translator import GoldStudyTranslator
-from nmdc_runtime.site.translation.neon_translator import NeonDataTranslator
+from nmdc_runtime.site.translation.neon_soil_translator import NeonSoilDataTranslator
+from nmdc_runtime.site.translation.neon_benthic_translator import (
+    NeonBenthicDataTranslator,
+)
 from nmdc_runtime.site.translation.submission_portal_translator import (
     SubmissionPortalTranslator,
 )
@@ -776,6 +779,11 @@ def get_neon_pipeline_sls_data_product(context: OpExecutionContext) -> dict:
     return context.op_config["sls_data_product"]
 
 
+@op(config_schema={"benthic_data_product": dict})
+def get_neon_pipeline_benthic_data_product(context: OpExecutionContext) -> dict:
+    return context.op_config["benthic_data_product"]
+
+
 @op(required_resource_keys={"neon_api_client"})
 def neon_data_by_product(
     context: OpExecutionContext, data_product: dict
@@ -803,10 +811,38 @@ def neon_data_by_product(
 
 
 @op(required_resource_keys={"runtime_api_site_client"})
-def nmdc_schema_database_from_neon_data(
+def nmdc_schema_database_from_neon_soil_data(
     context: OpExecutionContext,
     mms_data: Dict[str, pd.DataFrame],
     sls_data: Dict[str, pd.DataFrame],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
+) -> nmdc.Database:
+    client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
+
+    def id_minter(*args, **kwargs):
+        response = client.mint_id(*args, **kwargs)
+        return response.json()
+
+    translator = NeonSoilDataTranslator(
+        mms_data,
+        sls_data,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
+
+    database = translator.get_database()
+    return database
+
+
+@op(required_resource_keys={"runtime_api_site_client"})
+def nmdc_schema_database_from_neon_benthic_data(
+    context: OpExecutionContext,
+    benthic_data: Dict[str, pd.DataFrame],
+    site_code_mapping: Dict[str, str],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
 ) -> nmdc.Database:
     client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
 
@@ -814,12 +850,34 @@ def nmdc_schema_database_from_neon_data(
         response = client.mint_id(*args, **kwargs)
         return response.json()
 
-    translator = NeonDataTranslator(mms_data, sls_data, id_minter=id_minter)
+    translator = NeonBenthicDataTranslator(
+        benthic_data,
+        site_code_mapping,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
 
     database = translator.get_database()
     return database
 
 
+@op(
+    out={
+        "neon_envo_mappings_file_url": Out(),
+        "neon_raw_data_file_mappings_file_url": Out(),
+    }
+)
+def get_neon_pipeline_inputs(
+    neon_envo_mappings_file_url: str,
+    neon_raw_data_file_mappings_file_url: str,
+) -> Tuple[str, str]:
+    return (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    )
+
+
 @op
 def nmdc_schema_database_export_filename_neon() -> str:
     return "database_from_neon_metadata.json"
@@ -847,3 +905,41 @@ def get_csv_rows_from_url(url: Optional[str]) -> List[Dict]:
     # Collect all the rows into a list of dicts while stripping out (valfilter) cells where the
     # value is an empty string (identity returns a Falsy value).
     return [valfilter(identity, row) for row in reader]
+
+
+@op
+def get_df_from_url(url: str) -> pd.DataFrame:
+    """Download and return a pandas DataFrame from the URL of a TSV file.
+
+    :param url: raw URL of the TSV file to be downloaded as a DataFrame
+    :return: pandas DataFrame of TSV data
+    """
+    if not url:
+        return pd.DataFrame()
+
+    response = requests.get(url)
+    response.raise_for_status()
+
+    # Using Pandas read_csv to directly read the file-like object
+    df = pd.read_csv(url, delimiter="\t")
+
+    return df
+
+
+@op
+def site_code_mapping() -> dict:
+    endpoint = "https://data.neonscience.org/api/v0/sites/"
+    response = requests.get(endpoint)
+    if response.status_code == 200:
+        sites_data = response.json()
+        site_code_mapping = {
+            site["siteCode"]: f"USA: {site['stateName']}, {site['siteName']}".replace(
+                " NEON", ""
+            )
+            for site in sites_data["data"]
+        }
+        return site_code_mapping
+    else:
+        raise Exception(
+            f"Failed to fetch site data from {endpoint}. Status code: {response.status_code}, Content: {response.content}"
+        )

nmdc_runtime/site/repository.py

@@ -1,5 +1,7 @@
 import json
 
+from typing import Any
+
 from dagster import (
     repository,
     ScheduleDefinition,
@@ -34,8 +36,10 @@ from nmdc_runtime.site.graphs import (
     apply_changesheet,
     apply_metadata_in,
     hello_graph,
-    translate_neon_api_metadata_to_nmdc_schema_database,
-    ingest_neon_metadata,
+    translate_neon_api_soil_metadata_to_nmdc_schema_database,
+    translate_neon_api_benthic_metadata_to_nmdc_schema_database,
+    ingest_neon_soil_metadata,
+    ingest_neon_benthic_metadata,
 )
 from nmdc_runtime.site.resources import (
     get_mongo,
@@ -578,7 +582,7 @@ def biosample_submission_ingest():
                 },
             },
         ),
-        translate_neon_api_metadata_to_nmdc_schema_database.to_job(
+        translate_neon_api_soil_metadata_to_nmdc_schema_database.to_job(
             description="This job fetches the metadata associated with a given NEON data product code and translates it into an equivalent nmdc:Database object. The object is serialized to JSON and stored in DRS. This can be considered a dry-run for the `ingest_neon_metadata` job.",
             resource_defs=resource_defs,
             config={
@@ -590,10 +594,27 @@ def biosample_submission_ingest():
                                 "base_url": {"env": "NEON_API_BASE_URL"},
                                 "api_token": {"env": "NEON_API_TOKEN"},
                             },
-                        }
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
                     },
                 ),
                 "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
                     "get_neon_pipeline_mms_data_product": {
                         "config": {
                             "mms_data_product": {
@@ -610,11 +631,16 @@ def biosample_submission_ingest():
                             }
                         }
                     },
-                    "export_json_to_drs": {"config": {"username": ""}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
                 },
             },
         ),
-        ingest_neon_metadata.to_job(
+        ingest_neon_soil_metadata.to_job(
             description="This job fetches the metadata associated with a given data product code and translates it into an equivalent nmdc:Database object. This object is validated and ingested into Mongo via a `POST /metadata/json:submit` request.",
             resource_defs=resource_defs,
             config={
@@ -646,6 +672,95 @@ def biosample_submission_ingest():
                             }
                         }
                     },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                },
+            },
+        ),
+        translate_neon_api_benthic_metadata_to_nmdc_schema_database.to_job(
+            description="This job fetches the metadata associated with a given NEON data product code and translates it into an equivalent nmdc:Database object. The object is serialized to JSON and stored in DRS. This can be considered a dry-run for the `ingest_neon_metadata` job.",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
+                    },
+                ),
+                "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                    "get_neon_pipeline_benthic_data_product": {
+                        "config": {
+                            "benthic_data_product": {
+                                "product_id": "DP1.20279.001",
+                                "product_tables": "mms_benthicMetagenomeSequencing, mms_benthicMetagenomeDnaExtraction, mms_benthicRawDataFiles, amb_fieldParent",
+                            }
+                        }
+                    },
+                },
+            },
+        ),
+        ingest_neon_benthic_metadata.to_job(
+            description="",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        }
+                    },
+                ),
+                "ops": {
+                    "get_neon_pipeline_benthic_data_product": {
+                        "config": {
+                            "benthic_data_product": {
+                                "product_id": "DP1.20279.001",
+                                "product_tables": "mms_benthicMetagenomeSequencing, mms_benthicMetagenomeDnaExtraction, mms_benthicRawDataFiles, amb_fieldParent",
+                            }
+                        }
+                    },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
                 },
             },
         ),

nmdc_runtime/site/translation/gold_translator.py

@@ -212,7 +212,7 @@ class GoldStudyTranslator(Translator):
                 return None
             elif minimum_numeric_value is not None and maximum_numeric_value is None:
                 return nmdc.QuantityValue(
-                    has_raw_value=field_value,
+                    has_raw_value=minimum_numeric_value,
                     has_numeric_value=nmdc.Double(minimum_numeric_value),
                     has_unit=unit,
                 )