nmdc-runtime 1.5.0__tar.gz → 1.6.0__tar.gz

{nmdc_runtime-1.5.0 → nmdc_runtime-1.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nmdc_runtime
-Version: 1.5.0
+Version: 1.6.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston

{nmdc_runtime-1.5.0 → nmdc_runtime-1.6.0}/nmdc_runtime/api/analytics.py

@@ -49,11 +49,17 @@ class Analytics(BaseHTTPMiddleware):
         start = time()
         response = await call_next(request)
 
+        # Build a dictionary that describes the incoming request.
+        #
+        # Note: `request.headers` is an instance of `MultiDict`. References:
+        #       - https://www.starlette.io/requests/#headers
+        #       - https://multidict.aio-libs.org/en/stable/multidict/
+        #
         request_data = {
             "hostname": request.url.hostname,
             "ip_address": request.client.host,
             "path": request.url.path,
-            "user_agent": request.headers["user-agent"],
+            "user_agent": request.headers.get("user-agent"),
             "method": request.method,
             "status": response.status_code,
             "response_time": int((time() - start) * 1000),

{nmdc_runtime-1.5.0 → nmdc_runtime-1.6.0}/nmdc_runtime/site/graphs.py

@@ -38,9 +38,11 @@ from nmdc_runtime.site.ops import (
     neon_data_by_product,
     nmdc_schema_database_from_neon_soil_data,
     nmdc_schema_database_from_neon_benthic_data,
+    nmdc_schema_database_from_neon_surface_water_data,
     nmdc_schema_database_export_filename_neon,
     get_neon_pipeline_mms_data_product,
     get_neon_pipeline_sls_data_product,
+    get_neon_pipeline_surface_water_data_product,
     get_submission_portal_pipeline_inputs,
     get_csv_rows_from_url,
     get_neon_pipeline_benthic_data_product,
@@ -317,3 +319,65 @@ def ingest_neon_benthic_metadata():
     )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)
+
+
+@graph
+def translate_neon_api_surface_water_metadata_to_nmdc_schema_database():
+    mms_surface_water_data_product = get_neon_pipeline_surface_water_data_product()
+
+    mms_surface_water = neon_data_by_product(mms_surface_water_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_surface_water_data(
+        mms_surface_water,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+
+    database_dict = nmdc_schema_object_to_dict(database)
+    filename = nmdc_schema_database_export_filename_neon()
+
+    outputs = export_json_to_drs(database_dict, filename)
+    add_output_run_event(outputs)
+
+
+@graph
+def ingest_neon_surface_water_metadata():
+    mms_surface_water_data_product = get_neon_pipeline_surface_water_data_product()
+
+    mms_surface_water = neon_data_by_product(mms_surface_water_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_surface_water,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+    run_id = submit_metadata_to_db(database)
+    poll_for_run_completion(run_id)

{nmdc_runtime-1.5.0 → nmdc_runtime-1.6.0}/nmdc_runtime/site/ops.py

@@ -68,6 +68,9 @@ from nmdc_runtime.site.translation.neon_soil_translator import NeonSoilDataTrans
 from nmdc_runtime.site.translation.neon_benthic_translator import (
     NeonBenthicDataTranslator,
 )
+from nmdc_runtime.site.translation.neon_surface_water_translator import (
+    NeonSurfaceWaterDataTranslator,
+)
 from nmdc_runtime.site.translation.submission_portal_translator import (
     SubmissionPortalTranslator,
 )
@@ -784,6 +787,11 @@ def get_neon_pipeline_benthic_data_product(context: OpExecutionContext) -> dict:
     return context.op_config["benthic_data_product"]
 
 
+@op(config_schema={"surface_water_data_product": dict})
+def get_neon_pipeline_surface_water_data_product(context: OpExecutionContext) -> dict:
+    return context.op_config["surface_water_data_product"]
+
+
 @op(required_resource_keys={"neon_api_client"})
 def neon_data_by_product(
     context: OpExecutionContext, data_product: dict
@@ -862,6 +870,32 @@ def nmdc_schema_database_from_neon_benthic_data(
     return database
 
 
+@op(required_resource_keys={"runtime_api_site_client"})
+def nmdc_schema_database_from_neon_surface_water_data(
+    context: OpExecutionContext,
+    surface_water_data: Dict[str, pd.DataFrame],
+    site_code_mapping: Dict[str, str],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
+) -> nmdc.Database:
+    client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
+
+    def id_minter(*args, **kwargs):
+        response = client.mint_id(*args, **kwargs)
+        return response.json()
+
+    translator = NeonSurfaceWaterDataTranslator(
+        surface_water_data,
+        site_code_mapping,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
+
+    database = translator.get_database()
+    return database
+
+
 @op(
     out={
         "neon_envo_mappings_file_url": Out(),

{nmdc_runtime-1.5.0 → nmdc_runtime-1.6.0}/nmdc_runtime/site/repository.py

@@ -38,8 +38,10 @@ from nmdc_runtime.site.graphs import (
     hello_graph,
     translate_neon_api_soil_metadata_to_nmdc_schema_database,
     translate_neon_api_benthic_metadata_to_nmdc_schema_database,
+    translate_neon_api_surface_water_metadata_to_nmdc_schema_database,
     ingest_neon_soil_metadata,
     ingest_neon_benthic_metadata,
+    ingest_neon_surface_water_metadata,
 )
 from nmdc_runtime.site.resources import (
     get_mongo,
@@ -764,6 +766,89 @@ def biosample_submission_ingest():
                 },
             },
         ),
+        translate_neon_api_surface_water_metadata_to_nmdc_schema_database.to_job(
+            description="This job fetches the metadata associated with a given NEON data product code and translates it into an equivalent nmdc:Database object. The object is serialized to JSON and stored in DRS. This can be considered a dry-run for the `ingest_neon_metadata` job.",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
+                    },
+                ),
+                "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                    "get_neon_pipeline_surface_water_data_product": {
+                        "config": {
+                            "surface_water_data_product": {
+                                "product_id": "DP1.20281.001",
+                                "product_tables": "mms_swMetagenomeSequencing, mms_swMetagenomeDnaExtraction, amc_fieldGenetic, amc_fieldSuperParent",
+                            }
+                        }
+                    },
+                },
+            },
+        ),
+        ingest_neon_surface_water_metadata.to_job(
+            description="",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        }
+                    },
+                ),
+                "ops": {
+                    "get_neon_pipeline_surface_water_data_product": {
+                        "config": {
+                            "surface_water_data_product": {
+                                "product_id": "DP1.20281.001",
+                                "product_tables": "mms_swMetagenomeSequencing, mms_swMetagenomeDnaExtraction, amc_fieldGenetic, amc_fieldSuperParent",
+                            }
+                        }
+                    },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                },
+            },
+        ),
     ]