nmdc-runtime 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

nmdc_runtime/containers.py

@@ -9,7 +9,6 @@ from nmdc_runtime.infrastructure.database.impl.mongo.models.user import (
 
 
 class Container(containers.DeclarativeContainer):
-
     user_queries = providers.Singleton(UserQueries)
 
     user_service = providers.Factory(UserService, user_queries=user_queries)

nmdc_runtime/lib/nmdc_etl_class.py

@@ -196,7 +196,6 @@ class NMDC_ETL:
         print_df=False,
         print_dict=False,
     ) -> list:
-
         ## used for testing
         if test_rows != 0:
             nmdc_df = nmdc_df.head(test_rows)

nmdc_runtime/lib/transform_nmdc_data.py

@@ -995,7 +995,6 @@ def make_quantity_value(nmdc_objs: list, tx_attributes: list, **kwargs) -> list:
     for attribute in tx_attributes:
         for obj in nmdc_objs:
             if has_raw_value(obj, attribute):
-
                 val = getattr(obj, attribute)
 
                 ## split raw value after first space

nmdc_runtime/site/graphs.py

@@ -44,6 +44,9 @@ from nmdc_runtime.site.ops import (
     get_submission_portal_pipeline_inputs,
     get_csv_rows_from_url,
     get_neon_pipeline_benthic_data_product,
+    get_neon_pipeline_inputs,
+    get_df_from_url,
+    site_code_mapping,
 )
 
 
@@ -207,7 +210,20 @@ def translate_neon_api_soil_metadata_to_nmdc_schema_database():
     mms_data = neon_data_by_product(mms_data_product)
     sls_data = neon_data_by_product(sls_data_product)
 
-    database = nmdc_schema_database_from_neon_soil_data(mms_data, sls_data)
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_soil_data(
+        mms_data, sls_data, neon_envo_mappings_file, neon_raw_data_file_mappings_file
+    )
 
     database_dict = nmdc_schema_object_to_dict(database)
     filename = nmdc_schema_database_export_filename_neon()
@@ -224,18 +240,48 @@ def ingest_neon_soil_metadata():
     mms_data = neon_data_by_product(mms_data_product)
     sls_data = neon_data_by_product(sls_data_product)
 
-    database = nmdc_schema_database_from_neon_soil_data(mms_data, sls_data)
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_soil_data(
+        mms_data, sls_data, neon_envo_mappings_file, neon_raw_data_file_mappings_file
+    )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)
 
 
 @graph
 def translate_neon_api_benthic_metadata_to_nmdc_schema_database():
-    mms_benthic_data_product = get_neon_pipeline_benthic_data_product()
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
 
+    mms_benthic_data_product = get_neon_pipeline_benthic_data_product()
     mms_benthic = neon_data_by_product(mms_benthic_data_product)
 
-    database = nmdc_schema_database_from_neon_benthic_data(mms_benthic)
+    sites_mapping_dict = site_code_mapping()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_benthic,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
 
     database_dict = nmdc_schema_object_to_dict(database)
     filename = nmdc_schema_database_export_filename_neon()
@@ -250,6 +296,24 @@ def ingest_neon_benthic_metadata():
 
     mms_benthic = neon_data_by_product(mms_benthic_data_product)
 
-    database = nmdc_schema_database_from_neon_benthic_data(mms_benthic)
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_benthic,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)

nmdc_runtime/site/ops.py

@@ -6,7 +6,7 @@ import subprocess
 import tempfile
 from collections import defaultdict
 from datetime import datetime, timezone
-from io import BytesIO
+from io import BytesIO, StringIO
 from typing import Tuple
 from zipfile import ZipFile
 import pandas as pd
@@ -815,6 +815,8 @@ def nmdc_schema_database_from_neon_soil_data(
     context: OpExecutionContext,
     mms_data: Dict[str, pd.DataFrame],
     sls_data: Dict[str, pd.DataFrame],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
 ) -> nmdc.Database:
     client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
 
@@ -822,7 +824,13 @@ def nmdc_schema_database_from_neon_soil_data(
         response = client.mint_id(*args, **kwargs)
         return response.json()
 
-    translator = NeonSoilDataTranslator(mms_data, sls_data, id_minter=id_minter)
+    translator = NeonSoilDataTranslator(
+        mms_data,
+        sls_data,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
 
     database = translator.get_database()
     return database
@@ -832,6 +840,9 @@ def nmdc_schema_database_from_neon_soil_data(
 def nmdc_schema_database_from_neon_benthic_data(
     context: OpExecutionContext,
     benthic_data: Dict[str, pd.DataFrame],
+    site_code_mapping: Dict[str, str],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
 ) -> nmdc.Database:
     client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
 
@@ -839,12 +850,34 @@ def nmdc_schema_database_from_neon_benthic_data(
         response = client.mint_id(*args, **kwargs)
         return response.json()
 
-    translator = NeonBenthicDataTranslator(benthic_data, id_minter=id_minter)
+    translator = NeonBenthicDataTranslator(
+        benthic_data,
+        site_code_mapping,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
 
     database = translator.get_database()
     return database
 
 
+@op(
+    out={
+        "neon_envo_mappings_file_url": Out(),
+        "neon_raw_data_file_mappings_file_url": Out(),
+    }
+)
+def get_neon_pipeline_inputs(
+    neon_envo_mappings_file_url: str,
+    neon_raw_data_file_mappings_file_url: str,
+) -> Tuple[str, str]:
+    return (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    )
+
+
 @op
 def nmdc_schema_database_export_filename_neon() -> str:
     return "database_from_neon_metadata.json"
@@ -872,3 +905,41 @@ def get_csv_rows_from_url(url: Optional[str]) -> List[Dict]:
     # Collect all the rows into a list of dicts while stripping out (valfilter) cells where the
     # value is an empty string (identity returns a Falsy value).
     return [valfilter(identity, row) for row in reader]
+
+
+@op
+def get_df_from_url(url: str) -> pd.DataFrame:
+    """Download and return a pandas DataFrame from the URL of a TSV file.
+
+    :param url: raw URL of the TSV file to be downloaded as a DataFrame
+    :return: pandas DataFrame of TSV data
+    """
+    if not url:
+        return pd.DataFrame()
+
+    response = requests.get(url)
+    response.raise_for_status()
+
+    # Using Pandas read_csv to directly read the file-like object
+    df = pd.read_csv(url, delimiter="\t")
+
+    return df
+
+
+@op
+def site_code_mapping() -> dict:
+    endpoint = "https://data.neonscience.org/api/v0/sites/"
+    response = requests.get(endpoint)
+    if response.status_code == 200:
+        sites_data = response.json()
+        site_code_mapping = {
+            site["siteCode"]: f"USA: {site['stateName']}, {site['siteName']}".replace(
+                " NEON", ""
+            )
+            for site in sites_data["data"]
+        }
+        return site_code_mapping
+    else:
+        raise Exception(
+            f"Failed to fetch site data from {endpoint}. Status code: {response.status_code}, Content: {response.content}"
+        )

nmdc_runtime/site/repository.py

@@ -1,5 +1,7 @@
 import json
 
+from typing import Any
+
 from dagster import (
     repository,
     ScheduleDefinition,
@@ -592,10 +594,27 @@ def biosample_submission_ingest():
                                 "base_url": {"env": "NEON_API_BASE_URL"},
                                 "api_token": {"env": "NEON_API_TOKEN"},
                             },
-                        }
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
                     },
                 ),
                 "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
                     "get_neon_pipeline_mms_data_product": {
                         "config": {
                             "mms_data_product": {
@@ -612,7 +631,12 @@ def biosample_submission_ingest():
                             }
                         }
                     },
-                    "export_json_to_drs": {"config": {"username": ""}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
                 },
             },
         ),
@@ -648,6 +672,12 @@ def biosample_submission_ingest():
                             }
                         }
                     },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
                 },
             },
         ),
@@ -663,10 +693,33 @@ def biosample_submission_ingest():
                                 "base_url": {"env": "NEON_API_BASE_URL"},
                                 "api_token": {"env": "NEON_API_TOKEN"},
                             },
-                        }
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
                     },
                 ),
                 "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
                     "get_neon_pipeline_benthic_data_product": {
                         "config": {
                             "benthic_data_product": {
@@ -675,7 +728,6 @@ def biosample_submission_ingest():
                             }
                         }
                     },
-                    "export_json_to_drs": {"config": {"username": ""}},
                 },
             },
         ),
@@ -703,6 +755,12 @@ def biosample_submission_ingest():
                             }
                         }
                     },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
                 },
             },
         ),

nmdc_runtime/site/translation/gold_translator.py

@@ -212,7 +212,7 @@ class GoldStudyTranslator(Translator):
                 return None
             elif minimum_numeric_value is not None and maximum_numeric_value is None:
                 return nmdc.QuantityValue(
-                    has_raw_value=field_value,
+                    has_raw_value=minimum_numeric_value,
                     has_numeric_value=nmdc.Double(minimum_numeric_value),
                     has_unit=unit,
                 )

nmdc_runtime/site/translation/neon_benthic_translator.py

@@ -2,13 +2,20 @@ import re
 import sqlite3
 
 import pandas as pd
-import requests
 import requests_cache
 
 from nmdc_schema import nmdc
 from nmdc_runtime.site.translation.translator import Translator
 from nmdc_runtime.site.util import get_basename
-from nmdc_runtime.site.translation.neon_utils import _get_value_or_none, _create_controlled_identified_term_value, _create_controlled_term_value, _create_geolocation_value, _create_quantity_value, _create_timestamp_value, _create_text_value
+from nmdc_runtime.site.translation.neon_utils import (
+    _get_value_or_none,
+    _create_controlled_identified_term_value,
+    _create_controlled_term_value,
+    _create_geolocation_value,
+    _create_quantity_value,
+    _create_timestamp_value,
+    _create_text_value,
+)
 
 
 BENTHIC_BROAD_SCALE_MAPPINGS = {
@@ -34,7 +41,15 @@ BENTHIC_ENV_MEDIUM_MAPPINGS = {
 
 
 class NeonBenthicDataTranslator(Translator):
-    def __init__(self, benthic_data: dict, *args, **kwargs) -> None:
+    def __init__(
+        self,
+        benthic_data: dict,
+        site_code_mapping: dict,
+        neon_envo_mappings_file: pd.DataFrame,
+        neon_raw_data_file_mappings_file: pd.DataFrame,
+        *args,
+        **kwargs,
+    ) -> None:
         super().__init__(*args, **kwargs)
 
         self.conn = sqlite3.connect("neon.db")
@@ -67,29 +82,16 @@ class NeonBenthicDataTranslator(Translator):
                 f"You are missing one of the aquatic benthic microbiome tables: {neon_amb_data_tables}"
             )
 
-        neon_envo_mappings_file = "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv"
-        neon_envo_terms = pd.read_csv(neon_envo_mappings_file, delimiter="\t")
-        neon_envo_terms.to_sql(
+        neon_envo_mappings_file.to_sql(
             "neonEnvoTerms", self.conn, if_exists="replace", index=False
         )
 
-        neon_raw_data_file_mappings_file = "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv"
-        self.neon_raw_data_file_mappings_df = pd.read_csv(
-            neon_raw_data_file_mappings_file, delimiter="\t"
-        )
+        self.neon_raw_data_file_mappings_df = neon_raw_data_file_mappings_file
         self.neon_raw_data_file_mappings_df.to_sql(
             "neonRawDataFile", self.conn, if_exists="replace", index=False
         )
 
-    def get_site_by_code(self, site_code: str) -> str:
-        site_response = requests.get(
-            f"https://data.neonscience.org/api/v0/sites/{site_code}"
-        )
-        if site_response.status_code == 200:
-            site_response = site_response.json()
-            return f"USA: {site_response['data']['stateName']}, {site_response['data']['siteName']}".replace(
-                " NEON", ""
-            )
+        self.site_code_mapping = site_code_mapping
 
     def _translate_biosample(
         self, neon_id: str, nmdc_id: str, biosample_row: pd.DataFrame
@@ -134,7 +136,7 @@ class NeonBenthicDataTranslator(Translator):
                 biosample_row["fieldSampleVolume"].values[0], "mL"
             ),
             geo_loc_name=_create_text_value(
-                self.get_site_by_code(biosample_row["siteID"].values[0])
+                self.site_code_mapping[biosample_row["siteID"].values[0]]
                 if biosample_row["siteID"].values[0]
                 else None
             ),
@@ -210,9 +212,7 @@ class NeonBenthicDataTranslator(Translator):
         :return: Object that using LibraryPreparation process model.
         """
         processing_institution = None
-        laboratory_name = _get_value_or_none(
-            library_preparation_row, "laboratoryName"
-        )
+        laboratory_name = _get_value_or_none(library_preparation_row, "laboratoryName")
         if laboratory_name is not None:
             if re.search("Battelle", laboratory_name, re.IGNORECASE):
                 processing_institution = "Battelle"
@@ -263,9 +263,7 @@ class NeonBenthicDataTranslator(Translator):
             has_input=processed_sample_id,
             has_output=raw_data_file_data,
             processing_institution=processing_institution,
-            ncbi_project_name=_get_value_or_none(
-                omics_processing_row, "ncbiProjectID"
-            ),
+            ncbi_project_name=_get_value_or_none(omics_processing_row, "ncbiProjectID"),
             omics_type=_create_controlled_term_value(
                 omics_processing_row["investigation_type"].values[0]
             ),
@@ -434,9 +432,7 @@ class NeonBenthicDataTranslator(Translator):
         )
 
         for neon_id, nmdc_id in neon_to_nmdc_biosample_ids.items():
-            biosample_row = benthic_samples[
-                benthic_samples["sampleID"] == neon_id
-            ]
+            biosample_row = benthic_samples[benthic_samples["sampleID"] == neon_id]
 
             database.biosample_set.append(
                 self._translate_biosample(neon_id, nmdc_id, biosample_row)
@@ -458,7 +454,9 @@ class NeonBenthicDataTranslator(Translator):
                     )
                 )
 
-                genomics_sample_id = _get_value_or_none(extraction_row, "genomicsSampleID")
+                genomics_sample_id = _get_value_or_none(
+                    extraction_row, "genomicsSampleID"
+                )
 
                 database.processed_sample_set.append(
                     self._translate_processed_sample(

nmdc_runtime/site/translation/neon_soil_translator.py

@@ -1,17 +1,34 @@
 import re
 import sqlite3
-from typing import Union, List
+from typing import List
 
 import pandas as pd
 
 from nmdc_schema import nmdc
 from nmdc_runtime.site.translation.translator import Translator
 from nmdc_runtime.site.util import get_basename
-from nmdc_runtime.site.translation.neon_utils import _get_value_or_none, _create_controlled_identified_term_value, _create_controlled_term_value, _create_geolocation_value, _create_quantity_value, _create_timestamp_value, _create_text_value, _create_double_value
+from nmdc_runtime.site.translation.neon_utils import (
+    _get_value_or_none,
+    _create_controlled_identified_term_value,
+    _create_controlled_term_value,
+    _create_geolocation_value,
+    _create_quantity_value,
+    _create_timestamp_value,
+    _create_text_value,
+    _create_double_value,
+)
 
 
 class NeonSoilDataTranslator(Translator):
-    def __init__(self, mms_data: dict, sls_data: dict, *args, **kwargs) -> None:
+    def __init__(
+        self,
+        mms_data: dict,
+        sls_data: dict,
+        neon_envo_mappings_file: pd.DataFrame,
+        neon_raw_data_file_mappings_file: pd.DataFrame,
+        *args,
+        **kwargs,
+    ) -> None:
         super().__init__(*args, **kwargs)
 
         self.conn = sqlite3.connect("neon.db")
@@ -73,16 +90,11 @@ class NeonSoilDataTranslator(Translator):
                 f"You are missing one of the soil periodic tables: {neon_sls_data_tables}"
             )
 
-        neon_envo_mappings_file = "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv"
-        neon_envo_terms = pd.read_csv(neon_envo_mappings_file, delimiter="\t")
-        neon_envo_terms.to_sql(
+        neon_envo_mappings_file.to_sql(
             "neonEnvoTerms", self.conn, if_exists="replace", index=False
         )
 
-        neon_raw_data_file_mappings_file = "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv"
-        self.neon_raw_data_file_mappings_df = pd.read_csv(
-            neon_raw_data_file_mappings_file, delimiter="\t"
-        )
+        self.neon_raw_data_file_mappings_df = neon_raw_data_file_mappings_file
         self.neon_raw_data_file_mappings_df.to_sql(
             "neonRawDataFile", self.conn, if_exists="replace", index=False
         )
@@ -124,9 +136,7 @@ class NeonSoilDataTranslator(Translator):
             collection_date=_create_timestamp_value(
                 biosample_row["collectDate"].values[0]
             ),
-            temp=_create_quantity_value(
-                biosample_row["soilTemp"].values[0], "Celsius"
-            ),
+            temp=_create_quantity_value(biosample_row["soilTemp"].values[0], "Celsius"),
             depth=nmdc.QuantityValue(
                 has_minimum_numeric_value=_get_value_or_none(
                     biosample_row, "sampleTopDepth"
@@ -136,13 +146,9 @@ class NeonSoilDataTranslator(Translator):
                 ),
                 has_unit="m",
             ),
-            samp_collec_device=_get_value_or_none(
-                biosample_row, "soilSamplingDevice"
-            ),
+            samp_collec_device=_get_value_or_none(biosample_row, "soilSamplingDevice"),
             soil_horizon=_get_value_or_none(biosample_row, "horizon"),
-            analysis_type=_get_value_or_none(
-                biosample_row, "sequenceAnalysisType"
-            ),
+            analysis_type=_get_value_or_none(biosample_row, "sequenceAnalysisType"),
             env_package=_create_text_value(biosample_row["sampleType"].values[0]),
             nitro=_create_quantity_value(
                 biosample_row["nitrogenPercent"].values[0], "percent"
@@ -301,9 +307,7 @@ class NeonSoilDataTranslator(Translator):
         :return: Object that using LibraryPreparation process model.
         """
         processing_institution = None
-        laboratory_name = _get_value_or_none(
-            library_preparation_row, "laboratoryName"
-        )
+        laboratory_name = _get_value_or_none(library_preparation_row, "laboratoryName")
         if laboratory_name is not None:
             if re.search("Battelle", laboratory_name, re.IGNORECASE):
                 processing_institution = "Battelle"
@@ -354,9 +358,7 @@ class NeonSoilDataTranslator(Translator):
             has_input=processed_sample_id,
             has_output=raw_data_file_data,
             processing_institution=processing_institution,
-            ncbi_project_name=_get_value_or_none(
-                omics_processing_row, "ncbiProjectID"
-            ),
+            ncbi_project_name=_get_value_or_none(omics_processing_row, "ncbiProjectID"),
             omics_type=_create_controlled_term_value(
                 omics_processing_row["investigation_type"].values[0]
             ),

nmdc_runtime/site/translation/neon_utils.py

@@ -5,35 +5,34 @@ import pandas as pd
 from nmdc_schema import nmdc
 
 
-def _get_value_or_none(
-        data: pd.DataFrame, column_name: str
-    ) -> Union[str, float, None]:
-        """
-        Get the value from the specified column in the data DataFrame.
-        If the column value is NaN, return None. However, there are handlers
-        for a select set of columns - horizon, qaqcStatus, sampleTopDepth,
-        and sampleBottomDepth.
-
-        :param data: DataFrame to read the column value from.
-        :return: Either a string, float or None depending on the column/column values.
-        """
-        if (
-            column_name in data
-            and not data[column_name].isna().any()
-            and not data[column_name].empty
-        ):
-            if column_name == "horizon":
-                return f"{data[column_name].values[0]} horizon"
-            elif column_name == "qaqcStatus":
-                return data[column_name].values[0].lower()
-            elif column_name == "sampleTopDepth":
-                return float(data[column_name].values[0]) / 100
-            elif column_name == "sampleBottomDepth":
-                return float(data[column_name].values[0]) / 100
-            else:
-                return data[column_name].values[0]
+def _get_value_or_none(data: pd.DataFrame, column_name: str) -> Union[str, float, None]:
+    """
+    Get the value from the specified column in the data DataFrame.
+    If the column value is NaN, return None. However, there are handlers
+    for a select set of columns - horizon, qaqcStatus, sampleTopDepth,
+    and sampleBottomDepth.
+
+    :param data: DataFrame to read the column value from.
+    :return: Either a string, float or None depending on the column/column values.
+    """
+    if (
+        column_name in data
+        and not data[column_name].isna().any()
+        and not data[column_name].empty
+    ):
+        if column_name == "horizon":
+            return f"{data[column_name].values[0]} horizon"
+        elif column_name == "qaqcStatus":
+            return data[column_name].values[0].lower()
+        elif column_name == "sampleTopDepth":
+            return float(data[column_name].values[0]) / 100
+        elif column_name == "sampleBottomDepth":
+            return float(data[column_name].values[0]) / 100
+        else:
+            return data[column_name].values[0]
+
+    return None
 
-        return None
 
 def _create_controlled_identified_term_value(
     id: str = None, name: str = None
@@ -47,13 +46,10 @@ def _create_controlled_identified_term_value(
     """
     if id is None or name is None:
         return None
-    return nmdc.ControlledIdentifiedTermValue(
-        term=nmdc.OntologyClass(id=id, name=name)
-    )
+    return nmdc.ControlledIdentifiedTermValue(term=nmdc.OntologyClass(id=id, name=name))
 
-def _create_controlled_term_value(
-    name: str = None
-) -> nmdc.ControlledTermValue:
+
+def _create_controlled_term_value(name: str = None) -> nmdc.ControlledTermValue:
     """
     Create a ControlledIdentifiedTermValue object with the specified id and name.
 
@@ -66,6 +62,7 @@ def _create_controlled_term_value(
         return None
     return nmdc.ControlledTermValue(has_raw_value=name)
 
+
 def _create_timestamp_value(value: str = None) -> nmdc.TimestampValue:
     """
     Create a TimestampValue object with the specified value.
@@ -78,6 +75,7 @@ def _create_timestamp_value(value: str = None) -> nmdc.TimestampValue:
         return None
     return nmdc.TimestampValue(has_raw_value=value)
 
+
 def _create_quantity_value(
     numeric_value: Union[str, int, float] = None, unit: str = None
 ) -> nmdc.QuantityValue:
@@ -94,6 +92,7 @@ def _create_quantity_value(
         return None
     return nmdc.QuantityValue(has_numeric_value=float(numeric_value), has_unit=unit)
 
+
 def _create_text_value(value: str = None) -> nmdc.TextValue:
     """
     Create a TextValue object with the specified value.
@@ -105,6 +104,7 @@ def _create_text_value(value: str = None) -> nmdc.TextValue:
         return None
     return nmdc.TextValue(has_raw_value=value)
 
+
 def _create_double_value(value: str = None) -> nmdc.Double:
     """
     Create a Double object with the specified value.
@@ -117,6 +117,7 @@ def _create_double_value(value: str = None) -> nmdc.Double:
         return None
     return nmdc.Double(value)
 
+
 def _create_geolocation_value(
     latitude: str = None, longitude: str = None
 ) -> nmdc.GeolocationValue:
@@ -142,4 +143,4 @@ def _create_geolocation_value(
     return nmdc.GeolocationValue(
         latitude=nmdc.DecimalDegree(latitude),
         longitude=nmdc.DecimalDegree(longitude),
-    )
+    )

{nmdc_runtime-1.2.0.dist-info → nmdc_runtime-1.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: nmdc-runtime
-Version: 1.2.0
+Name: nmdc_runtime
+Version: 1.3.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston

{nmdc_runtime-1.2.0.dist-info → nmdc_runtime-1.3.0.dist-info}/RECORD

@@ -1,5 +1,5 @@
 nmdc_runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nmdc_runtime/containers.py,sha256=WBzPue0PRoyKXFxgLR-aQcuHetTa8yC5JjI0dGBblYA,419
+nmdc_runtime/containers.py,sha256=8m_S1wiFu8VOWvY7tyqzf-02X9gXY83YGc8FgjWzLGA,418
 nmdc_runtime/main.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/util.py,sha256=o74ZKOmSD79brPFAcQFsYpA6wh9287m0hDhDlIpn9VM,19872
 nmdc_runtime/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,8 +23,8 @@ nmdc_runtime/lib/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,
 nmdc_runtime/lib/extract_nmdc_data.py,sha256=xDFPoYsgkauN48R4v-tJIF0cP_p3J-sBjnyHd0InD9Y,1177
 nmdc_runtime/lib/load_nmdc_data.py,sha256=KO2cIqkY3cBCVcFIwsGokZNOKntOejZVG8ecq43NjFM,3934
 nmdc_runtime/lib/nmdc_dataframes.py,sha256=rVTczY2Jey1yE3x3nZ-RTgtdc2XkzLtKhB_PM3FIb-E,28849
-nmdc_runtime/lib/nmdc_etl_class.py,sha256=x83nKGi7peATlPLf3LGGU6cf42ZmpE__s1XQV2DpkPc,13523
-nmdc_runtime/lib/transform_nmdc_data.py,sha256=3w8Em5byzzOSgjC1LX2nq_d2O4FsTpkDC3UBiZYi9EM,39907
+nmdc_runtime/lib/nmdc_etl_class.py,sha256=tVh3rKVMkBHQE65_LhKeIjCsaCZQk_HJzbc9K4xUNCs,13522
+nmdc_runtime/lib/transform_nmdc_data.py,sha256=hij4lR3IMQRJQdL-rsP_I-m_WyFPsBMchV2MNFUkh0M,39906
 nmdc_runtime/minter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/minter/bootstrap.py,sha256=5Ej6pJVBRryRIi0ZwEloY78Zky7iE2okF6tPwRI2axM,822
 nmdc_runtime/minter/config.py,sha256=mq_s0xjLZK-zwjwk3IGgnk9ZIvvejyyZ7_qZkLt3V-c,1409
@@ -35,9 +35,9 @@ nmdc_runtime/minter/domain/model.py,sha256=WMOuKub3dVzkOt_EZSRDLeTsJPqFbKx01SMQ5
 nmdc_runtime/minter/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/minter/entrypoints/fastapi_app.py,sha256=JC4thvzfFwRc1mhWQ-kHy3yvs0SYxF6ktE7LXNCwqlI,4031
 nmdc_runtime/site/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nmdc_runtime/site/graphs.py,sha256=Ipv6AKz_Itdcs3bu6QQ-wybwdzzBFVsh5AJP1DM16jY,7709
-nmdc_runtime/site/ops.py,sha256=knMVKlEaaQ2mx_-KRayEMcuIPvmjWdGwcyeSFqgtyyo,30538
-nmdc_runtime/site/repository.py,sha256=OXr5z9EH_2ZYmsbkwPhXPXg-ID9NpV_a4-zQEv5zxGs,27150
+nmdc_runtime/site/graphs.py,sha256=siHlRnD2eS9nw3Ne049TcGG6I6IYFvjgWQuuSHzEOqc,9492
+nmdc_runtime/site/ops.py,sha256=YzDm7Dm2sELptwTew8DTOcS3nYBH_JegXhu3wzZuuiY,32482
+nmdc_runtime/site/repository.py,sha256=UgY9eMnNgZxa-Y0QeDyENh4KHtxuBWkYCjxltM4mTzA,30938
 nmdc_runtime/site/resources.py,sha256=pQSwg1dRpL_D91gYLzzaOIDZ3qa69rPqSlsq5dS9i_M,17783
 nmdc_runtime/site/util.py,sha256=6hyVPpb6ZkWEG8Nm7uQxnZ-QmuPOG9hgWvl0mUBr5JU,1303
 nmdc_runtime/site/backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -60,11 +60,11 @@ nmdc_runtime/site/terminusdb/schema.py,sha256=3e39rHUSZsNbN_F0SHHNsvcEGRWtYa6O9K
 nmdc_runtime/site/translation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/site/translation/emsl.py,sha256=l6Q9Jj3RNJFQNYAU_TtKTJ7cyFcR93xBRs_lLdX0bMQ,1244
 nmdc_runtime/site/translation/gold.py,sha256=R3W99sdQb7Pgu_esN7ruIC-tyREQD_idJ4xCzkqWuGw,1622
-nmdc_runtime/site/translation/gold_translator.py,sha256=vqTmZ5dqmaExIgA8nXpBYCs0d2OedNZgwbWJrt5Tm_M,26441
+nmdc_runtime/site/translation/gold_translator.py,sha256=8i5FxrgAG4rLbM0mcCSBaZEzyReht6xwmpm4xeX4HwI,26451
 nmdc_runtime/site/translation/jgi.py,sha256=bh73r0uq5BT3ywXwIa1OEKKtz9LbFsSng472tdr-xtg,875
-nmdc_runtime/site/translation/neon_benthic_translator.py,sha256=DON0ssFVE11fAHssSteP9lQdaRuu-Rfzi9V90vy27n8,23737
-nmdc_runtime/site/translation/neon_soil_translator.py,sha256=fzKvXfMv9biYgNRi_hqns8hZMED1Fj2Jgt1uBTPQwRQ,38036
-nmdc_runtime/site/translation/neon_utils.py,sha256=aBx1TKA1rUpA3pZR3LbL7bIrJ8beci7_pZSCgmPAzII,5208
+nmdc_runtime/site/translation/neon_benthic_translator.py,sha256=e_7tXFrP0PpdhqUCxXmOaFViSuG36IIMDqyj3FHLcgQ,23069
+nmdc_runtime/site/translation/neon_soil_translator.py,sha256=x-FfNKsIv0efgxty9v4wOxNu5nrrS-N8phx12IqfLOI,37624
+nmdc_runtime/site/translation/neon_utils.py,sha256=k8JYMnm-L981BTOdAMomR1CulS_Hz5v7aYxrJ94KEJc,5086
 nmdc_runtime/site/translation/submission_portal_translator.py,sha256=lHcrfPR5wk3BcZ0Uw5zUyWu5XRVikgOzdzSb5nFVS9I,27964
 nmdc_runtime/site/translation/translator.py,sha256=xM9dM-nTgSWwu5HFoUVNHf8kqk9iiH4PgWdSx4OKxEk,601
 nmdc_runtime/site/translation/util.py,sha256=w_l3SiExGsl6cXRqto0a_ssDmHkP64ITvrOVfPxmNpY,4366
@@ -73,9 +73,9 @@ nmdc_runtime/site/validation/emsl.py,sha256=TgckqKkFquHDLso77sn-jZRu5ZaBevGCt5p8
 nmdc_runtime/site/validation/gold.py,sha256=kJ1L081SZb-8qKpF731r5aQOueM206SUfUYMTTNTFMc,802
 nmdc_runtime/site/validation/jgi.py,sha256=lBo-FCtEYedT74CpW-Kdj512Ib963ik-4YIYmY5puDo,1298
 nmdc_runtime/site/validation/util.py,sha256=GGbMDSwR090sr_E_fHffCN418gpYESaiot6XghS7OYk,3349
-nmdc_runtime-1.2.0.dist-info/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
-nmdc_runtime-1.2.0.dist-info/METADATA,sha256=MFVvY4uNvHv32d9XVbMk25MolCbsPYfilS85DTvQQtQ,7424
-nmdc_runtime-1.2.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-nmdc_runtime-1.2.0.dist-info/entry_points.txt,sha256=nfH6-K9tDKv7va8ENfShsBnxVQoYJdEe7HHdwtkbh1Y,289
-nmdc_runtime-1.2.0.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
-nmdc_runtime-1.2.0.dist-info/RECORD,,
+nmdc_runtime-1.3.0.dist-info/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
+nmdc_runtime-1.3.0.dist-info/METADATA,sha256=XiOhRGoaBESF48sWor9SMpTdCL8X9yPKIh6mnA9xZtY,7424
+nmdc_runtime-1.3.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+nmdc_runtime-1.3.0.dist-info/entry_points.txt,sha256=nfH6-K9tDKv7va8ENfShsBnxVQoYJdEe7HHdwtkbh1Y,289
+nmdc_runtime-1.3.0.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
+nmdc_runtime-1.3.0.dist-info/RECORD,,