nmdc-runtime 1.4.2__py3-none-any.whl → 1.6.0__py3-none-any.whl

nmdc_runtime/site/graphs.py

@@ -38,9 +38,11 @@ from nmdc_runtime.site.ops import (
     neon_data_by_product,
     nmdc_schema_database_from_neon_soil_data,
     nmdc_schema_database_from_neon_benthic_data,
+    nmdc_schema_database_from_neon_surface_water_data,
     nmdc_schema_database_export_filename_neon,
     get_neon_pipeline_mms_data_product,
     get_neon_pipeline_sls_data_product,
+    get_neon_pipeline_surface_water_data_product,
     get_submission_portal_pipeline_inputs,
     get_csv_rows_from_url,
     get_neon_pipeline_benthic_data_product,
@@ -317,3 +319,65 @@ def ingest_neon_benthic_metadata():
     )
     run_id = submit_metadata_to_db(database)
     poll_for_run_completion(run_id)
+
+
+@graph
+def translate_neon_api_surface_water_metadata_to_nmdc_schema_database():
+    mms_surface_water_data_product = get_neon_pipeline_surface_water_data_product()
+
+    mms_surface_water = neon_data_by_product(mms_surface_water_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_surface_water_data(
+        mms_surface_water,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+
+    database_dict = nmdc_schema_object_to_dict(database)
+    filename = nmdc_schema_database_export_filename_neon()
+
+    outputs = export_json_to_drs(database_dict, filename)
+    add_output_run_event(outputs)
+
+
+@graph
+def ingest_neon_surface_water_metadata():
+    mms_surface_water_data_product = get_neon_pipeline_surface_water_data_product()
+
+    mms_surface_water = neon_data_by_product(mms_surface_water_data_product)
+
+    sites_mapping_dict = site_code_mapping()
+
+    (
+        neon_envo_mappings_file_url,
+        neon_raw_data_file_mappings_file_url,
+    ) = get_neon_pipeline_inputs()
+
+    neon_envo_mappings_file = get_df_from_url(neon_envo_mappings_file_url)
+
+    neon_raw_data_file_mappings_file = get_df_from_url(
+        neon_raw_data_file_mappings_file_url
+    )
+
+    database = nmdc_schema_database_from_neon_benthic_data(
+        mms_surface_water,
+        sites_mapping_dict,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+    )
+    run_id = submit_metadata_to_db(database)
+    poll_for_run_completion(run_id)

nmdc_runtime/site/ops.py

@@ -68,6 +68,9 @@ from nmdc_runtime.site.translation.neon_soil_translator import NeonSoilDataTrans
 from nmdc_runtime.site.translation.neon_benthic_translator import (
     NeonBenthicDataTranslator,
 )
+from nmdc_runtime.site.translation.neon_surface_water_translator import (
+    NeonSurfaceWaterDataTranslator,
+)
 from nmdc_runtime.site.translation.submission_portal_translator import (
     SubmissionPortalTranslator,
 )
@@ -784,6 +787,11 @@ def get_neon_pipeline_benthic_data_product(context: OpExecutionContext) -> dict:
     return context.op_config["benthic_data_product"]
 
 
+@op(config_schema={"surface_water_data_product": dict})
+def get_neon_pipeline_surface_water_data_product(context: OpExecutionContext) -> dict:
+    return context.op_config["surface_water_data_product"]
+
+
 @op(required_resource_keys={"neon_api_client"})
 def neon_data_by_product(
     context: OpExecutionContext, data_product: dict
@@ -862,6 +870,32 @@ def nmdc_schema_database_from_neon_benthic_data(
     return database
 
 
+@op(required_resource_keys={"runtime_api_site_client"})
+def nmdc_schema_database_from_neon_surface_water_data(
+    context: OpExecutionContext,
+    surface_water_data: Dict[str, pd.DataFrame],
+    site_code_mapping: Dict[str, str],
+    neon_envo_mappings_file: pd.DataFrame,
+    neon_raw_data_file_mappings_file: pd.DataFrame,
+) -> nmdc.Database:
+    client: RuntimeApiSiteClient = context.resources.runtime_api_site_client
+
+    def id_minter(*args, **kwargs):
+        response = client.mint_id(*args, **kwargs)
+        return response.json()
+
+    translator = NeonSurfaceWaterDataTranslator(
+        surface_water_data,
+        site_code_mapping,
+        neon_envo_mappings_file,
+        neon_raw_data_file_mappings_file,
+        id_minter=id_minter,
+    )
+
+    database = translator.get_database()
+    return database
+
+
 @op(
     out={
         "neon_envo_mappings_file_url": Out(),

nmdc_runtime/site/repository.py

@@ -38,8 +38,10 @@ from nmdc_runtime.site.graphs import (
     hello_graph,
     translate_neon_api_soil_metadata_to_nmdc_schema_database,
     translate_neon_api_benthic_metadata_to_nmdc_schema_database,
+    translate_neon_api_surface_water_metadata_to_nmdc_schema_database,
     ingest_neon_soil_metadata,
     ingest_neon_benthic_metadata,
+    ingest_neon_surface_water_metadata,
 )
 from nmdc_runtime.site.resources import (
     get_mongo,
@@ -764,6 +766,89 @@ def biosample_submission_ingest():
                 },
             },
         ),
+        translate_neon_api_surface_water_metadata_to_nmdc_schema_database.to_job(
+            description="This job fetches the metadata associated with a given NEON data product code and translates it into an equivalent nmdc:Database object. The object is serialized to JSON and stored in DRS. This can be considered a dry-run for the `ingest_neon_metadata` job.",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        },
+                        "mongo": {
+                            "config": {
+                                "dbname": {"env": "MONGO_DBNAME"},
+                                "host": {"env": "MONGO_HOST"},
+                                "password": {"env": "MONGO_PASSWORD"},
+                                "username": {"env": "MONGO_USERNAME"},
+                            },
+                        },
+                        "runtime_api_site_client": {
+                            "config": {
+                                "base_url": {"env": "API_HOST"},
+                                "client_id": {"env": "API_SITE_CLIENT_ID"},
+                                "client_secret": {"env": "API_SITE_CLIENT_SECRET"},
+                                "site_id": {"env": "API_SITE_ID"},
+                            },
+                        },
+                    },
+                ),
+                "ops": {
+                    "export_json_to_drs": {"config": {"username": "..."}},
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                    "get_neon_pipeline_surface_water_data_product": {
+                        "config": {
+                            "surface_water_data_product": {
+                                "product_id": "DP1.20281.001",
+                                "product_tables": "mms_swMetagenomeSequencing, mms_swMetagenomeDnaExtraction, amc_fieldGenetic, amc_fieldSuperParent",
+                            }
+                        }
+                    },
+                },
+            },
+        ),
+        ingest_neon_surface_water_metadata.to_job(
+            description="",
+            resource_defs=resource_defs,
+            config={
+                "resources": merge(
+                    unfreeze(normal_resources),
+                    {
+                        "neon_api_client": {
+                            "config": {
+                                "base_url": {"env": "NEON_API_BASE_URL"},
+                                "api_token": {"env": "NEON_API_TOKEN"},
+                            },
+                        }
+                    },
+                ),
+                "ops": {
+                    "get_neon_pipeline_surface_water_data_product": {
+                        "config": {
+                            "surface_water_data_product": {
+                                "product_id": "DP1.20281.001",
+                                "product_tables": "mms_swMetagenomeSequencing, mms_swMetagenomeDnaExtraction, amc_fieldGenetic, amc_fieldSuperParent",
+                            }
+                        }
+                    },
+                    "get_neon_pipeline_inputs": {
+                        "inputs": {
+                            "neon_envo_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/neon_mixs_env_triad_mappings/neon-nlcd-local-broad-mappings.tsv",
+                            "neon_raw_data_file_mappings_file_url": "https://raw.githubusercontent.com/microbiomedata/nmdc-schema/main/assets/misc/neon_raw_data_file_mappings.tsv",
+                        }
+                    },
+                },
+            },
+        ),
     ]
 
 

nmdc_runtime/site/translation/neon_surface_water_translator.py

@@ -0,0 +1,620 @@
+import re
+import sqlite3
+from typing import Dict, Optional
+
+import pandas as pd
+import requests
+import requests_cache
+
+from nmdc_schema import nmdc
+from nmdc_runtime.site.translation.translator import Translator
+from nmdc_runtime.site.util import get_basename
+from nmdc_runtime.site.translation.neon_utils import (
+    _get_value_or_none,
+    _create_controlled_identified_term_value,
+    _create_controlled_term_value,
+    _create_geolocation_value,
+    _create_quantity_value,
+    _create_timestamp_value,
+    _create_text_value,
+)
+
+
+SURFACE_WATER_BROAD_SCALE_MAPPINGS = {
+    "lake": {"term_id": "ENVO:01000252", "term_name": "freshwater lake biome"},
+    "river": {"term_id": "ENVO:01000253", "term_name": "freshwater river biome"},
+    "stream": {"term_id": "ENVO:03605008", "term_name": "freshwater stream biome"},
+}
+
+SURFACE_WATER_LOCAL_SCALE_MAPPINGS = {
+    "lake": {
+        "buoy.c0": {"term_id": "ENVO:01001191", "term_name": "water surface"},
+        "buoy.c1": {"term_id": "ENVO:00002131", "term_name": "epilimnion"},
+        "buoy.c2": {"term_id": "ENVO:00002269", "term_name": "thermocline"},
+        "buoy.c3": {"term_id": "ENVO:00002130", "term_name": "hypolimnion"},
+        "littoral": {
+            "term_id": "ENVO:01000409",
+            "term_name": "freshwater littoral zone",
+        },
+    },
+    "river": {"term_id": "ENVO:01000297", "term_name": "freshwater river"},
+    "stream": {"term_id": "ENVO:03605007", "term_name": "freshwater stream"},
+}
+
+SURFACE_WATER_MEDIUM_MAPPINGS = {
+    "lake": {
+        "term_id": "ENVO:04000007",
+        "term_name": "lake water",
+    },
+    "river": {"term_id": "ENVO:01000599", "term_name": "river water"},
+    "stream": {"term_id": "ENVO:03605006", "term_name": "stream water"},
+}
+
+
+class NeonSurfaceWaterDataTranslator(Translator):
+    def __init__(
+        self,
+        surface_water_data: dict,
+        site_code_mapping: dict,
+        neon_envo_mappings_file: pd.DataFrame,
+        neon_raw_data_file_mappings_file: pd.DataFrame,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+
+        self.conn = sqlite3.connect("neon.db")
+        requests_cache.install_cache("neon_api_cache")
+
+        neon_amb_data_tables = (
+            "mms_swMetagenomeSequencing",
+            "mms_swMetagenomeDnaExtraction",
+            "amc_fieldGenetic",
+            "amc_fieldSuperParent",
+        )
+
+        if all(k in surface_water_data for k in neon_amb_data_tables):
+            surface_water_data["mms_swMetagenomeSequencing"].to_sql(
+                "mms_swMetagenomeSequencing",
+                self.conn,
+                if_exists="replace",
+                index=False,
+            )
+            surface_water_data["mms_swMetagenomeDnaExtraction"].to_sql(
+                "mms_swMetagenomeDnaExtraction",
+                self.conn,
+                if_exists="replace",
+                index=False,
+            )
+            surface_water_data["amc_fieldGenetic"].to_sql(
+                "amc_fieldGenetic", self.conn, if_exists="replace", index=False
+            )
+            surface_water_data["amc_fieldSuperParent"].to_sql(
+                "amc_fieldSuperParent", self.conn, if_exists="replace", index=False
+            )
+        else:
+            raise ValueError(
+                f"You are missing one of the aquatic benthic microbiome tables: {neon_amb_data_tables}"
+            )
+
+        neon_envo_mappings_file.to_sql(
+            "neonEnvoTerms", self.conn, if_exists="replace", index=False
+        )
+
+        self.neon_raw_data_file_mappings_df = neon_raw_data_file_mappings_file
+        self.neon_raw_data_file_mappings_df.to_sql(
+            "neonRawDataFile", self.conn, if_exists="replace", index=False
+        )
+
+        self.site_code_mapping = site_code_mapping
+
+    def _translate_biosample(
+        self, neon_id: str, nmdc_id: str, biosample_row: pd.DataFrame
+    ) -> nmdc.Biosample:
+        def map_local_scale(
+            aquatic_site_type: str, named_location: Optional[str] = None
+        ) -> Dict[str, str]:
+            if aquatic_site_type == "lake":
+                for key in SURFACE_WATER_LOCAL_SCALE_MAPPINGS.get(
+                    aquatic_site_type, {}
+                ):
+                    if key in named_location:
+                        return SURFACE_WATER_LOCAL_SCALE_MAPPINGS[aquatic_site_type][
+                            key
+                        ]
+            elif aquatic_site_type == "river" or aquatic_site_type == "stream":
+                return SURFACE_WATER_LOCAL_SCALE_MAPPINGS.get(aquatic_site_type, {})
+            return {}
+
+        depth = None
+        minimum_depth = biosample_row["lakeSampleDepth1"].values[0]
+        maximum_depth = biosample_row["lakeSampleDepth2"].values[0]
+
+        if not pd.isna(minimum_depth):
+            if not pd.isna(maximum_depth):
+                depth = nmdc.QuantityValue(
+                    has_minimum_numeric_value=nmdc.Float(minimum_depth),
+                    has_maximum_numeric_value=nmdc.Float(maximum_depth),
+                    has_unit="m",
+                )
+            else:
+                depth = nmdc.QuantityValue(
+                    has_numeric_value=nmdc.Float(minimum_depth),
+                    has_unit="m",
+                )
+
+        return nmdc.Biosample(
+            id=nmdc_id,
+            part_of="nmdc:sty-11-hht5sb92",
+            env_broad_scale=_create_controlled_identified_term_value(
+                SURFACE_WATER_BROAD_SCALE_MAPPINGS.get(
+                    biosample_row["aquaticSiteType"].values[0]
+                ).get("term_id"),
+                SURFACE_WATER_BROAD_SCALE_MAPPINGS.get(
+                    biosample_row["aquaticSiteType"].values[0]
+                ).get("term_name"),
+            ),
+            env_local_scale=_create_controlled_identified_term_value(
+                map_local_scale(
+                    biosample_row["aquaticSiteType"].values[0],
+                    biosample_row["namedLocation"].values[0],
+                ).get("term_id"),
+                map_local_scale(
+                    biosample_row["aquaticSiteType"].values[0],
+                    biosample_row["namedLocation"].values[0],
+                ).get("term_name"),
+            ),
+            env_medium=_create_controlled_identified_term_value(
+                SURFACE_WATER_MEDIUM_MAPPINGS.get(
+                    biosample_row["aquaticSiteType"].values[0]
+                ).get("term_id"),
+                SURFACE_WATER_MEDIUM_MAPPINGS.get(
+                    biosample_row["aquaticSiteType"].values[0]
+                ).get("term_name"),
+            ),
+            name=neon_id,
+            lat_lon=_create_geolocation_value(
+                biosample_row["decimalLatitude"].values[0],
+                biosample_row["decimalLongitude"].values[0],
+            ),
+            elev=nmdc.Float(biosample_row["elevation"].values[0]),
+            collection_date=_create_timestamp_value(
+                biosample_row["seqCollectDate"].values[0]
+            ),
+            geo_loc_name=_create_text_value(
+                self.site_code_mapping[biosample_row["siteID"].values[0]]
+                if biosample_row["siteID"].values[0]
+                else None
+            ),
+            samp_collec_device=biosample_row["samplerType"].values[0],
+            diss_oxygen=_create_quantity_value(
+                biosample_row["dissolvedOxygen"].values[0], "mg/L"
+            ),
+            conduc=_create_quantity_value(
+                biosample_row["specificConductance"].values[0], "uS/cm"
+            ),
+            temp=_create_quantity_value(biosample_row["waterTemp"].values[0], "Cel"),
+            type="nmdc:Biosample",
+            analysis_type="metagenomics",
+            biosample_categories="NEON",
+            depth=depth,
+            samp_size=_create_quantity_value(
+                biosample_row["geneticFilteredSampleVolume"].values[0], "mL"
+            ),
+            env_package=nmdc.TextValue(has_raw_value="water"),
+        )
+
+    def _translate_extraction_process(
+        self,
+        extraction_id: str,
+        extraction_input: str,
+        processed_sample_id: str,
+        extraction_row: pd.DataFrame,
+    ) -> nmdc.Extraction:
+        """
+        Create an nmdc Extraction process, which is a process to model the DNA extraction in
+        a metagenome sequencing experiment. The input to an Extraction process is the
+        output from a Pooling process.
+
+        :param extraction_id: Minted id for Extraction process.
+        :param extraction_input: Input to an Extraction process is the output from a Pooling process.
+        :param processed_sample_id: Output of Extraction process is a ProcessedSample.
+        :param extraction_row: DataFrame with Extraction process metadata.
+        :return: Extraction process object.
+        """
+        processing_institution = None
+        laboratory_name = _get_value_or_none(extraction_row, "laboratoryName")
+        if laboratory_name is not None:
+            if re.search("Battelle", laboratory_name, re.IGNORECASE):
+                processing_institution = "Battelle"
+            elif re.search("Argonne", laboratory_name, re.IGNORECASE):
+                processing_institution = "ANL"
+
+        return nmdc.Extraction(
+            id=extraction_id,
+            has_input=extraction_input,
+            has_output=processed_sample_id,
+            start_date=_get_value_or_none(extraction_row, "extrCollectDate"),
+            end_date=_get_value_or_none(extraction_row, "extrProcessedDate"),
+            input_mass=_create_quantity_value(
+                _get_value_or_none(extraction_row, "sampleMass"), "g"
+            ),
+            qc_status=nmdc.StatusEnum(
+                _get_value_or_none(extraction_row, "extrQaqcStatus")
+            ),
+            processing_institution=processing_institution,
+        )
+
+    def _translate_library_preparation(
+        self,
+        library_preparation_id: str,
+        library_preparation_input: str,
+        processed_sample_id: str,
+        library_preparation_row: pd.DataFrame,
+    ):
+        """
+        Create LibraryPreparation process object. The input to LibraryPreparation process
+        is the output ProcessedSample from an Extraction process. The output of LibraryPreparation
+        process is fed as input to an OmicsProcessing object.
+
+        :param library_preparation_id: Minted id for LibraryPreparation process.
+        :param library_preparation_input: Input to LibraryPreparation process is output from
+        Extraction process.
+        :param processed_sample_id: Minted ProcessedSample id which is output of LibraryPreparation
+        is also input to OmicsProcessing.
+        :param library_preparation_row: Metadata required to populate LibraryPreparation.
+        :return: Object that using LibraryPreparation process model.
+        """
+        processing_institution = None
+        laboratory_name = _get_value_or_none(library_preparation_row, "laboratoryName")
+        if laboratory_name is not None:
+            if re.search("Battelle", laboratory_name, re.IGNORECASE):
+                processing_institution = "Battelle"
+            elif re.search("Argonne", laboratory_name, re.IGNORECASE):
+                processing_institution = "ANL"
+
+        return nmdc.LibraryPreparation(
+            id=library_preparation_id,
+            has_input=library_preparation_input,
+            has_output=processed_sample_id,
+            start_date=_get_value_or_none(library_preparation_row, "seqCollectDate"),
+            end_date=_get_value_or_none(library_preparation_row, "seqProcessedDate"),
+            processing_institution=processing_institution,
+        )
+
+    def _translate_omics_processing(
+        self,
+        omics_processing_id: str,
+        processed_sample_id: str,
+        raw_data_file_data: str,
+        omics_processing_row: pd.DataFrame,
+    ) -> nmdc.OmicsProcessing:
+        """Create nmdc OmicsProcessing object. This class typically models the run of a
+        Bioinformatics workflow on sequence data from a biosample. The input to an OmicsProcessing
+        process is the output from a LibraryPreparation process, and the output of OmicsProcessing
+        is a DataObject which has the FASTQ sequence file URLs embedded in them.
+
+        :param omics_processing_id: Minted id for an OmicsProcessing process.
+        :param processed_sample_id: ProcessedSample that is the output of LibraryPreparation.
+        :param raw_data_file_data: R1/R2 DataObjects which have links to workflow processed output
+        files embedded in them.
+        :param omics_processing_row: DataFrame with metadata for an OmicsProcessing workflow
+        process/run.
+        :return: OmicsProcessing object that models a Bioinformatics workflow process/run.
+        """
+        processing_institution = None
+        sequencing_facility = _get_value_or_none(
+            omics_processing_row, "sequencingFacilityID"
+        )
+        if sequencing_facility is not None:
+            if re.search("Battelle", sequencing_facility, re.IGNORECASE):
+                processing_institution = "Battelle"
+            elif re.search("Argonne", sequencing_facility, re.IGNORECASE):
+                processing_institution = "ANL"
+
+        return nmdc.OmicsProcessing(
+            id=omics_processing_id,
+            has_input=processed_sample_id,
+            has_output=raw_data_file_data,
+            processing_institution=processing_institution,
+            ncbi_project_name=_get_value_or_none(omics_processing_row, "ncbiProjectID"),
+            omics_type=_create_controlled_term_value(
+                omics_processing_row["investigation_type"].values[0]
+            ),
+            instrument_name=f"{_get_value_or_none(omics_processing_row, 'sequencingMethod')} {_get_value_or_none(omics_processing_row, 'instrument_model')}",
+            part_of="nmdc:sty-11-hht5sb92",
+            name=f"Surface water microbial communities - {_get_value_or_none(omics_processing_row, 'dnaSampleID')}",
+            type="nmdc:OmicsProcessing",
+        )
+
+    def _translate_processed_sample(
+        self, processed_sample_id: str, sample_id: str
+    ) -> nmdc.ProcessedSample:
+        """
+        Create an nmdc ProcessedSample. ProcessedSample is typically the output of a PlannedProcess
+        like Pooling, Extraction, LibraryPreparation, etc. We are using this to create a
+        reference for the nmdc minted ProcessedSample ids in `processed_sample_set`. We are
+        associating the minted ids with the name of the sample it is coming from which can be
+        a value from either the `genomicsSampleID` column or from the `dnaSampleID` column.
+
+        :param processed_sample_id: NMDC minted ProcessedSampleID.
+        :param sample_id: Value from `genomicsSampleID` or `dnaSampleID` column.
+        :return: ProcessedSample objects to be stored in `processed_sample_set`.
+        """
+        return nmdc.ProcessedSample(id=processed_sample_id, name=sample_id)
+
+    def _translate_data_object(
+        self, do_id: str, url: str, do_type: str, checksum: str
+    ) -> nmdc.DataObject:
+        """Create nmdc DataObject which is the output of an OmicsProcessing process. This
+        object mainly contains information about the sequencing file that was generated as
+        the result of running a Bioinformatics workflow on a certain ProcessedSample, which
+        is the result of a LibraryPreparation process.
+
+        :param do_id: NMDC minted DataObject id.
+        :param url: URL of zipped FASTQ file on NEON file server. Retrieved from file provided
+        by Hugh Cross at NEON.
+        :param do_type: Indicate whether it is FASTQ for Read 1 or Read 2 (paired end sequencing).
+        :param checksum: Checksum value for FASTQ in zip file, once again provided by Hugh Cross
+        at NEON.
+        :return: DataObject with all the sequencing file metadata.
+        """
+        file_name = get_basename(url)
+        basename = file_name.split(".", 1)[0]
+
+        return nmdc.DataObject(
+            id=do_id,
+            name=file_name,
+            url=url,
+            description=f"sequencing results for {basename}",
+            type="nmdc:DataObject",
+            md5_checksum=checksum,
+            data_object_type=do_type,
+        )
+
+    def get_database(self):
+        database = nmdc.Database()
+
+        query = """
+                SELECT
+                    merged.laboratoryName,
+                    merged.sequencingFacilityID,
+                    merged.extrProcessedDate,
+                    merged.seqProcessedDate,
+                    merged.dnaSampleID,
+                    merged.internalLabID,
+                    merged.instrument_model,
+                    merged.sequencingMethod,
+                    merged.investigation_type,
+                    merged.extrQaqcStatus,
+                    merged.ncbiProjectID,
+                    merged.genomicsSampleID,
+                    merged.sequenceAnalysisType,
+                    merged.sampleMass,
+                    merged.nucleicAcidConcentration,
+                    merged.siteID,
+                    merged.seqCollectDate,
+                    merged.extrCollectDate,
+                    afg.geneticSampleID,
+                    afg.geneticFilteredSampleVolume,
+                    afg.sampleMaterial,
+                    afs.parentSampleID,
+                    afs.namedLocation,
+                    afs.decimalLatitude,
+                    afs.decimalLongitude,
+                    afs.elevation,
+                    afs.aquaticSiteType,
+                    afs.samplerType,
+                    afs.dissolvedOxygen,
+                    afs.specificConductance,
+                    afs.waterTemp,
+                    afs.lakeSampleDepth1,
+                    afs.lakeSampleDepth2
+                FROM 
+                    (
+                        SELECT
+                            msq.collectDate AS seqCollectDate,
+                            mde.collectDate AS extrCollectDate,
+                            msq.laboratoryName,
+                            msq.sequencingFacilityID,
+                            msq.processedDate AS seqProcessedDate,
+                            mde.processedDate AS extrProcessedDate,
+                            msq.dnaSampleID,
+                            msq.internalLabID,
+                            msq.instrument_model,
+                            msq.sequencingMethod,
+                            msq.investigation_type,
+                            mde.qaqcStatus AS extrQaqcStatus,
+                            msq.ncbiProjectID,
+                            msq.siteID,
+                            msq.labPrepMethod,
+                            mde.genomicsSampleID,
+                            mde.sequenceAnalysisType,
+                            mde.sampleMass,
+                            mde.nucleicAcidConcentration,
+                            mde.nucleicAcidQuantMethod,
+                            mde.nucleicAcidPurity
+                        FROM 
+                            mms_swMetagenomeSequencing AS msq
+                        JOIN 
+                            mms_swMetagenomeDnaExtraction AS mde
+                        ON 
+                            msq.dnaSampleID = mde.dnaSampleID
+                    ) AS merged
+                JOIN amc_fieldGenetic AS afg
+                ON
+                    merged.genomicsSampleID = afg.geneticSampleID
+                JOIN amc_fieldSuperParent AS afs
+                ON
+                    afg.parentSampleID = afs.parentSampleID
+        """
+        surface_water_samples = pd.read_sql_query(query, self.conn)
+
+        neon_biosample_ids = surface_water_samples["parentSampleID"]
+        nmdc_biosample_ids = self._id_minter("nmdc:Biosample", len(neon_biosample_ids))
+        neon_to_nmdc_biosample_ids = dict(zip(neon_biosample_ids, nmdc_biosample_ids))
+
+        neon_extraction_ids = surface_water_samples["parentSampleID"]
+        nmdc_extraction_ids = self._id_minter(
+            "nmdc:Extraction", len(neon_extraction_ids)
+        )
+        neon_to_nmdc_extraction_ids = dict(
+            zip(neon_extraction_ids, nmdc_extraction_ids)
+        )
+
+        neon_extraction_processed_ids = surface_water_samples["parentSampleID"]
+        nmdc_extraction_processed_ids = self._id_minter(
+            "nmdc:ProcessedSample", len(neon_extraction_processed_ids)
+        )
+        neon_to_nmdc_extraction_processed_ids = dict(
+            zip(neon_extraction_processed_ids, nmdc_extraction_processed_ids)
+        )
+
+        neon_lib_prep_ids = surface_water_samples["parentSampleID"]
+        nmdc_lib_prep_ids = self._id_minter(
+            "nmdc:LibraryPreparation", len(neon_lib_prep_ids)
+        )
+        neon_to_nmdc_lib_prep_ids = dict(zip(neon_lib_prep_ids, nmdc_lib_prep_ids))
+
+        neon_lib_prep_processed_ids = surface_water_samples["parentSampleID"]
+        nmdc_lib_prep_processed_ids = self._id_minter(
+            "nmdc:ProcessedSample", len(neon_lib_prep_processed_ids)
+        )
+        neon_to_nmdc_lib_prep_processed_ids = dict(
+            zip(neon_lib_prep_processed_ids, nmdc_lib_prep_processed_ids)
+        )
+
+        neon_omprc_ids = surface_water_samples["parentSampleID"]
+        nmdc_omprc_ids = self._id_minter("nmdc:OmicsProcessing", len(neon_omprc_ids))
+        neon_to_nmdc_omprc_ids = dict(zip(neon_omprc_ids, nmdc_omprc_ids))
+
+        neon_raw_data_file_mappings_df = self.neon_raw_data_file_mappings_df
+        neon_raw_file_paths = neon_raw_data_file_mappings_df["rawDataFilePath"]
+        nmdc_data_object_ids = self._id_minter(
+            "nmdc:DataObject", len(neon_raw_file_paths)
+        )
+        neon_to_nmdc_data_object_ids = dict(
+            zip(neon_raw_file_paths, nmdc_data_object_ids)
+        )
+
+        for neon_id, nmdc_id in neon_to_nmdc_biosample_ids.items():
+            biosample_row = surface_water_samples[
+                surface_water_samples["parentSampleID"] == neon_id
+            ]
+
+            database.biosample_set.append(
+                self._translate_biosample(neon_id, nmdc_id, biosample_row)
+            )
+
+        for neon_id, nmdc_id in neon_to_nmdc_extraction_ids.items():
+            extraction_row = surface_water_samples[
+                surface_water_samples["parentSampleID"] == neon_id
+            ]
+
+            extraction_input = neon_to_nmdc_biosample_ids.get(neon_id)
+            processed_sample_id = neon_to_nmdc_extraction_processed_ids.get(neon_id)
+
+            if extraction_input is not None and processed_sample_id is not None:
+                database.extraction_set.append(
+                    self._translate_extraction_process(
+                        nmdc_id,
+                        extraction_input,
+                        processed_sample_id,
+                        extraction_row,
+                    )
+                )
+
+                genomics_sample_id = _get_value_or_none(
+                    extraction_row, "genomicsSampleID"
+                )
+
+                database.processed_sample_set.append(
+                    self._translate_processed_sample(
+                        processed_sample_id,
+                        f"Extracted DNA from {genomics_sample_id}",
+                    )
+                )
+
+        query = """
+            SELECT dnaSampleID, GROUP_CONCAT(rawDataFilePath, '|') AS rawDataFilePaths
+            FROM neonRawDataFile
+            GROUP BY dnaSampleID
+        """
+        neon_raw_data_files = pd.read_sql_query(query, self.conn)
+        neon_raw_data_files_dict = (
+            neon_raw_data_files.set_index("dnaSampleID")["rawDataFilePaths"]
+            .str.split("|")
+            .to_dict()
+        )
+        filtered_neon_raw_data_files_dict = {
+            key: value
+            for key, value in neon_raw_data_files_dict.items()
+            if len(value) <= 2
+        }
+
+        for neon_id, nmdc_id in neon_to_nmdc_lib_prep_ids.items():
+            lib_prep_row = surface_water_samples[
+                surface_water_samples["parentSampleID"] == neon_id
+            ]
+
+            lib_prep_input = neon_to_nmdc_extraction_processed_ids.get(neon_id)
+            processed_sample_id = neon_to_nmdc_lib_prep_processed_ids.get(neon_id)
+
+            if lib_prep_input is not None and processed_sample_id is not None:
+                database.library_preparation_set.append(
+                    self._translate_library_preparation(
+                        nmdc_id,
+                        lib_prep_input,
+                        processed_sample_id,
+                        lib_prep_row,
+                    )
+                )
+
+                dna_sample_id = _get_value_or_none(lib_prep_row, "dnaSampleID")
+
+                database.processed_sample_set.append(
+                    self._translate_processed_sample(
+                        processed_sample_id,
+                        f"Library preparation for {dna_sample_id}",
+                    )
+                )
+
+                has_output = None
+                has_output_do_ids = []
+
+                if dna_sample_id in filtered_neon_raw_data_files_dict:
+                    has_output = filtered_neon_raw_data_files_dict[dna_sample_id]
+                    for item in has_output:
+                        if item in neon_to_nmdc_data_object_ids:
+                            has_output_do_ids.append(neon_to_nmdc_data_object_ids[item])
+
+                        checksum = None
+                        do_type = None
+
+                        checksum = neon_raw_data_file_mappings_df[
+                            neon_raw_data_file_mappings_df["rawDataFilePath"] == item
+                        ]["checkSum"].values[0]
+                        if "_R1.fastq.gz" in item:
+                            do_type = "Metagenome Raw Read 1"
+                        elif "_R2.fastq.gz" in item:
+                            do_type = "Metagenome Raw Read 2"
+
+                        database.data_object_set.append(
+                            self._translate_data_object(
+                                neon_to_nmdc_data_object_ids.get(item),
+                                item,
+                                do_type,
+                                checksum,
+                            )
+                        )
+
+                    database.omics_processing_set.append(
+                        self._translate_omics_processing(
+                            neon_to_nmdc_omprc_ids.get(neon_id),
+                            processed_sample_id,
+                            has_output_do_ids,
+                            lib_prep_row,
+                        )
+                    )
+
+        return database

nmdc_runtime/site/translation/neon_utils.py

@@ -22,7 +22,11 @@ def _get_value_or_none(data: pd.DataFrame, column_name: str) -> Union[str, float
     ):
         if column_name == "horizon":
             return f"{data[column_name].values[0]} horizon"
-        elif column_name == "qaqcStatus":
+        elif (
+            column_name == "qaqcStatus"
+            or column_name == "extrQaqcStatus"
+            or column_name == "seqQaqcStatus"
+        ):
             return data[column_name].values[0].lower()
         elif column_name == "sampleTopDepth":
             return float(data[column_name].values[0]) / 100

{nmdc_runtime-1.4.2.dist-info → nmdc_runtime-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nmdc_runtime
-Version: 1.4.2
+Version: 1.6.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston

{nmdc_runtime-1.4.2.dist-info → nmdc_runtime-1.6.0.dist-info}/RECORD

@@ -35,9 +35,9 @@ nmdc_runtime/minter/domain/model.py,sha256=WMOuKub3dVzkOt_EZSRDLeTsJPqFbKx01SMQ5
 nmdc_runtime/minter/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nmdc_runtime/minter/entrypoints/fastapi_app.py,sha256=JC4thvzfFwRc1mhWQ-kHy3yvs0SYxF6ktE7LXNCwqlI,4031
 nmdc_runtime/site/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nmdc_runtime/site/graphs.py,sha256=siHlRnD2eS9nw3Ne049TcGG6I6IYFvjgWQuuSHzEOqc,9492
-nmdc_runtime/site/ops.py,sha256=YzDm7Dm2sELptwTew8DTOcS3nYBH_JegXhu3wzZuuiY,32482
-nmdc_runtime/site/repository.py,sha256=UgY9eMnNgZxa-Y0QeDyENh4KHtxuBWkYCjxltM4mTzA,30938
+nmdc_runtime/site/graphs.py,sha256=mOWZvT2Rk4X96RmVAvHQwur-FhNuMWAko3jjRLGygEE,11455
+nmdc_runtime/site/ops.py,sha256=YjaH2zqzd01cRcqV0E93RoaWt8T4ExESx4SSszmczZ8,33620
+nmdc_runtime/site/repository.py,sha256=QI9Gcjr68-DT2MPwOx87Vkxcwp3ZIOVaFZ9uCO13w9U,35502
 nmdc_runtime/site/resources.py,sha256=pQSwg1dRpL_D91gYLzzaOIDZ3qa69rPqSlsq5dS9i_M,17783
 nmdc_runtime/site/util.py,sha256=6hyVPpb6ZkWEG8Nm7uQxnZ-QmuPOG9hgWvl0mUBr5JU,1303
 nmdc_runtime/site/backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -64,7 +64,8 @@ nmdc_runtime/site/translation/gold_translator.py,sha256=8i5FxrgAG4rLbM0mcCSBaZEz
 nmdc_runtime/site/translation/jgi.py,sha256=qk878KhIw674TkrVfbl2x1QJrKi3zlvE0vesIpe9slM,876
 nmdc_runtime/site/translation/neon_benthic_translator.py,sha256=e_7tXFrP0PpdhqUCxXmOaFViSuG36IIMDqyj3FHLcgQ,23069
 nmdc_runtime/site/translation/neon_soil_translator.py,sha256=cJJ_QPva5G5SIT_7DjCSsqbDvgbiKGqUYrxK3nx7_Lw,37634
-nmdc_runtime/site/translation/neon_utils.py,sha256=k8JYMnm-L981BTOdAMomR1CulS_Hz5v7aYxrJ94KEJc,5086
+nmdc_runtime/site/translation/neon_surface_water_translator.py,sha256=6LaFwBnVx6TN9v1D-G6LFrDxY0TK05AvMklx0E1tTeQ,26590
+nmdc_runtime/site/translation/neon_utils.py,sha256=mdxJVPb3zbD4DiKW3Fwgk22kjczKMwkcozvy7fwteTE,5203
 nmdc_runtime/site/translation/submission_portal_translator.py,sha256=lHcrfPR5wk3BcZ0Uw5zUyWu5XRVikgOzdzSb5nFVS9I,27964
 nmdc_runtime/site/translation/translator.py,sha256=xM9dM-nTgSWwu5HFoUVNHf8kqk9iiH4PgWdSx4OKxEk,601
 nmdc_runtime/site/translation/util.py,sha256=w_l3SiExGsl6cXRqto0a_ssDmHkP64ITvrOVfPxmNpY,4366
@@ -73,9 +74,9 @@ nmdc_runtime/site/validation/emsl.py,sha256=OG20mv_3E2rkQqTQtYO0_SVRqFb-Z_zKCiAV
 nmdc_runtime/site/validation/gold.py,sha256=Z5ZzYdjERbrJ2Tu06d0TDTBSfwaFdL1Z23Rl-YkZ2Ow,803
 nmdc_runtime/site/validation/jgi.py,sha256=LdJfhqBVHWCDp0Kzyk8eJZMwEI5NQ-zuTda31BcGwOA,1299
 nmdc_runtime/site/validation/util.py,sha256=GGbMDSwR090sr_E_fHffCN418gpYESaiot6XghS7OYk,3349
-nmdc_runtime-1.4.2.dist-info/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
-nmdc_runtime-1.4.2.dist-info/METADATA,sha256=Y-wVXewG_8q6XgG2JMsAUyJJ_bdvadDKRUCWkmP-u4U,7424
-nmdc_runtime-1.4.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-nmdc_runtime-1.4.2.dist-info/entry_points.txt,sha256=nfH6-K9tDKv7va8ENfShsBnxVQoYJdEe7HHdwtkbh1Y,289
-nmdc_runtime-1.4.2.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
-nmdc_runtime-1.4.2.dist-info/RECORD,,
+nmdc_runtime-1.6.0.dist-info/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
+nmdc_runtime-1.6.0.dist-info/METADATA,sha256=hKgDLZfx14AX3IWIi3C9vHa9YAP-agU7tsmKZ_kg8JY,7424
+nmdc_runtime-1.6.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+nmdc_runtime-1.6.0.dist-info/entry_points.txt,sha256=nfH6-K9tDKv7va8ENfShsBnxVQoYJdEe7HHdwtkbh1Y,289
+nmdc_runtime-1.6.0.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
+nmdc_runtime-1.6.0.dist-info/RECORD,,