nmdc-runtime 2.8.0__py3-none-any.whl → 2.10.0__py3-none-any.whl

nmdc_runtime/minter/domain/model.py

@@ -1,9 +1,11 @@
 from enum import Enum
+import re
 from typing import Optional
 
+from base32_lib import base32
 from pydantic import BaseModel, PositiveInt
 
-from nmdc_runtime.minter.config import schema_classes
+from nmdc_runtime.minter.config import schema_classes, typecodes
 
 
 class Entity(BaseModel):
@@ -71,3 +73,35 @@ class Identifier(Entity):
 class Typecode(Entity):
     schema_class: str
     name: str
+
+
+id_prefix_pattern = rf"(?P<prefix>nmdc)"
+id_typecode_pattern = rf"(?P<typecode>[a-z]{{1,6}})"
+id_shoulder_pattern = rf"(?P<shoulder>[0-9][a-z]{{0,6}}[0-9])"
+id_blade_pattern = rf"(?P<blade>[A-Za-z0-9]+)"
+id_version_pattern = rf"(?P<version>(\.[A-Za-z0-9]+)*)"
+id_locus_pattern = rf"(?P<locus>_[A-Za-z0-9_\.-]+)?"
+id_pattern = (
+    rf"^{id_prefix_pattern}:{id_typecode_pattern}-{id_shoulder_pattern}-"
+    rf"{id_blade_pattern}{id_version_pattern}{id_locus_pattern}$"
+)
+ID_TYPECODE_VALUES = [t["name"] for t in typecodes()]
+id_typecode_pattern_strict = rf"(?P<typecode_strict>({'|'.join(ID_TYPECODE_VALUES)}))"
+id_blade_pattern_strict = rf"(?P<blade_strict>[{base32.ENCODING_CHARS}]+)"
+id_pattern_strict = (
+    rf"^{id_prefix_pattern}:{id_typecode_pattern_strict}-{id_shoulder_pattern}-"
+    rf"{id_blade_pattern_strict}{id_version_pattern}{id_locus_pattern}$"
+)
+id_pattern_strict_compiled = re.compile(id_pattern_strict)
+
+
+def check_valid_ids(ids: list[str]):
+    for id_ in ids:
+        if not re.match(id_pattern, id_):
+            raise ValueError(
+                (
+                    f"Invalid ID format for given ID: '{id_}'.\n\nAn ID must match the pattern: '{id_pattern}'.\n\n"
+                    "See: <https://microbiomedata.github.io/nmdc-schema/identifiers/#ids-minted-for-use-within-nmdc>"
+                )
+            )
+    return ids

nmdc_runtime/minter/entrypoints/fastapi_app.py

@@ -8,7 +8,7 @@ from nmdc_runtime.api.core.util import raise404_if_none
 from nmdc_runtime.api.db.mongo import get_mongo_db
 from nmdc_runtime.api.models.site import get_current_client_site, Site
 from nmdc_runtime.minter.adapters.repository import MongoIDStore, MinterError
-from nmdc_runtime.minter.config import minting_service_id, schema_classes
+from nmdc_runtime.minter.config import minting_service_id
 from nmdc_runtime.minter.domain.model import (
     Identifier,
     AuthenticatedMintingRequest,

nmdc_runtime/mongo_util.py

@@ -1,7 +1,6 @@
-from pymongo import MongoClient
 from pymongo.database import Database
 from pymongo.collection import Collection
-from typing import Any, Mapping, Optional, Type, Callable
+from typing import Any, Optional
 from pymongo.client_session import ClientSession
 import inspect
 

nmdc_runtime/site/backup/nmdcdb_mongodump.py

@@ -6,7 +6,7 @@ $ nmdcdb-mongodump
 
 import os
 import subprocess
-from datetime import datetime, timezone
+from datetime import datetime
 from pathlib import Path
 from zoneinfo import ZoneInfo
 

nmdc_runtime/site/backup/nmdcdb_mongoexport.py

@@ -16,9 +16,7 @@ from toolz import assoc
 
 from nmdc_runtime.api.core.util import pick
 from nmdc_runtime.api.db.mongo import get_mongo_db
-from nmdc_runtime.site.repository import run_config_frozen__normal_env
-from nmdc_runtime.site.resources import get_mongo
-from nmdc_runtime.util import nmdc_jsonschema, schema_collection_names_with_id_field
+from nmdc_runtime.util import schema_collection_names_with_id_field
 
 
 def collection_stats(mdb: MongoDatabase):

nmdc_runtime/site/export/ncbi_xml.py

@@ -4,7 +4,7 @@ import datetime
 import xml.etree.ElementTree as ET
 import xml.dom.minidom
 
-from typing import Any, List, Union
+from typing import Any, List
 from urllib.parse import urlparse
 from nmdc_runtime.site.export.ncbi_xml_utils import (
     handle_controlled_identified_term_value,
@@ -16,7 +16,6 @@ from nmdc_runtime.site.export.ncbi_xml_utils import (
     handle_float_value,
     handle_string_value,
     load_mappings,
-    validate_xml,
 )
 
 

nmdc_runtime/site/export/ncbi_xml_utils.py

@@ -1,5 +1,5 @@
 from io import BytesIO, StringIO
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, List
 
 from nmdc_runtime.api.endpoints.util import strip_oid
 from nmdc_runtime.minter.config import typecodes

nmdc_runtime/site/graphs.py

@@ -1,7 +1,6 @@
-from dagster import graph, GraphIn
+from dagster import graph
 
 from nmdc_runtime.site.ops import (
-    build_merged_db,
     generate_biosample_set_for_nmdc_study_from_gold,
     nmdc_schema_database_export_filename,
     nmdc_schema_database_from_gold_study,
@@ -12,8 +11,6 @@ from nmdc_runtime.site.ops import (
     gold_projects_by_study,
     gold_study,
     poll_for_run_completion,
-    run_etl,
-    local_file_to_api_object,
     get_operation,
     produce_curated_db,
     delete_operations,
@@ -70,24 +67,6 @@ from nmdc_runtime.site.ops import (
 from nmdc_runtime.site.export.study_metadata import get_biosamples_by_study_id
 
 
-@graph
-def gold_translation():
-    """
-    Translating an export of the JGI GOLD [1] SQL database to the NMDC database JSON schema.
-
-    [1] Genomes OnLine Database (GOLD) <https://gold.jgi.doe.gov/>.
-    """
-    local_file_to_api_object(run_etl(build_merged_db()))
-
-
-@graph()
-def gold_translation_curation():
-    # TODO
-    #   - have produce_curated_db do actual curation (see notebook), persisting to db.
-    #   - more steps in pipeline? Or handoff via run_status_sensor on DagsterRunStatus.SUCCESS.
-    produce_curated_db(get_operation())
-
-
 @graph()
 def create_objects_from_site_object_puts():
     delete_operations(
@@ -160,6 +139,7 @@ def gold_study_to_database():
         study_type,
         gold_nmdc_instrument_mapping_file_url,
         include_field_site_info,
+        enable_biosample_filtering,
     ) = get_gold_study_pipeline_inputs()
 
     projects = gold_projects_by_study(study_id)
@@ -176,6 +156,7 @@ def gold_study_to_database():
         analysis_projects,
         gold_nmdc_instrument_map_df,
         include_field_site_info,
+        enable_biosample_filtering,
     )
     database_dict = nmdc_schema_object_to_dict(database)
     filename = nmdc_schema_database_export_filename(study)
@@ -506,11 +487,19 @@ def nmdc_study_to_ncbi_submission_export():
 
 @graph
 def generate_data_generation_set_for_biosamples_in_nmdc_study():
-    (study_id, gold_nmdc_instrument_mapping_file_url) = get_database_updater_inputs()
+    (
+        study_id,
+        gold_nmdc_instrument_mapping_file_url,
+        include_field_site_info,
+        enable_biosample_filtering,
+    ) = get_database_updater_inputs()
     gold_nmdc_instrument_map_df = get_df_from_url(gold_nmdc_instrument_mapping_file_url)
 
     database = generate_data_generation_set_post_biosample_ingest(
-        study_id, gold_nmdc_instrument_map_df
+        study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
 
     database_dict = nmdc_schema_object_to_dict(database)
@@ -523,11 +512,19 @@ def generate_data_generation_set_for_biosamples_in_nmdc_study():
 
 @graph
 def generate_biosample_set_from_samples_in_gold():
-    (study_id, gold_nmdc_instrument_mapping_file_url) = get_database_updater_inputs()
+    (
+        study_id,
+        gold_nmdc_instrument_mapping_file_url,
+        include_field_site_info,
+        enable_biosample_filtering,
+    ) = get_database_updater_inputs()
     gold_nmdc_instrument_map_df = get_df_from_url(gold_nmdc_instrument_mapping_file_url)
 
     database = generate_biosample_set_for_nmdc_study_from_gold(
-        study_id, gold_nmdc_instrument_map_df
+        study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
     database_dict = nmdc_schema_object_to_dict(database)
     filename = post_submission_portal_biosample_ingest_record_stitching_filename(
@@ -545,10 +542,18 @@ def generate_update_script_for_insdc_biosample_identifiers():
     to generate a script for updating biosample records with INSDC identifiers obtained from GOLD.
     The script is returned as a dictionary that can be executed against MongoDB.
     """
-    (study_id, gold_nmdc_instrument_mapping_file_url) = get_database_updater_inputs()
+    (
+        study_id,
+        gold_nmdc_instrument_mapping_file_url,
+        include_field_site_info,
+        enable_biosample_filtering,
+    ) = get_database_updater_inputs()
     gold_nmdc_instrument_map_df = get_df_from_url(gold_nmdc_instrument_mapping_file_url)
 
     update_script = run_script_to_update_insdc_biosample_identifiers(
-        study_id, gold_nmdc_instrument_map_df
+        study_id,
+        gold_nmdc_instrument_map_df,
+        include_field_site_info,
+        enable_biosample_filtering,
     )
     render_text(update_script)