nmdc-runtime 1.3.1__py3-none-any.whl → 2.12.0__py3-none-any.whl

nmdc_runtime/site/translation/translator.py

@@ -1,9 +1,14 @@
+import logging
+import re
 from abc import ABC, abstractmethod
-from typing import Any, Callable, Dict, List, Optional
+from decimal import Decimal
+from typing import Any, Callable, Dict, List, Optional, Union
 from nmdc_schema import nmdc
 
 JSON_OBJECT = Dict[str, Any]
 
+logger = logging.getLogger(__name__)
+
 
 class Translator(ABC):
     def __init__(
@@ -14,9 +19,74 @@ class Translator(ABC):
     def _index_by_id(self, collection, id):
         return {item[id]: item for item in collection}
 
-    def _get_curie(self, prefix: str, local: str) -> str:
-        return f"{prefix}:{local}"
+    @staticmethod
+    def _ensure_curie(identifier: str, *, default_prefix: str) -> str:
+        identifier_parts = identifier.split(":", 1)
+
+        # Don't add prefix if identifier is already a CURIE
+        if len(identifier_parts) == 2:
+            return identifier
+
+        return f"{default_prefix}:{identifier_parts[0]}"
 
     @abstractmethod
     def get_database(self) -> nmdc.Database:
         pass
+
+    def _parse_quantity_value(
+        self, raw_value: Optional[str], unit: Optional[str] = None
+    ) -> Union[nmdc.QuantityValue, None]:
+        """Construct a nmdc:QuantityValue from a raw value string
+
+        The regex pattern minimally matches on a single numeric value (possibly
+        floating point). The pattern can also identify a range represented by
+        two numeric values separated by a hyphen. It can also identify non-numeric
+        characters at the end of the string which are interpreted as a unit. A unit
+        may also be explicitly provided as an argument to this function. If parsing
+        identifies a unit and a unit argument is provided, the unit argument is used.
+        If the pattern is not matched at all None is returned.
+
+        :param raw_value: string to parse
+        :param unit: optional unit, defaults to None. If None, the unit is extracted from the
+            raw_value. If a unit is provided, it will override the unit extracted from the
+            raw_value.
+        :return: nmdc:QuantityValue
+        """
+        if raw_value is None:
+            return None
+
+        match = re.fullmatch(
+            "([+-]?(?=\.\d|\d)(?:\d+)?(?:\.?\d*)(?:[eE][+-]?\d+)?)(?: *- *([+-]?(?=\.\d|\d)(?:\d+)?(?:\.?\d*)(?:[eE][+-]?\d+)?))?(?: *(\S+))?",
+            raw_value,
+        )
+        if not match:
+            return None
+
+        quantity_value_kwargs = {
+            "has_raw_value": raw_value,
+            "type": "nmdc:QuantityValue",
+        }
+        if match.group(2):
+            # having group 2 means the value is a range like "0 - 1". Either
+            # group 1 or group 2 might be the minimum especially when handling
+            # negative ranges like "0 - -1"
+            num_1 = Decimal(match.group(1))
+            num_2 = Decimal(match.group(2))
+            quantity_value_kwargs["has_minimum_numeric_value"] = min(num_1, num_2)
+            quantity_value_kwargs["has_maximum_numeric_value"] = max(num_1, num_2)
+        else:
+            # otherwise we just have a single numeric value
+            quantity_value_kwargs["has_numeric_value"] = Decimal(match.group(1))
+
+        if unit:
+            # a unit was manually specified
+            if match.group(3) and unit != match.group(3):
+                # a unit was also found in the raw string; issue a warning
+                # if they don't agree, but keep the manually specified one
+                logger.warning(f'Unit mismatch: "{unit}" and "{match.group(3)}"')
+            quantity_value_kwargs["has_unit"] = unit
+        elif match.group(3):
+            # a unit was found in the raw string
+            quantity_value_kwargs["has_unit"] = match.group(3)
+
+        return nmdc.QuantityValue(**quantity_value_kwargs)

nmdc_runtime/site/util.py

@@ -1,10 +1,13 @@
 import os
-from functools import lru_cache
-from subprocess import Popen, PIPE, STDOUT, CalledProcessError
 
+from functools import lru_cache
 from pymongo.database import Database as MongoDatabase
+from subprocess import Popen, PIPE, STDOUT, CalledProcessError
+from refscan.lib.helpers import get_collection_names_from_schema
 
 from nmdc_runtime.site.resources import mongo_resource
+from nmdc_runtime.util import nmdc_schema_view
+
 
 mode_test = {
     "resource_defs": {"mongo": mongo_resource}
@@ -34,14 +37,30 @@ def run_and_log(shell_cmd, context):
 
 
 @lru_cache
-def collection_indexed_on_id(mdb: MongoDatabase) -> dict:
-    set_collection_names = [
-        name for name in mdb.list_collection_names() if name.endswith("_set")
-    ]
+def schema_collection_has_index_on_id(mdb: MongoDatabase) -> dict:
+    """
+    TODO: Document this function.
+    """
+    schema_view = nmdc_schema_view()
+    present_collection_names = set(mdb.list_collection_names())
     return {
-        name: ("id_1" in mdb[name].index_information()) for name in set_collection_names
+        name: (
+            name in present_collection_names and "id_1" in mdb[name].index_information()
+        )
+        for name in get_collection_names_from_schema(schema_view)
     }
 
 
 def get_basename(filename: str) -> str:
     return os.path.basename(filename)
+
+
+def nmdc_study_id_to_filename(nmdc_study_id: str) -> str:
+    return nmdc_study_id.replace(":", "_").replace("-", "_")
+
+
+def get_instruments_by_id(mdb: MongoDatabase) -> dict[str, dict]:
+    """Get all documents from the instrument_set collection in a dict keyed by id."""
+    return {
+        instrument["id"]: instrument for instrument in mdb["instrument_set"].find({})
+    }

nmdc_runtime/site/validation/emsl.py

@@ -1,6 +1,7 @@
 """
 Validates data in the EMSL collection in the nmdc_etl_staging database.
 """
+
 from dagster import op, graph
 from nmdc_runtime.site.validation.util import (
     preset_prod,

nmdc_runtime/site/validation/gold.py

@@ -1,6 +1,7 @@
 """
 Validates data in the GOLD collection in the nmdc_etl_staging database.
 """
+
 from dagster import op, graph
 from nmdc_runtime.site.validation.util import (
     preset_prod,

nmdc_runtime/site/validation/util.py

@@ -1,6 +1,5 @@
-from dagster import op, AssetMaterialization, AssetKey, EventMetadata
-from jsonschema import Draft7Validator
-from nmdc_runtime.util import get_nmdc_jsonschema_dict
+from dagster import op, AssetMaterialization, AssetKey, MetadataValue
+from nmdc_runtime.util import get_nmdc_schema_validator
 from toolz import dissoc
 
 from nmdc_runtime.site.resources import mongo_resource
@@ -61,19 +60,19 @@ def validate_mongo_collection(context, collection_name: str):
     collection = mongo_db[collection_name]  # get mongo collection
     db_set = collection_name.split(".")[0]
 
-    validator = Draft7Validator(get_nmdc_jsonschema_dict())
+    validator = get_nmdc_schema_validator()
     validation_errors = []
 
     for count, doc in enumerate(collection.find()):
         # add logging for progress?
         # e.g.: if count % 1000 == 0: context.log.info(“done X of Y")
         doc = dissoc(doc, "_id")  # dissoc _id
-        errors = list(validator.iter_errors({f"{db_set}": [doc]}))
-        if len(errors) > 0:
+        report = validator.validate({f"{db_set}": [doc]}, target_class="Database")
+        if len(report.results) > 0:
             if "id" in doc.keys():
-                errors = {doc["id"]: [e.message for e in errors]}
+                errors = {doc["id"]: [r.message for r in report.results]}
             else:
-                errors = {f"missing id ({count})": [e.message for e in errors]}
+                errors = {f"missing id ({count})": [r.message for r in report.results]}
             validation_errors.append(errors)
 
     return {"collection_name": collection_name, "errors": validation_errors}
@@ -92,10 +91,15 @@ def announce_validation_report(context, report, api_object):
         asset_key=AssetKey(["validation", f"{collection_name}_validation"]),
         description=f"{collection_name} translation validation",
         metadata={
-            # https://docs.dagster.io/_apidocs/solids#event-metadata
-            # also .json, .md, .path, .url, .python_artifact, ...
-            "n_errors": EventMetadata.int(len(report["errors"])),
-            "object_id": EventMetadata.text(api_object["id"]),
+            # Note: When this code was originally written, it used Dagster's `EventMetadata` class,
+            #       which has since been replaced by Dagster's `MetadataValue` class.
+            #
+            #       Reference:
+            #       - https://docs.dagster.io/api/dagster/ops#dagster.MetadataValue
+            #       - https://docs.dagster.io/api/dagster/metadata#dagster.MetadataValue
+            #
+            "n_errors": MetadataValue.int(len(report["errors"])),
+            "object_id": MetadataValue.text(api_object["id"]),
         },
     )
 

nmdc_runtime/site/workspace.yaml

@@ -0,0 +1,13 @@
+load_from:
+  - python_package:
+      package_name: nmdc_runtime.site.repository
+      attribute: repo
+  - python_package:
+      package_name: nmdc_runtime.site.repository
+      attribute: biosample_submission_ingest
+  - python_package:
+      package_name: nmdc_runtime.site.repository
+      attribute: biosample_export
+  - python_package:
+      package_name: nmdc_runtime.site.repository
+      attribute: database_records_stitching