nmdc-schema 11.12.0rc2__tar.gz → 11.12.1__tar.gz

{nmdc_schema-11.12.0rc2 → nmdc_schema-11.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nmdc_schema
-Version: 11.12.0rc2
+Version: 11.12.1
 Summary: Schema resources for the National Microbiome Data Collaborative (NMDC)
 License: MIT
 License-File: LICENSE

nmdc_schema-11.12.1/nmdc_schema/__init__.py

@@ -0,0 +1 @@
+from nmdc_schema.nmdc_schema_validation_plugin import NmdcSchemaValidationPlugin as NmdcSchemaValidationPlugin

{nmdc_schema-11.12.0rc2 → nmdc_schema-11.12.1}/nmdc_schema/migration_recursion.py

@@ -14,6 +14,8 @@ from nmdc_schema.migrators.adapters.dictionary_adapter import DictionaryAdapter
 logger = logging.getLogger(__name__)
 click_log.basic_config(logger)
 
+# TODO: Is this script somewhat redundant with nmdc_schema/migrators/cli/run_migrator.py? Can they
+#       be consolidated?
 
 # todo: log before and after states of migration
 

{nmdc_schema-11.12.0rc2 → nmdc_schema-11.12.1}/nmdc_schema/migrators/cli/run_migrator.py

@@ -17,6 +17,7 @@ Environment variables (loaded from .env file if present):
     MONGO_PASSWORD - MongoDB password
     MONGO_AUTH_DB - Authentication database name
 """
+import importlib
 
 import click
 import sys
@@ -25,21 +26,15 @@ from dotenv import load_dotenv
 from pymongo import MongoClient
 from nmdc_schema.migrators.adapters.mongo_adapter import MongoAdapter
 
+# TODO: Is this script somewhat redundant with nmdc_schema/migration_recursion.py? Can they be
+#       consolidated?
+
 # Load environment variables from .env file if it exists
 load_dotenv()
 
-# Import all migrators
-from nmdc_schema.migrators.partials.migrator_from_11_9_1_to_11_10_0 import migrator_from_11_9_1_to_11_10_0_part_1
-
-# Map migrator names to their classes
-MIGRATORS = {
-    "migrator_from_11_9_1_to_11_10_0": migrator_from_11_9_1_to_11_10_0_part_1.Migrator,
-    # Add other migrators here as needed
-}
-
 
 @click.command()
-@click.argument("migrator_name", required=True, type=click.Choice(list(MIGRATORS.keys())))
+@click.argument("migrator", required=True)
 @click.argument("action", required=False, default="rollback", type=click.Choice(["commit", "rollback"]))
 @click.option("--host", default=lambda: os.getenv("MONGO_HOST", "localhost"), help="MongoDB host (default: from MONGO_HOST env var or localhost)")
 @click.option("--port", default=lambda: int(os.getenv("MONGO_PORT", "27022")), type=int, help="MongoDB port (default: from MONGO_PORT env var or 27022)")
@@ -47,10 +42,10 @@ MIGRATORS = {
 @click.option("--auth-db", default=lambda: os.getenv("MONGO_AUTH_DB"), help="Authentication database name (default: from MONGO_AUTH_DB env var)")
 @click.option("--username", default=lambda: os.getenv("MONGO_USERNAME"), help="MongoDB username (default: from MONGO_USERNAME env var)")
 @click.option("--password", default=lambda: os.getenv("MONGO_PASSWORD"), help="MongoDB password (default: from MONGO_PASSWORD env var)")
-def run_migrator(migrator_name, action, host, port, database, auth_db, username, password):
+def run_migrator(migrator, action, host, port, database, auth_db, username, password):
     """Run a specific migrator against MongoDB using PyMongo.
     
-    MIGRATOR_NAME should be one of the available migrators.
+    MIGRATOR should the fully qualified module name for module containing a Migrator class
     ACTION should be either 'commit' to save changes or 'rollback' to discard changes (default: rollback).
     """
     
@@ -70,12 +65,13 @@ def run_migrator(migrator_name, action, host, port, database, auth_db, username,
     
     commit_changes = action == "commit"
     
-    click.echo(f"Running migrator: {migrator_name}")
+    click.echo(f"Running migrator: {migrator}")
     click.echo(f"Action: {action.upper()} (changes will {'be saved' if commit_changes else 'be discarded'})")
     click.echo(f"MongoDB connection: {host}:{port}/{database}")
     
     # Get the migrator class
-    migrator_class = MIGRATORS[migrator_name]
+    migrator_module = importlib.import_module(migrator)
+    migrator_class = getattr(migrator_module, "Migrator")
     
     # Connect to MongoDB using PyMongo
     try:

{nmdc_schema-11.12.0rc2 → nmdc_schema-11.12.1}/nmdc_schema/migrators/helpers.py

@@ -113,3 +113,27 @@ def get_classes_with_slots_by_range(schema_view: SchemaView, range_constraint: s
     
     return classes_with_slots
 
+
+# TODO: Check whether this function can be replaced by a similar function from `refscan`.
+def get_database_collections_for_class(schema_view: SchemaView, class_name: str) -> List[str]:
+    """
+    Returns the list of database collection names associated with the specified class.
+    This function retrieves the collection names from the slots of the Database class
+    that have the specified class as their range.
+
+    Args:
+        schema_view: SchemaView instance
+        class_name: The name of the class to find collections for
+
+    Returns:
+        List[str]: List of collection names which permit storage of instances of the specified class
+    """
+    collection_names = set()
+
+    for database_slot_name in schema_view.class_slots(DATABASE_CLASS_NAME):
+        database_slot_def = schema_view.get_slot(database_slot_name, strict=True)
+        database_slot_range_descendants = schema_view.class_descendants(database_slot_def.range, reflexive=True)
+        if class_name in database_slot_range_descendants:
+            collection_names.add(database_slot_name)
+
+    return list(collection_names)

nmdc_schema-11.12.1/nmdc_schema/migrators/migrator_from_11_11_0_to_11_12_0.py

@@ -0,0 +1,35 @@
+from nmdc_schema.migrators.migrator_base import MigratorBase
+from nmdc_schema.migrators.partials.migrator_from_11_11_0_to_11_12_0 import (
+    get_migrator_classes,
+)
+
+
+class Migrator(MigratorBase):
+    r"""
+    Migrates a database between two schemas.
+    """
+
+    _from_version = "11.11.0"
+    _to_version = "11.12.0"
+
+    def upgrade(self, commit_changes: bool = False) -> None:
+        r"""
+        Migrates the database from conforming to the original schema, to conforming to the new schema.
+
+        This migrator uses partial migrators. It runs them in the order in which they are returned by
+        the `get_migrator_classes` function.
+        
+        Args:
+            commit_changes: If True, commits the changes. If False (default), performs a dry run or rollback.
+        """
+
+        migrator_classes = get_migrator_classes()
+        num_migrators = len(migrator_classes)
+        for idx, migrator_class in enumerate(migrator_classes):
+            self.logger.info(f"Running migrator {idx + 1} of {num_migrators}")
+            self.logger.debug(
+                f"Migrating from {migrator_class.get_origin_version()} "
+                f"to {migrator_class.get_destination_version()}"
+            )
+            migrator = migrator_class(adapter=self.adapter, logger=self.logger)
+            migrator.upgrade(commit_changes=commit_changes)

nmdc_schema-11.12.1/nmdc_schema/migrators/partials/migrator_from_11_11_0_to_11_12_0/__init__.py

@@ -0,0 +1,28 @@
+from typing import List, Type
+
+from nmdc_schema.migrators.migrator_base import MigratorBase
+from nmdc_schema.migrators.partials.migrator_from_11_11_0_to_11_12_0 import (
+    migrator_from_11_11_0_to_11_12_0_part_1,
+    migrator_from_11_11_0_to_11_12_0_part_2,
+)
+
+
+def get_migrator_classes() -> List[Type[MigratorBase]]:
+    r"""
+    Returns a list of migrator classes in the order in which they (i.e. their `upgrade` methods)
+    were designed to be run.
+
+    >>> migrator_classes = get_migrator_classes()
+    >>> type(migrator_classes) is list and len(migrator_classes) > 0  # the function returns a list
+    True
+    >>> from inspect import isclass
+    >>> all(isclass(c) for c in migrator_classes)  # each list item is a class
+    True
+    >>> all(callable(getattr(c, "upgrade")) for c in migrator_classes)  # each class has an `upgrade` method
+    True
+    """
+
+    return [
+        migrator_from_11_11_0_to_11_12_0_part_1.Migrator,
+        migrator_from_11_11_0_to_11_12_0_part_2.Migrator,
+    ]

nmdc_schema-11.12.1/nmdc_schema/migrators/partials/migrator_from_11_11_0_to_11_12_0/migrator_from_11_11_0_to_11_12_0_part_1.py

@@ -0,0 +1,187 @@
+import sys
+from pathlib import Path
+
+from nmdc_schema.migrators.migrator_base import MigratorBase
+from nmdc_schema.migrators.adapters.mongo_adapter import MongoAdapter
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+class Migrator(MigratorBase):
+    r'''
+    Migrates a database between two schemas.
+    '''
+
+    _from_version = '11.11.0'
+    _to_version = '11.12.0.part_1'
+
+    def upgrade(self, commit_changes: bool = False) -> None:
+        r'''
+        Migrates the database from conforming to the original schema, to conforming to the new schema.
+        '''
+
+        if isinstance(self.adapter, MongoAdapter):
+            try:
+                self.adapter.execute_in_transaction(
+                    operations_callback=self.perform_all_migration_operations,
+                    commit_changes=commit_changes
+                )
+
+                if commit_changes:
+                    self.logger.info("Transaction committed (changes have been saved)")
+                else:
+                    self.logger.info("Transaction rolled back (no changes were committed)")
+
+            except Exception as e:
+                self.logger.error(f"Migration failed: {e}")
+                raise
+        else:
+            raise NotImplementedError("This migrator currently only supports MongoAdapter.")
+
+
+    def perform_all_migration_operations(self, _, __) -> None:
+        self.logger.info("Moving problematic QuantityValue entries to misc_param")
+        self.adapter.process_each_document("biosample_set", [
+            self.migrate_misc_param_to_PropertyAssertion_range,
+            self.move_problematic_values_to_misc_param,
+        ])
+
+
+    def migrate_misc_param_to_PropertyAssertion_range(self, biosample: dict) -> dict:
+        r"""The misc_param slot on Biosample used to have range TextValue, but now has range
+        PropertyAssertion. This migrator changes misc_param values to conform to the new range.
+
+        >>> m = Migrator()
+
+        # Test: Biosample with no misc_param
+        >>> biosample_no_misc = {
+        ...     "id": "test1",
+        ...     "type": "nmdc:Biosample"
+        ... }
+        >>> m.migrate_misc_param_to_PropertyAssertion_range(biosample_no_misc)
+        {'id': 'test1', 'type': 'nmdc:Biosample'}
+
+        # Test: Biosample with misc_param as TextValue
+        >>> biosample_with_misc = {
+        ...     "id": "test2",
+        ...     "type": "nmdc:Biosample",
+        ...     "misc_param": [{"type": "nmdc:TextValue", "has_raw_value": "biomass_yield;2.565699 Mg/ha"}]
+        ... }
+        >>> m.migrate_misc_param_to_PropertyAssertion_range(biosample_with_misc)
+        {'id': 'test2', 'type': 'nmdc:Biosample', 'misc_param': [{'type': 'nmdc:PropertyAssertion', 'has_raw_value': 'biomass_yield;2.565699 Mg/ha', 'has_attribute_label': 'biomass_yield', 'has_numeric_value': 2.565699, 'has_unit': 'Mg/ha'}]}
+        """
+        misc_param = biosample.get("misc_param")
+        if not misc_param:
+            return biosample
+
+        property_assertions = []
+        for item in misc_param:
+            if isinstance(item, dict) and item.get("type") == "nmdc:TextValue":
+                # Convert TextValue to PropertyAssertion
+                raw_value = item.get("has_raw_value")
+                if not raw_value:
+                    continue
+                pa = {
+                    "type": "nmdc:PropertyAssertion",
+                    "has_raw_value": raw_value
+                }
+                if ";" in raw_value:
+                    [label, _, value] = item.get("has_raw_value", "").partition(";")
+                    pa["has_attribute_label"] = label.strip()
+                    if " " in value:
+                        num_str, _, unit = value.strip().partition(" ")
+                        try:
+                            pa["has_numeric_value"] = float(num_str)
+                        except ValueError:
+                            pass  # If conversion fails, skip adding has_numeric_value
+                        if unit:
+                            pa["has_unit"] = unit.strip()
+                property_assertions.append(pa)
+            else:
+                raise ValueError(
+                    f"Biosample {biosample.get('id')} has misc_param item that is not a TextValue: {item}"
+                )
+        biosample["misc_param"] = property_assertions
+        return biosample
+
+
+    def move_problematic_values_to_misc_param(self, biosample: dict) -> dict:
+        r"""Move QuantityValue entries with invalid units to misc_param as PropertyAssertion.
+
+        >>> m = Migrator()
+
+        # Test: Biosample with valid QuantityValue entries
+        >>> biosample_valid = {
+        ...     "id": "test1",
+        ...     "type": "nmdc:Biosample",
+        ...     "abs_air_humidity": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "kg/kg",
+        ...         "has_numeric_value": 75.0
+        ...     },
+        ...     "diss_oxygen": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "umol/L",
+        ...         "has_numeric_value": 8.0
+        ...     },
+        ...     "solar_irradiance": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "kW/m2/d",
+        ...         "has_numeric_value": 200.0
+        ...     }
+        ... }
+        >>> m.move_problematic_values_to_misc_param(biosample_valid)
+        {'id': 'test1', 'type': 'nmdc:Biosample', 'abs_air_humidity': {'type': 'nmdc:QuantityValue', 'has_unit': 'kg/kg', 'has_numeric_value': 75.0}, 'diss_oxygen': {'type': 'nmdc:QuantityValue', 'has_unit': 'umol/L', 'has_numeric_value': 8.0}, 'solar_irradiance': {'type': 'nmdc:QuantityValue', 'has_unit': 'kW/m2/d', 'has_numeric_value': 200.0}}
+
+        # Test: Biosample with problematic QuantityValue entries
+        >>> biosample_problematic = {
+        ...     "id": "test2",
+        ...     "type": "nmdc:Biosample",
+        ...     "abs_air_humidity": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "kPa",
+        ...         "has_numeric_value": 75.0
+        ...     },
+        ...     "diss_oxygen": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "mL/L",
+        ...         "has_numeric_value": 8.0
+        ...     },
+        ...     "solar_irradiance": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "W/m2",
+        ...         "has_numeric_value": 200.0
+        ...     }
+        ... }
+        >>> m.move_problematic_values_to_misc_param(biosample_problematic)
+        {'id': 'test2', 'type': 'nmdc:Biosample', 'misc_param': [{'type': 'nmdc:PropertyAssertion', 'has_attribute_label': 'abs_air_humidity', 'has_attribute_id': 'MIXS:0000122', 'has_numeric_value': 75.0, 'has_unit': 'kPa'}, {'type': 'nmdc:PropertyAssertion', 'has_attribute_label': 'diss_oxygen', 'has_attribute_id': 'MIXS:0000119', 'has_numeric_value': 8.0, 'has_unit': 'mL/L'}, {'type': 'nmdc:PropertyAssertion', 'has_attribute_label': 'solar_irradiance', 'has_attribute_id': 'MIXS:0000112', 'has_numeric_value': 200.0, 'has_unit': 'W/m2'}]}
+        """
+        problematic_units = (
+            ("abs_air_humidity", "MIXS:0000122", "kPa"),
+            ("diss_oxygen", "MIXS:0000119", "mL/L"),
+            ("solar_irradiance", "MIXS:0000112", "W/m2"),
+        )
+        for slot_name, slot_uri, bad_unit in problematic_units:
+            qv = biosample.get(slot_name)
+            if isinstance(qv, dict) and qv.get("type") == "nmdc:QuantityValue":
+                if qv.get("has_unit") == bad_unit:
+                    # Move to misc_param as PropertyAssertion
+                    pa = {
+                        "type": "nmdc:PropertyAssertion",
+                        "has_attribute_label": slot_name,
+                        "has_attribute_id": slot_uri,
+                    }
+                    if "has_numeric_value" in qv:
+                        pa["has_numeric_value"] = qv["has_numeric_value"]
+                    if "has_unit" in qv:
+                        pa["has_unit"] = qv["has_unit"]
+                    if "has_raw_value" in qv:
+                        pa["has_raw_value"] = qv["has_raw_value"]
+
+                    # Ensure misc_param is a list and append the new PropertyAssertion
+                    if "misc_param" not in biosample or not isinstance(biosample["misc_param"], list):
+                        biosample["misc_param"] = []
+                    biosample["misc_param"].append(pa)
+                    # Remove the problematic QuantityValue from its original slot
+                    del biosample[slot_name]
+        return biosample

nmdc_schema-11.12.1/nmdc_schema/migrators/partials/migrator_from_11_11_0_to_11_12_0/migrator_from_11_11_0_to_11_12_0_part_2.py

@@ -0,0 +1,113 @@
+from linkml.validator import Validator
+
+from nmdc_schema.migrators.migrator_base import MigratorBase
+import sys
+from pathlib import Path
+from nmdc_schema.migrators.helpers import create_schema_view, get_classes_with_slots_by_range, \
+    get_database_collections_for_class
+from nmdc_schema import NmdcSchemaValidationPlugin
+
+project_root = Path(__file__).parent.parent.parent
+sys.path.insert(0, str(project_root))
+
+
+class Migrator(MigratorBase):
+    r"""A check-only migrator that raises an exception if any QuantityValue's has_unit slot
+    is not valid against the slot's storage_unit or UnitEnum constraints."""
+
+    _from_version = '11.11.0'
+    _to_version = '11.12.0.part_2'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self.schema_view = create_schema_view()
+        self.validator = Validator(
+            self.schema_view.schema,
+            # This is intentionally *only* using the NMDC plugin, because this migrator is *only*
+            # concerned with validating the QuantityValue-related constraints that the plugin
+            # implements.
+            validation_plugins=[
+                NmdcSchemaValidationPlugin()
+            ]
+        )
+
+    def upgrade(self, commit_changes: bool = False) -> None:
+        r'''
+        Migrates the database from conforming to the original schema, to conforming to the new schema.
+        '''
+
+        # Get the schema classes which have slots with range QuantityValue
+        classes_with_qv_slots = get_classes_with_slots_by_range(self.schema_view, 'QuantityValue')
+
+        # Get the Database collection names that can hold these classes
+        eligible_collection_names = set()
+        for class_name in classes_with_qv_slots.keys():
+            collection_names = get_database_collections_for_class(self.schema_view, class_name)
+            eligible_collection_names.update(collection_names)
+
+        # Apply migrator through collections
+        self.logger.info("Checking QuantityValue units against UnitEnum and storage_units constraints")
+        for collection_name in eligible_collection_names:
+            self.logger.info(f"  Checking collection '{collection_name}'")
+            self.adapter.do_for_each_document(collection_name, self.confirm_units_fit_unitenum_and_storage_units)
+
+    def confirm_units_fit_unitenum_and_storage_units(self, document: dict) -> None:
+        r'''
+        Raise an exception if the QuantityValue's has_unit slot is not valid against slot's storage_unit or UnitEnum constraints.
+
+        >>> m = Migrator()
+
+        # Test: valid QuantityValue with proper units in biosample
+        >>> valid_biosample = {
+        ...     "id": "test1",
+        ...     "type": "nmdc:Biosample",
+        ...     "temp": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "Cel",
+        ...         "has_numeric_value": 25.0
+        ...     }
+        ... }
+        >>> m.confirm_units_fit_unitenum_and_storage_units(valid_biosample)  # Should not raise
+
+        # Test: invalid unit not in UnitEnum
+        >>> invalid_biosample_enum = {
+        ...     "id": "test2",
+        ...     "type": "nmdc:Biosample",
+        ...     "temp": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "invalid_unit",
+        ...         "has_numeric_value": 25.0
+        ...     }
+        ... }
+        >>> m.confirm_units_fit_unitenum_and_storage_units(invalid_biosample_enum)
+        Traceback (most recent call last):
+            ...
+        ValueError: In test2:
+          QuantityValue at /temp has unit 'invalid_unit' which is not allowed for slot 'temp' (allowed: Cel)
+
+        # Test: unit not allowed for specific slot's storage_units
+        >>> invalid_biosample_storage = {
+        ...     "id": "test3",
+        ...     "type": "nmdc:Biosample",
+        ...     "temp": {
+        ...         "type": "nmdc:QuantityValue",
+        ...         "has_unit": "m",  # Wrong unit type for temperature
+        ...         "has_numeric_value": 25.0
+        ...     }
+        ... }
+        >>> m.confirm_units_fit_unitenum_and_storage_units(invalid_biosample_storage)
+        Traceback (most recent call last):
+            ...
+        ValueError: In test3:
+          QuantityValue at /temp has unit 'm' which is not allowed for slot 'temp' (allowed: Cel)
+        '''
+
+        document_id = document.get('id', '<unknown id>')
+        document_type = document.get('type')
+        if not document_type:
+            raise ValueError(f"Unable to infer target_class for document with id '{document_id}'")
+        target_class = document_type.replace('nmdc:', '')
+        report = self.validator.validate(document, target_class)
+        if report.results:
+            raise ValueError(f"In {document_id}:\n" + "\n".join("  " + result.message for result in report.results))