nmdc-runtime 1.3.1__py3-none-any.whl → 2.12.0__py3-none-any.whl

nmdc_runtime/static/ORCID-iD_icon_vector.svg

@@ -0,0 +1,4 @@
+<svg width="32" height="32" fill="none" xmlns="http://www.w3.org/2000/svg">
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M32 16c0 8.837-7.163 16-16 16-8.838 0-16-7.163-16-16C0 7.162 7.162 0 16 0c8.837 0 16 7.162 16 16Z" fill="#A6CE39"/>
+  <path fill-rule="evenodd" clip-rule="evenodd" d="M18.813 9.637h-5.45v13.9h5.474c4.555 0 7.35-3.378 7.35-6.95 0-1.635-.562-3.372-1.77-4.704-1.215-1.336-3.065-2.246-5.605-2.246ZM18.6 21.3h-2.813v-9.425H18.5c1.823 0 3.12.552 3.96 1.4.842.849 1.252 2.021 1.252 3.312 0 .784-.239 1.967-.993 2.948-.745.969-2.01 1.765-4.119 1.765Zm5.311-4.026c-.251 1.74-1.494 4.276-5.311 4.276h-3.063H18.6c3.817 0 5.06-2.536 5.311-4.276Zm1.812-2.405c-.657-2.601-2.85-4.982-6.91-4.982h-5.2 5.2c4.06 0 6.253 2.38 6.91 4.982Zm.215 1.718ZM8.363 9.675v13.887h2.425V9.675H8.363Zm2.175 13.637H8.612h1.925ZM9.575 8.65c.84 0 1.513-.689 1.513-1.513 0-.823-.673-1.512-1.513-1.512-.838 0-1.512.674-1.512 1.513 0 .823.672 1.512 1.512 1.512Z" fill="#fff"/>
+</svg>

nmdc_runtime/static/README.md

@@ -0,0 +1,5 @@
+# Static
+
+This document contains information about the origins of the other files in this directory.
+
+- `ORCID-iD_icon_vector.svg`: On September 27, 2025, we downloaded this SVG file from ORCID's [Brand Library](https://orcid.filecamp.com/s/o/3CCuLloCl73Knntn/VU19wHSMUnX9TD4R), which we found a link to on the [Brand Guidelines](https://info.orcid.org/brand-guidelines/) page of ORCID's website.

nmdc_runtime/util.py

@@ -1,31 +1,34 @@
+import importlib.resources
 import json
 import mimetypes
 import os
-import pkgutil
+from collections import defaultdict
 from collections.abc import Iterable
-from contextlib import AbstractContextManager
-from copy import deepcopy
 from datetime import datetime, timezone
 from functools import lru_cache
-from io import BytesIO
+from itertools import chain
 from pathlib import Path
-from uuid import uuid4
-from typing import List, Optional, Set, Dict
+from typing import Callable, List, Optional, Set, Dict
 
-import fastjsonschema
 import requests
+from bson.son import SON
 from frozendict import frozendict
-from jsonschema.validators import Draft7Validator
-from nmdc_schema.nmdc_schema_accepting_legacy_ids import Database as NMDCDatabase
+from linkml.validator import Validator
+from linkml.validator.plugins import JsonschemaValidationPlugin
+from linkml_runtime import SchemaView
+from nmdc_schema import NmdcSchemaValidationPlugin
 from nmdc_schema.get_nmdc_view import ViewGetter
-from pydantic import Field, BaseModel
 from pymongo.database import Database as MongoDatabase
 from pymongo.errors import OperationFailure
-from toolz import merge, unique
+from refscan.lib.helpers import (
+    identify_references,
+    get_collection_name_to_class_names_map,
+)
+from refscan.lib.ReferenceList import ReferenceList
+from toolz import merge
 
 from nmdc_runtime.api.core.util import sha256hash_from_file
 from nmdc_runtime.api.models.object import DrsObjectIn
-from typing_extensions import Annotated
 
 
 def get_class_names_from_collection_spec(
@@ -75,6 +78,23 @@ def get_class_names_from_collection_spec(
     return class_names
 
 
+@lru_cache
+def get_allowed_references() -> ReferenceList:
+    r"""
+    Returns a `ReferenceList` of all the inter-document references that
+    the NMDC Schema allows a schema-compliant MongoDB database to contain.
+    """
+
+    # Identify the inter-document references that the schema allows a database to contain.
+    print("Identifying schema-allowed references.")
+    references = identify_references(
+        schema_view=nmdc_schema_view(),
+        collection_name_to_class_names=collection_name_to_class_names,
+    )
+
+    return references
+
+
 @lru_cache
 def get_type_collections() -> dict:
     """Returns a dictionary mapping class names to Mongo collection names."""
@@ -91,41 +111,23 @@ def get_type_collections() -> dict:
     return mappings
 
 
-def without_id_patterns(nmdc_jsonschema):
-    rv = deepcopy(nmdc_jsonschema)
-    for cls_, spec in rv["$defs"].items():
-        if "properties" in spec:
-            if "id" in spec["properties"]:
-                spec["properties"]["id"].pop("pattern", None)
-    return rv
-
-
 @lru_cache
-def get_nmdc_jsonschema_dict(enforce_id_patterns=True):
-    """Get NMDC JSON Schema with materialized patterns (for identifier regexes)."""
-    d = json.loads(
-        BytesIO(
-            pkgutil.get_data("nmdc_schema", "nmdc_materialized_patterns.schema.json")
-        )
-        .getvalue()
-        .decode("utf-8")
-    )
-    return d if enforce_id_patterns else without_id_patterns(d)
+def get_nmdc_jsonschema_path() -> Path:
+    """Get path to NMDC JSON Schema file."""
+    with importlib.resources.path(
+        "nmdc_schema", "nmdc_materialized_patterns.schema.json"
+    ) as p:
+        return p
 
 
-@lru_cache
-def get_nmdc_jsonschema_validator(enforce_id_patterns=True):
-    return fastjsonschema.compile(
-        get_nmdc_jsonschema_dict(enforce_id_patterns=enforce_id_patterns)
-    )
+@lru_cache()
+def get_nmdc_jsonschema_dict() -> dict:
+    """Get NMDC JSON Schema with materialized patterns (for identifier regexes)."""
+    with open(get_nmdc_jsonschema_path(), "r") as f:
+        return json.load(f)
 
 
 nmdc_jsonschema = get_nmdc_jsonschema_dict()
-nmdc_jsonschema_validator = get_nmdc_jsonschema_validator()
-nmdc_jsonschema_noidpatterns = get_nmdc_jsonschema_dict(enforce_id_patterns=False)
-nmdc_jsonschema_validator_noidpatterns = get_nmdc_jsonschema_validator(
-    enforce_id_patterns=False
-)
 
 REPO_ROOT_DIR = Path(__file__).parent.parent
 
@@ -266,9 +268,9 @@ def find_one(k_v: dict, entities: Iterable[dict]):
     """Find the first entity with key-value pair k_v, if any?
 
     >>> find_one({"id": "foo"}, [{"id": "foo"}])
+    {'id': 'foo'}
+    >>> find_one({"id": "foo"}, [{"id": "bar"}]) is None
     True
-    >>> find_one({"id": "foo"}, [{"id": "bar"}])
-    False
     """
     if len(k_v) > 1:
         raise Exception("Supports only one key-value pair")
@@ -294,6 +296,49 @@ def nmdc_schema_view():
     return ViewGetter().get_view()
 
 
+@lru_cache()
+def get_nmdc_schema_validator() -> Validator:
+    schema_view = nmdc_schema_view()
+    return Validator(
+        schema_view.schema,
+        validation_plugins=[
+            JsonschemaValidationPlugin(
+                closed=True,
+                # Since the `nmdc-schema` package exports a pre-built JSON Schema file, use that
+                # instead of relying on the plugin to generate one on the fly.
+                json_schema_path=get_nmdc_jsonschema_path(),
+            ),
+            NmdcSchemaValidationPlugin(),
+        ],
+    )
+
+
+@lru_cache
+def get_class_name_to_collection_names_map(
+    schema_view: SchemaView,
+) -> Dict[str, List[str]]:
+    """
+    Returns a mapping of class names to the names of the collections that can store instances of those classes/types,
+    according to the specified `SchemaView`.
+
+    Example output:
+    ```
+    {
+        "Study": ["study_set"],
+        "Biosample": ["biosample_set"],
+        ...
+    }
+    ```
+    """
+    class_name_to_collection_names = defaultdict(list)
+    for collection_name, class_names in get_collection_name_to_class_names_map(
+        schema_view
+    ).items():
+        for class_name in class_names:
+            class_name_to_collection_names[class_name].append(collection_name)
+    return class_name_to_collection_names
+
+
 @lru_cache
 def nmdc_database_collection_instance_class_names():
     names = []
@@ -308,6 +353,14 @@ def nmdc_database_collection_instance_class_names():
 
 @lru_cache
 def nmdc_database_collection_names():
+    r"""
+    TODO: Document this function.
+
+    TODO: Assuming this function was designed to return a list of names of all Database slots that represents database
+          collections, import/use the function named `get_collection_names_from_schema` from `refscan.lib.helpers`
+          instead, since (a) it includes documentation and (b) it performs the additional checks the lead schema
+          maintainer expects (e.g. checking whether a slot is `multivalued` and `inlined_as_list`).
+    """
     names = []
     view = nmdc_schema_view()
     all_classes = set(view.all_classes())
@@ -340,6 +393,12 @@ def all_docs_have_unique_id(coll) -> bool:
 
 
 def specialize_activity_set_docs(docs):
+    """
+    TODO: Document this function.
+
+    TODO: Check whether this function is still necessary, given that the `Database` class
+          in `nmdc-schema` does not have a slot named `activity_set`.
+    """
     validation_errors = {}
     type_collections = get_type_collections()
     if "activity_set" in docs:
@@ -369,13 +428,38 @@ def specialize_activity_set_docs(docs):
 
 # Define a mapping from collection name to a list of class names allowable for that collection's documents.
 collection_name_to_class_names: Dict[str, List[str]] = {
-    collection_name: get_class_names_from_collection_spec(spec)
+    collection_name: list(
+        set(
+            chain.from_iterable(
+                nmdc_schema_view().class_descendants(cls_name)
+                for cls_name in get_class_names_from_collection_spec(spec)
+            )
+        )
+    )
     for collection_name, spec in nmdc_jsonschema["$defs"]["Database"][
         "properties"
     ].items()
 }
 
 
+def class_hierarchy_as_list(obj) -> list[str]:
+    """
+    get list of inherited classes for each concrete class
+    """
+    rv = []
+    current_class = obj.__class__
+
+    def recurse_through_bases(cls):
+        if cls.__name__ == "YAMLRoot":
+            return rv
+        rv.append(cls.__name__)
+        for base in cls.__bases__:
+            recurse_through_bases(base)
+        return rv
+
+    return recurse_through_bases(current_class)
+
+
 @lru_cache
 def schema_collection_names_with_id_field() -> Set[str]:
     """
@@ -393,169 +477,129 @@ def schema_collection_names_with_id_field() -> Set[str]:
     return target_collection_names
 
 
-def ensure_unique_id_indexes(mdb: MongoDatabase):
-    """Ensure that any collections with an "id" field have an index on "id"."""
-    candidate_names = (
-        set(mdb.list_collection_names()) | schema_collection_names_with_id_field()
-    )
-    for collection_name in candidate_names:
-        if collection_name.startswith("system."):  # reserved by mongodb
-            continue
+def populated_schema_collection_names_with_id_field(mdb: MongoDatabase) -> List[str]:
+    collection_names = sorted(schema_collection_names_with_id_field())
+    return [n for n in collection_names if mdb[n].find_one({"id": {"$exists": True}})]
 
-        if (
-            collection_name in schema_collection_names_with_id_field()
-            or all_docs_have_unique_id(mdb[collection_name])
-        ):
-            mdb[collection_name].create_index("id", unique=True)
 
+def does_collection_have_unique_index_on_id_field(
+    collection_name: str, db: MongoDatabase
+) -> bool:
+    """Check whether the specified MongoDB collection has a unique index on its `id` field (not `_id`).
 
-class UpdateStatement(BaseModel):
-    q: dict
-    u: dict
-    upsert: bool = False
-    multi: bool = False
+    Note: If the specified MongoDB collection either does not exist or is a _view_ instead of a collection,
+          this function will return `False`.
 
+    References:
+    - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.list_indexes
+    - https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html#pymongo.collection.Collection.index_information
+    """
+    # Check whether the specified collection actually exists in the database; and, if it does,
+    # whether it is really a _collection_ (as opposed to being a _view_). If it doesn't exist,
+    # or it is a view, return `False` right away.
+    collection_infos_cursor = db.list_collections(filter={"name": collection_name})
+    collection_infos = list(collection_infos_cursor)
+    if len(collection_infos) == 0:
+        return False
+    collection_info = collection_infos[0]
+    if collection_info["type"] != "collection":
+        return False
 
-class DeleteStatement(BaseModel):
-    q: dict
-    limit: Annotated[int, Field(ge=0, le=1)] = 1
+    # Now that we know we're dealing with a collection, get information about each of its indexes.
+    collection = db.get_collection(collection_name)
+    for index_information in collection.list_indexes():
+        # Get the "field_name-direction" pairs that make up this index.
+        field_name_and_direction_pairs: SON = index_information["key"]
 
+        # If this index involves a number of fields other than one, skip it.
+        # We're only interested in indexes that involve the `id` field by itself.
+        if len(field_name_and_direction_pairs.keys()) != 1:
+            continue
 
-class OverlayDBError(Exception):
-    pass
+        # Check whether the field this index involves is the `id` field,
+        # and whether this index is `unique`.
+        field_name = list(field_name_and_direction_pairs.keys())[0]
+        if field_name == "id" and index_information.get("unique", False):
+            return True
 
+    return False
 
-class OverlayDB(AbstractContextManager):
-    """Provides a context whereby a base Database is overlaid with a temporary one.
 
-    If you need to run basic simulations of updates to a base database,
-    you don't want to actually commit transactions to the base database.
+def ensure_unique_id_indexes(mdb: MongoDatabase):
+    """Ensure that any collections with an "id" field have an index on "id"."""
 
-    For example, to insert or replace (matching on "id") many documents into a collection in order
-    to then validate the resulting total set of collection documents, an OverlayDB writes to
-    an overlay collection that "shadows" the base collection during a "find" query
-    (the "merge_find" method of an OverlayDB object): if a document with `id0` is found in the
-    overlay collection, that id is marked as "seen" and will not also be returned when
-    subsequently scanning the (unmodified) base-database collection.
+    # Note: The pipe (i.e. `|`) operator performs a union of the two sets. In this case,
+    #       it creates a set (i.e. `candidate_names`) consisting of the names of both
+    #       (a) all collections in the real database, and (b) all collections that
+    #       the NMDC schema says can contain instances of classes that have an "id" slot.
+    candidate_names = (
+        set(mdb.list_collection_names()) | schema_collection_names_with_id_field()
+    )
+    for collection_name in candidate_names:
+        if collection_name.startswith("system."):  # reserved by mongodb
+            continue
 
-    Mongo "update" commands (as the "apply_updates" method) are simulated by first copying affected
-    documents from a base collection to the overlay, and then applying the updates to the overlay,
-    so that again, base collections are unmodified, and a "merge_find" call will produce a result
-    *as if* the base collection(s) were modified.
+        # If the collection already has a unique index on `id`, there's no need
+        # to check anything else about the collection.
+        if does_collection_have_unique_index_on_id_field(collection_name, mdb):
+            continue
 
-    Mongo deletions (as the "delete" method) also copy affected documents from the base collection
-    to the overlay collection, and flag them using the "_deleted" field. In this way, a `merge_find`
-    call will match a relevant document given a suitable filter, and will mark the document's id
-    as "seen" *without* returning the document. Thus, the result is as if the document were deleted.
+        if (
+            collection_name in schema_collection_names_with_id_field()
+            or all_docs_have_unique_id(mdb[collection_name])
+        ):
+            # Check if index already exists, and if so, drop it if not unique
+            try:
+                existing_indexes = list(mdb[collection_name].list_indexes())
+                id_index = next(
+                    (idx for idx in existing_indexes if idx["name"] == "id_1"), None
+                )
 
-    Usage:
-    ````
-    with OverlayDB(mdb) as odb:
-        # do stuff, e.g. `odb.replace_or_insert_many(...)`
-    ```
+                if id_index:
+                    # If index exists but isn't unique, drop it so we can recreate
+                    if not id_index.get("unique", False):
+                        mdb[collection_name].drop_index("id_1")
+
+                # Create index with unique constraint
+                mdb[collection_name].create_index("id", unique=True)
+            except OperationFailure as e:
+                # If error is about index with same name, just continue
+                if "An existing index has the same name" in str(e):
+                    continue
+                else:
+                    # Re-raise other errors
+                    raise
+
+
+def decorate_if(condition: bool = False) -> Callable:
+    r"""
+    Decorator that applies another decorator only when `condition` is `True`.
+
+    Note: We implemented this so we could conditionally register
+          endpoints with FastAPI's `@router`.
+
+    Example usages:
+    A. Apply the `@router.get` decorator:
+       ```python
+       @decorate_if(True)(router.get("/me"))
+       def get_me(...):
+           ...
+       ```
+    B. Bypass the `@router.get` decorator:
+       ```python
+       @decorate_if(False)(router.get("/me"))
+       def get_me(...):
+           ...
+       ```
     """
 
-    def __init__(self, mdb: MongoDatabase):
-        self._bottom_db = mdb
-        self._top_db = self._bottom_db.client.get_database(f"overlay-{uuid4()}")
-        ensure_unique_id_indexes(self._top_db)
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, exc_type, exc_value, traceback):
-        self._bottom_db.client.drop_database(self._top_db.name)
-
-    def replace_or_insert_many(self, coll_name, documents: list):
-        try:
-            self._top_db[coll_name].insert_many(documents)
-        except OperationFailure as e:
-            raise OverlayDBError(str(e.details))
-
-    def apply_updates(self, coll_name, updates: list):
-        """prepare overlay db and apply updates to it."""
-        assert all(UpdateStatement(**us) for us in updates)
-        for update_spec in updates:
-            for bottom_doc in self._bottom_db[coll_name].find(update_spec["q"]):
-                self._top_db[coll_name].insert_one(bottom_doc)
-        try:
-            self._top_db.command({"update": coll_name, "updates": updates})
-        except OperationFailure as e:
-            raise OverlayDBError(str(e.details))
-
-    def delete(self, coll_name, deletes: list):
-        """ "apply" delete command by flagging docs in overlay database"""
-        assert all(DeleteStatement(**us) for us in deletes)
-        for delete_spec in deletes:
-            for bottom_doc in self._bottom_db[coll_name].find(
-                delete_spec["q"], limit=delete_spec.get("limit", 1)
-            ):
-                bottom_doc["_deleted"] = True
-                self._top_db[coll_name].insert_one(bottom_doc)
-
-    def merge_find(self, coll_name, find_spec: dict):
-        """Yield docs first from overlay and then from base db, minding deletion flags."""
-        # ensure projection of "id" and "_deleted"
-        if "projection" in find_spec:
-            proj = find_spec["projection"]
-            if isinstance(proj, dict):
-                proj = merge(proj, {"id": 1, "_deleted": 1})
-            elif isinstance(proj, list):
-                proj = list(unique(proj + ["id", "_deleted"]))
-
-        top_docs = self._top_db[coll_name].find(**find_spec)
-        bottom_docs = self._bottom_db[coll_name].find(**find_spec)
-        top_seen_ids = set()
-        for doc in top_docs:
-            if not doc.get("_deleted"):
-                yield doc
-            top_seen_ids.add(doc["id"])
-
-        for doc in bottom_docs:
-            if doc["id"] not in top_seen_ids:
-                yield doc
-
-
-def validate_json(in_docs: dict, mdb: MongoDatabase):
-    validator = Draft7Validator(get_nmdc_jsonschema_dict())
-    docs = deepcopy(in_docs)
-    validation_errors = {}
-
-    known_coll_names = set(nmdc_database_collection_names())
-    for coll_name, coll_docs in docs.items():
-        if coll_name not in known_coll_names:
-            if coll_name == "@type" and coll_docs in ("Database", "nmdc:Database"):
-                continue
+    def apply_original_decorator(original_decorator: Callable) -> Callable:
+        def check_condition(original_function: Callable) -> Callable:
+            if condition:
+                return original_decorator(original_function)
             else:
-                validation_errors[coll_name] = [
-                    f"'{coll_name}' is not a known schema collection name"
-                ]
-                continue
+                return original_function
 
-        errors = list(validator.iter_errors({coll_name: coll_docs}))
-        validation_errors[coll_name] = [e.message for e in errors]
-        if coll_docs:
-            if not isinstance(coll_docs, list):
-                validation_errors[coll_name].append("value must be a list")
-            elif not all(isinstance(d, dict) for d in coll_docs):
-                validation_errors[coll_name].append(
-                    "all elements of list must be dicts"
-                )
-            if not validation_errors[coll_name]:
-                try:
-                    with OverlayDB(mdb) as odb:
-                        odb.replace_or_insert_many(coll_name, coll_docs)
-                except OverlayDBError as e:
-                    validation_errors[coll_name].append(str(e))
-
-    if all(len(v) == 0 for v in validation_errors.values()):
-        # Second pass. Try instantiating linkml-sourced dataclass
-        in_docs.pop("@type", None)
-        try:
-            NMDCDatabase(**in_docs)
-        except Exception as e:
-            return {"result": "errors", "detail": str(e)}
-
-        return {"result": "All Okay!"}
-    else:
-        return {"result": "errors", "detail": validation_errors}
+        return check_condition
+
+    return apply_original_decorator

nmdc_runtime-2.12.0.dist-info/METADATA

@@ -0,0 +1,45 @@
+Metadata-Version: 2.4
+Name: nmdc-runtime
+Version: 2.12.0
+Summary: A runtime system for NMDC data management and orchestration
+Project-URL: Changelog, https://github.com/microbiomedata/nmdc-runtime/releases
+Project-URL: Documentation, https://docs.microbiomedata.org/runtime
+Project-URL: Issues, https://github.com/microbiomedata/nmdc-runtime/issues
+Project-URL: Repository, https://github.com/microbiomedata/nmdc-runtime
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: base32-lib
+Requires-Dist: boto3
+Requires-Dist: click
+Requires-Dist: dagit
+Requires-Dist: dagster
+Requires-Dist: dagster-graphql
+Requires-Dist: dagster-postgres
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: frozendict
+Requires-Dist: git-root
+Requires-Dist: jq
+Requires-Dist: jsonasobj2
+Requires-Dist: linkml
+Requires-Dist: linkml-runtime
+Requires-Dist: lxml
+Requires-Dist: nmdc-schema==11.13.0
+Requires-Dist: ontology-loader==0.2.2
+Requires-Dist: pandas
+Requires-Dist: passlib[bcrypt]
+Requires-Dist: pydantic[email]>=1.10.0
+Requires-Dist: pyinstrument
+Requires-Dist: pymongo
+Requires-Dist: python-dotenv
+Requires-Dist: python-jose[cryptography]
+Requires-Dist: python-multipart>=0.0.18
+Requires-Dist: pyyaml
+Requires-Dist: refscan==0.3.2
+Requires-Dist: requests
+Requires-Dist: requests-cache
+Requires-Dist: scalar-fastapi<2.0.0,>=1.4.1
+Requires-Dist: tenacity
+Requires-Dist: toolz
+Requires-Dist: tqdm
+Requires-Dist: unidecode
+Requires-Dist: uvicorn[standard]