nmdc-runtime 2.9.0__py3-none-any.whl → 2.10.0__py3-none-any.whl

nmdc_runtime/api/core/util.py

@@ -0,0 +1,109 @@
+import hashlib
+import json
+import os
+import secrets
+import string
+from datetime import datetime, timezone, timedelta
+from importlib import import_module
+
+from fastapi import HTTPException, status
+from pydantic import BaseModel
+from toolz import keyfilter
+
+API_SITE_ID = os.getenv("API_SITE_ID")
+API_SITE_CLIENT_ID = os.getenv("API_SITE_CLIENT_ID")
+
+
+def omit(blacklist, d):
+    return keyfilter(lambda k: k not in blacklist, d)
+
+
+def pick(whitelist, d):
+    return keyfilter(lambda k: k in whitelist, d)
+
+
+def hash_from_str(s: str, algo="sha256") -> str:
+    if algo not in hashlib.algorithms_guaranteed:
+        raise ValueError(f"desired algorithm {algo} not supported")
+    return getattr(hashlib, algo)(s.encode("utf-8")).hexdigest()
+
+
+def sha256hash_from_file(file_path: str, timestamp: str):
+    # https://stackoverflow.com/a/55542529
+    h = hashlib.sha256()
+
+    timestamp_bytes = timestamp.encode("utf-8")
+    h.update(timestamp_bytes)
+
+    with open(file_path, "rb") as file:
+        while True:
+            # Reading is buffered, so we can read smaller chunks.
+            chunk = file.read(h.block_size)
+            if not chunk:
+                break
+            h.update(chunk)
+
+    return h.hexdigest()
+
+
+def raise404_if_none(doc, detail="Not found"):
+    if doc is None:
+        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=detail)
+    return doc
+
+
+def now(as_str=False):
+    dt = datetime.now(timezone.utc)
+    return dt.isoformat() if as_str else dt
+
+
+def expiry_dt_from_now(days=0, hours=0, minutes=0, seconds=0):
+    return now() + timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds)
+
+
+def has_passed(dt):
+    return now() > dt
+
+
+def import_via_dotted_path(dotted_path: str):
+    module_name, _, member_name = dotted_path.rpartition(".")
+    return getattr(import_module(module_name), member_name)
+
+
+def dotted_path_for(member):
+    return f"{member.__module__}.{member.__name__}"
+
+
+def generate_secret(length=12):
+    """Generate a secret.
+
+    With
+    - at least one lowercase character,
+    - at least one uppercase character, and
+    - at least three digits
+
+    """
+    if length < 8:
+        raise ValueError(f"{length=} must be >=8.")
+    alphabet = string.ascii_letters + string.digits + "!@#$%^*-_+="
+    # based on https://docs.python.org/3.8/library/secrets.html#recipes-and-best-practices
+    while True:
+        _secret = "".join(secrets.choice(alphabet) for i in range(length))
+        if (
+            any(c.islower() for c in _secret)
+            and any(c.isupper() for c in _secret)
+            and sum(c.isdigit() for c in _secret) >= 3
+        ):
+            break
+    return _secret
+
+
+def json_clean(data, model, exclude_unset=False) -> dict:
+    """Run data through a JSON serializer for a pydantic model."""
+    if not isinstance(data, (dict, BaseModel)):
+        raise TypeError("`data` must be a pydantic model or its .model_dump()")
+    m = model(**data) if isinstance(data, dict) else data
+
+    # Note: Between Pydantic v1 and v2, the `json` method was renamed to `model_dump_json`.
+    #       Reference: https://docs.pydantic.dev/2.11/migration/#changes-to-pydanticbasemodel
+    return json.loads(m.model_dump_json(exclude_unset=exclude_unset))

nmdc_runtime/api/db/mongo.py

@@ -0,0 +1,447 @@
+import gzip
+import os
+from contextlib import AbstractContextManager
+from copy import deepcopy
+from functools import lru_cache
+from typing import Set
+from uuid import uuid4
+
+import bson
+from jsonschema import Draft7Validator
+from nmdc_schema.nmdc import Database as NMDCDatabase
+from pymongo.errors import AutoReconnect, OperationFailure
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase
+from refscan.lib.Finder import Finder
+from refscan.scanner import scan_outgoing_references
+from tenacity import wait_random_exponential, retry, retry_if_exception_type
+from toolz import merge, unique
+from refscan.lib.helpers import get_collection_names_from_schema
+
+from nmdc_runtime.api.models.query import UpdateStatement, DeleteStatement
+from nmdc_runtime.mongo_util import SessionBoundDatabase
+from nmdc_runtime.util import (
+    nmdc_schema_view,
+    collection_name_to_class_names,
+    ensure_unique_id_indexes,
+    get_nmdc_jsonschema_dict,
+    nmdc_database_collection_names,
+    get_allowed_references,
+)
+from pymongo import MongoClient
+from pymongo.database import Database as MongoDatabase
+
+
+@retry(
+    retry=retry_if_exception_type(AutoReconnect),
+    wait=wait_random_exponential(multiplier=0.5, max=60),
+)
+def check_mongo_ok_autoreconnect(mdb: MongoDatabase):
+    r"""
+    Check whether the application can write to the database.
+    """
+    collection = mdb.get_collection("_runtime.healthcheck")
+    collection.insert_one({"status": "ok"})
+    collection.delete_many({"status": "ok"})
+    return True
+
+
+@lru_cache
+def get_mongo_client() -> MongoClient:
+    r"""
+    Returns a `MongoClient` instance you can use to access the MongoDB server specified via environment variables.
+    Reference: https://pymongo.readthedocs.io/en/stable/api/pymongo/mongo_client.html#pymongo.mongo_client.MongoClient
+    """
+    return MongoClient(
+        host=os.getenv("MONGO_HOST"),
+        username=os.getenv("MONGO_USERNAME"),
+        password=os.getenv("MONGO_PASSWORD"),
+        directConnection=True,
+    )
+
+
+@lru_cache
+def get_mongo_db() -> MongoDatabase:
+    r"""
+    Returns a `Database` instance you can use to access the MongoDB database specified via an environment variable.
+    Reference: https://pymongo.readthedocs.io/en/stable/api/pymongo/database.html#pymongo.database.Database
+    """
+    _client = get_mongo_client()
+    mdb = _client[os.getenv("MONGO_DBNAME")]
+    check_mongo_ok_autoreconnect(mdb)
+    return mdb
+
+
+@lru_cache
+def get_session_bound_mongo_db(session=None) -> MongoDatabase:
+    r"""
+    Returns a `Database` instance you can use to access the MongoDB database specified via an environment variable.
+    Reference: https://pymongo.readthedocs.io/en/stable/api/pymongo/database.html#pymongo.database.Database
+    """
+    _client = get_mongo_client()
+    mdb = _client[os.getenv("MONGO_DBNAME")]
+    check_mongo_ok_autoreconnect(mdb)
+    return SessionBoundDatabase(mdb, session) if session is not None else mdb
+
+
+@lru_cache
+def get_async_mongo_db() -> AsyncIOMotorDatabase:
+    _client = AsyncIOMotorClient(
+        host=os.getenv("MONGO_HOST"),
+        username=os.getenv("MONGO_USERNAME"),
+        password=os.getenv("MONGO_PASSWORD"),
+        directConnection=True,
+    )
+    return _client[os.getenv("MONGO_DBNAME")]
+
+
+def get_nonempty_nmdc_schema_collection_names(mdb: MongoDatabase) -> Set[str]:
+    """
+    Returns the names of the collections that (a) exist in the database,
+    (b) are described by the schema, and (c) contain at least one document.
+
+    Note: The ampersand (`&`) is the "set intersection" operator.
+    """
+    collection_names_from_database = mdb.list_collection_names()
+    schema_view = nmdc_schema_view()
+    collection_names_from_schema = get_collection_names_from_schema(schema_view)
+    names = set(collection_names_from_database) & set(collection_names_from_schema)
+    return {name for name in names if mdb[name].estimated_document_count() > 0}
+
+
+@lru_cache
+def activity_collection_names(mdb: MongoDatabase) -> Set[str]:
+    r"""
+    TODO: Document this function.
+    """
+    return get_nonempty_nmdc_schema_collection_names(mdb) - {
+        "biosample_set",
+        "study_set",
+        "data_object_set",
+        "functional_annotation_set",
+        "genome_feature_set",
+    }
+
+
+@lru_cache
+def get_planned_process_collection_names() -> Set[str]:
+    r"""
+    Returns the names of all collections that the schema says can contain documents
+    that represent instances of the `PlannedProcess` class or any of its subclasses.
+    """
+    schema_view = nmdc_schema_view()
+    collection_names = set()
+    planned_process_descendants = set(schema_view.class_descendants("PlannedProcess"))
+
+    for collection_name, class_names in collection_name_to_class_names.items():
+        for class_name in class_names:
+            # If the name of this class is the name of the `PlannedProcess` class
+            # or any of its subclasses, add it to the result set.
+            if class_name in planned_process_descendants:
+                collection_names.add(collection_name)
+
+    return collection_names
+
+
+def mongodump_excluded_collections() -> str:
+    """
+    TODO: Document this function.
+    """
+    _mdb = get_mongo_db()
+    schema_view = nmdc_schema_view()
+    collection_names_from_database = _mdb.list_collection_names()
+    collection_names_from_schema = get_collection_names_from_schema(schema_view)
+    excluded_collections = " ".join(
+        f"--excludeCollection={c}"
+        for c in sorted(
+            set(collection_names_from_database) - set(collection_names_from_schema)
+        )
+    )
+    return excluded_collections
+
+
+def mongorestore_collection(mdb, collection_name, bson_file_path):
+    """
+    Replaces the specified collection with one that reflects the contents of the
+    specified BSON file.
+    """
+    with gzip.open(bson_file_path, "rb") as bson_file:
+        data = bson.decode_all(bson_file.read())
+        if data:
+            mdb.drop_collection(collection_name)
+            mdb[collection_name].insert_many(data)
+            print(
+                f"mongorestore_collection: inserted {len(data)} documents into {collection_name} after drop"
+            )
+        else:
+            print(f"mongorestore_collection: no {collection_name} documents found")
+
+
+def mongorestore_from_dir(mdb, dump_directory, skip_collections=None):
+    """
+    Effectively runs a `mongorestore` command in pure Python.
+    Helpful in a container context that does not have the `mongorestore` command available.
+    """
+    skip_collections = skip_collections or []
+    for root, dirs, files in os.walk(dump_directory):
+        for file in files:
+            if file.endswith(".bson.gz"):
+                collection_name = file.replace(".bson.gz", "")
+                if collection_name in skip_collections:
+                    continue
+                bson_file_path = os.path.join(root, file)
+                mongorestore_collection(mdb, collection_name, bson_file_path)
+
+    print("mongorestore_from_dir completed successfully.")
+
+
+class OverlayDBError(Exception):
+    pass
+
+
+class OverlayDB(AbstractContextManager):
+    """Provides a context whereby a base Database is overlaid with a temporary one.
+
+    If you need to run basic simulations of updates to a base database,
+    you don't want to actually commit transactions to the base database.
+
+    For example, to insert or replace (matching on "id") many documents into a collection in order
+    to then validate the resulting total set of collection documents, an OverlayDB writes to
+    an overlay collection that "shadows" the base collection during a "find" query
+    (the "merge_find" method of an OverlayDB object): if a document with `id0` is found in the
+    overlay collection, that id is marked as "seen" and will not also be returned when
+    subsequently scanning the (unmodified) base-database collection.
+
+    Note: The OverlayDB object does not provide a means to perform arbitrary MongoDB queries on the virtual "merged"
+          database. Callers can access the real database via `overlay_db._bottom_db` and the overlaying database via
+          `overlay_db._top_db` and perform arbitrary MongoDB queries on the individual databases that way. Access to
+          the virtual "merged" database is limited to the methods of the `OverlayDB` class, which simulates the
+          "merging" just-in-time to process the method invocation. You can see an example of this in the implementation
+          of the `merge_find` method, which internally accesses both the real database and the overlaying database.
+
+    Mongo "update" commands (as the "apply_updates" method) are simulated by first copying affected
+    documents from a base collection to the overlay, and then applying the updates to the overlay,
+    so that again, base collections are unmodified, and a "merge_find" call will produce a result
+    *as if* the base collection(s) were modified.
+
+    Mongo deletions (as the "delete" method) also copy affected documents from the base collection
+    to the overlay collection, and flag them using the "_deleted" field. In this way, a `merge_find`
+    call will match a relevant document given a suitable filter, and will mark the document's id
+    as "seen" *without* returning the document. Thus, the result is as if the document were deleted.
+
+    Usage:
+    ````
+    with OverlayDB(mdb) as odb:
+        # do stuff, e.g. `odb.replace_or_insert_many(...)`
+    ```
+    """
+
+    def __init__(self, mdb: MongoDatabase):
+        self._bottom_db = mdb
+        self._top_db = self._bottom_db.client.get_database(f"overlay-{uuid4()}")
+        ensure_unique_id_indexes(self._top_db)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._bottom_db.client.drop_database(self._top_db.name)
+
+    def replace_or_insert_many(self, coll_name, documents: list):
+        try:
+            self._top_db[coll_name].insert_many(documents)
+        except OperationFailure as e:
+            raise OverlayDBError(str(e.details))
+
+    def apply_updates(self, coll_name, updates: list):
+        """prepare overlay db and apply updates to it."""
+        assert all(UpdateStatement(**us) for us in updates)
+        for update_spec in updates:
+            for bottom_doc in self._bottom_db[coll_name].find(update_spec["q"]):
+                self._top_db[coll_name].insert_one(bottom_doc)
+        try:
+            self._top_db.command({"update": coll_name, "updates": updates})
+        except OperationFailure as e:
+            raise OverlayDBError(str(e.details))
+
+    def delete(self, coll_name, deletes: list):
+        """ "apply" delete command by flagging docs in overlay database"""
+        assert all(DeleteStatement(**us) for us in deletes)
+        for delete_spec in deletes:
+            for bottom_doc in self._bottom_db[coll_name].find(
+                delete_spec["q"], limit=delete_spec["limit"]
+            ):
+                bottom_doc["_deleted"] = True
+                self._top_db[coll_name].insert_one(bottom_doc)
+
+    def merge_find(self, coll_name, find_spec: dict):
+        """Yield docs first from overlay and then from base db, minding deletion flags."""
+        # ensure projection of "id" and "_deleted"
+        if "projection" in find_spec:
+            proj = find_spec["projection"]
+            if isinstance(proj, dict):
+                proj = merge(proj, {"id": 1, "_deleted": 1})
+            elif isinstance(proj, list):
+                proj = list(unique(proj + ["id", "_deleted"]))
+
+        top_docs = self._top_db[coll_name].find(**find_spec)
+        bottom_docs = self._bottom_db[coll_name].find(**find_spec)
+        top_seen_ids = set()
+        for doc in top_docs:
+            if not doc.get("_deleted"):
+                yield doc
+            top_seen_ids.add(doc["id"])
+
+        for doc in bottom_docs:
+            if doc["id"] not in top_seen_ids:
+                yield doc
+
+
+def validate_json(
+    in_docs: dict, mdb: MongoDatabase, check_inter_document_references: bool = False
+):
+    r"""
+    Checks whether the specified dictionary represents a valid instance of the `Database` class
+    defined in the NMDC Schema. Referential integrity checking is performed on an opt-in basis.
+
+    Example dictionary:
+    {
+        "biosample_set": [
+            {"id": "nmdc:bsm-00-000001", ...},
+            {"id": "nmdc:bsm-00-000002", ...}
+        ],
+        "study_set": [
+            {"id": "nmdc:sty-00-000001", ...},
+            {"id": "nmdc:sty-00-000002", ...}
+        ]
+    }
+
+    :param in_docs: The dictionary you want to validate
+    :param mdb: A reference to a MongoDB database
+    :param check_inter_document_references: Whether you want this function to check whether every document that
+                                            is referenced by any of the documents passed in would, indeed, exist
+                                            in the database, if the documents passed in were to be inserted into
+                                            the database. In other words, set this to `True` if you want this
+                                            function to perform referential integrity checks.
+    """
+    validator = Draft7Validator(get_nmdc_jsonschema_dict())
+    docs = deepcopy(in_docs)
+    validation_errors = {}
+
+    known_coll_names = set(nmdc_database_collection_names())
+    for coll_name, coll_docs in docs.items():
+        if coll_name not in known_coll_names:
+            # We expect each key in `in_docs` to be a known schema collection name. However, `@type` is a special key
+            # for JSON-LD, used for JSON serialization of e.g. LinkML objects. That is, the value of `@type` lets a
+            # client know that the JSON object (a dict in Python) should be interpreted as a
+            # <https://w3id.org/nmdc/Database>. If `@type` is present as a key, and its value indicates that
+            # `in_docs` is indeed a nmdc:Database, that's fine, and we don't want to raise an exception.
+            #
+            # prompted by: https://github.com/microbiomedata/nmdc-runtime/discussions/858
+            if coll_name == "@type" and coll_docs in ("Database", "nmdc:Database"):
+                continue
+            else:
+                validation_errors[coll_name] = [
+                    f"'{coll_name}' is not a known schema collection name"
+                ]
+                continue
+
+        errors = list(validator.iter_errors({coll_name: coll_docs}))
+        validation_errors[coll_name] = [e.message for e in errors]
+        if coll_docs:
+            if not isinstance(coll_docs, list):
+                validation_errors[coll_name].append("value must be a list")
+            elif not all(isinstance(d, dict) for d in coll_docs):
+                validation_errors[coll_name].append(
+                    "all elements of list must be dicts"
+                )
+            if not validation_errors[coll_name]:
+                try:
+                    with OverlayDB(mdb) as odb:
+                        odb.replace_or_insert_many(coll_name, coll_docs)
+                except OverlayDBError as e:
+                    validation_errors[coll_name].append(str(e))
+
+    if all(len(v) == 0 for v in validation_errors.values()):
+        # Second pass. Try instantiating linkml-sourced dataclass
+        in_docs.pop("@type", None)
+        try:
+            NMDCDatabase(**in_docs)
+        except Exception as e:
+            return {"result": "errors", "detail": str(e)}
+
+        # Third pass (if enabled): Check inter-document references.
+        if check_inter_document_references is True:
+            # Prepare to use `refscan`.
+            #
+            # Note: We check the inter-document references in two stages, which are:
+            #       1. For each document in the JSON payload, check whether each document it references already exists
+            #          (in the collections the schema says it can exist in) in the database. We use the
+            #          `refscan` package to do this, which returns violation details we'll use in the second stage.
+            #       2. For each violation found in the first stage (i.e. each reference to a not-found document), we
+            #          check whether that document exists (in the collections the schema says it can exist in) in the
+            #          JSON payload. If it does, then we "waive" (i.e. discard) that violation.
+            #       The violations that remain after those two stages are the ones we return to the caller.
+            #
+            # Note: The reason we do not insert documents into an `OverlayDB` and scan _that_, is that the `OverlayDB`
+            #       does not provide a means to perform arbitrary queries against its virtual "merged" database. It
+            #       is not a drop-in replacement for a pymongo's `Database` class, which is the only thing that
+            #       `refscan`'s `Finder` class accepts.
+            #
+            finder = Finder(database=mdb)
+            references = get_allowed_references()
+
+            # Iterate over the collections in the JSON payload.
+            for source_collection_name, documents in in_docs.items():
+                for document in documents:
+                    # Add an `_id` field to the document, since `refscan` requires the document to have one.
+                    source_document = dict(document, _id=None)
+                    violations = scan_outgoing_references(
+                        document=source_document,
+                        schema_view=nmdc_schema_view(),
+                        references=references,
+                        finder=finder,
+                        source_collection_name=source_collection_name,
+                        user_wants_to_locate_misplaced_documents=False,
+                    )
+
+                    # For each violation, check whether the misplaced document is in the JSON payload, itself.
+                    for violation in violations:
+                        can_waive_violation = False
+                        # Determine which collections can contain the referenced document, based upon
+                        # the schema class of which this source document is an instance.
+                        target_collection_names = (
+                            references.get_target_collection_names(
+                                source_class_name=violation.source_class_name,
+                                source_field_name=violation.source_field_name,
+                            )
+                        )
+                        # Check whether the referenced document exists in any of those collections in the JSON payload.
+                        for json_coll_name, json_coll_docs in in_docs.items():
+                            if json_coll_name in target_collection_names:
+                                for json_coll_doc in json_coll_docs:
+                                    if json_coll_doc["id"] == violation.target_id:
+                                        can_waive_violation = True
+                                        break  # stop checking
+                            if can_waive_violation:
+                                break  # stop checking
+                        if not can_waive_violation:
+                            violation_as_str = (
+                                f"Document '{violation.source_document_id}' "
+                                f"in collection '{violation.source_collection_name}' "
+                                f"has a field '{violation.source_field_name}' that "
+                                f"references a document having id "
+                                f"'{violation.target_id}', but the latter document "
+                                f"does not exist in any of the collections the "
+                                f"NMDC Schema says it can exist in."
+                            )
+                            validation_errors[source_collection_name].append(
+                                violation_as_str
+                            )
+
+            # If any collection's error list is not empty, return an error response.
+            if any(len(v) > 0 for v in validation_errors.values()):
+                return {"result": "errors", "detail": validation_errors}
+
+        return {"result": "All Okay!"}
+    else:
+        return {"result": "errors", "detail": validation_errors}

nmdc_runtime/api/db/s3.py

@@ -0,0 +1,37 @@
+from functools import lru_cache
+import os
+
+import boto3
+
+API_SITE_BUCKET = os.getenv("API_SITE_ID")
+S3_ID_NS = "do"  # Namespace for Drs Objects in Site S3-bucket store.
+
+
+@lru_cache
+def get_s3_client():
+    _session = boto3.session.Session()
+    return _session.client(
+        "s3",
+        region_name=os.getenv("DO_REGION_NAME"),
+        endpoint_url=os.getenv("DO_ENDPOINT_URL"),
+        aws_access_key_id=os.getenv("DO_SPACES_KEY"),
+        aws_secret_access_key=os.getenv("DO_SPACES_SECRET"),
+    )
+
+
+def presigned_url_to_put(
+    key, client=None, mime_type=None, bucket=API_SITE_BUCKET, expires_in=300
+):
+    return client.generate_presigned_url(
+        ClientMethod="put_object",
+        Params={"Bucket": bucket, "Key": key, "ContentType": mime_type},
+        ExpiresIn=expires_in,
+    )
+
+
+def presigned_url_to_get(key, client=None, bucket=API_SITE_BUCKET, expires_in=300):
+    return client.generate_presigned_url(
+        ClientMethod="get_object",
+        Params={"Bucket": bucket, "Key": key},
+        ExpiresIn=expires_in,
+    )

nmdc_runtime/api/endpoints/capabilities.py

@@ -0,0 +1,25 @@
+from typing import List
+
+import pymongo
+from fastapi import APIRouter, Depends
+
+from nmdc_runtime.api.core.util import raise404_if_none
+from nmdc_runtime.api.db.mongo import get_mongo_db
+from nmdc_runtime.api.models.capability import Capability
+
+router = APIRouter()
+
+
+@router.get("/capabilities", response_model=List[Capability])
+def list_capabilities(
+    mdb: pymongo.database.Database = Depends(get_mongo_db),
+):
+    return list(mdb.capabilities.find())
+
+
+@router.get("/capabilities/{capability_id}", response_model=Capability)
+def get_capability(
+    capability_id: str,
+    mdb: pymongo.database.Database = Depends(get_mongo_db),
+):
+    return raise404_if_none(mdb.capabilities.find_one({"id": capability_id}))