nmdc-runtime 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

nmdc_runtime/api/endpoints/lib/linked_instances.py

@@ -0,0 +1,180 @@
+"""
+
+This module houses logic for the `GET /nmdcschema/linked_instances` endpoint, defined as
+`nmdc_runtime.api.endpoints.nmdcschema.linked_instances`, to avoid (further) bloating the
+`nmdc_runtime.api.endpoints.nmdcschema` module.
+
+"""
+
+from typing import Literal, Any
+
+from bson import ObjectId
+from pymongo.collection import Collection as MongoCollection
+from pymongo.database import Database as MongoDatabase
+from toolz import merge
+
+from nmdc_runtime.api.core.util import hash_from_str
+from nmdc_runtime.util import get_class_name_to_collection_names_map, nmdc_schema_view
+
+
+def hash_from_ids_and_types(ids: list[str], types: list[str]) -> str:
+    """A quick hash as a function of `ids` and `types`.
+
+    This will serve as part of a temporary mongo collection name.
+    Because it will only be "part of" the name, avoiding hash collisions isn't a priority.
+
+    Returns a hex digest truncated to 8 characters, so 16**8 ≈ 4M possible values.
+    """
+    return hash_from_str(
+        ",".join(sorted(ids)) + "." + ",".join(sorted(types)), algo="md5"
+    )[:8]
+
+
+def temp_linked_instances_collection_name(ids: list[str], types: list[str]) -> str:
+    """A name for a temporary mongo collection to store linked instances in service of an API request."""
+    return f"_runtime.tmp.linked_instances.{hash_from_ids_and_types(ids=ids,types=types)}.{ObjectId()}"
+
+
+def gather_linked_instances(
+    alldocs_collection: MongoCollection,
+    ids: list[str],
+    types: list[str],
+) -> str:
+    """Collect linked instances and stores them in a new temporary collection.
+
+    Run an aggregation pipeline over `alldocs_collection` that collects ∈`types` instances linked to `ids`.
+    The pipeline is run twice, once for each of {"downstream", "upstream"} directions.
+    """
+    merge_into_collection_name = temp_linked_instances_collection_name(
+        ids=ids, types=types
+    )
+    for direction in ["downstream", "upstream"]:
+        _ = list(
+            alldocs_collection.aggregate(
+                pipeline_for_direction(
+                    ids=ids,
+                    types=types,
+                    direction=direction,
+                    merge_into_collection_name=merge_into_collection_name,
+                ),
+                allowDiskUse=True,
+            )
+        )
+    return merge_into_collection_name
+
+
+def pipeline_for_direction(
+    ids: list[str],
+    types: list[str],
+    direction: Literal["downstream", "upstream"],
+    merge_into_collection_name: str,
+    alldocs_collection_name: str = "alldocs",
+) -> list:
+    """A pure function that returns the aggregation pipeline for `direction`.
+
+    The pipeline
+    - collects ∈`types` instances linked to `ids` along `direction`,
+    - retains only those document fields essential to the caller, and
+    - ensures the collected instances are present, and properly updated if applicable, in a merge-target collection.
+    """
+    return pipeline_for_instances_linked_to_ids_by_direction(
+        ids=ids,
+        types=types,
+        direction=direction,
+        alldocs_collection_name=alldocs_collection_name,
+    ) + [
+        {"$project": {"id": 1, "type": 1, f"_{direction}_of": 1}},
+        pipeline_stage_for_merging_instances_and_grouping_link_provenance_by_direction(
+            merge_into_collection_name=merge_into_collection_name, direction=direction
+        ),
+    ]
+
+
+def pipeline_for_instances_linked_to_ids_by_direction(
+    ids: list[str],
+    types: list[str],
+    direction: Literal["downstream", "upstream"],
+    alldocs_collection_name: str = "alldocs",
+    slim: bool = True,
+) -> list[dict[str, Any]]:
+    """
+    Returns an aggregation pipeline that:
+    - traverses the graph of documents in the alldocs collection, following `direction`-specific relationships
+      to discover documents linked to the documents given by `ids`.
+    - `$unwind`s the collected (via `$graphLookup`) docs,
+    - filters them by given `types` of interest,
+     - adds bookkeeping information about `direction`ality, and
+     - (optionally) projects only essential fields to reduce response latency and size.
+    """
+    return [
+        {"$match": {"id": {"$in": ids}}},
+        {
+            "$graphLookup": {
+                "from": alldocs_collection_name,
+                "startWith": f"$_{direction}.id",
+                "connectFromField": f"_{direction}.id",
+                "connectToField": "id",
+                "as": f"{direction}_docs",
+            }
+        },
+        {"$unwind": {"path": f"${direction}_docs"}},
+        {"$match": {f"{direction}_docs._type_and_ancestors": {"$in": types}}},
+        {"$addFields": {f"{direction}_docs._{direction}_of": ["$id"]}},
+        {"$replaceRoot": {"newRoot": f"${direction}_docs"}},
+    ] + ([{"$project": {"id": 1, "type": 1, f"_{direction}_of": 1}}] if slim else [])
+
+
+def pipeline_stage_for_merging_instances_and_grouping_link_provenance_by_direction(
+    merge_into_collection_name: str,
+    direction: Literal["downstream", "upstream"],
+) -> dict[str, Any]:
+    """
+    Returns an aggregation-pipeline step that merges its input document stream to a collection dedicated to serving
+    the caller in a manner amenable to pagination across multiple HTTP requests.
+    """
+    return {
+        "$merge": {
+            "into": merge_into_collection_name,
+            "on": "_id",
+            "whenMatched": [
+                {
+                    "$set": {
+                        f"_{direction}_of": {
+                            "$setUnion": [
+                                f"$_{direction}_of",
+                                f"$$new._{direction}_of",
+                            ]
+                        }
+                    }
+                }
+            ],
+            "whenNotMatched": "insert",
+        }
+    }
+
+
+def hydrated(resources: list[dict], mdb: MongoDatabase) -> list[dict]:
+    """Replace each `dict` in `resources` with a hydrated version.
+
+    Instead of returning the retrieved "full" documents as is, we merge each one with (a copy of) the corresponding
+    original document in *resources*, which includes additional fields, e.g. `_upstream_of` and `_downstream_of`.
+    """
+    class_name_to_collection_names_map = get_class_name_to_collection_names_map(
+        nmdc_schema_view()
+    )
+    types_of_resources = {r["type"] for r in resources}
+    full_docs_by_id = {}
+
+    for type in types_of_resources:
+        resource_ids_of_type = [d["id"] for d in resources if d["type"] == type]
+        schema_collection = mdb.get_collection(
+            # Note: We are assuming that documents of a given type are only allowed (by the schema) to reside in one
+            # collection. Based on that assumption, we will query only the _first_ collection whose name we get from
+            # the map. This assumption is continuously verified prior to code deployment via
+            # `test_get_class_name_to_collection_names_map_has_one_and_only_one_collection_name_per_class_name`.
+            class_name_to_collection_names_map[type.removeprefix("nmdc:")][0]
+        )
+        for doc in schema_collection.find({"id": {"$in": resource_ids_of_type}}):
+            full_docs_by_id[doc["id"]] = doc
+
+    return [merge(r, full_docs_by_id[r["id"]]) for r in resources]

nmdc_runtime/api/endpoints/nmdcschema.py

@@ -10,6 +10,10 @@ from refscan.lib.helpers import (
     get_names_of_classes_eligible_for_collection,
 )
 
+from nmdc_runtime.api.endpoints.lib.linked_instances import (
+    gather_linked_instances,
+    hydrated,
+)
 from nmdc_runtime.config import IS_LINKED_INSTANCES_ENDPOINT_ENABLED
 from nmdc_runtime.minter.config import typecodes
 from nmdc_runtime.minter.domain.model import check_valid_ids
@@ -118,7 +122,7 @@ def get_nmdc_database_collection_stats(
 @decorate_if(condition=IS_LINKED_INSTANCES_ENDPOINT_ENABLED)(
     router.get(
         "/nmdcschema/linked_instances",
-        response_model=ListResponse,
+        response_model=ListResponse[Doc],
         response_model_exclude_unset=True,
     )
 )
@@ -147,23 +151,54 @@ def get_linked_instances(
             examples=["nmdc:bsm-11-abc123"],
         ),
     ] = None,
+    hydrate: Annotated[
+        bool,
+        Query(
+            title="Hydrate",
+            description="Whether to include full documents in the response. The default is to include slim documents.",
+        ),
+    ] = False,
+    page_token: Annotated[
+        str | None,
+        Query(
+            title="Next page token",
+            description="""A bookmark you can use to fetch the _next_ page of resources. You can get this from the
+                    `next_page_token` field in a previous response from this endpoint.\n\n_Example_: 
+                    `nmdc:sys0zr0fbt71`""",
+            examples=[
+                "nmdc:sys0zr0fbt71",
+            ],
+        ),
+    ] = None,
+    max_page_size: Annotated[
+        int,
+        Query(
+            title="Resources per page",
+            description="How many resources you want _each page_ to contain, formatted as a positive integer.",
+            examples=[20],
+        ),
+    ] = 20,
     mdb: MongoDatabase = Depends(get_mongo_db),
 ):
     """
     Retrieves database instances that are both (a) linked to any of `ids`, and (b) of a type in `types`.
 
-    An [instance](https://linkml.io/linkml-model/latest/docs/specification/02instances/) is an object conforming to
-    a class definition ([linkml:ClassDefinition](https://w3id.org/linkml/ClassDefinition))
-    in our database ([nmdc:Database](https://w3id.org/nmdc/Database)).
-    While a [nmdc:Database](https://w3id.org/nmdc/Database) is organized into collections,
-    every item in every database collection -- that is, every instance -- knows its `type`, so we can
-    (and here do)<sup>&dagger;</sup>
-    return a simple list of instances
-    ([a LinkML CollectionInstance](https://linkml.io/linkml-model/latest/docs/specification/02instances/#collections)),
-    which a client may use to construct a corresponding [nmdc:Database](https://w3id.org/nmdc/Database).
-
-    From the nexus instance IDs given in `ids`, both "upstream" and "downstream" links are followed (transitively) in
-    order to collect the set of all instances linked to these `ids`.
+    An [instance](https://linkml.io/linkml-model/latest/docs/specification/02instances/) is an object conforming to a
+    class definition ([linkml:ClassDefinition](https://w3id.org/linkml/ClassDefinition)) in our database ([
+    nmdc:Database](https://w3id.org/nmdc/Database)). While a [nmdc:Database](https://w3id.org/nmdc/Database) is
+    organized into collections, every item in every database collection -- that is, every instance -- knows its
+    `type`, so we can (and here do) return a simple list of instances ([a LinkML CollectionInstance](
+    https://linkml.io/linkml-model/latest/docs/specification/02instances/#collections)). If hydrate is `False` (the
+    default), then the returned list contains "slim" documents that include only the `id` and `type` of each
+    instance. If hydrate is `True`, then the returned list contains "full" (aka <a
+    href="https://en.wikipedia.org/wiki/Hydration_(web_development)">"hydrated"</a>) documents of each instance,
+    suitable e.g. for a client to subsequently use to construct a corresponding
+    [nmdc:Database](https://w3id.org/nmdc/Database) instance with schema-compliant documents.
+    Both "slim" and "full" documents include (optional) `_upstream_of` and `_downstream_of` fields,
+    to indicate the returned document's relationship to `ids`.
+
+    From the nexus instance IDs given in `ids`, both "upstream" and "downstream" links are followed (transitively)
+    to collect the set of all instances linked to these `ids`.
 
     * A link "upstream" is represented by a slot ([linkml:SlotDefinition](https://w3id.org/linkml/SlotDefinition))
     for which the
@@ -186,16 +221,15 @@ def get_linked_instances(
     [nmdc:InformationObject](https://w3id.org/nmdc/InformationObject),
     [nmdc:Sample](https://w3id.org/nmdc/Sample), etc. -- may be given.
     If no value for `types` is given, then all [nmdc:NamedThing](https://w3id.org/nmdc/NamedThing)s are returned.
-
-    <sup>&dagger;</sup>: actually, we do not (yet).
-    For now (see [microbiomedata/nmdc-runtime#1118](https://github.com/microbiomedata/nmdc-runtime/issues/1118)),
-    we return a short list of "fat" documents,  each of which represents one of the `ids` and presents
-    representations of that id's downstream and upstream instances (currently just each instance's `id` and `type`)
-    as separate subdocument array fields.
     """
-    # TODO move logic from endpoint to unit-testable handler
-    # TODO ListResponse[SimplifiedNMDCDatabase]
-    # TODO ensure pagination for responses
+    if page_token is not None:
+        rv = list_resources(
+            req=ListRequest(page_token=page_token, max_page_size=max_page_size), mdb=mdb
+        )
+        rv["resources"] = hydrated(rv["resources"], mdb) if hydrate else rv["resources"]
+        rv["resources"] = [strip_oid(d) for d in rv["resources"]]
+        return rv
+
     ids_found = [d["id"] for d in mdb.alldocs.find({"id": {"$in": ids}}, {"id": 1})]
     ids_not_found = list(set(ids) - set(ids_found))
     if ids_not_found:
@@ -217,131 +251,18 @@ def get_linked_instances(
             ),
         )
 
-    # This aggregation pipeline traverses the graph of documents in the alldocs collection, following upstream
-    # relationships (_upstream.id) to discover upstream documents for entities that originated, or helped produce,
-    # the entities with documents identified by `ids`. It unwinds the collected (via `$graphLookup`) upstream docs,
-    # filters them by given `types` of interest, projects only essential fields to reduce response latency and size,
-    # and groups them by each of the given `ids`, i.e. re-winding the `$unwind`-ed upstream docs into an array for each
-    # given ID.
-    upstream_docs = list(
-        mdb.alldocs.aggregate(
-            [
-                {"$match": {"id": {"$in": ids}}},
-                {
-                    "$graphLookup": {
-                        "from": "alldocs",
-                        "startWith": "$_upstream.id",
-                        "connectFromField": "_upstream.id",
-                        "connectToField": "id",
-                        "as": "upstream_docs",
-                    }
-                },
-                {"$unwind": {"path": "$upstream_docs"}},
-                {"$match": {"upstream_docs._type_and_ancestors": {"$in": types}}},
-                {"$project": {"id": 1, "upstream_docs": "$upstream_docs"}},
-                {
-                    "$group": {
-                        "_id": "$id",
-                        "upstream_docs": {
-                            "$addToSet": {
-                                "id": "$upstream_docs.id",
-                                "type": "$upstream_docs.type",
-                            }
-                        },
-                    }
-                },
-                {
-                    "$lookup": {
-                        "from": "alldocs",
-                        "localField": "_id",
-                        "foreignField": "id",
-                        "as": "selves",
-                    }
-                },
-                {
-                    "$project": {
-                        "_id": 0,
-                        "id": "$_id",
-                        "upstream_docs": 1,
-                        "type": {"$arrayElemAt": ["$selves.type", 0]},
-                    }
-                },
-            ],
-            allowDiskUse=True,
-        )
+    merge_into_collection_name = gather_linked_instances(
+        alldocs_collection=mdb.alldocs, ids=ids, types=types
     )
 
-    # This aggregation pipeline traverses the graph of documents in the alldocs collection, following downstream
-    # relationships (_downstream.id) to discover downstream documents for entities that originated from,
-    # or are considered part of, the entities with documents identified by `ids`. It unwinds the collected (via
-    # `$graphLookup`) downstream docs, filters them by given `types` of interest, projects only essential fields to
-    # reduce response latency and size, and groups them by each of the given `ids`, i.e. re-winding the `$unwind`-ed
-    # downstream docs into an array for each given ID.
-    downstream_docs = list(
-        mdb.alldocs.aggregate(
-            [
-                {"$match": {"id": {"$in": ids}}},
-                {
-                    "$graphLookup": {
-                        "from": "alldocs",
-                        "startWith": "$_downstream.id",
-                        "connectFromField": "_downstream.id",
-                        "connectToField": "id",
-                        "as": "downstream_docs",
-                    }
-                },
-                {"$unwind": {"path": "$downstream_docs"}},
-                {"$match": {"downstream_docs._type_and_ancestors": {"$in": types}}},
-                {
-                    "$group": {
-                        "_id": "$id",
-                        "downstream_docs": {
-                            "$addToSet": {
-                                "id": "$downstream_docs.id",
-                                "type": "$downstream_docs.type",
-                            }
-                        },
-                    }
-                },
-                {
-                    "$lookup": {
-                        "from": "alldocs",
-                        "localField": "_id",
-                        "foreignField": "id",
-                        "as": "selves",
-                    }
-                },
-                {
-                    "$project": {
-                        "_id": 0,
-                        "id": "$_id",
-                        "downstream_docs": 1,
-                        "type": {"$arrayElemAt": ["$selves.type", 0]},
-                    }
-                },
-            ],
-            allowDiskUse=True,
-        )
+    rv = list_resources(
+        ListRequest(page_token=page_token, max_page_size=max_page_size),
+        mdb,
+        merge_into_collection_name,
     )
-
-    relations_by_id = {
-        id_: {
-            "id": id_,
-            "upstream_docs": [],
-            "downstream_docs": [],
-        }
-        for id_ in ids
-    }
-
-    # For each subject document that was upstream of or downstream of any documents, create a dictionary
-    # containing that subject document's `id`, its `type`, and the list of `id`s of the
-    # documents that it for upstream or or downstream of.
-    for d in upstream_docs + downstream_docs:
-        relations_by_id[d["id"]]["type"] = d["type"]
-        relations_by_id[d["id"]]["upstream_docs"] += d.get("upstream_docs", [])
-        relations_by_id[d["id"]]["downstream_docs"] += d.get("downstream_docs", [])
-
-    return {"resources": list(relations_by_id.values())}
+    rv["resources"] = hydrated(rv["resources"], mdb) if hydrate else rv["resources"]
+    rv["resources"] = [strip_oid(d) for d in rv["resources"]]
+    return rv
 
 
 @router.get(

nmdc_runtime/api/endpoints/objects.py

@@ -124,9 +124,8 @@ def get_object_info(
         )
     if object_id.startswith("sty-"):
         url_to_try = f"https://data.microbiomedata.org/api/study/nmdc:{object_id}"
-        rv = requests.get(
-            url_to_try, allow_redirects=True
-        )  # TODO use HEAD when enabled upstream
+        # TODO: Update this HTTP request to use the HTTP "HEAD" method once the upstream endpoint supports that method.
+        rv = requests.get(url_to_try, allow_redirects=True)
         if rv.status_code != 404:
             return RedirectResponse(
                 f"https://data.microbiomedata.org/details/study/nmdc:{object_id}",
@@ -134,9 +133,8 @@ def get_object_info(
             )
     elif object_id.startswith("bsm-"):
         url_to_try = f"https://data.microbiomedata.org/api/biosample/nmdc:{object_id}"
-        rv = requests.get(
-            url_to_try, allow_redirects=True
-        )  # TODO use HEAD when enabled upstream
+        # TODO: Update this HTTP request to use the HTTP "HEAD" method once the upstream endpoint supports that method.
+        rv = requests.get(url_to_try, allow_redirects=True)
         if rv.status_code != 404:
             return RedirectResponse(
                 f"https://data.microbiomedata.org/details/sample/nmdc:{object_id}",
@@ -270,8 +268,3 @@ def update_object(
     doc_object_patched = merge(doc, object_patch.model_dump(exclude_unset=True))
     mdb.operations.replace_one({"id": object_id}, doc_object_patched)
     return doc_object_patched
-
-
-@router.put("/objects/{object_id}", response_model=DrsObject)
-def replace_object():
-    pass

nmdc_runtime/api/endpoints/operations.py

@@ -76,30 +76,3 @@ def update_operation(
     )
     mdb.operations.replace_one({"id": op_id}, doc_op_patched)
     return doc_op_patched
-
-
-@router.post(
-    "/operations/{op_id}:wait",
-    description=(
-        "Wait until the operation is resolved or rejected before returning the result."
-        " This is a 'blocking' alternative to client-side polling, and may not be available"
-        " for operation types know to be particularly long-running."
-    ),
-)
-def wait_operation():
-    pass
-
-
-@router.post("/operations/{op_id}:cancel")
-def cancel_operation():
-    pass
-
-
-@router.post("/operations/{op_id}:pause")
-def pause_operation():
-    pass
-
-
-@router.post("/operations/{op_id}:resume")
-def resume_operation():
-    pass

nmdc_runtime/api/endpoints/queries.py

@@ -175,6 +175,28 @@ def run_query(
     }
     ```
 
+    Get a specific study and all the biosamples associated with that study.
+    ```
+    {
+      "aggregate": "study_set",
+      "pipeline": [
+        {
+          "$match": {
+            "id": "nmdc:sty-11-8fb6t785"
+          }
+        },
+        {
+          "$lookup": {
+            "from": "biosample_set",
+            "localField": "id",
+            "foreignField": "associated_studies",
+            "as": "biosamples_of_study"
+          }
+        }
+      ]
+    }
+    ```
+
     Use the `cursor.id` from a previous response to get the next batch of results,
     whether that batch is empty or non-empty.
     ```

nmdc_runtime/api/endpoints/sites.py

@@ -87,30 +87,6 @@ def get_site(
     return raise404_if_none(mdb.sites.find_one({"id": site_id}))
 
 
-@router.patch("/sites/{site_id}", include_in_schema=False)
-def update_site():
-    """Not yet implemented"""
-    pass
-
-
-@router.put("/sites/{site_id}", include_in_schema=False)
-def replace_site():
-    """Not yet implemented"""
-    pass
-
-
-@router.get("/sites/{site_id}/capabilities", include_in_schema=False)
-def list_site_capabilities(site_id: str):
-    """Not yet implemented"""
-    pass
-
-
-@router.put("/sites/{site_id}/capabilities", include_in_schema=False)
-def replace_site_capabilities(site_id: str, capability_ids: List[str]):
-    """Not yet implemented"""
-    pass
-
-
 def verify_client_site_pair(
     site_id: str,
     mdb: pymongo.database.Database = Depends(get_mongo_db),