nmdc-runtime 2.8.0__py3-none-any.whl → 2.10.0__py3-none-any.whl

nmdc_runtime/api/endpoints/ids.py

@@ -0,0 +1,192 @@
+"""
+`router` here is deprecated in favor of `nmdc_runtime.minter.entrypoints.fastapi_app.router`
+"""
+
+import re
+from typing import List, Dict, Any
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import ValidationError
+from pymongo.database import Database as MongoDatabase
+from starlette import status
+from toolz import dissoc
+
+from nmdc_runtime.api.core.idgen import (
+    generate_ids,
+    decode_id,
+    collection_name,
+)
+from nmdc_runtime.api.core.util import raise404_if_none, pick
+from nmdc_runtime.api.db.mongo import get_mongo_db
+from nmdc_runtime.api.models.id import (
+    MintRequest,
+    pattern_shoulder,
+    AssignedBaseName,
+    pattern_assigned_base_name,
+    IdBindingRequest,
+    pattern_base_object_name,
+    IdThreeParts,
+    pattern_naa,
+)
+from nmdc_runtime.api.models.site import get_current_client_site, Site
+
+router = APIRouter()
+
+
+@router.post("/ids/mint", response_model=List[str])
+def mint_ids(
+    mint_req: MintRequest,
+    mdb: MongoDatabase = Depends(get_mongo_db),
+    site: Site = Depends(get_current_client_site),
+):
+    """Generate one or more identifiers.
+
+    Leaving `populator` blank will set it to the site ID of the request client.
+    """
+    ids = generate_ids(
+        mdb,
+        owner=site.id,
+        populator=(mint_req.populator or site.id),
+        number=mint_req.number,
+        naa=mint_req.naa,
+        shoulder=mint_req.shoulder,
+    )
+    return ids
+
+
+@router.post("/ids/bindings", response_model=List[Dict[str, Any]])
+def set_id_bindings(
+    binding_requests: List[IdBindingRequest],
+    mdb: MongoDatabase = Depends(get_mongo_db),
+    site: Site = Depends(get_current_client_site),
+):
+    bons = [r.i for r in binding_requests]
+    ids: List[IdThreeParts] = []
+    for bon in bons:
+        m = re.match(pattern_base_object_name, bon)
+        ids.append(
+            IdThreeParts(
+                naa=m.group("naa"),
+                shoulder=m.group("shoulder"),
+                blade=m.group("blade"),
+            )
+        )
+    # Ensure that user owns all supplied identifiers.
+    for id_, r in zip(ids, binding_requests):
+        collection = mdb.get_collection(collection_name(id_.naa, id_.shoulder))
+        doc = collection.find_one({"_id": decode_id(str(id_.blade))}, ["__ao"])
+        if doc is None:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=f"id {r.i} not found",
+            )
+        elif doc.get("__ao") != site.id:
+            raise HTTPException(
+                status_code=status.HTTP_403_FORBIDDEN,
+                detail=(
+                    f"authenticated site client does not manage {r.i} "
+                    f"(client represents site {site.id}).",
+                ),
+            )
+    # Ensure no attempts to set reserved attributes.
+    if any(r.a.startswith("__a") for r in binding_requests):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Cannot set attribute names beginning with '__a'.",
+        )
+    # Process binding requests
+    docs = []
+    for id_, r in zip(ids, binding_requests):
+        collection = mdb.get_collection(collection_name(id_.naa, id_.shoulder))
+
+        filter_ = {"_id": decode_id(id_.blade)}
+        if r.o == "purge":
+            docs.append(collection.find_one_and_delete(filter_))
+        elif r.o == "rm":
+            docs.append(collection.find_one_and_update(filter_, {"$unset": {r.a: ""}}))
+        elif r.o == "set":
+            docs.append(collection.find_one_and_update(filter_, {"$set": {r.a: r.v}}))
+        elif r.o == "addToSet":
+            docs.append(
+                collection.find_one_and_update(filter_, {"$addToSet": {r.a: r.v}})
+            )
+        else:
+            # Note: IdBindingRequest root_validator methods should preclude this.
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid operation 'o'."
+            )
+
+        return [dissoc(d, "_id") for d in docs]
+
+
+@router.get("/ids/bindings/{rest:path}", response_model=Dict[str, Any])
+def get_id_bindings(
+    rest: str,
+    mdb: MongoDatabase = Depends(get_mongo_db),
+):
+    cleaned = rest.replace("-", "")
+    parts = cleaned.split(":")
+    if len(parts) != 2:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=(
+                "Invalid ID - needs both name assigning authority (NAA) part"
+                "(e.g. 'nmdc') and name part (e.g. 'fk4ra92'), separated by a colon (':')."
+            ),
+        )
+    naa = parts[0]
+    suffix_parts = parts[1].split("/")
+    if len(suffix_parts) == 2 and suffix_parts[-1] != "":  # one '/', or ends with '/'
+        assigned_base_name, attribute = suffix_parts
+    else:
+        assigned_base_name = suffix_parts[0]
+        attribute = None
+
+    if re.match(pattern_naa, naa) is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=f"Invalid ID - invalid name assigning authority (NAA) '{naa}'.",
+        )
+    print(assigned_base_name)
+    if re.match(pattern_shoulder, assigned_base_name) is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail=(
+                "Invalid ID - invalid shoulder. "
+                "Every name part begins with a 'shoulder', a "
+                "sequence of letters followed by a number, "
+                "for example 'fk4'. "
+                "Did you forget to include the shoulder?",
+            ),
+        )
+    try:
+        m = re.match(pattern_assigned_base_name, AssignedBaseName(assigned_base_name))
+        shoulder, blade = m.group("shoulder"), m.group("blade")
+        id_decoded = decode_id(blade)
+    except (AttributeError, ValidationError):
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid ID - characters used outside of base32.",
+        )
+    except ValueError:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Invalid ID - failed checksum. Did you copy it incorrectly?",
+        )
+
+    collection = mdb.get_collection(collection_name(naa, shoulder))
+    d = raise404_if_none(collection.find_one({"_id": id_decoded}))
+    d = dissoc(d, "_id")
+    if attribute is not None:
+        if attribute not in d:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail=(
+                    f"attribute '{attribute}' not found in "
+                    f"{naa}:{assigned_base_name}."
+                ),
+            )
+        rv = pick(["where", attribute], d)
+    else:
+        rv = d
+    return rv

nmdc_runtime/api/endpoints/jobs.py

@@ -0,0 +1,143 @@
+import json
+from typing import Optional, Annotated
+
+from pymongo.database import Database
+from fastapi import APIRouter, Depends, Query, HTTPException, Path
+from pymongo.errors import ConnectionFailure, OperationFailure
+from starlette import status
+
+from nmdc_runtime.api.core.util import (
+    raise404_if_none,
+)
+from nmdc_runtime.api.db.mongo import get_mongo_db
+from nmdc_runtime.api.endpoints.util import list_resources, _claim_job
+from nmdc_runtime.api.models.job import Job, JobClaim
+from nmdc_runtime.api.models.operation import Operation, MetadataT
+from nmdc_runtime.api.models.site import (
+    Site,
+    maybe_get_current_client_site,
+    get_current_client_site,
+)
+from nmdc_runtime.api.models.util import ListRequest, ListResponse, ResultT
+
+router = APIRouter()
+
+
+@router.get(
+    "/jobs", response_model=ListResponse[Job], response_model_exclude_unset=True
+)
+def list_jobs(
+    req: Annotated[ListRequest, Query()],
+    mdb: Database = Depends(get_mongo_db),
+    maybe_site: Optional[Site] = Depends(maybe_get_current_client_site),
+):
+    """List pre-configured workflow jobs.
+
+    If authenticated as a site client, `req.filter` defaults to fetch unclaimed jobs
+    that are claimable by the site client. This default can be overridden to view all jobs
+    by explicitly passing a `req.filter` of `{}`.
+    """
+    if isinstance(maybe_site, Site) and req.filter is None:
+        req.filter = json.dumps({"claims.site_id": {"$ne": maybe_site.id}})
+    return list_resources(req, mdb, "jobs")
+
+
+@router.get("/jobs/{job_id}", response_model=Job, response_model_exclude_unset=True)
+def get_job_info(
+    job_id: str,
+    mdb: Database = Depends(get_mongo_db),
+):
+    return raise404_if_none(mdb.jobs.find_one({"id": job_id}))
+
+
+@router.post("/jobs/{job_id}:claim", response_model=Operation[ResultT, MetadataT])
+def claim_job(
+    job_id: str,
+    mdb: Database = Depends(get_mongo_db),
+    site: Site = Depends(get_current_client_site),
+):
+    return _claim_job(job_id, mdb, site)
+
+
+@router.post("/jobs/{job_id}:release")
+def release_job(
+    job_id: Annotated[
+        str,
+        Path(
+            title="Job ID",
+            description="The `id` of the job.\n\n_Example_: `nmdc:f81d4fae-7dec-11d0-a765-00a0c91e6bf6`",
+            examples=["nmdc:f81d4fae-7dec-11d0-a765-00a0c91e6bf6"],
+        ),
+    ],
+    mdb: Database = Depends(get_mongo_db),
+    site: Site = Depends(get_current_client_site),
+) -> Optional[Job]:
+    r"""
+    Release the specified job.
+
+    Releasing a job cancels all the unfinished operations (of that job)
+    claimed by the `site` associated with the logged-in site client.
+
+    Return the updated job, reflecting that the aforementioned operations have been cancelled.
+    """
+    job = Job(**raise404_if_none(mdb.jobs.find_one({"id": job_id})))
+    active_job_claims_by_this_site = list(
+        mdb.operations.find(
+            {
+                "metadata.job.id": job_id,
+                "metadata.site_id": site.id,
+                "done": False,
+            },
+            ["id"],
+        )
+    )
+    job_claims_by_this_site_post_release = [
+        JobClaim(op_id=claim["id"], site_id=site.id, done=True, cancelled=True)
+        for claim in active_job_claims_by_this_site
+    ]
+    job_claims_not_by_this_site = [
+        claim for claim in job.claims if (claim.site_id != site.id)
+    ]
+
+    # Execute MongoDB transaction to ensure atomic change of job document plus relevant set of operations documents.
+    def transactional_update(session):
+        mdb.operations.update_many(
+            {"id": {"$in": [claim["id"] for claim in active_job_claims_by_this_site]}},
+            {"$set": {"metadata.cancelled": True, "metadata.done": True}},
+            session=session,
+        )
+        job_claim_subdocuments_post_release = [
+            claim.model_dump(exclude_unset=True)
+            for claim in (
+                job_claims_not_by_this_site + job_claims_by_this_site_post_release
+            )
+        ]
+        mdb.jobs.update_one(
+            {"id": job_id},
+            {"$set": {"claims": job_claim_subdocuments_post_release}},
+            session=session,
+        )
+
+    try:
+        with mdb.client.start_session() as session:
+            with session.start_transaction():
+                transactional_update(session)
+    except (ConnectionFailure, OperationFailure) as e:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail=f"Transaction failed: {e}",
+        )
+
+    # Return the updated `jobs` document.
+    #
+    # TODO: Consider retrieving the document within the transaction
+    #       to ensure it still exists.
+    #
+    updated_job = mdb.jobs.find_one({"id": job_id})
+    if updated_job is None:
+        # Note: We return `None` in this case because that's what the
+        #       endpoint originally did in this case, and we don't want
+        #       to introduce a breaking change as part of this refactor.
+        return None
+    else:
+        return Job(**updated_job)

nmdc_runtime/api/endpoints/lib/helpers.py

@@ -0,0 +1,274 @@
+import json
+import bson.json_util
+from typing import List
+
+from pymongo.database import Database
+from refscan.lib.Finder import Finder
+from refscan.lib.helpers import derive_schema_class_name_from_document
+from refscan.scanner import identify_referring_documents, scan_outgoing_references
+
+from nmdc_runtime.api.models.lib.helpers import derive_update_specs
+from nmdc_runtime.api.models.query import UpdateCommand, UpdateSpecs
+from nmdc_runtime.util import get_allowed_references, nmdc_schema_view
+
+
+def make_violation_message(
+    collection_name: str,
+    source_document_id: str,
+    source_field_name: str,
+    target_document_id: str,
+) -> str:
+    r"""
+    Constructs a violation message that indicates that a document would contain a broken reference.
+
+    :param collection_name: The name of the collection containing the document containing the broken reference
+    :param source_document_id: The `id` of the document containing the broken reference
+    :param source_field_name: The name of the field containing the broken reference
+    :param target_document_id: The `id` of the document that is being referenced
+
+    :return: A formatted string describing the violation
+    """
+    return (
+        f"The document having id='{source_document_id}' in "
+        f"the collection '{collection_name}' contains a "
+        f"reference (in its '{source_field_name}' field, "
+        f"referring to the document having id='{target_document_id}') "
+        f"which would be broken."
+    )
+
+
+def simulate_updates_and_check_references(
+    db: Database, update_cmd: UpdateCommand
+) -> List[str]:
+    r"""
+    Checks whether, if the specified updates were performed on the specified database,
+    both of the following things would be true afterward:
+    1. (Regarding outgoing references): The updated documents do not contain any
+       broken references.
+    2. (Regarding incoming references): The documents that originally _referenced_
+       any of the updated documents do not contain any broken references.
+       This check is necessary because update operations can currently change `id`
+       and `type` values, which can affect what can legally reference those documents.
+
+    This function checks those things by performing the updates within a MongoDB
+    transaction, leaving the transaction in the _pending_ (i.e. not committed) state,
+    and then performing various checks on the database in that _pending_ state.
+
+    :param db: The database on which to simulate performing the updates
+    :param update_cmd: The command that specifies the updates
+
+    :return: List of violation messages. If the list is empty, it means that—if
+             the updates had been performed (instead of only simulated) here—they
+             would not have left behind any broken references.
+    """
+
+    # Initialize the list of violation messages that we will return.
+    violation_messages: List[str] = []
+
+    # Instantiate a `Finder` bound to the Mongo database. This will be
+    # used later, to identify and check inter-document references.
+    finder = Finder(database=db)
+
+    # Extract the collection name from the command.
+    collection_name = update_cmd.update
+
+    # Derive the update specifications from the command.
+    update_specs: UpdateSpecs = derive_update_specs(update_cmd)
+
+    # Get a reference to a `SchemaView` bound to the NMDC schema, so we can
+    # use it to, for example, map `type` field values to schema class names.
+    schema_view = nmdc_schema_view()
+
+    # Get some data structures that indicate which fields of which documents
+    # can legally contain references, according to the NMDC schema.
+    legal_references = get_allowed_references()
+    reference_field_names_by_source_class_name = (
+        legal_references.get_reference_field_names_by_source_class_name()
+    )
+
+    # Start a "throwaway" MongoDB transaction so we can simulate the updates.
+    with db.client.start_session() as session:
+        with session.start_transaction():
+
+            # Make a list of the `_id`, `id`, and `type` values of the documents that
+            # the user wants to update.
+            projection = {"_id": 1, "id": 1, "type": 1}
+            subject_document_summaries_pre_update = list(
+                db[collection_name].find(
+                    filter={"$or": [spec["filter"] for spec in update_specs]},
+                    projection=projection,
+                    session=session,
+                )
+            )
+
+            # Make a set of the `_id` values of the subject documents so that (later) we can
+            # check whether a given _referring_ document is also one of the _subject_
+            # documents (i.e. is among the documents the user wants to update).
+            subject_document_object_ids = set(
+                tdd["_id"] for tdd in subject_document_summaries_pre_update
+            )
+
+            # Identify _all_ documents that reference any of the subject documents.
+            all_referring_document_descriptors_pre_update = []
+            for subject_document_summary in subject_document_summaries_pre_update:
+                # If the document summary lacks the "id" field, we already know that no
+                # documents reference it (since they would have to _use_ that "id" value to
+                # do so); so, we abort this iteration and move on to the next subject document.
+                if "id" not in subject_document_summary:
+                    continue
+
+                referring_document_descriptors = identify_referring_documents(
+                    document=subject_document_summary,  # expects at least "id" and "type"
+                    schema_view=schema_view,
+                    references=legal_references,
+                    finder=finder,
+                    client_session=session,
+                )
+                all_referring_document_descriptors_pre_update.extend(
+                    referring_document_descriptors
+                )
+
+            # Simulate the updates (i.e. apply them within the context of the transaction).
+            db.command(
+                # Note: This expression was copied from the `_run_mdb_cmd` function in `queries.py`.
+                # TODO: Document this expression (i.e. the Pydantic->JSON->BSON chain).
+                bson.json_util.loads(
+                    json.dumps(update_cmd.model_dump(exclude_unset=True))
+                ),
+                session=session,
+            )
+            # For each referring document, check whether any of its outgoing references
+            # is broken (in the context of the transaction).
+            for descriptor in all_referring_document_descriptors_pre_update:
+                referring_document_oid = descriptor["source_document_object_id"]
+                referring_document_id = descriptor["source_document_id"]
+                referring_collection_name = descriptor["source_collection_name"]
+                # If the referring document is among the documents that the user wanted to
+                # update, we skip it for now. We will check its outgoing references later
+                # (i.e. when we check the outgoing references of _all_ updated documents).
+                if referring_document_oid in subject_document_object_ids:
+                    continue
+                # Get the referring document, so we can check its outgoing references.
+                # Note: We project only the fields that can legally contain references,
+                #       plus other fields involved in referential integrity checking.
+                referring_document_reference_field_names = (
+                    reference_field_names_by_source_class_name[
+                        descriptor["source_class_name"]
+                    ]
+                )
+                projection = {
+                    field_name: 1
+                    for field_name in referring_document_reference_field_names
+                } | {
+                    "_id": 1,
+                    "id": 1,
+                    "type": 1,
+                }  # note: `|` unions the dicts
+                referring_document = db[referring_collection_name].find_one(
+                    {"_id": referring_document_oid},
+                    projection=projection,
+                    session=session,
+                )
+                # Note: We assert that the referring document exists (to satisfy the type checker).
+                assert (
+                    referring_document is not None
+                ), "A referring document has vanished."
+                violations = scan_outgoing_references(
+                    document=referring_document,
+                    source_collection_name=referring_collection_name,
+                    schema_view=schema_view,
+                    references=legal_references,
+                    finder=finder,
+                    client_session=session,  # so it uses the pending transaction's session
+                )
+                # For each violation (i.e. broken reference) that exists, add a violation message
+                # to the list of violation messages.
+                #
+                # TODO: The violation might not involve a reference to one of the
+                #       subject documents. The `scan_outgoing_references` function
+                #       scans _all_ references emanating from the document.
+                #
+                for violation in violations:
+                    source_field_name = violation.source_field_name
+                    target_id = violation.target_id
+                    violation_messages.append(
+                        make_violation_message(
+                            collection_name=referring_collection_name,
+                            source_document_id=referring_document_id,
+                            source_field_name=source_field_name,
+                            target_document_id=target_id,
+                        )
+                    )
+
+            # For each updated document, check whether any of its outgoing references
+            # is broken (in the context of the transaction).
+            for subject_document_summary in subject_document_summaries_pre_update:
+                subject_document_oid = subject_document_summary["_id"]
+                subject_document_id = subject_document_summary["id"]
+                subject_document_class_name = derive_schema_class_name_from_document(
+                    document=subject_document_summary,
+                    schema_view=schema_view,
+                )
+                assert (
+                    subject_document_class_name is not None
+                ), "The updated document does not represent a valid schema class instance."
+                subject_collection_name = (
+                    collection_name  # makes a disambiguating alias
+                )
+                # Get the updated document, so we can check its outgoing references.
+                # Note: We project only the fields that can legally contain references,
+                #       plus other fields involved in referential integrity checking.
+                updated_document_reference_field_names = (
+                    reference_field_names_by_source_class_name[
+                        subject_document_class_name
+                    ]
+                )
+                projection = {
+                    field_name: 1
+                    for field_name in updated_document_reference_field_names
+                } | {
+                    "_id": 1,
+                    "id": 1,
+                    "type": 1,
+                }  # note: `|` unions the dicts
+                updated_document = db[subject_collection_name].find_one(
+                    {"_id": subject_document_oid},
+                    projection=projection,
+                    session=session,
+                )
+                # Note: We assert that the updated document exists (to satisfy the type checker).
+                assert updated_document is not None, "An updated document has vanished."
+                violations = scan_outgoing_references(
+                    document=updated_document,
+                    source_collection_name=subject_collection_name,
+                    schema_view=schema_view,
+                    references=legal_references,
+                    finder=finder,
+                    client_session=session,  # so it uses the pending transaction's session
+                )
+                # For each violation (i.e. broken reference) that exists, add a violation message
+                # to the list of violation messages.
+                for violation in violations:
+                    source_field_name = violation.source_field_name
+                    target_id = violation.target_id
+                    violation_messages.append(
+                        make_violation_message(
+                            collection_name=subject_collection_name,
+                            source_document_id=subject_document_id,
+                            source_field_name=source_field_name,
+                            target_document_id=target_id,
+                        )
+                    )
+
+            # Whatever happens (i.e. whether there are violations or not), abort the transaction.
+            #
+            # Note: If an exception was raised within this `with` block, the transaction
+            #       will already have been aborted automatically (and execution will not
+            #       have reached this statement). On the other hand, if no exception
+            #       was raised, we explicitly abort the transaction so that the updates
+            #       that we "simulated" in this block do not get applied to the real database.
+            #       Reference: https://pymongo.readthedocs.io/en/stable/api/pymongo/client_session.html
+            #
+            session.abort_transaction()
+
+    return violation_messages