nmdc-runtime 2.10.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

nmdc_runtime/api/endpoints/util.py

@@ -6,7 +6,7 @@ from functools import lru_cache
 from json import JSONDecodeError
 from pathlib import Path
 from time import time_ns
-from typing import Dict, List, Optional, Set, Tuple
+from typing import List, Optional, Set, Tuple
 from zoneinfo import ZoneInfo
 
 from bson import json_util
@@ -55,18 +55,23 @@ BASE_URL_EXTERNAL = os.getenv("API_HOST_EXTERNAL")
 HOSTNAME_EXTERNAL = BASE_URL_EXTERNAL.split("://", 1)[-1]
 
 
-def does_num_matching_docs_exceed_threshold(
-    collection: MongoCollection, filter_: dict, threshold: int
+def is_num_matching_docs_within_limit(
+    collection: MongoCollection, filter_: dict, limit: int
 ) -> bool:
-    """Check whether a MongoDB collection contains more than `threshold` documents matching the filter."""
-    if threshold < 0:
-        raise ValueError("Threshold must be at least 0.")
+    """
+    Check whether the number of documents in a MongoDB collection that match
+    the filter is within (i.e. is no greater than) the specified limit.
+    """
+    if limit < 0:
+        raise ValueError("Limit must be at least 0.")
 
+    # Count the number of documents matching the filter, but only count up to limit + 1,
+    # since that's enough to determine whether the number exceeds the limit.
     limited_num_matching_docs = collection.count_documents(
         filter=filter_,
-        limit=threshold + 1,
+        limit=limit + 1,
     )
-    return limited_num_matching_docs > threshold
+    return limited_num_matching_docs <= limit
 
 
 def check_filter(filter_: str):
@@ -87,22 +92,44 @@ def check_filter(filter_: str):
     return filter_
 
 
-def list_resources(req: ListRequest, mdb: MongoDatabase, collection_name: str):
-    r"""
+def list_resources(
+    req: ListRequest, mdb: MongoDatabase, collection_name: str = ""
+) -> dict:
+    """
     Returns a dictionary containing the requested MongoDB documents, maybe alongside pagination information.
 
-    Note: If the specified page size (`req.max_page_size`) is non-zero and more documents match the filter
-          criteria than can fit on a page of that size, this function will paginate the resources.
+    `mdb.page_tokens` docs are `{"_id": req.page_token, "ns": collection_name}`, Because `page_token` is globally
+    unique, and because the `mdb.page_tokens.find_one({"_id": req.page_token})` document stores `collection_name` in
+    the "ns" (namespace) field, the value for `collection_name` stored there takes precedence over any value supplied
+    as an argument to this function's `collection_name` parameter.
+
+    If the specified page size (`req.max_page_size`) is non-zero and more documents match the filter criteria than
+    can fit on a page of that size, this function will paginate the resources.
     """
+    if collection_name == "" and req.page_token is None:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Must specify a collection name if no page token is supplied.",
+        )
+    if req.page_token:
+        doc = mdb.page_tokens.find_one({"_id": req.page_token})
+        if doc is None:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST, detail="`page_token` not found"
+            )
+        collection_name = doc["ns"]
+        last_id = doc["last_id"]
+        mdb.page_tokens.delete_one({"_id": req.page_token})
+    else:
+        last_id = None
 
     id_field = "id"
     if "id_1" not in mdb[collection_name].index_information():
         logging.warning(
             f"list_resources: no index set on 'id' for collection {collection_name}"
         )
-        id_field = (
-            "_id"  # currently expected for `functional_annotation_agg` collection
-        )
+        id_field = "_id"  # expected for `functional_annotation_agg` collection
+
     max_page_size = req.max_page_size
     filter_ = json_util.loads(check_filter(req.filter)) if req.filter else {}
     projection = (
@@ -110,16 +137,6 @@ def list_resources(req: ListRequest, mdb: MongoDatabase, collection_name: str):
         if req.projection
         else None
     )
-    if req.page_token:
-        doc = mdb.page_tokens.find_one({"_id": req.page_token, "ns": collection_name})
-        if doc is None:
-            raise HTTPException(
-                status_code=status.HTTP_400_BAD_REQUEST, detail="Bad page_token"
-            )
-        last_id = doc["last_id"]
-        mdb.page_tokens.delete_one({"_id": req.page_token})
-    else:
-        last_id = None
     if last_id is not None:
         if id_field in filter_:
             filter_[id_field] = merge(filter_[id_field], {"$gt": last_id})
@@ -128,17 +145,12 @@ def list_resources(req: ListRequest, mdb: MongoDatabase, collection_name: str):
 
     # Determine whether we will paginate the results.
     #
-    # Note: We will paginate them unless either:
-    #       - the `max_page_size` is not a positive integer
-    #       - the number of documents matching the filter does not exceed `max_page_size`
+    # Note: We will paginate them unless either (a) the `max_page_size` is less than 1,
+    #       or (b) the number of documents matching the filter can fit on a single page.
     #
     will_paginate = True
-    if not isinstance(max_page_size, int):
-        will_paginate = False
-    elif max_page_size < 1:
-        will_paginate = False
-    elif not does_num_matching_docs_exceed_threshold(
-        collection=mdb[collection_name], filter_=filter_, threshold=max_page_size
+    if max_page_size < 1 or is_num_matching_docs_within_limit(
+        collection=mdb[collection_name], filter_=filter_, limit=max_page_size
     ):
         will_paginate = False
 
@@ -304,9 +316,19 @@ def find_resources(req: FindRequest, mdb: MongoDatabase, collection_name: str):
     if req.page:
         skip = (req.page - 1) * req.per_page
         if skip > 10_000:
+            # Note: because _page number_-based pagination is currently implemented via MongoDB's `skip` and `limit`
+            # parameters, a full (slow) collection scan is performed to skip to the requested page. This scan takes
+            # longer and longer as `skip` increases, which is why cursor-based pagination is preferred for large
+            # collections.
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
-                detail="Use cursor-based pagination for paging beyond 10,000 items",
+                detail=(
+                    "Use cursor-based pagination for paging beyond 10,000 items. "
+                    "That is, instead of specifying the `page` query parameter for this endpoint, "
+                    "specify the `cursor` query parameter. In particular, set `cursor` to `*` to get the first page, "
+                    "and use the value of `meta.next_cursor` in the response, if not `null`, as the value to which "
+                    "you set `cursor` in the next request."
+                ),
             )
         limit = req.per_page
         results, db_response_time_ms = timeit(

nmdc_runtime/api/entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -euo pipefail
+
+exec gunicorn --worker-tmp-dir /dev/shm --workers=2 \
+              --threads=4 --worker-class gthread \
+              --log-file=- --bind 0.0.0.0:8000 nmdc_runtime.api.main:app

nmdc_runtime/api/main.py

@@ -1,5 +1,6 @@
 import os
 from contextlib import asynccontextmanager
+from html import escape
 from importlib import import_module
 from importlib.metadata import version
 from typing import Annotated
@@ -12,7 +13,6 @@ from fastapi import APIRouter, FastAPI, Cookie
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.openapi.docs import get_swagger_ui_html
 from fastapi.staticfiles import StaticFiles
-from setuptools_scm import get_version
 from starlette import status
 from starlette.responses import RedirectResponse, HTMLResponse, FileResponse
 from refscan.lib.helpers import get_collection_names_from_schema
@@ -55,29 +55,32 @@ from nmdc_runtime.api.endpoints.util import BASE_URL_EXTERNAL
 from nmdc_runtime.api.models.site import SiteClientInDB, SiteInDB
 from nmdc_runtime.api.models.user import UserInDB
 from nmdc_runtime.api.models.util import entity_attributes_to_index
-from nmdc_runtime.api.openapi import ordered_tag_descriptors, make_api_description
-from nmdc_runtime.api.v1.router import router_v1
+from nmdc_runtime.api.openapi import (
+    OpenAPITag,
+    ordered_tag_descriptors,
+    make_api_description,
+)
+from nmdc_runtime.api.swagger_ui.swagger_ui import base_swagger_ui_parameters
 from nmdc_runtime.minter.bootstrap import bootstrap as minter_bootstrap
 from nmdc_runtime.minter.entrypoints.fastapi_app import router as minter_router
 
 
 api_router = APIRouter()
-api_router.include_router(users.router, tags=["users"])
-api_router.include_router(operations.router, tags=["operations"])
-api_router.include_router(sites.router, tags=["sites"])
-api_router.include_router(jobs.router, tags=["jobs"])
-api_router.include_router(objects.router, tags=["objects"])
-api_router.include_router(capabilities.router, tags=["capabilities"])
-api_router.include_router(triggers.router, tags=["triggers"])
-api_router.include_router(workflows.router, tags=["workflows"])
-api_router.include_router(object_types.router, tags=["object types"])
-api_router.include_router(queries.router, tags=["queries"])
-api_router.include_router(metadata.router, tags=["metadata"])
-api_router.include_router(nmdcschema.router, tags=["metadata"])
-api_router.include_router(find.router, tags=["find"])
-api_router.include_router(runs.router, tags=["runs"])
-api_router.include_router(router_v1, tags=["v1"])
-api_router.include_router(minter_router, prefix="/pids", tags=["minter"])
+api_router.include_router(find.router, tags=[OpenAPITag.METADATA_ACCESS.value])
+api_router.include_router(nmdcschema.router, tags=[OpenAPITag.METADATA_ACCESS.value])
+api_router.include_router(queries.router, tags=[OpenAPITag.METADATA_ACCESS.value])
+api_router.include_router(metadata.router, tags=[OpenAPITag.METADATA_ACCESS.value])
+api_router.include_router(sites.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(workflows.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(capabilities.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(object_types.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(triggers.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(jobs.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(objects.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(operations.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(runs.router, tags=[OpenAPITag.WORKFLOWS.value])
+api_router.include_router(minter_router, prefix="/pids", tags=[OpenAPITag.MINTER.value])
+api_router.include_router(users.router, tags=[OpenAPITag.USERS.value])
 
 
 def ensure_initial_resources_on_boot():
@@ -219,9 +222,6 @@ async def lifespan(app: FastAPI):
     From the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/events/#lifespan-function):
     > You can define logic (code) that should be executed before the application starts up. This means that
     > this code will be executed once, before the application starts receiving requests.
-
-    Note: Based on my own observations, I think this function gets called when the first request starts coming in,
-          but not before that (i.e. not when the application is idle before any requests start coming in).
     """
     ensure_initial_resources_on_boot()
     ensure_attribute_indexes()
@@ -242,21 +242,24 @@ async def root():
     )
 
 
-@api_router.get("/version")
+@api_router.get("/version", tags=[OpenAPITag.SYSTEM_ADMINISTRATION.value])
 async def get_versions():
     return {
-        "nmdc-runtime": get_version(),
+        "nmdc-runtime": version("nmdc_runtime"),
         "fastapi": fastapi.__version__,
         "nmdc-schema": version("nmdc_schema"),
     }
 
 
+# Build an ORCID Login URL for the Swagger UI page, based upon some environment variables.
+orcid_login_url = f"{ORCID_BASE_URL}/oauth/authorize?client_id={ORCID_NMDC_CLIENT_ID}&response_type=code&scope=openid&redirect_uri={BASE_URL_EXTERNAL}/orcid_code"
+
+
 app = FastAPI(
     title="NMDC Runtime API",
-    version=get_version(),
+    version=version("nmdc_runtime"),
     description=make_api_description(
-        schema_version=version("nmdc_schema"),
-        orcid_login_url=f"{ORCID_BASE_URL}/oauth/authorize?client_id={ORCID_NMDC_CLIENT_ID}&response_type=code&scope=openid&redirect_uri={BASE_URL_EXTERNAL}/orcid_code",
+        api_version=version("nmdc_runtime"), schema_version=version("nmdc_schema")
     ),
     openapi_tags=ordered_tag_descriptors,
     lifespan=lifespan,
@@ -309,6 +312,14 @@ async def get_scalar_html():
 def custom_swagger_ui_html(
     user_id_token: Annotated[str | None, Cookie()] = None,
 ):
+    r"""Returns the HTML markup for an interactive API docs web page powered by Swagger UI.
+
+    If the `user_id_token` cookie is present and not empty, this function will send its value to
+    the `/token` endpoint in an attempt to get an access token. If it gets one, this function will
+    inject that access token into the web page so Swagger UI will consider the user to be logged in.
+
+    Reference: https://fastapi.tiangolo.com/tutorial/cookie-params/
+    """
     access_token = None
     if user_id_token:
         # get bearer token
@@ -329,32 +340,9 @@ def custom_swagger_ui_html(
             rv.raise_for_status()
         access_token = rv.json()["access_token"]
 
-    swagger_ui_parameters = {"withCredentials": True}
     onComplete = ""
     if access_token is not None:
-        onComplete += f"""
-            ui.preauthorizeApiKey('bearerAuth', '{access_token}');
-            
-            token_info = document.createElement('section');
-            token_info.classList.add('nmdc-info', 'nmdc-info-token', 'block', 'col-12');
-            token_info.innerHTML = <double-quote>
-                <p>You are now authorized. Prefer a command-line interface (CLI)? Use this header for HTTP requests:</p>
-                <p>
-                    <code>
-                        <span>Authorization: Bearer </span>
-                        <span id='token' data-token-value='{access_token}' data-state='masked'>***</span>
-                    </code>
-                </p>
-                <p>
-                    <button id='token-mask-toggler'>Show token</button>
-                    <button id='token-copier'>Copy token</button>
-                    <span id='token-copier-message'></span>
-                </p>
-            </double-quote>;
-            document.querySelector('.information-container').append(token_info);
-        """.replace(
-            "\n", " "
-        )
+        onComplete += f"ui.preauthorizeApiKey('bearerAuth', '{access_token}');"
     if os.getenv("INFO_BANNER_INNERHTML"):
         info_banner_innerhtml = os.getenv("INFO_BANNER_INNERHTML")
         onComplete += f"""
@@ -365,14 +353,14 @@ def custom_swagger_ui_html(
         """.replace(
             "\n", " "
         )
-    if onComplete:
-        # Note: The `nmdcInit` JavaScript event is a custom event we use to trigger anything that is listening for it.
-        #       Reference: https://developer.mozilla.org/en-US/docs/Web/Events/Creating_and_triggering_events
-        swagger_ui_parameters.update(
-            {
-                "onComplete": f"""<unquote-safe>() => {{ {onComplete}; dispatchEvent(new Event('nmdcInit')); }}</unquote-safe>""",
-            }
-        )
+    swagger_ui_parameters = base_swagger_ui_parameters.copy()
+    # Note: The `nmdcInit` JavaScript event is a custom event we use to trigger anything that is listening for it.
+    #       Reference: https://developer.mozilla.org/en-US/docs/Web/Events/Creating_and_triggering_events
+    swagger_ui_parameters.update(
+        {
+            "onComplete": f"""<unquote-safe>() => {{ {onComplete}; dispatchEvent(new Event('nmdcInit')); }}</unquote-safe>""",
+        }
+    )
     response = get_swagger_ui_html(
         openapi_url=app.openapi_url,
         title=app.title,
@@ -383,15 +371,51 @@ def custom_swagger_ui_html(
     assets_dir_path = Path(__file__).parent / "swagger_ui" / "assets"
     style_css: str = Path(assets_dir_path / "style.css").read_text()
     script_js: str = Path(assets_dir_path / "script.js").read_text()
+    custom_elements_js: str = Path(assets_dir_path / "custom-elements.js").read_text()
     content = (
         response.body.decode()
         .replace('"<unquote-safe>', "")
         .replace('</unquote-safe>"', "")
         .replace("<double-quote>", '"')
         .replace("</double-quote>", '"')
+        # TODO: Consider using a "custom layout" implemented as a React component.
+        #       Reference: https://github.com/swagger-api/swagger-ui/blob/master/docs/customization/custom-layout.md
+        #
+        #       Note: Custom layouts are specified via the Swagger UI parameter named `layout`, whose value identifies
+        #             a component that is specified via the Swagger UI parameter named `plugins`. The Swagger UI
+        #             JavaScript code expects each item in the `plugins` array to be a JavaScript function,
+        #             but FastAPI's `get_swagger_ui_html` function serializes each parameter's value into JSON,
+        #             preventing us from specifying a JavaScript function as a value in the `plugins` array.
+        #
+        #             As a workaround, we could use the string `replace`-ment technique shown below to put the literal
+        #             JavaScript characters into place in the final HTML document. Using that approach, I _have_ been
+        #             able to display a custom layout (a custom React component), but I have _not_ been able to get
+        #             that custom layout to display Swagger UI's `BaseLayout` component (which includes the core
+        #             Swagger UI functionality). That's a deal breaker.
+        #
+        .replace(r'"{{ NMDC_SWAGGER_UI_PARAMETERS_PLUGINS_PLACEHOLDER }}"', r"[]")
+        # Inject HTML elements containing data that can be read via JavaScript (e.g., `swagger_ui/assets/script.js`).
+        # Note: We escape the values here so they can be safely used as HTML attribute values.
+        .replace(
+            "</head>",
+            f"""
+            </head>
+            <div
+                id="nmdc-access-token"
+                data-token="{escape(access_token if access_token is not None else '')}"
+                style="display: none"
+            ></div>
+            <div
+                id="nmdc-orcid-login-url"
+                data-url="{escape(orcid_login_url)}"
+                style="display: none"
+            ></div>
+            """,
+        )
         # Inject a custom CSS stylesheet immediately before the closing `</head>` tag.
         .replace("</head>", f"<style>\n{style_css}\n</style>\n</head>")
-        # Inject a custom JavaScript script immediately before the closing `</body>` tag.
+        # Inject custom JavaScript scripts immediately before the closing `</body>` tag.
+        .replace("</body>", f"<script>\n{custom_elements_js}\n</script>\n</body>")
         .replace("</body>", f"<script>\n{script_js}\n</script>\n</body>")
     )
     return HTMLResponse(content=content)

nmdc_runtime/api/models/util.py

@@ -30,8 +30,13 @@ class ListRequest(BaseModel):
             r'{"lat_lon.latitude": {"$gt": 45.0}, "ecosystem_category": "Plants"}',
         ],
     )
-    # TODO: Document why the optional type here is `int` as opposed to `PerPageRange` (`FindRequest` uses the latter).
-    max_page_size: Optional[int] = Field(
+    # TODO: Document the following things about this type hint and `Field` definition:
+    #       (a) why the type here is `int` as opposed to `PerPageRange` (`FindRequest` uses the latter),
+    #       (b) why the default value here is 20 as opposed to 25 (the default value in `FindRequest`), and
+    #       (c) why there is no upper limit on the value (the `PerPageRange` type has an upper limit of 2000).
+    #
+    # Note: If the HTTP request lacks a value for this parameter, Pydantic will fall back to the default value specified here.
+    max_page_size: int = Field(
         default=20,
         title="Resources per page",
         description="How many resources you want _each page_ to contain, formatted as a positive integer.",
@@ -120,10 +125,12 @@ class FindRequest(BaseModel):
         default=None,
         title="Page number",
         description="""_Which page_ of resources you want to retrieve, when using page number-based pagination.
-                    This is the page number formatted as an integer ≥ 1.""",
+                    This is the page number formatted as an integer ≥ 1.
+                    **Limitation:** When using _page number_-based pagination, only the first 10,000 resources
+                    are accessible. You can access resources beyond that by using _cursor_-based pagination.""",
         examples=[1],
     )
-    per_page: Optional[PerPageRange] = Field(
+    per_page: PerPageRange = Field(
         default=25,
         title="Resources per page",
         description="How many resources you want _each page_ to contain, formatted as a positive integer ≤ 2000.",
@@ -133,7 +140,7 @@ class FindRequest(BaseModel):
         default=None,
         title="Cursor",
         description="""A bookmark you can use to fetch the _next_ page of resources, when using cursor-based pagination.
-                    To use cursor-based pagination, set the `cursor` parameter to `*`. The response's `meta` object will
+                    To begin using cursor-based pagination, set the `cursor` parameter to `*`. The response's `meta` object will
                     include a `next_cursor` field, whose value can be used as the `cursor` parameter in a subsequent
                     request.\n\n_Example_: `nmdc:sys0zr0fbt71`""",
         examples=[