nmdc-runtime 2.9.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

nmdc_runtime/api/models/util.py

@@ -0,0 +1,260 @@
+from typing import TypeVar, List, Optional, Generic, Annotated
+
+from pydantic import model_validator, Field, BaseModel
+
+ResultT = TypeVar("ResultT")
+
+
+class ListResponse(BaseModel, Generic[ResultT]):
+    resources: List[ResultT]
+    next_page_token: Optional[str] = None
+
+
+class ListRequest(BaseModel):
+    r"""
+    An encapsulation of a set of parameters accepted by API endpoints related to listing things.
+
+    Note: This class was documented after the `FindRequest` class was documented. You can refer to the documentation of
+          the latter class for additional context about the usage of Pydantic's `Field` constructor in this class.
+    """
+
+    filter: Optional[str] = Field(
+        default=None,
+        title="Filter",
+        description="""The criteria by which you want to filter the resources, in the same format as the [`query`
+                    parameter](https://www.mongodb.com/docs/manual/reference/method/db.collection.find/#std-label-method-find-query)
+                    of MongoDB's `db.collection.find()` method.\n\n_Example:_
+                    `{"lat_lon.latitude": {"$gt": 45.0}, "ecosystem_category": "Plants"}`""",
+        examples=[
+            r'{"ecosystem_type": "Freshwater"}',
+            r'{"lat_lon.latitude": {"$gt": 45.0}, "ecosystem_category": "Plants"}',
+        ],
+    )
+    # TODO: Document the following things about this type hint and `Field` definition:
+    #       (a) why the type here is `int` as opposed to `PerPageRange` (`FindRequest` uses the latter),
+    #       (b) why the default value here is 20 as opposed to 25 (the default value in `FindRequest`), and
+    #       (c) why there is no upper limit on the value (the `PerPageRange` type has an upper limit of 2000).
+    #
+    # Note: If the HTTP request lacks a value for this parameter, Pydantic will fall back to the default value specified here.
+    max_page_size: int = Field(
+        default=20,
+        title="Resources per page",
+        description="How many resources you want _each page_ to contain, formatted as a positive integer.",
+        examples=[20],
+    )
+    page_token: Optional[str] = Field(
+        default=None,
+        title="Next page token",
+        description="""A bookmark you can use to fetch the _next_ page of resources. You can get this from the
+                    `next_page_token` field in a previous response from this endpoint.\n\n_Example_: 
+                    `nmdc:sys0zr0fbt71`""",
+        examples=[
+            "nmdc:sys0zr0fbt71",
+        ],
+    )
+    # TODO: Document the endpoint's behavior when a projection includes a _nested_ field identifier (i.e. `foo.bar`),
+    #       and ensure the endpoint doesn't break when the projection includes field descriptors that contain commas.
+    projection: Optional[str] = Field(
+        default=None,
+        title="Projection",
+        description="""Comma-delimited list of the names of the fields you want the resources in the response to
+                    include. Note: In addition to those fields, the response will also include the `id`
+                    field.\n\n_Example_: `name, ecosystem_type`""",
+        examples=[
+            "name, ecosystem_type",
+        ],
+    )
+
+
+PerPageRange = Annotated[int, Field(gt=0, le=2_000)]
+
+
+class FindRequest(BaseModel):
+    r"""
+    An encapsulation of a set of parameters accepted by API endpoints related to finding things.
+
+    Notes:
+    - The "Query Parameter Models" section of the FastAPI docs says that this way of encapsulating
+      a set of query parameter definitions in a Pydantic model — so that Swagger UI displays a given
+      parameter's _description_ — was introduced in FastAPI 0.115.0.
+      Reference: https://fastapi.tiangolo.com/tutorial/query-param-models/
+    - While Swagger UI does show the parameter's _description_, specifically, it does not currently show the
+      parameter's _title_ or example value(s). The approach shown in the "Classes as Dependencies" section
+      of the FastAPI docs (i.e. https://fastapi.tiangolo.com/tutorial/dependencies/classes-as-dependencies/)
+      does result in Swagger UI showing those additional things, but the approach involves not inheriting
+      from Pydantic's `BaseModel` class and involves defining an `__init__` method for the class. That is
+      further than I want to take these classes from their existing selves at this point. To compensate
+      for that, I have included examples _within_ some of the descriptions.
+      Reference: https://github.com/fastapi/fastapi/issues/318#issuecomment-507043221
+    - The "Fields" section of the Pydantic docs says:
+      > "The `Field` function is used to customize and add metadata to fields of models."
+      References: https://docs.pydantic.dev/latest/concepts/fields/
+    """
+
+    filter: Optional[str] = Field(
+        default=None,
+        title="Filter",
+        description="""The criteria by which you want to filter the resources, formatted as a comma-separated list of
+                    `attribute:value` pairs. The `value` can include a comparison operator (e.g. `>=`). If the attribute
+                    is of type _string_ and you append `.search` to its name, the server will perform a full-text
+                    search.\n\n_Example:_ `ecosystem_category:Plants, lat_lon.latitude:>35.0`""",
+        examples=[
+            "ecosystem_category:Plants",
+            "ecosystem_category:Plants, lat_lon.latitude:>35.0",
+        ],
+    )
+    search: Optional[str] = Field(
+        default=None,
+        title="Search",
+        description="N/A _(not implemented yet)_",
+    )
+    sort: Optional[str] = Field(
+        default=None,
+        title="Sort",
+        description="""How you want the resources to be ordered in the response, formatted as a comma-separated list of
+                    `attribute:value` pairs. Each `attribute` is the name of a field you want the resources to be
+                    ordered by, and each `value` is the direction you want the values in that field to be ordered
+                    (i.e. `asc` or no value for _ascending_ order, and `desc` for _descending_ order).\n\n_Example:_
+                    `depth.has_numeric_value:desc, ecosystem_type`""",
+        examples=[
+            "depth.has_numeric_value:desc",
+            "depth.has_numeric_value:desc, ecosystem_type",
+        ],
+    )
+    page: Optional[int] = Field(
+        default=None,
+        title="Page number",
+        description="""_Which page_ of resources you want to retrieve, when using page number-based pagination.
+                    This is the page number formatted as an integer ≥ 1.
+                    **Limitation:** When using _page number_-based pagination, only the first 10,000 resources
+                    are accessible. You can access resources beyond that by using _cursor_-based pagination.""",
+        examples=[1],
+    )
+    per_page: PerPageRange = Field(
+        default=25,
+        title="Resources per page",
+        description="How many resources you want _each page_ to contain, formatted as a positive integer ≤ 2000.",
+        examples=[25],
+    )
+    cursor: Optional[str] = Field(
+        default=None,
+        title="Cursor",
+        description="""A bookmark you can use to fetch the _next_ page of resources, when using cursor-based pagination.
+                    To begin using cursor-based pagination, set the `cursor` parameter to `*`. The response's `meta` object will
+                    include a `next_cursor` field, whose value can be used as the `cursor` parameter in a subsequent
+                    request.\n\n_Example_: `nmdc:sys0zr0fbt71`""",
+        examples=[
+            "*",
+            "nmdc:sys0zr0fbt71",
+        ],
+    )
+    group_by: Optional[str] = Field(
+        default=None,
+        title="Group by",
+        description="N/A _(not implemented yet)_",
+    )
+    fields: Optional[str] = Field(
+        default=None,
+        title="Fields",
+        description="""The fields you want the resources to include in the response, formatted as a comma-separated list
+                    of field names. This can be used to reduce the size and complexity of the response.\n\n_Example:_
+                    `name, ess_dive_datasets`""",
+        examples=[
+            "name",
+            "name, ess_dive_datasets",
+        ],
+    )
+
+    # Reference: https://docs.pydantic.dev/latest/concepts/validators/#model-validators
+    @model_validator(mode="before")
+    def set_page_if_cursor_unset(cls, values):
+        page, cursor = values.get("page"), values.get("cursor")
+        if page is not None and cursor is not None:
+            raise ValueError("cannot use cursor- and page-based pagination together")
+        if page is None and cursor is None:
+            values["page"] = 1
+        return values
+
+
+class FindResponse(BaseModel):
+    meta: dict
+    results: List[dict]
+    group_by: List[dict]
+
+
+class DeleteResponse(BaseModel):
+    r"""
+    Response model for "delete" operations. It summarizes the result of the
+    operation and it lists identifiers of the documents that were deleted.
+    """
+
+    message: str = Field(
+        description="Success message describing the deletion operation"
+    )
+    deleted_workflow_execution_ids: List[str] = Field(
+        # Note: `default_factory=list` sets this to an empty list by default.
+        default_factory=list,
+        description="The `id`s of the `WorkflowExecution`s that were deleted",
+    )
+    deleted_data_object_ids: List[str] = Field(
+        default_factory=list,
+        description="The `id`s of the `DataObject`s that were deleted",
+    )
+    deleted_functional_annotation_agg_oids: List[str] = Field(
+        default_factory=list,
+        description="The internal MongoDB `ObjectId`s of the `FunctionalAnnotationAggMember`s that were deleted",
+    )
+    deleted_job_ids: List[str] = Field(
+        default_factory=list,
+        description="The `id`s of the `jobs` documents that were deleted",
+    )
+
+
+# Note: For MongoDB, a single collection can have no more than 64 indexes
+# Note: Each collection has a unique index set on "id" elsewhere.
+entity_attributes_to_index = {
+    "biosample_set": {
+        "alternative_identifiers",
+        "env_broad_scale.has_raw_value",
+        "env_local_scale.has_raw_value",
+        "env_medium.has_raw_value",
+        "collection_date.has_raw_value",
+        "ecosystem",
+        "ecosystem_category",
+        "ecosystem_type",
+        "ecosystem_subtype",
+        "specific_ecosystem",
+        # Note: if `lat_lon` was GeoJSON, i.e. {type,coordinates}, MongoDB has a "2dsphere" index
+        "lat_lon.latitude",
+        "lat_lon.longitude",
+    },
+    "study_set": {
+        "has_credit_associations.applied_roles",
+        "has_credit_associations.applies_to_person.name",
+        "has_credit_associations.applies_to_person.orcid",
+    },
+    "data_object_set": {
+        "data_object_type",
+        "file_size_bytes",
+        "md5_checksum",
+        "url",
+    },
+    # TODO: Refrain from ensuring indexes exist in the `omics_processing_set` collection,
+    #       since that collection was deleted as part of the "Berkeley schema" refactor.
+    #       Reference: https://microbiomedata.github.io/nmdc-schema/v10-vs-v11-retrospective/#slots-removed-from-database
+    "omics_processing_set": {
+        "has_input",
+        "has_output",
+        "instrument_name",
+        "alternative_identifiers",
+    },
+    "functional_annotation_agg": {"was_generated_by"},
+    "workflow_execution_set": {
+        "has_input",
+        "has_output",
+    },
+    # Note: The `jobs` collection is not described by the NMDC schema.
+    "jobs": {
+        "config.activity_id",
+    },
+}

nmdc_runtime/api/models/workflow.py

@@ -0,0 +1,15 @@
+import datetime
+from typing import Optional, List
+
+from pydantic import BaseModel
+
+
+class WorkflowBase(BaseModel):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    capability_ids: Optional[List[str]] = None
+
+
+class Workflow(WorkflowBase):
+    id: str
+    created_at: Optional[datetime.datetime] = None

nmdc_runtime/api/openapi.py

@@ -0,0 +1,178 @@
+r"""
+This module contains the definitions of constants and functions related to
+generating the API's OpenAPI schema (a.k.a. Swagger schema).
+
+References:
+- FastAPI Documentation: https://fastapi.tiangolo.com/tutorial/metadata/
+
+Notes:
+- The tag descriptions in this file were cut/pasted from `nmdc_runtime/api/main.py`.
+  Now that they are in a separate module, we will be able to edit them more easily.
+"""
+
+from typing import List, Dict
+from enum import Enum
+
+
+class OpenAPITag(str, Enum):
+    r"""A tag you can use to group related API endpoints together in an OpenAPI schema."""
+
+    MINTER = "Persistent identifiers"
+    SYSTEM_ADMINISTRATION = "System administration"
+    WORKFLOWS = "Workflow management"
+    METADATA_ACCESS = "Metadata access"
+    USERS = "User accounts"
+
+
+# Mapping from tag names to their (Markdown-formatted) descriptions.
+tag_descriptions: Dict[str, str] = {}
+
+tag_descriptions[
+    OpenAPITag.METADATA_ACCESS.value
+] = r"""
+Retrieve and manage metadata.
+
+The metadata access endpoints fall into several subcategories:
+
+- **Find**: Find a few types of metadata, using a simplified syntax.
+    - Each endpoint deals with a predetermined type of metadata; i.e., [studies](https://w3id.org/nmdc/Study/), [biosamples](https://w3id.org/nmdc/Biosample/), [data objects](https://w3id.org/nmdc/DataObject/), [planned processes](https://w3id.org/nmdc/PlannedProcess/), or [workflow executions](https://w3id.org/nmdc/WorkflowExecution/).
+- **NMDC schema**: Examine the [NMDC schema](https://microbiomedata.github.io/nmdc-schema/), itself, and use schema-related terminology to find metadata of any type.
+- **Queries**: Find, update, and delete metadata using [MongoDB commands](https://www.mongodb.com/docs/manual/reference/command/#user-commands).
+- **Changesheets**: Modify metadata by uploading [changesheets](https://docs.microbiomedata.org/runtime/howto-guides/author-changesheets/).
+- **JSON operations**: Insert or update metadata by submitting a JSON document representing a [Database](https://w3id.org/nmdc/Database/).
+"""
+
+tag_descriptions[
+    OpenAPITag.WORKFLOWS.value
+] = r"""
+Manage workflows and their execution.
+
+The workflow management endpoints fall into several subcategories:
+
+- **Sites**: Register compute sites that can execute workflows, and generate credentials for them.
+    - A site corresponds to a physical place that may participate in job execution.
+    - A site may register data objects and capabilities with the Runtime. It may claim jobs to execute, and it may update job operations with execution info.
+    - A site must be able to service requests for any data objects it has registered.
+    - A site may expose a "put object" custom method for authorized users. This method facilitates an operation to upload an object to the site and have the site register that object with the Runtime system.
+- **Workflows**: Manage workflow templates, which serve as blueprints for job execution.
+    - A workflow is a template for creating jobs.
+    - Workflow jobs are typically created by the system via triggers, which are associations between workflows and data object types.
+- **Capabilities**: Manage the technical requirements that sites must meet to execute specific workflows.
+    - A workflow may require a site that executes it to have specific capabilities.
+    - These capabilities may go beyond the simple ability to access the data objects registered with the Runtime system.
+    - Sites register their capabilities, and sites are only able to claim workflow jobs if those sites have the capabilities required by the workflow.
+- **Object types**: Manage the types of data objects whose creation can trigger job creation and, eventually, workflow execution.
+    - A data object type is an annotation that can be applied to data objects.
+    - A data object may have one or more types. Those types can be associated with workflows, through triggers.
+- **Triggers**: Define associations between workflows and object types to enable automatic job creation.
+    - A [trigger](https://docs.microbiomedata.org/runtime/howto-guides/create-triggers/) is an association between a workflow and a data object type.
+    - When a data object is [annotated with a type](https://docs.microbiomedata.org/runtime/nb/queue_and_trigger_data_jobs/#use-case-annotate-a-known-object-with-a-type-that-will-trigger-a-workflow)—which may occur shortly after object registration—the Runtime will check—via trigger associations—whether it is due to create any jobs.
+- **Jobs**: Manage the [claiming](https://docs.microbiomedata.org/runtime/howto-guides/claim-and-run-jobs/) and status of workflow executions.
+    - A job is a resource that decouples the configuration of a workflow, from execution of that workflow.
+    - Rather than directly creating a workflow operation, the Runtime creates a job that pairs a workflow with its configuration. Then, a site can claim the job—by its ID—and execute the associated workflow without doing additional configuration.
+    - A job can have multiple executions. All executions of all jobs of a given workflow, make up that workflow's executions.
+    - A site that already has a compatible job execution result can preempt the unnecessary creation of a job by _pre-claiming_ it. This will return like a claim, and now the site can register known data object inputs for the job without the risk of the Runtime creating a claimable job of the pre-claimed type.
+- **Objects**: Manage the Data Repository Service (DRS) objects that are inputs and outputs of workflow executions.
+    - A [Data Repository Service (DRS) object](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.1.0/docs/#_drs_datatypes) represents content necessary for—or content produced by—job execution.
+    - An object may be a *blob* (analogous to a file) or a *bundle* (analogous to a folder). Sites register objects, and sites must ensure that these objects are accessible to the "NMDC data broker."
+    - An object may be annotated with one or more object types, useful for triggering workflows.
+- **Operations**: Track and monitor the real-time execution status of claimed jobs, including progress updates and error handling.
+    - An operation is a resource for tracking the execution of a job.
+    - When a job is claimed by a site for execution, an operation resource is created.
+    - An operation is like a "promise," in that it should eventually resolve to either a successful result—i.e., an execution resource—or to an error.
+    - An operation is parameterized to return a result type, and a metadata type for storing progress information, that are both particular to the job type.
+    - Operations may be paused, resumed, and/or cancelled.
+    - Operations may expire, i.e. not be stored indefinitely. In this case, it is recommended that execution resources have longer lifetimes/not expire, so that information about successful results of operations are available.
+- **Runs**: _(work in progress)_ Execute simple jobs and report execution events back to the Runtime.
+    - Run simple jobs.
+    - For off-site job runs, keep the Runtime appraised of run events.
+"""
+
+tag_descriptions[
+    OpenAPITag.USERS.value
+] = r"""
+Create and manage user accounts.
+"""
+
+tag_descriptions[
+    OpenAPITag.MINTER.value
+] = r"""
+Mint and manage persistent identifiers.
+"""
+
+tag_descriptions[
+    OpenAPITag.SYSTEM_ADMINISTRATION.value
+] = r"""
+Retrieve information about the software components that make up the Runtime.
+"""
+
+# Remove leading and trailing whitespace from each description.
+for name, description in tag_descriptions.items():
+    tag_descriptions[name] = description.strip()
+
+ordered_tag_descriptors: List[Dict] = [
+    {
+        "name": OpenAPITag.METADATA_ACCESS.value,
+        "description": tag_descriptions[OpenAPITag.METADATA_ACCESS.value],
+    },
+    {
+        "name": OpenAPITag.WORKFLOWS.value,
+        "description": tag_descriptions[OpenAPITag.WORKFLOWS.value],
+    },
+    {
+        "name": OpenAPITag.MINTER.value,
+        "description": tag_descriptions[OpenAPITag.MINTER.value],
+    },
+    {
+        "name": OpenAPITag.USERS.value,
+        "description": tag_descriptions[OpenAPITag.USERS.value],
+    },
+    {
+        "name": OpenAPITag.SYSTEM_ADMINISTRATION.value,
+        "description": tag_descriptions[OpenAPITag.SYSTEM_ADMINISTRATION.value],
+    },
+]
+
+
+def make_api_description(api_version: str, schema_version: str) -> str:
+    r"""
+    Returns an API description into which the specified schema version string has been incorporated.
+
+    Args:
+        api_version (str): The version of this Runtime instance.
+        schema_version (str): The version of `nmdc-schema` the Runtime is using.
+
+    Returns:
+        str: The Markdown-formatted API description.
+    """
+    result = f"""
+Welcome to the **NMDC Runtime API**, an API you can use to [access metadata](https://docs.microbiomedata.org/howto_guides/api_gui/) residing in the NMDC database.
+
+Users having adequate permissions can also use it to generate identifiers, submit metadata,
+and manage workflow executions.
+
+##### Quick start
+
+The endpoints of the NMDC Runtime API are listed below.
+They are organized into sections, each of which can be opened and closed.
+The endpoints, themselves, can also be opened and closed.
+
+Each endpoint—when opened—has a "Try it out" button, which you can press in order to send a request
+to the endpoint directly from this web page. Each endpoint can also be
+[accessed programmatically](https://docs.microbiomedata.org/runtime/nb/api_access_via_python/).
+
+Some endpoints have a padlock icon, which means that the endpoint is only accessible to logged-in users.
+You can log in by clicking the "Authorize" button located directly above the list of endpoints.
+
+##### Contact us
+
+You can [contact us](https://microbiomedata.org/contact/) anytime.
+We continuously refine the API and may be able to streamline your use case.
+
+##### Versions
+
+[NMDC Runtime](https://docs.microbiomedata.org/runtime/) version: `{api_version}`
+
+[NMDC Schema](https://microbiomedata.github.io/nmdc-schema/) version: `{schema_version}`
+""".strip()
+    return result