nmdc-runtime 2.8.0__py3-none-any.whl → 2.10.0__py3-none-any.whl

nmdc_runtime/api/analytics.py

@@ -0,0 +1,70 @@
+"""
+Based on <https://github.com/tom-draper/api-analytics/tree/main/analytics/python/fastapi>
+under MIT License <https://github.com/tom-draper/api-analytics/blob/main/analytics/python/fastapi/LICENSE>
+"""
+
+from datetime import datetime
+import threading
+from time import time
+from typing import Dict, List
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.types import ASGIApp
+from toolz import merge
+
+from nmdc_runtime.api.db.mongo import get_mongo_db
+
+_requests = []
+_last_posted = datetime.now()
+
+
+def _post_requests(collection: str, requests_data: List[Dict], source: str):
+    mdb = get_mongo_db()
+    mdb[collection].insert_many([merge(d, {"source": source}) for d in requests_data])
+
+
+def log_request(collection: str, request_data: Dict, source: str = "FastAPI"):
+    global _requests, _last_posted
+    _requests.append(request_data)
+    now = datetime.now()
+    # flush queue every minute at most
+    if (now - _last_posted).total_seconds() > 60.0:
+        threading.Thread(
+            target=_post_requests, args=(collection, _requests, source)
+        ).start()
+        _requests = []
+        _last_posted = now
+
+
+class Analytics(BaseHTTPMiddleware):
+    def __init__(self, app: ASGIApp, collection: str = "_runtime.analytics"):
+        super().__init__(app)
+        self.collection = collection
+
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        start = time()
+        response = await call_next(request)
+
+        # Build a dictionary that describes the incoming request.
+        #
+        # Note: `request.headers` is an instance of `MultiDict`. References:
+        #       - https://www.starlette.io/requests/#headers
+        #       - https://multidict.aio-libs.org/en/stable/multidict/
+        #
+        request_data = {
+            "hostname": request.url.hostname,
+            "ip_address": request.client.host,
+            "path": request.url.path,
+            "user_agent": request.headers.get("user-agent"),
+            "method": request.method,
+            "status": response.status_code,
+            "response_time": int((time() - start) * 1000),
+            "created_at": datetime.now().isoformat(),
+        }
+
+        log_request(self.collection, request_data, "FastAPI")
+        return response

nmdc_runtime/api/boot/capabilities.py

@@ -0,0 +1,9 @@
+from nmdc_runtime.api.models.capability import Capability
+import nmdc_runtime.api.boot.workflows as workflows_boot
+
+# Include 1-to-1 "I can run this workflow" capabilities.
+_raw = [item for item in workflows_boot._raw]
+
+
+def construct():
+    return [Capability(**kwargs) for kwargs in _raw]

nmdc_runtime/api/boot/object_types.py

@@ -0,0 +1,126 @@
+from datetime import datetime, timezone
+
+from toolz import get_in
+
+from nmdc_runtime.api.models.object_type import ObjectType
+from nmdc_runtime.util import nmdc_jsonschema
+
+_raw = [
+    {
+        "id": "read_qc_analysis_activity_set",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        # "description": "JSON documents satisfying schema for readqc analysis activity",
+    },
+    {
+        "id": "metagenome_sequencing_activity_set",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        # "description": "JSON documents satisfying schema for metagenome sequencing activity",
+    },
+    {
+        "id": "mags_activity_set",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        # "description": "JSON documents satisfying schema for mags activity",
+    },
+    {
+        "id": "metagenome_annotation_activity_set",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        # "description": "JSON documents satisfying schema for metagenome annotation activity",
+    },
+    {
+        "id": "metagenome_assembly_set",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        # "description": "JSON documents satisfying schema for metagenome assembly activity",
+    },
+    {
+        "id": "read_based_taxonomy_analysis_activity_set",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        # "description": "JSON documents satisfying schema for read based analysis activity",
+    },
+    {
+        "id": "metadata-in",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "metadata submission",
+        "description": "Input to the portal ETL process",
+    },
+    {
+        "id": "metaproteomics_analysis_activity_set",
+        "created_at": datetime(2021, 8, 23, tzinfo=timezone.utc),
+        "name": "metaP analysis activity",
+        "description": "JSON documents satisfying schema for metaproteomics analysis activity",
+    },
+    {
+        "id": "metagenome_raw_paired_end_reads",
+        "created_at": datetime(2021, 8, 24, tzinfo=timezone.utc),
+        "name": "Metagenome Raw Paired-End Reads Workflow Input",
+        "description": "workflow input",
+    },
+    {
+        "id": "metatranscriptome_raw_paired_end_reads",
+        "created_at": datetime(2021, 9, 7, tzinfo=timezone.utc),
+        "name": "Metatranscriptome Raw Paired-End Reads Workflow Input",
+        "description": "workflow input 2",
+    },
+    {
+        "id": "gcms-metab-input",
+        "created_at": datetime(2021, 9, 7, tzinfo=timezone.utc),
+        "name": "Raw GCMS MetaB Input",
+        "description": "",
+    },
+    {
+        "id": "gcms-metab-calibration",
+        "created_at": datetime(2021, 9, 7, tzinfo=timezone.utc),
+        "name": "Raw GCMS MetaB Calibration",
+        "description": "",
+    },
+    {
+        "id": "nom-input",
+        "created_at": datetime(2021, 9, 7, tzinfo=timezone.utc),
+        "name": "Raw FTMS MetaB Input",
+        "description": "",
+    },
+    {
+        "id": "test",
+        "created_at": datetime(2021, 9, 7, tzinfo=timezone.utc),
+        "name": "A test object type",
+        "description": "For use in unit and integration tests",
+    },
+    {
+        "id": "metadata-changesheet",
+        "created_at": datetime(2021, 9, 30, tzinfo=timezone.utc),
+        "name": "metadata changesheet",
+        "description": "Specification for changes to existing metadata",
+    },
+]
+
+_raw.extend(
+    [
+        {
+            "id": key,
+            "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+            "name": key,
+            # "description": spec["description"],
+        }
+        for key, spec in nmdc_jsonschema["properties"].items()
+        if key.endswith("_set")
+    ]
+)
+_raw.append(
+    {
+        "id": "schema#/definitions/Database",
+        "created_at": datetime(2021, 9, 14, tzinfo=timezone.utc),
+        "name": "Bundle of one or more metadata `*_set`s.",
+        "description": get_in(
+            ["definitions", "Database", "description"], nmdc_jsonschema
+        ),
+    }
+)
+
+
+def construct():
+    return [ObjectType(**kwargs) for kwargs in _raw]

nmdc_runtime/api/boot/triggers.py

@@ -0,0 +1,84 @@
+from datetime import datetime, timezone
+
+from nmdc_runtime.api.models.trigger import Trigger
+
+_raw = [
+    {
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "object_type_id": "metadata-in",
+        "workflow_id": "metadata-in-1.0.0",
+    },
+    {
+        "created_at": datetime(2021, 9, 1, tzinfo=timezone.utc),
+        "object_type_id": "metaproteomics_analysis_activity_set",
+        "workflow_id": "metap-metadata-1.0.0",
+    },
+    {
+        "created_at": datetime(2021, 9, 1, tzinfo=timezone.utc),
+        "object_type_id": "metagenome_raw_paired_end_reads",
+        "workflow_id": "metag-1.0.0",
+    },
+    {
+        "created_at": datetime(2021, 9, 7, tzinfo=timezone.utc),
+        "object_type_id": "metatranscriptome_raw_paired_end_reads",
+        "workflow_id": "metat-1.0.0",
+    },
+    {
+        "created_at": datetime(2021, 9, 9, tzinfo=timezone.utc),
+        "object_type_id": "test",
+        "workflow_id": "test",
+    },
+    {
+        "created_at": datetime(2021, 9, 20, tzinfo=timezone.utc),
+        "object_type_id": "nom-input",
+        "workflow_id": "nom-1.0.0",
+    },
+    {
+        "created_at": datetime(2021, 9, 20, tzinfo=timezone.utc),
+        "object_type_id": "gcms-metab-input",
+        "workflow_id": "gcms-metab-1.0.0",
+    },
+    {
+        "created_at": datetime(2021, 9, 30, tzinfo=timezone.utc),
+        "object_type_id": "metadata-changesheet",
+        "workflow_id": "apply-changesheet-1.0.0",
+    },
+    {
+        "created_at": datetime(2022, 1, 20, tzinfo=timezone.utc),
+        "object_type_id": "metagenome_sequencing_activity_set",
+        "workflow_id": "mgrc-1.0.6",
+    },
+    {
+        "created_at": datetime(2022, 1, 20, tzinfo=timezone.utc),
+        "object_type_id": "metagenome_sequencing_activity_set",
+        "workflow_id": "metag-1.0.0",
+    },
+    {
+        "created_at": datetime(2022, 1, 20, tzinfo=timezone.utc),
+        "object_type_id": "metagenome_annotation_activity_set",
+        "workflow_id": "mags-1.0.4",
+    },
+    {
+        "created_at": datetime(2022, 1, 20, tzinfo=timezone.utc),
+        "object_type_id": "metagenome_assembly_set",
+        "workflow_id": "mgann-1.0.0",
+    },
+    {
+        "created_at": datetime(2022, 1, 20, tzinfo=timezone.utc),
+        "object_type_id": "read_qc_analysis_activity_set",
+        "workflow_id": "mgasm-1.0.3",
+    },
+    {
+        "created_at": datetime(2022, 1, 20, tzinfo=timezone.utc),
+        "object_type_id": "read_qc_analysis_activity_set",
+        "workflow_id": "mgrba-1.0.2",
+    },
+]
+
+
+def construct():
+    models = []
+    for kwargs in _raw:
+        kwargs["id"] = f'{kwargs["object_type_id"]}--{kwargs["workflow_id"]}'
+        models.append(Trigger(**kwargs))
+    return models

nmdc_runtime/api/boot/workflows.py

@@ -0,0 +1,116 @@
+from datetime import datetime, timezone
+
+from nmdc_runtime.api.models.workflow import Workflow
+
+_raw = [
+    {
+        "id": "metag-1.0.0",
+        "created_at": datetime(2021, 8, 24, tzinfo=timezone.utc),
+        "name": "Metagenome Analysis Workflow (v1.0.0)",
+    },
+    {
+        "id": "readqc-1.0.6",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Reads QC Workflow (v1.0.1)",
+    },
+    {
+        "id": "mags-1.0.4",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Read-based Analysis (v1.0.1)",
+    },
+    {
+        "id": "mgrba-1.0.2",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Read-based Analysis (v1.0.1)",
+    },
+    {
+        "id": "mgasm-1.0.3",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Metagenome Assembly (v1.0.1)",
+    },
+    {
+        "id": "mgann-1.0.0",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Metagenome Annotation (v1.0.0)",
+    },
+    {
+        "id": "mgasmbgen-1.0.1",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Metagenome Assembled Genomes (v1.0.2)",
+    },
+    {
+        "id": "metat-0.0.2",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Metatranscriptome (v0.0.2)",
+    },
+    {
+        "id": "metap-1.0.0",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Metaproteomic (v1.0.0)",
+    },
+    {
+        "id": "metab-2.1.0",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "Metabolomics  (v2.1.0)",
+    },
+    {
+        "id": "gold-translation-1.0.0",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "GOLD db dump translation",
+        "description": "Transform metadata obtained from the JGI GOLD database.",
+    },
+    {
+        "id": "metap-metadata-1.0.0",
+        "created_at": datetime(2021, 6, 1, tzinfo=timezone.utc),
+        "name": "metaP metadata ETL",
+        "description": "Ingest and validate metaP metadata",
+    },
+    {
+        "id": "metadata-in-1.0.0",
+        "created_at": datetime(2021, 10, 12, tzinfo=timezone.utc),
+        "name": "general metadata ETL",
+        "description": "Validate and ingest metadata from JSON files",
+    },
+    {
+        "id": "test",
+        "created_at": datetime(2021, 9, 9, tzinfo=timezone.utc),
+        "name": "A test workflow",
+        "description": "For use in unit and integration tests",
+    },
+    {
+        "id": "gcms-metab-1.0.0",
+        "created_at": datetime(2021, 9, 20, tzinfo=timezone.utc),
+        "name": "GCMS-based metabolomics",
+    },
+    {
+        "id": "nom-1.0.0",
+        "created_at": datetime(2021, 9, 20, tzinfo=timezone.utc),
+        "name": "Natural Organic Matter characterization",
+    },
+    {
+        "id": "apply-changesheet-1.0.0",
+        "created_at": datetime(2021, 9, 30, tzinfo=timezone.utc),
+        "name": "apply metadata changesheet",
+        "description": "Validate and apply metadata changes from TSV/CSV files",
+    },
+    {
+        "id": "export-study-biosamples-as-csv-1.0.0",
+        "created_at": datetime(2022, 6, 8, tzinfo=timezone.utc),
+        "name": "export study biosamples metadata as CSV",
+        "description": "Export study biosamples metadata as CSV",
+    },
+    {
+        "id": "gold_study_to_database",
+        "created_at": datetime(2023, 2, 17, tzinfo=timezone.utc),
+        "name": "Get nmdc:Database for GOLD study",
+        "description": "For a given GOLD study ID, produce an nmdc:Database representing that study and related entities",
+    },
+]
+
+
+def construct():
+    models = []
+    for kwargs in _raw:
+        kwargs["capability_ids"] = []
+        models.append(Workflow(**kwargs))
+    return models

nmdc_runtime/api/core/auth.py

@@ -0,0 +1,208 @@
+import os
+from datetime import datetime, timedelta, timezone
+from typing import Optional, Dict
+
+from fastapi import Depends
+from fastapi.exceptions import HTTPException
+from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel
+from fastapi.param_functions import Form
+from fastapi.security import (
+    OAuth2,
+    HTTPBasic,
+    HTTPBasicCredentials,
+    HTTPBearer,
+    HTTPAuthorizationCredentials,
+)
+from fastapi.security.utils import get_authorization_scheme_param
+from jose import JWTError, jwt
+from passlib.context import CryptContext
+from pydantic import BaseModel
+from starlette.requests import Request
+from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED
+
+ORCID_PRODUCTION_BASE_URL = "https://orcid.org"
+
+SECRET_KEY = os.getenv("JWT_SECRET_KEY")
+ALGORITHM = "HS256"
+ORCID_NMDC_CLIENT_ID = os.getenv("ORCID_NMDC_CLIENT_ID")
+ORCID_NMDC_CLIENT_SECRET = os.getenv("ORCID_NMDC_CLIENT_SECRET")
+ORCID_BASE_URL = os.getenv("ORCID_BASE_URL", default=ORCID_PRODUCTION_BASE_URL)
+
+# Define the JSON Web Key Set (JWKS) for ORCID.
+#
+# Note: The URL from which we got this dictionary is: https://orcid.org/oauth/jwks
+#       We got _that_ URL from the dictionary at: https://orcid.org/.well-known/openid-configuration
+#
+# TODO: Consider _live-loading_ this dictionary from the Internet.
+#
+ORCID_JWK = {
+    "e": "AQAB",
+    "kid": "production-orcid-org-7hdmdswarosg3gjujo8agwtazgkp1ojs",
+    "kty": "RSA",
+    "n": "jxTIntA7YvdfnYkLSN4wk__E2zf_wbb0SV_HLHFvh6a9ENVRD1_rHK0EijlBzikb-1rgDQihJETcgBLsMoZVQqGj8fDUUuxnVHsuGav_bf41PA7E_58HXKPrB2C0cON41f7K3o9TStKpVJOSXBrRWURmNQ64qnSSryn1nCxMzXpaw7VUo409ohybbvN6ngxVy4QR2NCC7Fr0QVdtapxD7zdlwx6lEwGemuqs_oG5oDtrRuRgeOHmRps2R6gG5oc-JqVMrVRv6F9h4ja3UgxCDBQjOVT1BFPWmMHnHCsVYLqbbXkZUfvP2sO1dJiYd_zrQhi-FtNth9qrLLv3gkgtwQ",
+    "use": "sig",
+}
+# If the application is using a _non-production_ ORCID environment, overwrite
+# the "kid" and "n" values with those from the sandbox ORCID environment.
+#
+# Source: https://sandbox.orcid.org/oauth/jwks
+#
+if ORCID_BASE_URL != ORCID_PRODUCTION_BASE_URL:
+    ORCID_JWK["kid"] = "sandbox-orcid-org-3hpgosl3b6lapenh1ewsgdob3fawepoj"
+    ORCID_JWK["n"] = (
+        "pl-jp-kTAGf6BZUrWIYUJTvqqMVd4iAnoLS6vve-KNV0q8TxKvMre7oi9IulDcqTuJ1alHrZAIVlgrgFn88MKirZuTqHG6LCtEsr7qGD9XyVcz64oXrb9vx4FO9tLNQxvdnIWCIwyPAYWtPMHMSSD5oEVUtVL_5IaxfCJvU-FchdHiwfxvXMWmA-i3mcEEe9zggag2vUPPIqUwbPVUFNj2hE7UsZbasuIToEMFRZqSB6juc9zv6PEUueQ5hAJCEylTkzMwyBMibrt04TmtZk2w9DfKJR91555s2ZMstX4G_su1_FqQ6p9vgcuLQ6tCtrW77tta-Rw7McF_tyPmvnhQ"
+    )
+
+ORCID_JWS_VERITY_ALGORITHM = "RS256"
+
+
+class ClientCredentials(BaseModel):
+    client_id: str
+    client_secret: str
+
+
+class TokenExpires(BaseModel):
+    days: Optional[int] = 1
+    hours: Optional[int] = 0
+    minutes: Optional[int] = 0
+
+
+ACCESS_TOKEN_EXPIRES = TokenExpires(days=1, hours=0, minutes=0)
+
+
+class Token(BaseModel):
+    access_token: str
+    token_type: str
+    expires: Optional[TokenExpires] = None
+
+
+class TokenData(BaseModel):
+    subject: Optional[str] = None
+
+
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+credentials_exception = HTTPException(
+    status_code=HTTP_401_UNAUTHORIZED,
+    detail="Could not validate credentials",
+    headers={"WWW-Authenticate": "Bearer"},
+)
+
+
+def verify_password(plain_password, hashed_password):
+    return pwd_context.verify(plain_password, hashed_password)
+
+
+def get_password_hash(password):
+    return pwd_context.hash(password)
+
+
+def create_access_token(data: dict, expires_delta: Optional[timedelta] = None):
+    to_encode = data.copy()
+    if expires_delta:
+        expire = datetime.now(timezone.utc) + expires_delta
+    else:
+        expire = datetime.now(timezone.utc) + timedelta(minutes=15)
+    to_encode.update({"exp": expire})
+    encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
+    return encoded_jwt
+
+
+def get_access_token_expiration(token) -> datetime:
+    try:
+        payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
+        return payload.get("exp")
+    except JWTError:
+        raise credentials_exception
+
+
+class OAuth2PasswordOrClientCredentialsBearer(OAuth2):
+    def __init__(
+        self,
+        tokenUrl: str,
+        scheme_name: Optional[str] = None,
+        scopes: Optional[Dict[str, str]] = None,
+        auto_error: bool = True,
+    ):
+        if not scopes:
+            scopes = {}
+        flows = OAuthFlowsModel(
+            password={"tokenUrl": tokenUrl, "scopes": scopes},
+            clientCredentials={"tokenUrl": tokenUrl},
+        )
+        super().__init__(flows=flows, scheme_name=scheme_name, auto_error=auto_error)
+
+    async def __call__(self, request: Request) -> Optional[str]:
+        authorization: str = request.headers.get("Authorization")
+        scheme, param = get_authorization_scheme_param(authorization)
+        if not authorization or scheme.lower() != "bearer":
+            if self.auto_error:
+                raise HTTPException(
+                    status_code=HTTP_401_UNAUTHORIZED,
+                    detail="Not authenticated",
+                    headers={"WWW-Authenticate": "Bearer"},
+                )
+            else:
+                print(request.url)
+                return None
+        return param
+
+
+oauth2_scheme = OAuth2PasswordOrClientCredentialsBearer(
+    tokenUrl="token", auto_error=False
+)
+optional_oauth2_scheme = OAuth2PasswordOrClientCredentialsBearer(
+    tokenUrl="token", auto_error=False
+)
+
+bearer_scheme = HTTPBearer(scheme_name="bearerAuth", auto_error=False)
+
+
+async def basic_credentials(req: Request):
+    return await HTTPBasic(auto_error=False)(req)
+
+
+async def bearer_credentials(req: Request):
+    return await HTTPBearer(scheme_name="bearerAuth", auto_error=False)(req)
+
+
+class OAuth2PasswordOrClientCredentialsRequestForm:
+    def __init__(
+        self,
+        basic_creds: Optional[HTTPBasicCredentials] = Depends(basic_credentials),
+        bearer_creds: Optional[HTTPAuthorizationCredentials] = Depends(
+            bearer_credentials
+        ),
+        grant_type: str = Form(None, pattern="^password$|^client_credentials$"),
+        username: Optional[str] = Form(None),
+        password: Optional[str] = Form(None),
+        scope: str = Form(""),
+        client_id: Optional[str] = Form(None),
+        client_secret: Optional[str] = Form(None),
+    ):
+        if bearer_creds:
+            self.grant_type = "client_credentials"
+            self.username, self.password = None, None
+            self.scopes = scope.split()
+            self.client_id = bearer_creds.credentials
+            self.client_secret = None
+        elif grant_type == "password" and (username is None or password is None):
+            raise HTTPException(
+                status_code=HTTP_400_BAD_REQUEST,
+                detail="grant_type password requires username and password",
+            )
+        elif grant_type == "client_credentials" and (client_id is None):
+            if basic_creds:
+                client_id = basic_creds.username
+                client_secret = basic_creds.password
+            else:
+                raise HTTPException(
+                    status_code=HTTP_400_BAD_REQUEST,
+                    detail="grant_type client_credentials requires client_id and client_secret",
+                )
+        self.grant_type = grant_type
+        self.username = username
+        self.password = password
+        self.scopes = scope.split()
+        self.client_id = client_id
+        self.client_secret = client_secret