nmdc-runtime 1.2.0__tar.gz → 1.3.0__tar.gz

{nmdc_runtime-1.2.0 → nmdc_runtime-1.3.0}/.env.example

@@ -37,4 +37,5 @@ NEON_API_TOKEN=y
 NEON_API_BASE_URL=https://data.neonscience.org/api/v0
 
 NERSC_USERNAME=replaceme
-ORCID_CLIENT_ID=replaceme
+ORCID_NMDC_CLIENT_ID=replaceme
+ORCID_NMDC_CLIENT_SECRET=replaceme

nmdc_runtime-1.3.0/.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint-check + Style-normalize Python files
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/lint.yml'
+      - '**.py'
+
+
+jobs:
+  build:
+    name:  lint-check and style-normalize Python files
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.ref }}
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+    - name: Lint with flake8 and Reformat with black
+      run: |
+        make init-lint-and-black
+        make lint
+        make black
+    - name: commit and push if reformatted
+      run: |
+        git config user.name github-actions
+        git config user.email github-actions@github.com
+        if git status --short | grep -q '\.py$'; then git add '*.py' && git commit -m "style: reformat" && git push; fi
+        

{nmdc_runtime-1.2.0 → nmdc_runtime-1.3.0}/.gitignore

@@ -165,4 +165,7 @@ neon_cache.sqlite
 # output of changesheet generation
 nmdc_runtime/site/changesheets/changesheets_output/
 
-mdb-nmdc-dev.tar.gz
+mdb-nmdc-dev.tar.gz
+
+# superceded by .github/workflows/build-and-release-to-spin.yml
+docker-build.sh

{nmdc_runtime-1.2.0 → nmdc_runtime-1.3.0}/.gitpod.yml

@@ -9,4 +9,4 @@ tasks:
   - name: Start Dev on Fresh Gitpod
     before: cp .env.example .env
     init: docker compose up mongo --detach && make mongorestore-nmdc-dev
-    command: make up-dev && docker-compose logs -f fastapi
+    command: make up-dev && docker compose logs -f fastapi

{nmdc_runtime-1.2.0 → nmdc_runtime-1.3.0}/Makefile

@@ -18,14 +18,14 @@ update-deps:
 update: update-deps init
 
 up-dev:
-	docker-compose up --build --force-recreate --detach --remove-orphans
+	docker compose up --build --force-recreate --detach --remove-orphans
 
 dev-reset-db:
 	docker compose \
 		exec mongo /bin/bash -c "./app_tests/mongorestore-nmdc-testdb.sh"
 
 up-test:
-	docker-compose --file docker-compose.test.yml \
+	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
 test-build:
@@ -41,6 +41,9 @@ test-run:
 
 test: test-build test-run
 
+black:
+	black nmdc_runtime
+
 lint:
 	# Python syntax errors or undefined names
 	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F722
@@ -48,14 +51,21 @@ lint:
 	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 \
 		--statistics --extend-exclude="./build/" --extend-ignore=F722
 
+PIP_PINNED_FLAKE8 := $(shell grep 'flake8==' requirements/dev.txt)
+PIP_PINNED_BLACK := $(shell grep 'black==' requirements/dev.txt)
+
+init-lint-and-black:
+	pip install $(PIP_PINNED_FLAKE8)
+	pip install $(PIP_PINNED_BLACK)
+
 down-dev:
-	docker-compose down
+	docker compose down
 
 down-test:
-	docker-compose --file docker-compose.test.yml down
+	docker compose --file docker-compose.test.yml down
 
 follow-fastapi:
-	docker-compose logs fastapi -f
+	docker compose logs fastapi -f
 
 fastapi-deploy-spin:
 	rancher kubectl rollout restart deployment/runtime-fastapi --namespace=nmdc-dev

{nmdc_runtime-1.2.0 → nmdc_runtime-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nmdc_runtime
-Version: 1.2.0
+Version: 1.3.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston

nmdc_runtime-1.3.0/demo/metadata_migration/notebooks/migrate_9_3_2_to_10_0_0.ipynb

@@ -0,0 +1,480 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true
+   },
+   "source": [
+    "# Migrate MongoDB database from `nmdc-schema` `v9.3.2` to `v10.0.0`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f65ad4ab",
+   "metadata": {},
+   "source": [
+    "## Prerequisites"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "37d358ba",
+   "metadata": {},
+   "source": [
+    "### 1. Determine MongoDB collections involved.\n",
+    "\n",
+    "In this step, you will determine which MongoDB collections will be involved with this migration.\n",
+    "\n",
+    "1. In the [`nmdc-schema` repo](https://github.com/microbiomedata/nmdc-schema/tree/main/nmdc_schema/migrators), go to the `nmdc_schema/migrators` directory and open the Python module whose name reflects the initial and final schema versions.\n",
+    "2. In the `Migrator` class, note the collection names that are mentioned within the `upgrade` method.\n",
+    "3. Add their names to the `COLLECTION_NAMES` Python list below.\n",
+    "    - > TODO: Distinguish between colletions being read vs. collections being transformed; and consider what happens when a collection is created, deleted, or renamed."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "09966b0d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "COLLECTION_NAMES: list[str] = [\n",
+    "    \"extraction_set\",\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17f351e8",
+   "metadata": {},
+   "source": [
+    "### 2. Coordinate with stakeholders.\n",
+    "\n",
+    "In this step, you'll identify and reach out to the people that read/write to those collections; to agree on a migration schedule that works for you and them.\n",
+    "\n",
+    "Here's a table of MongoDB collections and the NMDC system components that write to them (according to [a conversation that occurred on September 11, 2023](https://nmdc-group.slack.com/archives/C01SVTKM8GK/p1694465755802979?thread_ts=1694216327.234519&cid=C01SVTKM8GK)).\n",
+    "\n",
+    "| Mongo collection                            | NMDC system components that write to it                  |\n",
+    "|---------------------------------------------|----------------------------------------------------------|\n",
+    "| `biosample_set`                             | Workflows (via manual entry via `nmdc-runtime` HTTP API) |\n",
+    "| `data_object_set`                           | Workflows (via `nmdc-runtime` HTTP API)                  |\n",
+    "| `mags_activity_set`                         | Workflows (via `nmdc-runtime` HTTP API)                  |\n",
+    "| `metagenome_annotation_activity_set`        | Workflows (via `nmdc-runtime` HTTP API)                  |\n",
+    "| `metagenome_assembly_set`                   | Workflows (via `nmdc-runtime` HTTP API)                  |\n",
+    "| `read_based_taxonomy_analysis_activity_set` | Workflows (via `nmdc-runtime` HTTP API)                  |\n",
+    "| `read_qc_analysis_activity_set`             | Workflows (via `nmdc-runtime` HTTP API)                  |\n",
+    "| `jobs`                                      | Scheduler (via MongoDB directly; e.g. `pymongo`)         |\n",
+    "| `*`                                         | `nmdc-runtime` (via MongoDB directly; e.g. `pymongo`)    |\n",
+    "\n",
+    "You can use that table to help determine which people read/write to those collections. You can then coordinate a migration time slot with them via Slack, email, etc."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "233a35c3",
+   "metadata": {},
+   "source": [
+    "### 3. Set up environment.\n",
+    "\n",
+    "In this step, you'll set up an environment in which you can run this notebook.\n",
+    "\n",
+    "1. Start a **MongoDB server** on your local machine (and ensure it does **not** already contain a database named `nmdc`).\n",
+    "    1. You can start a [Docker](https://hub.docker.com/_/mongo)-based MongoDB server at `localhost:27055` by running this command:\n",
+    "       ```shell\n",
+    "       # Run in any directory:\n",
+    "       docker run --rm --detach --name mongo-migration-transformer -p 27055:27017 mongo:6.0.4\n",
+    "       ```\n",
+    "       > Note: A MongoDB server started via that command will have no access control (i.e. you will be able to access it without a username or password).\n",
+    "2. Create and populate a **notebook configuration file** named `.notebook.env`.\n",
+    "    1. You can use the `.notebook.env.example` file as a template:\n",
+    "       ```shell\n",
+    "       # Run in the same directory as this notebook:\n",
+    "       $ cp .notebook.env.example .notebook.env\n",
+    "       ```\n",
+    "3. Create and populate **MongoDB configuration files** for connecting to the origin (typically, local) and transformer (typically, remote) MongoDB servers.\n",
+    "    1. You can use the `.mongo.yaml.example` file as a template:\n",
+    "       ```shell\n",
+    "       # Run in the same directory as this notebook:\n",
+    "       $ cp .mongo.yaml.example .mongo.origin.yaml\n",
+    "       $ cp .mongo.yaml.example .mongo.transformer.yaml\n",
+    "       ```\n",
+    "       > When populating the file for the origin MongoDB server, use credentials that have write access to the `nmdc` database."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "69937b18",
+   "metadata": {},
+   "source": [
+    "## Procedure"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fe81196a",
+   "metadata": {},
+   "source": [
+    "### Install Python dependencies\n",
+    "\n",
+    "In this step, you'll [install](https://saturncloud.io/blog/what-is-the-difference-between-and-in-jupyter-notebooks/) the Python packages upon which this notebook depends. You can do that by running this cell.\n",
+    "\n",
+    "> Note: If the output of this cell says \"Note: you may need to restart the kernel to use updated packages\", restart the kernel (not the notebook) now."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e25a0af308c3185b",
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "%pip install -r requirements.txt\n",
+    "%pip install nmdc-schema==10.0.0"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "a407c354",
+   "metadata": {},
+   "source": [
+    "### Import Python dependencies\n",
+    "\n",
+    "Import the Python objects upon which this notebook depends.\n",
+    "\n",
+    "> Note: One of the Python objects is a Python class that is specific to this migration."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dbecd561",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Third-party packages:\n",
+    "import pymongo\n",
+    "from nmdc_schema.nmdc_data import get_nmdc_jsonschema_dict\n",
+    "from nmdc_schema.migrators.adapters.mongo_adapter import MongoAdapter\n",
+    "from nmdc_schema.migrators.migrator_from_9_3_to_10_0 import Migrator\n",
+    "from jsonschema import Draft7Validator\n",
+    "\n",
+    "# First-party packages:\n",
+    "from helpers import Config"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "99b20ff4",
+   "metadata": {},
+   "source": [
+    "### Parse configuration files\n",
+    "\n",
+    "Parse the notebook and Mongo configuration files."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1eac645a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cfg = Config()\n",
+    "\n",
+    "# Define some aliases we can use to make the shell commands in this notebook easier to read.\n",
+    "mongodump = cfg.mongodump_path\n",
+    "mongorestore = cfg.mongorestore_path\n",
+    "\n",
+    "# Perform a sanity test of the application paths.\n",
+    "!{mongodump} --version\n",
+    "!{mongorestore} --version"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "68245d2b",
+   "metadata": {},
+   "source": [
+    "### Create MongoDB clients\n",
+    "\n",
+    "Create MongoDB clients you can use to access the \"origin\" MongoDB server (i.e. the one containing the database you want to migrate) and the \"transformer\" MongoDB server (i.e. the one you want to use to perform the data transformations)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8e95f559",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Mongo client for origin MongoDB server.\n",
+    "origin_mongo_client = pymongo.MongoClient(host=cfg.origin_mongo_server_uri, directConnection=True)\n",
+    "\n",
+    "# Mongo client for transformer MongoDB server.\n",
+    "transformer_mongo_client = pymongo.MongoClient(host=cfg.transformer_mongo_server_uri)\n",
+    "\n",
+    "# Perform sanity tests of those MongoDB clients' abilities to access their respective MongoDB servers.\n",
+    "with pymongo.timeout(3):\n",
+    "    # Display the MongoDB server version (running on the \"origin\" Mongo server).\n",
+    "    print(\"Origin Mongo server version:      \" + origin_mongo_client.server_info()[\"version\"])\n",
+    "\n",
+    "    # Sanity test: Ensure the origin database exists.\n",
+    "    assert \"nmdc\" in origin_mongo_client.list_database_names(), \"Origin database does not exist.\"\n",
+    "\n",
+    "    # Display the MongoDB server version (running on the \"transformer\" Mongo server).\n",
+    "    print(\"Transformer Mongo server version: \" + transformer_mongo_client.server_info()[\"version\"])\n",
+    "\n",
+    "    # Sanity test: Ensure the transformation database does not exist.\n",
+    "    assert \"nmdc\" not in transformer_mongo_client.list_database_names(), \"Transformation database already exists.\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3975ac24",
+   "metadata": {},
+   "source": [
+    "### Create JSON Schema validator\n",
+    "\n",
+    "In this step, you'll create a JSON Schema validator for the NMDC Schema."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9e2dbb92",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "nmdc_jsonschema: dict = get_nmdc_jsonschema_dict()\n",
+    "nmdc_jsonschema_validator = Draft7Validator(nmdc_jsonschema)\n",
+    "\n",
+    "# Perform sanity tests of the NMDC Schema dictionary and the JSON Schema validator.\n",
+    "# Reference: https://python-jsonschema.readthedocs.io/en/latest/api/jsonschema/protocols/#jsonschema.protocols.Validator.check_schema\n",
+    "print(\"NMDC Schema title:   \" + nmdc_jsonschema[\"title\"])\n",
+    "print(\"NMDC Schema version: \" + nmdc_jsonschema[\"version\"])\n",
+    "\n",
+    "nmdc_jsonschema_validator.check_schema(nmdc_jsonschema)  # raises exception if schema is invalid"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fd4994a0",
+   "metadata": {},
+   "source": [
+    "### Dump collections from the \"origin\" MongoDB server\n",
+    "\n",
+    "In this step, you'll use `mongodump` to dump the collections that will be impacted by this migration; from the \"origin\" MongoDB server.\n",
+    "\n",
+    "Since `mongodump` doesn't provide a CLI option that you can use to specify the collections you _want_ it to dump (unless you want it to dump only one collection), you can use a different CLI option to tell it all the collection you do _not_ want it to dump. \n",
+    "\n",
+    "The end result will be the same—there's just an extra step involved. That extra step is to generate an `--excludeCollection=\"{name}\"` CLI option for each collection that you do _not_ want it to dump; and then pass all those CLI options to the `mongodump` command."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cf8fa1ca",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Build a string containing zero or more `--excludeCollection=\"...\"` options, \n",
+    "# which can be included in a `mongodump` command.\n",
+    "all_collection_names: list[str] = origin_mongo_client[\"nmdc\"].list_collection_names()\n",
+    "non_agenda_collection_names = [name for name in all_collection_names if name not in COLLECTION_NAMES]\n",
+    "exclusion_options = [f\"--excludeCollection='{name}'\" for name in non_agenda_collection_names]\n",
+    "exclusion_options_str = \" \".join(exclusion_options)  # separates each option with a space\n",
+    "print(exclusion_options_str)\n",
+    "\n",
+    "# Dump the not-excluded collections from the origin database.\n",
+    "!{mongodump} \\\n",
+    "  --config=\"{cfg.origin_mongo_config_file_path}\" \\\n",
+    "  --db=\"nmdc\" \\\n",
+    "  --gzip \\\n",
+    "  --out=\"{cfg.origin_dump_folder_path}\" \\\n",
+    "  {exclusion_options_str}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c3e3c9c4",
+   "metadata": {},
+   "source": [
+    "### Load the collections into the \"transformer\" MongoDB server\n",
+    "\n",
+    "In this step, you'll load the collections dumped from the \"origin\" MongoDB server, into the \"transformer\" MongoDB server.\n",
+    "\n",
+    "Since it's possible that the dump includes more collections than are on the agenda (due to someone creating a collection between the time you generated the exclusion list and the time you ran `mongodump`), you will use one or more of `mongorestore`'s `--nsInclude` CLI options to indicate which collections you want to load.\n",
+    "\n",
+    "So, we'll first generate the `--nsInclude=\"nmdc.{name}\"` CLI options; then include them in the `mongorestore` command that follows."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "418571c5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Build a string containing zero or more `--nsInclude=\"...\"` options, \n",
+    "# which can be included in a `mongorestore` command.\n",
+    "inclusion_options = [f\"--nsInclude='nmdc.{name}'\" for name in COLLECTION_NAMES]\n",
+    "inclusion_options_str = \" \".join(inclusion_options)  # separates each option with a space\n",
+    "print(inclusion_options_str)\n",
+    "\n",
+    "# Restore the dumped collections to the transformer MongoDB server.\n",
+    "!{mongorestore} \\\n",
+    "  --config=\"{cfg.transformer_mongo_config_file_path}\" \\\n",
+    "  --gzip \\\n",
+    "  --drop \\\n",
+    "  --preserveUUID \\\n",
+    "  --dir=\"{cfg.origin_dump_folder_path}\" \\\n",
+    "  {inclusion_options_str}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4c090068",
+   "metadata": {},
+   "source": [
+    "### Transform the collections within the \"transformer\" MongoDB server\n",
+    "\n",
+    "Now that the transformer database contains a copy of each collection on the agenda, you can freely transform those copies. **The \"origin\" database is not involved with this step.**\n",
+    "\n",
+    "The database transformation functions are defined in the `nmdc-schema` Python package installed earlier.\n",
+    "\n",
+    "> Note: This step also includes validation. Reference: https://github.com/microbiomedata/nmdc-runtime/blob/main/metadata-translation/src/bin/validate_json.py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "05869340",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Instantiate a MongoAdapter bound to the transformer database.\n",
+    "adapter = MongoAdapter(database=transformer_mongo_client[\"nmdc\"])\n",
+    "\n",
+    "# Instantiate a Migrator bound to that adapter.\n",
+    "migrator = Migrator(adapter=adapter)\n",
+    "\n",
+    "# Execute the Migrator's `upgrade` method to perform the migration.\n",
+    "migrator.upgrade()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3edf77c7",
+   "metadata": {},
+   "source": [
+    "## Validate all documents in all collections involved\n",
+    "\n",
+    "> TODO: We could delegate this responsibility to the `Migrator` class; or have some `Migrator` methods accept a callback function to run on each document before and after transformation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "db6e432d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for collection_name in COLLECTION_NAMES:\n",
+    "    collection = transformer_mongo_client[\"nmdc\"][collection_name]\n",
+    "    for document in collection.find():\n",
+    "        # Validate the transformed document.\n",
+    "        #\n",
+    "        # Reference: https://github.com/microbiomedata/nmdc-schema/blob/main/src/docs/schema-validation.md\n",
+    "        #\n",
+    "        # Note: Dictionaries originating as Mongo documents include a Mongo-generated key named `_id`. However,\n",
+    "        #       the NMDC Schema does not describe that key and, indeed, data validators consider dictionaries\n",
+    "        #       containing that key to be invalid with respect to the NMDC Schema. So, here, we validate a\n",
+    "        #       copy (i.e. a shallow copy) of the document that lacks that specific key.\n",
+    "        #\n",
+    "        # Note: `root_to_validate` is a dictionary having the shape: { \"some_collection_name\": [ some_document ] }\n",
+    "        #       Reference: https://docs.python.org/3/library/stdtypes.html#dict (see the \"type constructor\" section)\n",
+    "        #\n",
+    "        document_without_underscore_id_key = {key: value for key, value in document.items() if key != \"_id\"}\n",
+    "        root_to_validate = dict([(collection_name, [document_without_underscore_id_key])])\n",
+    "        nmdc_jsonschema_validator.validate(root_to_validate)  # raises exception if invalid"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1e0c8891",
+   "metadata": {},
+   "source": [
+    "### Dump the collections from the \"transformer\" MongoDB server"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca49f61a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Dump the database from the transformer MongoDB server.\n",
+    "!{mongodump} \\\n",
+    "  --config=\"{cfg.transformer_mongo_config_file_path}\" \\\n",
+    "  --db=\"nmdc\" \\\n",
+    "  --gzip \\\n",
+    "  --out=\"{cfg.transformer_dump_folder_path}\" \\\n",
+    "  {exclusion_options_str}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d84bdc11",
+   "metadata": {},
+   "source": [
+    "### Load the collections into the \"origin\" MongoDB server\n",
+    "\n",
+    "In this step, you'll put the referenced collection(s) into the origin MongoDB server, replacing the original collection(s) that have the same name(s)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1dfbcf0a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Replace the same-named collection(s) on the origin server, with the transformed one(s).\n",
+    "!{mongorestore} \\\n",
+    "  --config=\"{cfg.origin_mongo_config_file_path}\" \\\n",
+    "  --gzip \\\n",
+    "  --verbose \\\n",
+    "  --dir=\"{cfg.transformer_dump_folder_path}\" \\\n",
+    "  --drop \\\n",
+    "  --preserveUUID \\\n",
+    "  {inclusion_options_str}"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

{nmdc_runtime-1.2.0 → nmdc_runtime-1.3.0}/nmdc_runtime/api/core/auth.py

@@ -6,17 +6,25 @@ from fastapi import Depends
 from fastapi.exceptions import HTTPException
 from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel
 from fastapi.param_functions import Form
-from fastapi.security import OAuth2, HTTPBasic, HTTPBasicCredentials
+from fastapi.security import (
+    OAuth2,
+    HTTPBasic,
+    HTTPBasicCredentials,
+    HTTPBearer,
+    HTTPAuthorizationCredentials,
+)
 from fastapi.security.utils import get_authorization_scheme_param
 from jose import JWTError, jwt
 from passlib.context import CryptContext
 from pydantic import BaseModel
+from starlette import status
 from starlette.requests import Request
 from starlette.status import HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED
 
 SECRET_KEY = os.getenv("JWT_SECRET_KEY")
 ALGORITHM = "HS256"
-ORCID_CLIENT_ID = os.getenv("ORCID_CLIENT_ID")
+ORCID_NMDC_CLIENT_ID = os.getenv("ORCID_NMDC_CLIENT_ID")
+ORCID_NMDC_CLIENT_SECRET = os.getenv("ORCID_NMDC_CLIENT_SECRET")
 
 # https://orcid.org/.well-known/openid-configuration
 # XXX do we want to live-load this?
@@ -129,15 +137,24 @@ optional_oauth2_scheme = OAuth2PasswordOrClientCredentialsBearer(
     tokenUrl="token", auto_error=False
 )
 
+bearer_scheme = HTTPBearer(scheme_name="bearerAuth", auto_error=False)
+
 
 async def basic_credentials(req: Request):
     return await HTTPBasic(auto_error=False)(req)
 
 
+async def bearer_credentials(req: Request):
+    return await HTTPBearer(scheme_name="bearerAuth", auto_error=False)(req)
+
+
 class OAuth2PasswordOrClientCredentialsRequestForm:
     def __init__(
         self,
         basic_creds: Optional[HTTPBasicCredentials] = Depends(basic_credentials),
+        bearer_creds: Optional[HTTPAuthorizationCredentials] = Depends(
+            bearer_credentials
+        ),
         grant_type: str = Form(None, regex="^password$|^client_credentials$"),
         username: Optional[str] = Form(None),
         password: Optional[str] = Form(None),
@@ -145,14 +162,18 @@ class OAuth2PasswordOrClientCredentialsRequestForm:
         client_id: Optional[str] = Form(None),
         client_secret: Optional[str] = Form(None),
     ):
-        if grant_type == "password" and (username is None or password is None):
+        if bearer_creds:
+            self.grant_type = "client_credentials"
+            self.username, self.password = None, None
+            self.scopes = scope.split()
+            self.client_id = bearer_creds.credentials
+            self.client_secret = None
+        elif grant_type == "password" and (username is None or password is None):
             raise HTTPException(
                 status_code=HTTP_400_BAD_REQUEST,
                 detail="grant_type password requires username and password",
             )
-        if grant_type == "client_credentials" and (
-            client_id is None or client_secret is None
-        ):
+        elif grant_type == "client_credentials" and (client_id is None):
             if basic_creds:
                 client_id = basic_creds.username
                 client_secret = basic_creds.password