nmdc-runtime 2.3.0__tar.gz → 2.5.0__tar.gz

nmdc_runtime-2.5.0/.docker/mongo_init/initialize_replica_set.sh

@@ -0,0 +1,44 @@
+#! /bin/sh
+
+###############################################################################
+# Overview
+# --------
+# This shell script uses `mongosh` to: (a) check whether the specified MongoDB
+# instance is a member of a replica set and, if is isn't, (b) creates a replica
+# set of which that MongoDB instance is the sole member.
+#
+# This shell script was designed to be run within a Docker container based upon
+# the `mongo:8.0.4` container image (i.e., https://hub.docker.com/_/mongo).
+#
+# Positional parameters
+# ---------------------
+# 1. MongoDB hostname (e.g., "mongo")
+# 2. MongoDB port     (e.g., "27017")
+# 3. MongoDB username (e.g., "admin")
+# 4. MongoDB password (e.g., "root")
+#
+# References
+# ----------
+# 1. https://www.mongodb.com/docs/manual/reference/method/rs.status/
+# 2. https://www.mongodb.com/docs/mongodb-shell/reference/options/
+# 3. https://www.warp.dev/terminus/docker-compose-health-check
+###############################################################################
+
+echo 'Setting up replica set.'
+
+echo '
+  try {
+    rs.status();
+  } catch (e) {
+    rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost" }] });
+  }
+' | mongosh \
+  --host "${1}" \
+  --port "${2}" \
+  --username "${3}" \
+  --password "${4}"
+
+echo 'Finished setting up replica set.'
+
+# Exit with a status code of 0.
+exit 0

nmdc_runtime-2.5.0/.dockerignore

@@ -0,0 +1,29 @@
+.env*
+!.env.example
+!.env.test
+.idea
+deployment
+.eggs
+.pytest_cache
+build
+dist
+
+# Omit the Docker configuration/specification files from the image build context,
+# so we can edit them without invalidating existing image layers.
+# Reference: https://docs.docker.com/build/concepts/context/#dockerignore-files
+/.dockerignore
+/.docker
+/.gitpod.Dockerfile
+/docker-compose.yml
+/docker-compose.test.yml
+/nmdc_runtime/Dockerfile
+
+# Omit any local Python virtual environment from the image build context. The
+# image build process includes its own package download/installation step.
+/venv
+/.venv
+
+# Omit residual files left behind by some tests.
+/neon.db
+/neon_api_cache.sqlite
+/neon_cache.sqlite

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/.env.example

@@ -51,4 +51,4 @@ INFO_BANNER_INNERHTML='Announcement: Something important is about to happen. If
 
 # To get the latest mongo dump directory, run e.g.
 # ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null
-MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2024-11-26_20-12-40
+MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2025-02-12_20-12-02

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/.env.test

@@ -20,4 +20,4 @@ API_SITE_ID=nmdc-runtime
 API_SITE_CLIENT_ID=chws-tk74-51
 API_SITE_CLIENT_SECRET=070d56033559e3096d4f82d30aabdd51c3b8a57ac20d24f743a27444d5edc3db
 
-MINTING_SERVICE_ID=nmdc:minter_service_11
+MINTING_SERVICE_ID=nmdc:minter_service_11

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/.github/workflows/build-and-release-to-spin.yml

@@ -9,7 +9,7 @@ on:
     paths:
       - '.github/workflows/build-and-release-to-spin.yml'
       - 'Makefile'
-      - '**.Dockerfile'
+      - '**/Dockerfile'
       - '**.py'
       - 'requirements/main.txt'
 
@@ -78,7 +78,10 @@ jobs:
         with:
           context: .
           push: ${{ env.IS_ORIGINAL_REPO }}
-          file: nmdc_runtime/${{ matrix.image }}.Dockerfile
+          file: nmdc_runtime/Dockerfile
+          # Note: Since our `Dockerfile` describes multiple stages, we use the `target` option to specify a stage.
+          #       Reference: https://github.com/marketplace/actions/build-and-push-docker-images#inputs
+          target: ${{ matrix.image }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 

nmdc_runtime-2.5.0/.github/workflows/python-app.yml

@@ -0,0 +1,91 @@
+# This GitHub Actions workflow spins up the "test" Docker Compose stack and runs automated tests within it.
+# Reference: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions
+name: Python application
+
+# Note: The following events will trigger this workflow:
+#       1. Someone pushes a commit to `main` that includes changes to any of the listed files.
+#       2. Someone opens a pull request that includes changes to any of the listed files.
+#       3. Someone clicks the "Run workflow" button on the "Actions" tab on GitHub.
+#
+#  References:
+#  - https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/triggering-a-workflow#example-including-paths
+#  - https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
+#
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/python-app.yml'
+      - 'Makefile'
+      - '**/Dockerfile'
+      - '**.py'
+      - 'requirements/main.txt'
+      # Every file in the `data` directory or in any of its subdirectories:
+      - 'metadata-translation/notebooks/data/**'
+  pull_request:
+    paths:
+      - '.github/workflows/python-app.yml'
+      - 'Makefile'
+      - '**/Dockerfile'
+      - '**.py'
+      - 'requirements/main.txt'
+      - 'metadata-translation/notebooks/data/**'
+  # Allow developers to trigger this workflow manually via the "Actions" page on GitHub.
+  # Reference: https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch: { }
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4 # update version to maintain consistency across workflows
+    # Prepare the MongoDB keyfile to be mounted by the `mongo` container.
+    #
+    # Note: This is to prevent MongoDB from reporting the error:
+    #       > "permissions on /path/to/keyfile are too open"
+    #
+    # Note: This involves making the file be readable _only_ by the OS user that MongoDB runs as.
+    #
+    #       In containers using the `mongo` image, that user is named `mongodb` and has a
+    #       UID of `999`. That user belongs to a group named `mongodb`, whose GID is `999`.
+    #       You can verify this by looking at the Dockerfile layers on Docker Hub.
+    #       Reference: https://hub.docker.com/layers/library/mongo/8.0.5/images/sha256-90bf5066fed8a3cd59345d963922bc5cb557d4b4b2a0e38dfd9ee299c405741b
+    #
+    #       The GHA Runner will allow me to `chmod 600` the file, but not `chown 999:999` it.
+    #       Since the GHA Runner will not allow me to `chown` the file, I use a Docker container
+    #       to (effectively) accomplish that. Since—after I use the Docker container to change
+    #       the file's owner—the GHA Runner will not allow me to then `chmod` the file, I opt to
+    #       accomplish that within the Docker container as well.
+    #
+    #       The reason—within the Docker container—I do not `chmod` or `chown` the original file
+    #       directly, is that I am under the impression that ownership/permission changes made
+    #       within a container to mounted files that already exist on the host will not be seen
+    #       by the host. I have not found official documentation supporting this yet.
+    #       TODO: Include a reference about changing mounted file's permission within container.
+    #
+    - name: Restrict access to MongoDB keyfile
+      run: |
+        mkdir _tmp
+        docker run --rm \
+          -v ./mongoKeyFile:/mongoKeyFile \
+          -v ./_tmp:/out \
+          alpine \
+          sh -c 'cp /mongoKeyFile /out/mongoKeyFile && chmod 600 /out/mongoKeyFile && chown 999:999 /out/mongoKeyFile'
+        mv _tmp/mongoKeyFile ./mongoKeyFile
+        rmdir _tmp
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+# deprecated: Consider merging python.app.yml and lint.yml
+#    - name: Lint with flake8
+#      run: |
+#        pip install flake8
+#        make lint
+    - name: Build and run containers upon which test runner depends
+      run: make up-test
+    - name: Build container image for test runner
+      run: make test-build
+    - name: Run tests
+      run: make test-run

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/.gitignore

@@ -15,6 +15,7 @@ downloads/
 eggs/
 .eggs/
 lib/
+!/tests/lib/
 lib64/
 parts/
 sdist/

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/Makefile

@@ -5,7 +5,7 @@ init:
 
 # Updates Python dependencies based upon the contents of the `requirements/main.in` and `requirements/dev.in` files.
 #
-# Note: To omit the `--upgrade` option (included by default) when running `pip-compile`:
+# Note: To omit the `--upgrade` option (included by default) from the constituent `pip-compile` commands:
 #       ```
 #       $ make update-deps UPDATE_DEPS_MAIN_UPGRADE_OPT='' UPDATE_DEPS_DEV_UPGRADE_OPT=''
 #       ```
@@ -14,6 +14,10 @@ init:
 #       ```sh
 #       $ docker compose run --rm --no-deps fastapi sh -c 'make update-deps'
 #       ```
+#       To omit the `--upgrade` option (included by default) from the constituent `pip-compile` commands:
+#       ```sh
+#       $ docker compose run --rm --no-deps fastapi sh -c 'make update-deps UPDATE_DEPS_MAIN_UPGRADE_OPT="" UPDATE_DEPS_DEV_UPGRADE_OPT=""'
+#       ```
 #
 UPDATE_DEPS_MAIN_UPGRADE_OPT ?= --upgrade
 UPDATE_DEPS_DEV_UPGRADE_OPT ?= --upgrade
@@ -34,25 +38,39 @@ update: update-deps init
 up-dev:
 	docker compose up --build --force-recreate --detach --remove-orphans
 
+# Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the dev stack.
 dev-reset-db:
 	docker compose \
-		exec mongo /bin/bash -c "./app_tests/mongorestore-nmdc-testdb.sh"
+		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
+# Uses Docker Compose to build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
+#
+# Note: This does not build or spin up the `test` container, itself, since the `test` container's Docker Compose service
+#       has the "donotstart" profile specified for it in `docker-compose.test.yml`.
+#
 up-test:
 	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
+# Uses Docker Compose to build the container image for the `test` container (i.e. the test runner).
 test-build:
 	docker compose --file docker-compose.test.yml build test
 
-test-dbinit:
+# Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the test stack.
+test-reset-db:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
-# Tip: If you append a file path to this "recipe", pytest will run only the tests defined in that file.
-#      For example, append `tests/test_api/test_endpoints.py` to have pytest only run the endpoint tests.
+# Uses Docker Compose to spin up the `test` container (i.e. the test runner), effectively running the tests.
+#
+# Tip: If you append `ARGS=` and a file path to the `make` command, pytest will run only the tests defined in that file.
+#      For example, to run only the tests defined in `tests/test_api/test_endpoints.py`:
+#      ```
+#      $ make test-run ARGS="tests/test_api/test_endpoints.py"
+#      ```
+#
 test-run:
-	docker compose --file docker-compose.test.yml run test
+	docker compose --file docker-compose.test.yml run test $(ARGS)
 
 test: test-build test-run
 

{nmdc_runtime-2.3.0/nmdc_runtime.egg-info → nmdc_runtime-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: nmdc_runtime
-Version: 2.3.0
+Version: 2.5.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston
@@ -17,6 +17,7 @@ Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
+Dynamic: license-file
 Dynamic: requires-python
 Dynamic: summary
 
@@ -37,8 +38,7 @@ houses the LinkML schema specification, as well as generated artifacts (e.g. JSO
 * [nmdc-server](https://github.com/microbiomedata/nmdc-server)
 houses code specific to the data portal -- its database, back-end API, and front-end application.
 
-* [workflow_documentation](https://nmdc-workflow-documentation.readthedocs.io/en/latest/index.html)
-references workflow code spread across several repositories, that take source data and produce computed data.
+* Workflows — documented in the [workflows](https://docs.microbiomedata.org/workflows/) section of the NMDC documentation website — take source data and produce computed data.
 
 * This repo (nmdc-runtime)
    * houses code that takes source data and computed data, and transforms it
@@ -156,6 +156,9 @@ Tests can be found in `tests` and are run with the following commands:
 ```bash
 make up-test
 make test
+
+# Run a Specific test file eg. tests/test_api/test_endpoints.py
+make test ARGS="tests/test_api/test_endpoints.py"
 ```
 
 As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
@@ -164,6 +167,16 @@ desired and does not break over time.
 [For hints on how to write tests for solids and pipelines in Dagster, see their documentation
 tutorial on Testing](https://docs.dagster.io/tutorial/testable).
 
+### RAM usage
+
+The `dagster-daemon` and `dagster-dagit` containers can consume a lot of RAM. If tests are failing and the console of
+the `test` container shows "Error 137," here is something you can try as a workaround: In Docker Desktop, go to 
+"Settings > Resources > Advanced," and increase the memory limit. One of our team members has
+found **12 GB** to be sufficient for running the tests.
+
+> Dedicating 12 GB of RAM to Docker may be prohibitive for some prospective developers.
+> There is an open [issue](https://github.com/microbiomedata/nmdc-runtime/issues/928) about the memory requirement.
+
 ## Publish to PyPI
 
 This repository contains a GitHub Actions workflow that publishes a Python package to [PyPI](https://pypi.org/project/nmdc-runtime/).

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/README.md

@@ -15,8 +15,7 @@ houses the LinkML schema specification, as well as generated artifacts (e.g. JSO
 * [nmdc-server](https://github.com/microbiomedata/nmdc-server)
 houses code specific to the data portal -- its database, back-end API, and front-end application.
 
-* [workflow_documentation](https://nmdc-workflow-documentation.readthedocs.io/en/latest/index.html)
-references workflow code spread across several repositories, that take source data and produce computed data.
+* Workflows — documented in the [workflows](https://docs.microbiomedata.org/workflows/) section of the NMDC documentation website — take source data and produce computed data.
 
 * This repo (nmdc-runtime)
    * houses code that takes source data and computed data, and transforms it
@@ -134,6 +133,9 @@ Tests can be found in `tests` and are run with the following commands:
 ```bash
 make up-test
 make test
+
+# Run a Specific test file eg. tests/test_api/test_endpoints.py
+make test ARGS="tests/test_api/test_endpoints.py"
 ```
 
 As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
@@ -142,6 +144,16 @@ desired and does not break over time.
 [For hints on how to write tests for solids and pipelines in Dagster, see their documentation
 tutorial on Testing](https://docs.dagster.io/tutorial/testable).
 
+### RAM usage
+
+The `dagster-daemon` and `dagster-dagit` containers can consume a lot of RAM. If tests are failing and the console of
+the `test` container shows "Error 137," here is something you can try as a workaround: In Docker Desktop, go to 
+"Settings > Resources > Advanced," and increase the memory limit. One of our team members has
+found **12 GB** to be sufficient for running the tests.
+
+> Dedicating 12 GB of RAM to Docker may be prohibitive for some prospective developers.
+> There is an open [issue](https://github.com/microbiomedata/nmdc-runtime/issues/928) about the memory requirement.
+
 ## Publish to PyPI
 
 This repository contains a GitHub Actions workflow that publishes a Python package to [PyPI](https://pypi.org/project/nmdc-runtime/).

{nmdc_runtime-2.3.0 → nmdc_runtime-2.5.0}/db/migrations/notebooks/.notebook.env.example

@@ -1,4 +1,4 @@
-# Paths to folders in which the notebook will store Mongo dumps.
+# Paths to existing folders in which the notebook will store Mongo dumps.
 PATH_TO_ORIGIN_MONGO_DUMP_FOLDER = "./mongodump.origin.out"
 PATH_TO_TRANSFORMER_MONGO_DUMP_FOLDER = "./mongodump.transformer.out"
 
@@ -12,7 +12,7 @@ PATH_TO_MONGOSH_BINARY = "__REPLACE_ME__"  # e.g. "/Users/Alice/Downloads/mongos
 #       to migrate. It is where the "E" and "L" in "ETL" will take place.
 ORIGIN_MONGO_HOST="__REPLACE_ME__"
 ORIGIN_MONGO_PORT="__REPLACE_ME__"
-ORIGIN_MONGO_USERNAME="__REPLACE_ME__"
+ORIGIN_MONGO_USERNAME="__REPLACE_ME__"  # e.g. "app.migrator"
 ORIGIN_MONGO_PASSWORD="__REPLACE_ME__"
 ORIGIN_MONGO_DATABASE_NAME="__REPLACE_ME__"  # e.g. "nmdc"
 

nmdc_runtime-2.5.0/db/migrations/notebooks/migrate_11_2_1_to_11_3_0.ipynb

@@ -0,0 +1,50 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "# Migrate MongoDB database from `nmdc-schema` `v11.2.1` to `v11.3.0`",
+   "id": "d05efc6327778f9c"
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "There are no migrators associated with any schema changes between schema versions `v11.2.1` and `v11.3.0`. So, this notebook is a \"no op\" (i.e. \"no operation\").",
+   "id": "b99d5924e825b9a2"
+  },
+  {
+   "cell_type": "code",
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "is_executing": true
+    }
+   },
+   "source": "# no op",
+   "outputs": [],
+   "execution_count": null
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}