nmdc-runtime 2.4.0__tar.gz → 2.6.0__tar.gz

nmdc_runtime-2.6.0/.docker/mongo_init/initialize_replica_set.sh

@@ -0,0 +1,44 @@
+#! /bin/sh
+
+###############################################################################
+# Overview
+# --------
+# This shell script uses `mongosh` to: (a) check whether the specified MongoDB
+# instance is a member of a replica set and, if is isn't, (b) creates a replica
+# set of which that MongoDB instance is the sole member.
+#
+# This shell script was designed to be run within a Docker container based upon
+# the `mongo:8.0.4` container image (i.e., https://hub.docker.com/_/mongo).
+#
+# Positional parameters
+# ---------------------
+# 1. MongoDB hostname (e.g., "mongo")
+# 2. MongoDB port     (e.g., "27017")
+# 3. MongoDB username (e.g., "admin")
+# 4. MongoDB password (e.g., "root")
+#
+# References
+# ----------
+# 1. https://www.mongodb.com/docs/manual/reference/method/rs.status/
+# 2. https://www.mongodb.com/docs/mongodb-shell/reference/options/
+# 3. https://www.warp.dev/terminus/docker-compose-health-check
+###############################################################################
+
+echo 'Setting up replica set.'
+
+echo '
+  try {
+    rs.status();
+  } catch (e) {
+    rs.initiate({ _id: "rs0", members: [{ _id: 0, host: "localhost" }] });
+  }
+' | mongosh \
+  --host "${1}" \
+  --port "${2}" \
+  --username "${3}" \
+  --password "${4}"
+
+echo 'Finished setting up replica set.'
+
+# Exit with a status code of 0.
+exit 0

nmdc_runtime-2.6.0/.dockerignore

@@ -0,0 +1,36 @@
+.env*
+!.env.example
+!.env.test
+.idea
+deployment
+.eggs
+.pytest_cache
+build
+dist
+
+# Omit the Docker configuration/specification files from the image build context,
+# so we can edit them without invalidating existing image layers.
+#
+# Note: Although we exclude the `.docker/` directory from the build context, we
+#       make an exception for the `wait-for-it.sh` file within it, since we want
+#       that specific file to be `COPY`-ed into the `test` container image during
+#       that container image's build process (which is defined in `Dockerfile`).
+#
+# Reference: https://docs.docker.com/build/concepts/context/#dockerignore-files
+#
+/.dockerignore
+/.docker/
+!/.docker/wait-for-it.sh
+/docker-compose.yml
+/docker-compose.test.yml
+/nmdc_runtime/Dockerfile
+
+# Omit any local Python virtual environment from the image build context. The
+# image build process includes its own package download/installation step.
+/venv
+/.venv
+
+# Omit residual files left behind by some tests.
+/neon.db
+/neon_api_cache.sqlite
+/neon_cache.sqlite

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/.env.example

@@ -51,4 +51,9 @@ INFO_BANNER_INNERHTML='Announcement: Something important is about to happen. If
 
 # To get the latest mongo dump directory, run e.g.
 # ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null
-MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2024-11-26_20-12-40
+MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2025-02-12_20-12-02
+
+# The path to an executable copy of the `sshproxy` program maintained by NERSC.
+# Note: This is currently only used by the `nersc-sshproxy` target in `Makefile`.
+# Reference: https://docs.nersc.gov/connect/mfa/#sshproxy
+PATH_TO_NERSC_SSHPROXY=/Users/Alice/Downloads/sshproxy

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/.env.test

@@ -20,4 +20,4 @@ API_SITE_ID=nmdc-runtime
 API_SITE_CLIENT_ID=chws-tk74-51
 API_SITE_CLIENT_SECRET=070d56033559e3096d4f82d30aabdd51c3b8a57ac20d24f743a27444d5edc3db
 
-MINTING_SERVICE_ID=nmdc:minter_service_11
+MINTING_SERVICE_ID=nmdc:minter_service_11

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/.github/workflows/build-and-release-to-spin.yml

@@ -9,7 +9,7 @@ on:
     paths:
       - '.github/workflows/build-and-release-to-spin.yml'
       - 'Makefile'
-      - '**.Dockerfile'
+      - '**/Dockerfile'
       - '**.py'
       - 'requirements/main.txt'
 
@@ -78,7 +78,10 @@ jobs:
         with:
           context: .
           push: ${{ env.IS_ORIGINAL_REPO }}
-          file: nmdc_runtime/${{ matrix.image }}.Dockerfile
+          file: nmdc_runtime/Dockerfile
+          # Note: Since our `Dockerfile` describes multiple stages, we use the `target` option to specify a stage.
+          #       Reference: https://github.com/marketplace/actions/build-and-push-docker-images#inputs
+          target: ${{ matrix.image }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/.github/workflows/mkdocs.yml

@@ -17,9 +17,9 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: 3.x
-      - run: pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git
-      - run: pip install mkdocs-mermaid2-plugin
-      - run: pip install mkdocs-jupyter
+      - name: Install Python package dependencies
+        run: pip install mkdocs-material mkdocs-mermaid2-plugin mkdocs-jupyter
+
       # Build the MkDocs site (but don't deploy it).
       #
       # Note: Until January 17, 2025, we would build and deploy the MkDocs website to GitHub Pages.
@@ -40,6 +40,5 @@ jobs:
       #       when the site is no longer buildable (e.g. due to someone inadvertently introducing an invalid source
       #       file, or due to one of the dependencies becoming unavailable), which can prompt remedial action.
       #
-      - run: mkdocs build
-env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN_MKDOCS_MATERIAL_INSIDERS }}
+      - name: Build MkDocs site
+        run: mkdocs build

nmdc_runtime-2.6.0/.github/workflows/python-app.yml

@@ -0,0 +1,91 @@
+# This GitHub Actions workflow spins up the "test" Docker Compose stack and runs automated tests within it.
+# Reference: https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions
+name: Python application
+
+# Note: The following events will trigger this workflow:
+#       1. Someone pushes a commit to `main` that includes changes to any of the listed files.
+#       2. Someone opens a pull request that includes changes to any of the listed files.
+#       3. Someone clicks the "Run workflow" button on the "Actions" tab on GitHub.
+#
+#  References:
+#  - https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/triggering-a-workflow#example-including-paths
+#  - https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
+#
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - '.github/workflows/python-app.yml'
+      - 'Makefile'
+      - '**/Dockerfile'
+      - '**.py'
+      - 'requirements/main.txt'
+      # Every file in the `data` directory or in any of its subdirectories:
+      - 'metadata-translation/notebooks/data/**'
+  pull_request:
+    paths:
+      - '.github/workflows/python-app.yml'
+      - 'Makefile'
+      - '**/Dockerfile'
+      - '**.py'
+      - 'requirements/main.txt'
+      - 'metadata-translation/notebooks/data/**'
+  # Allow developers to trigger this workflow manually via the "Actions" page on GitHub.
+  # Reference: https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/manually-running-a-workflow
+  workflow_dispatch: { }
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4 # update version to maintain consistency across workflows
+    # Prepare the MongoDB keyfile to be mounted by the `mongo` container.
+    #
+    # Note: This is to prevent MongoDB from reporting the error:
+    #       > "permissions on /path/to/keyfile are too open"
+    #
+    # Note: This involves making the file be readable _only_ by the OS user that MongoDB runs as.
+    #
+    #       In containers using the `mongo` image, that user is named `mongodb` and has a
+    #       UID of `999`. That user belongs to a group named `mongodb`, whose GID is `999`.
+    #       You can verify this by looking at the Dockerfile layers on Docker Hub.
+    #       Reference: https://hub.docker.com/layers/library/mongo/8.0.5/images/sha256-90bf5066fed8a3cd59345d963922bc5cb557d4b4b2a0e38dfd9ee299c405741b
+    #
+    #       The GHA Runner will allow me to `chmod 600` the file, but not `chown 999:999` it.
+    #       Since the GHA Runner will not allow me to `chown` the file, I use a Docker container
+    #       to (effectively) accomplish that. Since—after I use the Docker container to change
+    #       the file's owner—the GHA Runner will not allow me to then `chmod` the file, I opt to
+    #       accomplish that within the Docker container as well.
+    #
+    #       The reason—within the Docker container—I do not `chmod` or `chown` the original file
+    #       directly, is that I am under the impression that ownership/permission changes made
+    #       within a container to mounted files that already exist on the host will not be seen
+    #       by the host. I have not found official documentation supporting this yet.
+    #       TODO: Include a reference about changing mounted file's permission within container.
+    #
+    - name: Restrict access to MongoDB keyfile
+      run: |
+        mkdir _tmp
+        docker run --rm \
+          -v ./.docker/mongoKeyFile:/mongoKeyFile \
+          -v ./_tmp:/out \
+          alpine \
+          sh -c 'cp /mongoKeyFile /out/mongoKeyFile && chmod 600 /out/mongoKeyFile && chown 999:999 /out/mongoKeyFile'
+        mv _tmp/mongoKeyFile ./.docker/mongoKeyFile
+        rmdir _tmp
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.10'
+# deprecated: Consider merging python.app.yml and lint.yml
+#    - name: Lint with flake8
+#      run: |
+#        pip install flake8
+#        make lint
+    - name: Build and run containers upon which test runner depends
+      run: make up-test
+    - name: Build container image for test runner
+      run: make test-build
+    - name: Run tests
+      run: make test-run

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/.gitignore

@@ -15,6 +15,7 @@ downloads/
 eggs/
 .eggs/
 lib/
+!/tests/lib/
 lib64/
 parts/
 sdist/

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/Makefile

@@ -38,25 +38,39 @@ update: update-deps init
 up-dev:
 	docker compose up --build --force-recreate --detach --remove-orphans
 
+# Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the dev stack.
 dev-reset-db:
 	docker compose \
-		exec mongo /bin/bash -c "./app_tests/mongorestore-nmdc-testdb.sh"
+		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
+# Uses Docker Compose to build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
+#
+# Note: This does not build or spin up the `test` container, itself, since the `test` container's Docker Compose service
+#       has the "donotstart" profile specified for it in `docker-compose.test.yml`.
+#
 up-test:
 	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
+# Uses Docker Compose to build the container image for the `test` container (i.e. the test runner).
 test-build:
 	docker compose --file docker-compose.test.yml build test
 
-test-dbinit:
+# Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the test stack.
+test-reset-db:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
-# Tip: If you append a file path to this "recipe", pytest will run only the tests defined in that file.
-#      For example, append `tests/test_api/test_endpoints.py` to have pytest only run the endpoint tests.
+# Uses Docker Compose to spin up the `test` container (i.e. the test runner), effectively running the tests.
+#
+# Tip: If you append `ARGS=` and a file path to the `make` command, pytest will run only the tests defined in that file.
+#      For example, to run only the tests defined in `tests/test_api/test_endpoints.py`:
+#      ```
+#      $ make test-run ARGS="tests/test_api/test_endpoints.py"
+#      ```
+#
 test-run:
-	docker compose --file docker-compose.test.yml run test
+	docker compose --file docker-compose.test.yml run test $(ARGS)
 
 test: test-build test-run
 
@@ -99,8 +113,17 @@ publish:
 docs-dev:
 	mkdocs serve -a localhost:8080
 
+# 🙋 Prerequisites:
+#
+# 1. The `PATH_TO_NERSC_SSHPROXY` environment variable is set to the path of
+#    an executable copy of the `sshproxy` program maintained by NERSC.
+#    You can read about and download that program at the following URL:
+#    https://docs.nersc.gov/connect/mfa/#sshproxy
+#    
+# 2. The `NERSC_USERNAME` environment variable is set to your NERSC username.
+#
 nersc-sshproxy:
-	bash ./nersc-sshproxy.sh -u ${NERSC_USERNAME} # https://docs.nersc.gov/connect/mfa/#sshproxy
+	${PATH_TO_NERSC_SSHPROXY} -u ${NERSC_USERNAME}
 
 nersc-mongo-tunnels:
 	ssh -L27072:mongo-loadbalancer.nmdc.production.svc.spin.nersc.org:27017 \
@@ -116,7 +139,7 @@ mongorestore-nmdc-db:
 	# export MONGO_REMOTE_DUMP_DIR=$(ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null)
 	# ```
 	# Rsync the remote dump directory items of interest:
-	rsync -av --exclude='_*' --exclude='fs\.*' \
+	rsync -av --no-perms --exclude='_*' --exclude='fs\.*' \
 		-e "ssh -i ~/.ssh/nersc" \
 		${NERSC_USERNAME}@dtn01.nersc.gov:${MONGO_REMOTE_DUMP_DIR}/nmdc/ \
 		/tmp/remote-mongodump/nmdc

{nmdc_runtime-2.4.0/nmdc_runtime.egg-info → nmdc_runtime-2.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: nmdc_runtime
-Version: 2.4.0
+Version: 2.6.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston
@@ -17,6 +17,7 @@ Dynamic: classifier
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
+Dynamic: license-file
 Dynamic: requires-python
 Dynamic: summary
 
@@ -37,8 +38,7 @@ houses the LinkML schema specification, as well as generated artifacts (e.g. JSO
 * [nmdc-server](https://github.com/microbiomedata/nmdc-server)
 houses code specific to the data portal -- its database, back-end API, and front-end application.
 
-* [workflow_documentation](https://docs.microbiomedata.org/workflows/)
-references workflow code spread across several repositories, that take source data and produce computed data.
+* Workflows — documented in the [workflows](https://docs.microbiomedata.org/workflows/) section of the NMDC documentation website — take source data and produce computed data.
 
 * This repo (nmdc-runtime)
    * houses code that takes source data and computed data, and transforms it
@@ -106,10 +106,10 @@ docker compose version
 docker info
 ```
 
-Ensure the permissions of `./mongoKeyFile` are such that only the file's owner can read or write the file.
+Ensure the permissions of `./.docker/mongoKeyFile` are such that only the file's owner can read or write the file.
 
 ```shell
-chmod 600 ./mongoKeyFile
+chmod 600 ./.docker/mongoKeyFile
 ```
 
 Ensure you have a `.env` file for the Docker services to source from. You may copy `.env.example` to
@@ -156,6 +156,9 @@ Tests can be found in `tests` and are run with the following commands:
 ```bash
 make up-test
 make test
+
+# Run a Specific test file eg. tests/test_api/test_endpoints.py
+make test ARGS="tests/test_api/test_endpoints.py"
 ```
 
 As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
@@ -164,6 +167,16 @@ desired and does not break over time.
 [For hints on how to write tests for solids and pipelines in Dagster, see their documentation
 tutorial on Testing](https://docs.dagster.io/tutorial/testable).
 
+### RAM usage
+
+The `dagster-daemon` and `dagster-dagit` containers can consume a lot of RAM. If tests are failing and the console of
+the `test` container shows "Error 137," here is something you can try as a workaround: In Docker Desktop, go to 
+"Settings > Resources > Advanced," and increase the memory limit. One of our team members has
+found **12 GB** to be sufficient for running the tests.
+
+> Dedicating 12 GB of RAM to Docker may be prohibitive for some prospective developers.
+> There is an open [issue](https://github.com/microbiomedata/nmdc-runtime/issues/928) about the memory requirement.
+
 ## Publish to PyPI
 
 This repository contains a GitHub Actions workflow that publishes a Python package to [PyPI](https://pypi.org/project/nmdc-runtime/).

{nmdc_runtime-2.4.0 → nmdc_runtime-2.6.0}/README.md

@@ -15,8 +15,7 @@ houses the LinkML schema specification, as well as generated artifacts (e.g. JSO
 * [nmdc-server](https://github.com/microbiomedata/nmdc-server)
 houses code specific to the data portal -- its database, back-end API, and front-end application.
 
-* [workflow_documentation](https://docs.microbiomedata.org/workflows/)
-references workflow code spread across several repositories, that take source data and produce computed data.
+* Workflows — documented in the [workflows](https://docs.microbiomedata.org/workflows/) section of the NMDC documentation website — take source data and produce computed data.
 
 * This repo (nmdc-runtime)
    * houses code that takes source data and computed data, and transforms it
@@ -84,10 +83,10 @@ docker compose version
 docker info
 ```
 
-Ensure the permissions of `./mongoKeyFile` are such that only the file's owner can read or write the file.
+Ensure the permissions of `./.docker/mongoKeyFile` are such that only the file's owner can read or write the file.
 
 ```shell
-chmod 600 ./mongoKeyFile
+chmod 600 ./.docker/mongoKeyFile
 ```
 
 Ensure you have a `.env` file for the Docker services to source from. You may copy `.env.example` to
@@ -134,6 +133,9 @@ Tests can be found in `tests` and are run with the following commands:
 ```bash
 make up-test
 make test
+
+# Run a Specific test file eg. tests/test_api/test_endpoints.py
+make test ARGS="tests/test_api/test_endpoints.py"
 ```
 
 As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
@@ -142,6 +144,16 @@ desired and does not break over time.
 [For hints on how to write tests for solids and pipelines in Dagster, see their documentation
 tutorial on Testing](https://docs.dagster.io/tutorial/testable).
 
+### RAM usage
+
+The `dagster-daemon` and `dagster-dagit` containers can consume a lot of RAM. If tests are failing and the console of
+the `test` container shows "Error 137," here is something you can try as a workaround: In Docker Desktop, go to 
+"Settings > Resources > Advanced," and increase the memory limit. One of our team members has
+found **12 GB** to be sufficient for running the tests.
+
+> Dedicating 12 GB of RAM to Docker may be prohibitive for some prospective developers.
+> There is an open [issue](https://github.com/microbiomedata/nmdc-runtime/issues/928) about the memory requirement.
+
 ## Publish to PyPI
 
 This repository contains a GitHub Actions workflow that publishes a Python package to [PyPI](https://pypi.org/project/nmdc-runtime/).

nmdc_runtime-2.6.0/db/migrations/notebooks/migrate_11_4_0_to_11_5_0.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d05efc6327778f9c",
+   "metadata": {},
+   "source": [
+    "# Migrate MongoDB database from `nmdc-schema` `v11.4.0` to `v11.5.0`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b99d5924e825b9a2",
+   "metadata": {},
+   "source": [
+    "There are no migrators associated with any schema changes between schema versions `v11.4.0` and `v11.5.0`. So, this notebook is a \"no op\" (i.e. \"no operation\")."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "is_executing": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# no op"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

nmdc_runtime-2.6.0/db/migrations/notebooks/migrate_11_5_0_to_11_5_1.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d05efc6327778f9c",
+   "metadata": {},
+   "source": [
+    "# Migrate MongoDB database from `nmdc-schema` `v11.5.0` to `v11.5.1`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b99d5924e825b9a2",
+   "metadata": {},
+   "source": [
+    "There are no migrators associated with any schema changes between schema versions `v11.5.0` and `v11.5.1`. So, this notebook is a \"no op\" (i.e. \"no operation\")."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "is_executing": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# no op"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}