nmdc-runtime 2.5.0__tar.gz → 2.7.0__tar.gz

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/.dockerignore

@@ -10,10 +10,17 @@ dist
 
 # Omit the Docker configuration/specification files from the image build context,
 # so we can edit them without invalidating existing image layers.
+#
+# Note: Although we exclude the `.docker/` directory from the build context, we
+#       make an exception for the `wait-for-it.sh` file within it, since we want
+#       that specific file to be `COPY`-ed into the `test` container image during
+#       that container image's build process (which is defined in `Dockerfile`).
+#
 # Reference: https://docs.docker.com/build/concepts/context/#dockerignore-files
+#
 /.dockerignore
-/.docker
-/.gitpod.Dockerfile
+/.docker/
+!/.docker/wait-for-it.sh
 /docker-compose.yml
 /docker-compose.test.yml
 /nmdc_runtime/Dockerfile

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/.env.example

@@ -52,3 +52,8 @@ INFO_BANNER_INNERHTML='Announcement: Something important is about to happen. If
 # To get the latest mongo dump directory, run e.g.
 # ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null
 MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2025-02-12_20-12-02
+
+# The path to an executable copy of the `sshproxy` program maintained by NERSC.
+# Note: This is currently only used by the `nersc-sshproxy` target in `Makefile`.
+# Reference: https://docs.nersc.gov/connect/mfa/#sshproxy
+PATH_TO_NERSC_SSHPROXY=/Users/Alice/Downloads/sshproxy

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/.github/workflows/mkdocs.yml

@@ -5,8 +5,6 @@ on:
       - main
     paths:
       - 'docs/**'
-      - 'mkdocs_overrides/**'
-      - 'mkdocs.yml'
       - '.github/workflows/mkdocs.yml'
 jobs:
   deploy:
@@ -17,9 +15,9 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: 3.x
-      - run: pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git
-      - run: pip install mkdocs-mermaid2-plugin
-      - run: pip install mkdocs-jupyter
+      - name: Install Python package dependencies
+        run: pip install mkdocs-material mkdocs-mermaid2-plugin mkdocs-jupyter
+
       # Build the MkDocs site (but don't deploy it).
       #
       # Note: Until January 17, 2025, we would build and deploy the MkDocs website to GitHub Pages.
@@ -40,6 +38,5 @@ jobs:
       #       when the site is no longer buildable (e.g. due to someone inadvertently introducing an invalid source
       #       file, or due to one of the dependencies becoming unavailable), which can prompt remedial action.
       #
-      - run: mkdocs build
-env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN_MKDOCS_MATERIAL_INSIDERS }}
+      - name: Build MkDocs site
+        run: mkdocs build -f docs/mkdocs.yml

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/.github/workflows/python-app.yml

@@ -68,11 +68,11 @@ jobs:
       run: |
         mkdir _tmp
         docker run --rm \
-          -v ./mongoKeyFile:/mongoKeyFile \
+          -v ./.docker/mongoKeyFile:/mongoKeyFile \
           -v ./_tmp:/out \
           alpine \
           sh -c 'cp /mongoKeyFile /out/mongoKeyFile && chmod 600 /out/mongoKeyFile && chown 999:999 /out/mongoKeyFile'
-        mv _tmp/mongoKeyFile ./mongoKeyFile
+        mv _tmp/mongoKeyFile ./.docker/mongoKeyFile
         rmdir _tmp
     - name: Set up Python 3.10
       uses: actions/setup-python@v4
@@ -83,9 +83,7 @@ jobs:
 #      run: |
 #        pip install flake8
 #        make lint
-    - name: Build and run containers upon which test runner depends
+    - name: Build and start containers upon which test runner depends
       run: make up-test
-    - name: Build container image for test runner
-      run: make test-build
     - name: Run tests
-      run: make test-run
+      run: make run-test

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/.gitignore

@@ -119,7 +119,7 @@ venv.bak/
 .ropeproject
 
 # mkdocs documentation
-/site
+/docs/site
 
 # mypy
 .mypy_cache/

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/Makefile

@@ -38,41 +38,55 @@ update: update-deps init
 up-dev:
 	docker compose up --build --force-recreate --detach --remove-orphans
 
+down-dev:
+	docker compose down
+
 # Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the dev stack.
-dev-reset-db:
+reset-db-dev:
 	docker compose \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
 # Uses Docker Compose to build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
-#
-# Note: This does not build or spin up the `test` container, itself, since the `test` container's Docker Compose service
-#       has the "donotstart" profile specified for it in `docker-compose.test.yml`.
-#
 up-test:
 	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
-# Uses Docker Compose to build the container image for the `test` container (i.e. the test runner).
-test-build:
-	docker compose --file docker-compose.test.yml build test
+# Tears down the `test` stack, including removing data volumes such as that for the test MongoDB database.
+down-test:
+	docker compose --file docker-compose.test.yml down --volumes
 
 # Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the test stack.
-test-reset-db:
+reset-db-test:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
-# Uses Docker Compose to spin up the `test` container (i.e. the test runner), effectively running the tests.
+# Run tests on the started `test` stack, passing `ARGS` to `pytest` (see Tip below).
 #
 # Tip: If you append `ARGS=` and a file path to the `make` command, pytest will run only the tests defined in that file.
-#      For example, to run only the tests defined in `tests/test_api/test_endpoints.py`:
+#
+#      Some examples, using `make test`:
+#      (You can also use `make run-test` if are sure you don't need to reset global state)
+#
+#      To run only the tests defined in `tests/test_api/test_endpoints.py`:
+#      ```
+#      $ make test ARGS="tests/test_api/test_endpoints.py"
+#      ```
+#
+#      To run only the test `test_find_data_objects_for_study_having_one` in `tests/test_api/test_endpoints.py`:
 #      ```
-#      $ make test-run ARGS="tests/test_api/test_endpoints.py"
+#      $ make test ARGS="-k 'test_find_data_objects_for_study_having_one'"
 #      ```
 #
-test-run:
-	docker compose --file docker-compose.test.yml run test $(ARGS)
+run-test:
+	docker compose --file docker-compose.test.yml exec -it test \
+		./.docker/wait-for-it.sh fastapi:8000 --strict --timeout=300 -- pytest --cov=nmdc_runtime \
+		$(ARGS)
 
-test: test-build test-run
+# Uses Docker Compose to
+# 1. Ensure the `test` stack is torn down, including data volumes such as that of the test MongoDB database.
+# 2. Build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
+# 3. Run tests on the `test` container, passing `ARGS` to `pytest` (see Tip in comment above for `run-test` target).
+test: down-test up-test run-test
 
 black:
 	black nmdc_runtime
@@ -91,34 +105,28 @@ init-lint-and-black:
 	pip install $(PIP_PINNED_FLAKE8)
 	pip install $(PIP_PINNED_BLACK)
 
-down-dev:
-	docker compose down
-
-down-test:
-	docker compose --file docker-compose.test.yml down --volumes
-
 follow-fastapi:
 	docker compose logs fastapi -f
 
-fastapi-deploy-spin:
-	rancher kubectl rollout restart deployment/runtime-fastapi --namespace=nmdc-dev
-
-dagster-deploy-spin:
-	rancher kubectl rollout restart deployment/dagit --namespace=nmdc-dev
-	rancher kubectl rollout restart deployment/dagster-daemon --namespace=nmdc-dev
-
-publish:
-	invoke publish
-
+# Build the MkDocs documentation website and serve it at http://localhost:8080.
+# Docs: https://www.mkdocs.org/user-guide/cli/#mkdocs-serve
 docs-dev:
-	mkdocs serve -a localhost:8080
+	mkdocs serve --config-file docs/mkdocs.yml --dev-addr localhost:8080
 
+# 🙋 Prerequisites:
+#
+# 1. The `PATH_TO_NERSC_SSHPROXY` environment variable is set to the path of
+#    an executable copy of the `sshproxy` program maintained by NERSC.
+#    You can read about and download that program at the following URL:
+#    https://docs.nersc.gov/connect/mfa/#sshproxy
+#    
+# 2. The `NERSC_USERNAME` environment variable is set to your NERSC username.
+#
 nersc-sshproxy:
-	bash ./nersc-sshproxy.sh -u ${NERSC_USERNAME} # https://docs.nersc.gov/connect/mfa/#sshproxy
+	${PATH_TO_NERSC_SSHPROXY} -u ${NERSC_USERNAME}
 
 nersc-mongo-tunnels:
 	ssh -L27072:mongo-loadbalancer.nmdc.production.svc.spin.nersc.org:27017 \
-		-L28082:mongo-loadbalancer.nmdc-dev.development.svc.spin.nersc.org:27017 \
 		-L27092:mongo-loadbalancer.nmdc-dev.production.svc.spin.nersc.org:27017 \
 		-o ServerAliveInterval=60 \
 		${NERSC_USERNAME}@dtn02.nersc.gov
@@ -130,7 +138,14 @@ mongorestore-nmdc-db:
 	# export MONGO_REMOTE_DUMP_DIR=$(ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null)
 	# ```
 	# Rsync the remote dump directory items of interest:
-	rsync -av --exclude='_*' --exclude='fs\.*' \
+	rsync -av --no-perms \
+		--exclude='*_agg\.*' \
+		--exclude='operations\.*' \
+		--exclude='_*' \
+		--exclude='ids_nmdc_sys0\.*' \
+		--exclude='query_runs\.*' \
+		--exclude='fs\.*' \
+		--exclude="alldocs\.*" \
 		-e "ssh -i ~/.ssh/nersc" \
 		${NERSC_USERNAME}@dtn01.nersc.gov:${MONGO_REMOTE_DUMP_DIR}/nmdc/ \
 		/tmp/remote-mongodump/nmdc

{nmdc_runtime-2.5.0/nmdc_runtime.egg-info → nmdc_runtime-2.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nmdc_runtime
-Version: 2.5.0
+Version: 2.7.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston
@@ -106,10 +106,10 @@ docker compose version
 docker info
 ```
 
-Ensure the permissions of `./mongoKeyFile` are such that only the file's owner can read or write the file.
+Ensure the permissions of `./.docker/mongoKeyFile` are such that only the file's owner can read or write the file.
 
 ```shell
-chmod 600 ./mongoKeyFile
+chmod 600 ./.docker/mongoKeyFile
 ```
 
 Ensure you have a `.env` file for the Docker services to source from. You may copy `.env.example` to

{nmdc_runtime-2.5.0 → nmdc_runtime-2.7.0}/README.md

@@ -83,10 +83,10 @@ docker compose version
 docker info
 ```
 
-Ensure the permissions of `./mongoKeyFile` are such that only the file's owner can read or write the file.
+Ensure the permissions of `./.docker/mongoKeyFile` are such that only the file's owner can read or write the file.
 
 ```shell
-chmod 600 ./mongoKeyFile
+chmod 600 ./.docker/mongoKeyFile
 ```
 
 Ensure you have a `.env` file for the Docker services to source from. You may copy `.env.example` to

nmdc_runtime-2.7.0/db/migrations/notebooks/migrate_11_4_0_to_11_5_0.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d05efc6327778f9c",
+   "metadata": {},
+   "source": [
+    "# Migrate MongoDB database from `nmdc-schema` `v11.4.0` to `v11.5.0`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b99d5924e825b9a2",
+   "metadata": {},
+   "source": [
+    "There are no migrators associated with any schema changes between schema versions `v11.4.0` and `v11.5.0`. So, this notebook is a \"no op\" (i.e. \"no operation\")."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "is_executing": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# no op"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

nmdc_runtime-2.7.0/db/migrations/notebooks/migrate_11_5_0_to_11_5_1.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d05efc6327778f9c",
+   "metadata": {},
+   "source": [
+    "# Migrate MongoDB database from `nmdc-schema` `v11.5.0` to `v11.5.1`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b99d5924e825b9a2",
+   "metadata": {},
+   "source": [
+    "There are no migrators associated with any schema changes between schema versions `v11.5.0` and `v11.5.1`. So, this notebook is a \"no op\" (i.e. \"no operation\")."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "is_executing": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# no op"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}