nmdc-runtime 2.6.0__tar.gz → 2.8.0__tar.gz

{nmdc_runtime-2.6.0 → nmdc_runtime-2.8.0}/.env.example

@@ -13,6 +13,8 @@ DO_ENDPOINT_URL=https://nyc3.digitaloceanspaces.com
 DO_SPACES_KEY=generateme
 DO_SPACES_SECRET=generateme
 
+ENABLE_DB_TESTS=false
+
 # to get a string like this run:
 # openssl rand -hex 32
 JWT_SECRET_KEY=generateme
@@ -57,3 +59,16 @@ MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-pr
 # Note: This is currently only used by the `nersc-sshproxy` target in `Makefile`.
 # Reference: https://docs.nersc.gov/connect/mfa/#sshproxy
 PATH_TO_NERSC_SSHPROXY=/Users/Alice/Downloads/sshproxy
+
+# `make up-dev` and `make up-test` will map services to host-machine port numbers using the following environment
+# variables in your `.env` file. This way, different developers can use different local port numbers, without modifying
+# files that get stored in the repository.
+#
+# Note: the `.env` file specifically is used here, *not* the file set as `env_file` in a Docker Compose file.
+# This is why, e.g., `.env.test` does not include any of the below environment-variable settings.
+DEV_STACK_HOST_MACHINE_PORT_DAGIT=3000
+DEV_STACK_HOST_MACHINE_PORT_FASTAPI=8000
+DEV_STACK_HOST_MACHINE_PORT_MONGO=27018
+TEST_STACK_HOST_MACHINE_PORT_DAGIT=3005
+TEST_STACK_HOST_MACHINE_PORT_FASTAPI=8005
+TEST_STACK_HOST_MACHINE_PORT_MONGO=27022

{nmdc_runtime-2.6.0 → nmdc_runtime-2.8.0}/.github/workflows/mkdocs.yml

@@ -5,8 +5,6 @@ on:
       - main
     paths:
       - 'docs/**'
-      - 'mkdocs_overrides/**'
-      - 'mkdocs.yml'
       - '.github/workflows/mkdocs.yml'
 jobs:
   deploy:
@@ -41,4 +39,4 @@ jobs:
       #       file, or due to one of the dependencies becoming unavailable), which can prompt remedial action.
       #
       - name: Build MkDocs site
-        run: mkdocs build
+        run: mkdocs build -f docs/mkdocs.yml

{nmdc_runtime-2.6.0 → nmdc_runtime-2.8.0}/.github/workflows/python-app.yml

@@ -83,9 +83,7 @@ jobs:
 #      run: |
 #        pip install flake8
 #        make lint
-    - name: Build and run containers upon which test runner depends
+    - name: Build and start containers upon which test runner depends
       run: make up-test
-    - name: Build container image for test runner
-      run: make test-build
     - name: Run tests
-      run: make test-run
+      run: make run-test

{nmdc_runtime-2.6.0 → nmdc_runtime-2.8.0}/.gitignore

@@ -16,6 +16,7 @@ eggs/
 .eggs/
 lib/
 !/tests/lib/
+!/nmdc_runtime/api/endpoints/lib/
 lib64/
 parts/
 sdist/
@@ -119,7 +120,7 @@ venv.bak/
 .ropeproject
 
 # mkdocs documentation
-/site
+/docs/site
 
 # mypy
 .mypy_cache/
@@ -173,4 +174,5 @@ docker-build.sh
 
 neon.db
 neon_api_cache.sqlite
-nmdc-db.nt.gz
+nmdc-db.nt.gz
+**/.claude/settings.local.json

{nmdc_runtime-2.6.0 → nmdc_runtime-2.8.0}/Makefile

@@ -38,41 +38,55 @@ update: update-deps init
 up-dev:
 	docker compose up --build --force-recreate --detach --remove-orphans
 
+down-dev:
+	docker compose down
+
 # Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the dev stack.
-dev-reset-db:
+reset-db-dev:
 	docker compose \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
 # Uses Docker Compose to build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
-#
-# Note: This does not build or spin up the `test` container, itself, since the `test` container's Docker Compose service
-#       has the "donotstart" profile specified for it in `docker-compose.test.yml`.
-#
 up-test:
 	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
-# Uses Docker Compose to build the container image for the `test` container (i.e. the test runner).
-test-build:
-	docker compose --file docker-compose.test.yml build test
+# Tears down the `test` stack, including removing data volumes such as that for the test MongoDB database.
+down-test:
+	docker compose --file docker-compose.test.yml down --volumes
 
 # Restores the MongoDB dump residing in `./tests/nmdcdb` on the Docker host, into the MongoDB server in the test stack.
-test-reset-db:
+reset-db-test:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
-# Uses Docker Compose to spin up the `test` container (i.e. the test runner), effectively running the tests.
+# Run tests on the started `test` stack, passing `ARGS` to `pytest` (see Tip below).
 #
 # Tip: If you append `ARGS=` and a file path to the `make` command, pytest will run only the tests defined in that file.
-#      For example, to run only the tests defined in `tests/test_api/test_endpoints.py`:
+#
+#      Some examples, using `make test`:
+#      (You can also use `make run-test` if are sure you don't need to reset global state)
+#
+#      To run only the tests defined in `tests/test_api/test_endpoints.py`:
+#      ```
+#      $ make test ARGS="tests/test_api/test_endpoints.py"
+#      ```
+#
+#      To run only the test `test_find_data_objects_for_study_having_one` in `tests/test_api/test_endpoints.py`:
 #      ```
-#      $ make test-run ARGS="tests/test_api/test_endpoints.py"
+#      $ make test ARGS="-k 'test_find_data_objects_for_study_having_one'"
 #      ```
 #
-test-run:
-	docker compose --file docker-compose.test.yml run test $(ARGS)
+run-test:
+	docker compose --file docker-compose.test.yml exec -it test \
+		./.docker/wait-for-it.sh fastapi:8000 --strict --timeout=300 -- pytest --cov=nmdc_runtime \
+		$(ARGS)
 
-test: test-build test-run
+# Uses Docker Compose to
+# 1. Ensure the `test` stack is torn down, including data volumes such as that of the test MongoDB database.
+# 2. Build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
+# 3. Run tests on the `test` container, passing `ARGS` to `pytest` (see Tip in comment above for `run-test` target).
+test: down-test up-test run-test
 
 black:
 	black nmdc_runtime
@@ -91,27 +105,13 @@ init-lint-and-black:
 	pip install $(PIP_PINNED_FLAKE8)
 	pip install $(PIP_PINNED_BLACK)
 
-down-dev:
-	docker compose down
-
-down-test:
-	docker compose --file docker-compose.test.yml down --volumes
-
 follow-fastapi:
 	docker compose logs fastapi -f
 
-fastapi-deploy-spin:
-	rancher kubectl rollout restart deployment/runtime-fastapi --namespace=nmdc-dev
-
-dagster-deploy-spin:
-	rancher kubectl rollout restart deployment/dagit --namespace=nmdc-dev
-	rancher kubectl rollout restart deployment/dagster-daemon --namespace=nmdc-dev
-
-publish:
-	invoke publish
-
+# Build the MkDocs documentation website and serve it at http://localhost:8080.
+# Docs: https://www.mkdocs.org/user-guide/cli/#mkdocs-serve
 docs-dev:
-	mkdocs serve -a localhost:8080
+	mkdocs serve --config-file docs/mkdocs.yml --dev-addr localhost:8080
 
 # 🙋 Prerequisites:
 #
@@ -127,7 +127,6 @@ nersc-sshproxy:
 
 nersc-mongo-tunnels:
 	ssh -L27072:mongo-loadbalancer.nmdc.production.svc.spin.nersc.org:27017 \
-		-L28082:mongo-loadbalancer.nmdc-dev.development.svc.spin.nersc.org:27017 \
 		-L27092:mongo-loadbalancer.nmdc-dev.production.svc.spin.nersc.org:27017 \
 		-o ServerAliveInterval=60 \
 		${NERSC_USERNAME}@dtn02.nersc.gov
@@ -139,7 +138,14 @@ mongorestore-nmdc-db:
 	# export MONGO_REMOTE_DUMP_DIR=$(ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null)
 	# ```
 	# Rsync the remote dump directory items of interest:
-	rsync -av --no-perms --exclude='_*' --exclude='fs\.*' \
+	rsync -av --no-perms \
+		--exclude='*_agg\.*' \
+		--exclude='operations\.*' \
+		--exclude='_*' \
+		--exclude='ids_nmdc_sys0\.*' \
+		--exclude='query_runs\.*' \
+		--exclude='fs\.*' \
+		--exclude="alldocs\.*" \
 		-e "ssh -i ~/.ssh/nersc" \
 		${NERSC_USERNAME}@dtn01.nersc.gov:${MONGO_REMOTE_DUMP_DIR}/nmdc/ \
 		/tmp/remote-mongodump/nmdc
@@ -152,3 +158,15 @@ quick-blade:
 
 .PHONY: init update-deps update up-dev down-dev follow-fastapi \
 	publish docs
+
+docker-clean:
+	# Down the dev stack with volumes
+	-docker compose down --volumes --remove-orphans
+	# Down the test stack with volumes
+	-docker compose --file docker-compose.test.yml down --volumes --remove-orphans
+	# Remove any dangling images from this project
+	-docker image prune -f
+	# Remove the project-specific networks
+	-docker network rm nmdc-runtime-dev nmdc-runtime-test 2>/dev/null || true
+	# Remove the project-specific volumes
+	-docker volume rm nmdc-runtime-dev_mongo_data nmdc-runtime-dev_postgres_data nmdc-runtime-test_mongo_data nmdc-runtime-test_postgres_data 2>/dev/null || true

{nmdc_runtime-2.6.0/nmdc_runtime.egg-info → nmdc_runtime-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nmdc_runtime
-Version: 2.6.0
+Version: 2.8.0
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston
@@ -127,7 +127,8 @@ source .env
 set +a
 ```
 
-If you are connecting to resources that require an SSH tunnel—for example, a MongoDB server that is only accessible on the NERSC network—set up the SSH tunnel.
+If you are connecting to resources that require an SSH tunnel—for example, a MongoDB server that is only accessible on 
+the NERSC network—set up the SSH tunnel.
 
 The following command could be useful to you, either directly or as a template (see `Makefile`).
 
@@ -149,6 +150,19 @@ The Dagit web server is viewable at http://127.0.0.1:3000/.
 The FastAPI service is viewable at http://127.0.0.1:8000/ -- e.g., rendered documentation at
 http://127.0.0.1:8000/redoc/.
 
+
+*  NOTE: Any time you add or change requirements in requirements/main.in or requirements/dev.in, you must run:
+```
+pip-compile --build-isolation --allow-unsafe --resolver=backtracking --strip-extras --output-file requirements/[main|dev].txt requirements/[main|dev].in
+```
+to generate main.txt and dev.txt files respectively. main.in is kind of like a poetry dependency stanza, dev.in is kind 
+of like poetry dev.dependencies stanza. main.txt and dev.txt are kind of like poetry.lock files to specify the exact 
+versions of dependencies to use. main.txt and dev.txt are combined in the docker compose build process to create the 
+final requirements.txt file and import the dependencies into the Docker image.
+
+
+```bash
+
 ## Local Testing
 
 Tests can be found in `tests` and are run with the following commands:
@@ -160,12 +174,13 @@ make test
 # Run a Specific test file eg. tests/test_api/test_endpoints.py
 make test ARGS="tests/test_api/test_endpoints.py"
 ```
+docker compose --file docker-compose.test.yml run test
 
 As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
 desired and does not break over time.
 
 [For hints on how to write tests for solids and pipelines in Dagster, see their documentation
-tutorial on Testing](https://docs.dagster.io/tutorial/testable).
+tutorial on Testing](https://docs.dagster.io/guides/test/unit-testing-assets-and-ops).
 
 ### RAM usage
 

{nmdc_runtime-2.6.0 → nmdc_runtime-2.8.0}/README.md

@@ -104,7 +104,8 @@ source .env
 set +a
 ```
 
-If you are connecting to resources that require an SSH tunnel—for example, a MongoDB server that is only accessible on the NERSC network—set up the SSH tunnel.
+If you are connecting to resources that require an SSH tunnel—for example, a MongoDB server that is only accessible on 
+the NERSC network—set up the SSH tunnel.
 
 The following command could be useful to you, either directly or as a template (see `Makefile`).
 
@@ -126,6 +127,19 @@ The Dagit web server is viewable at http://127.0.0.1:3000/.
 The FastAPI service is viewable at http://127.0.0.1:8000/ -- e.g., rendered documentation at
 http://127.0.0.1:8000/redoc/.
 
+
+*  NOTE: Any time you add or change requirements in requirements/main.in or requirements/dev.in, you must run:
+```
+pip-compile --build-isolation --allow-unsafe --resolver=backtracking --strip-extras --output-file requirements/[main|dev].txt requirements/[main|dev].in
+```
+to generate main.txt and dev.txt files respectively. main.in is kind of like a poetry dependency stanza, dev.in is kind 
+of like poetry dev.dependencies stanza. main.txt and dev.txt are kind of like poetry.lock files to specify the exact 
+versions of dependencies to use. main.txt and dev.txt are combined in the docker compose build process to create the 
+final requirements.txt file and import the dependencies into the Docker image.
+
+
+```bash
+
 ## Local Testing
 
 Tests can be found in `tests` and are run with the following commands:
@@ -137,12 +151,13 @@ make test
 # Run a Specific test file eg. tests/test_api/test_endpoints.py
 make test ARGS="tests/test_api/test_endpoints.py"
 ```
+docker compose --file docker-compose.test.yml run test
 
 As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
 desired and does not break over time.
 
 [For hints on how to write tests for solids and pipelines in Dagster, see their documentation
-tutorial on Testing](https://docs.dagster.io/tutorial/testable).
+tutorial on Testing](https://docs.dagster.io/guides/test/unit-testing-assets-and-ops).
 
 ### RAM usage
 

nmdc_runtime-2.8.0/db/migrations/notebooks/migrate_11_6_0_to_11_6_1.ipynb

@@ -0,0 +1,56 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "d05efc6327778f9c",
+   "metadata": {},
+   "source": [
+    "# Migrate MongoDB database from `nmdc-schema` `v11.6.0` to `v11.6.1`"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "b99d5924e825b9a2",
+   "metadata": {},
+   "source": [
+    "There are no migrators associated with any schema changes between schema versions `v11.6.0` and `v11.6.1`. So, this notebook is a \"no op\" (i.e. \"no operation\")."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "jupyter": {
+     "is_executing": true
+    }
+   },
+   "outputs": [],
+   "source": [
+    "# no op"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}