nmdc-runtime 2.1.1__tar.gz → 2.2.1__tar.gz

{nmdc_runtime-2.1.1 → nmdc_runtime-2.2.1}/.env.example

@@ -47,4 +47,8 @@ ORCID_NMDC_CLIENT_SECRET=replaceme
 #       For the sandbox instance of ORCID, use: https://sandbox.orcid.org
 ORCID_BASE_URL=https://orcid.org
 
-INFO_BANNER_INNERHTML='Announcement: Something important is about to happen. If you have questions, please contact <a href="mailto:support@microbiomedata.org">support@microbiomedata.org</a>.'
+INFO_BANNER_INNERHTML='Announcement: Something important is about to happen. If you have questions, please contact <a href="mailto:support@microbiomedata.org">support@microbiomedata.org</a>.'
+
+# To get the latest mongo dump directory, run e.g.
+# ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null
+MONGO_REMOTE_DUMP_DIR=/global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2024-11-26_20-12-40

{nmdc_runtime-2.1.1 → nmdc_runtime-2.2.1}/.github/pull_request_template.md

@@ -55,6 +55,7 @@ In this branch, I...
 - [ ] Minter
 - [ ] Dagster
 - [ ] Project documentation (in the `docs` directory)
+- [ ] Translators (metadata ingest pipelines)
 - [ ] MongoDB migrations
 - [ ] Other
 

{nmdc_runtime-2.1.1 → nmdc_runtime-2.2.1}/Makefile

@@ -43,6 +43,8 @@ test-dbinit:
 	docker compose --file docker-compose.test.yml \
 		exec mongo /bin/bash -c "/mongorestore-nmdc-testdb.sh"
 
+# Tip: If you append a file path to this "recipe", pytest will run only the tests defined in that file.
+#      For example, append `tests/test_api/test_endpoints.py` to have pytest only run the endpoint tests.
 test-run:
 	docker compose --file docker-compose.test.yml run test
 
@@ -99,12 +101,18 @@ nersc-mongo-tunnels:
 
 mongorestore-nmdc-db:
 	mkdir -p /tmp/remote-mongodump/nmdc
-	# SSH into the remote server, stream the dump directory as a gzipped tar archive, and extract it locally.
-	ssh -i ~/.ssh/nersc ${NERSC_USERNAME}@dtn01.nersc.gov \
-		'tar -czf - -C /global/cfs/projectdirs/m3408/nmdc-mongodumps/dump_nmdc-prod_2024-07-29_20-12-07/nmdc .' \
-		| tar -xzv -C /tmp/remote-mongodump/nmdc
+	# Optionally, manually update MONGO_REMOTE_DUMP_DIR env var:
+	# ```bash
+	# export MONGO_REMOTE_DUMP_DIR=$(ssh -i ~/.ssh/nersc -q ${NERSC_USERNAME}@dtn01.nersc.gov 'bash -s ' < util/get_latest_nmdc_prod_dump_dir.sh 2>/dev/null)
+	# ```
+	# Rsync the remote dump directory items of interest:
+	rsync -av --exclude='_*' --exclude='fs\.*' \
+		-e "ssh -i ~/.ssh/nersc" \
+		${NERSC_USERNAME}@dtn01.nersc.gov:${MONGO_REMOTE_DUMP_DIR}/nmdc/ \
+		/tmp/remote-mongodump/nmdc
+	# Restore from `rsync`ed local directory:
 	mongorestore -v -h localhost:27018 -u admin -p root --authenticationDatabase=admin \
-		--drop --nsInclude='nmdc.*' --dir /tmp/remote-mongodump
+		--drop --nsInclude='nmdc.*' --gzip --dir /tmp/remote-mongodump
 
 quick-blade:
 	python -c "from nmdc_runtime.api.core.idgen import generate_id; print(f'nmdc:nt-11-{generate_id(length=8, split_every=0)}')"

{nmdc_runtime-2.1.1/nmdc_runtime.egg-info → nmdc_runtime-2.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nmdc_runtime
-Version: 2.1.1
+Version: 2.2.1
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston
@@ -145,8 +145,6 @@ http://127.0.0.1:8000/redoc/.
 
 Tests can be found in `tests` and are run with the following commands:
 
-On an M1 Mac? May need to `export DOCKER_DEFAULT_PLATFORM=linux/amd64`.
-
 ```bash
 make up-test
 make test

{nmdc_runtime-2.1.1 → nmdc_runtime-2.2.1}/README.md

@@ -131,8 +131,6 @@ http://127.0.0.1:8000/redoc/.
 
 Tests can be found in `tests` and are run with the following commands:
 
-On an M1 Mac? May need to `export DOCKER_DEFAULT_PLATFORM=linux/amd64`.
-
 ```bash
 make up-test
 make test

nmdc_runtime-2.2.1/db/README.md

@@ -0,0 +1,7 @@
+# Database
+
+This directory contains files related to the MongoDB database managed by the Runtime.
+
+It has the following subdirectories:
+
+- `./migrations`: files related to migrating the MongoDB database

nmdc_runtime-2.2.1/db/migrations/README.md

@@ -0,0 +1,3 @@
+# Migrations
+
+This directory contains files related to migrating the MongoDB database between schemas.

{nmdc_runtime-2.1.1/demo/metadata_migration → nmdc_runtime-2.2.1/db/migrations}/notebooks/bookkeeper.py

@@ -1,6 +1,6 @@
 from typing import Optional
 from enum import Enum
-from datetime import datetime
+from datetime import datetime, timezone
 
 from pymongo import MongoClient
 from nmdc_schema.migrators.migrator_base import MigratorBase
@@ -47,7 +47,7 @@ class Bookkeeper:
     @staticmethod
     def get_current_timestamp() -> str:
         r"""Returns an ISO 8601 timestamp (string) representing the current time in UTC."""
-        utc_now = datetime.utcnow()
+        utc_now = datetime.now(timezone.utc)
         iso_utc_now = utc_now.isoformat()
         return iso_utc_now  # e.g. "2024-02-21T04:31:03.115107"
 

{nmdc_runtime-2.1.1/demo/metadata_migration → nmdc_runtime-2.2.1/db/migrations}/notebooks/manual_test_bookkeeper.py

@@ -12,7 +12,7 @@ from pymongo import MongoClient, timeout
 from pymongo.database import Database
 from nmdc_schema.migrators.migrator_base import MigratorBase
 
-from demo.metadata_migration.notebooks.bookkeeper import Bookkeeper, MigrationEvent
+from db.migrations.notebooks.bookkeeper import Bookkeeper, MigrationEvent
 
 # Consume environment variables.
 MONGO_HOST: str = os.getenv("MONGO_HOST", "localhost")
@@ -36,7 +36,7 @@ class TestBookkeeper(unittest.TestCase):
     Tests targeting the `Bookkeeper` class.
 
     You can format this file like this:
-    $ python -m black demo/metadata_migration/notebooks/manual_test_bookkeeper.py
+    $ python -m black db/migrations/notebooks/manual_test_bookkeeper.py
 
     You can start up a containerized MongoDB server like this:
     $ docker run --rm --detach --name mongo-test-migration-bookkeeper -p 27017:27017 mongo
@@ -45,7 +45,7 @@ class TestBookkeeper(unittest.TestCase):
     - host.docker.internal:27017
 
     You can run these tests like this:
-    $ python -m unittest -v demo/metadata_migration/notebooks/manual_test_bookkeeper.py
+    $ python -m unittest -v db/migrations/notebooks/manual_test_bookkeeper.py
 
     Reference: https://docs.python.org/3/library/unittest.html#basic-example
     """