nmdc-runtime 1.0.9__tar.gz → 1.0.10__tar.gz

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/.dockerignore

@@ -2,7 +2,6 @@
 .idea
 docker-compose.yml
 .dockerignore
-docker-build.sh
 deployment
 .eggs
 .pytest_cache

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/.env.example

@@ -35,3 +35,5 @@ NMDC_PORTAL_API_BASE_URL=https://data-dev.microbiomedata.org/
 
 NEON_API_TOKEN=y
 NEON_API_BASE_URL=https://data.neonscience.org/api/v0
+
+NERSC_USERNAME=replaceme

nmdc_runtime-1.0.10/.github/workflows/build-and-release-to-spin.yml

@@ -0,0 +1,92 @@
+name: Build Docker images and release to Spin
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - 'v*'
+    paths:
+      - '.github/workflows/build-and-release-to-spin.yml'
+      - 'Makefile'
+      - '**.Dockerfile'
+      - '**.py'
+      - 'requirements/main.txt'
+
+env:
+  # We don't want to do certain steps if this is running in a fork
+  IS_ORIGINAL_REPO: ${{ github.repository == 'microbiomedata/nmdc-runtime' }}
+
+  # Used to determine if we add the special `prod` Docker image tag
+  IS_PROD_RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }}
+
+  # Used when sending redeploy action requests to Rancher
+  RANCHER_NAMESPACE: ${{ startsWith(github.ref, 'refs/tags/v') && 'nmdc' || 'nmdc-dev' }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        image: [ fastapi, dagster ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          # history for all branches and tags is needed for setuptools-scm (part of build and push step)
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            microbiomedata/nmdc-runtime-${{ matrix.image }}
+          flavor: |
+            latest=false
+          tags: |
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=raw,value=prod,enable=${{ env.IS_PROD_RELEASE }}
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: ${{ env.IS_ORIGINAL_REPO }}
+          file: nmdc_runtime/${{ matrix.image }}.Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  release:
+    needs: build
+
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        deployment: [ runtime-api, dagster-dagit, dagster-daemon ]
+
+    steps:
+      - name: Redeploy ${{ env.RANCHER_NAMESPACE }}:${{ matrix.deployment }}
+        if: ${{ env.IS_ORIGINAL_REPO }}
+        uses: fjogeleit/http-request-action@v1
+        with:
+          url: ${{ secrets.RANCHER_URL }}/v3/project/${{ secrets.RANCHER_CONTEXT }}/workloads/deployment:${{ env.RANCHER_NAMESPACE }}:${{ matrix.deployment }}?action=redeploy
+          method: POST
+          bearerToken: ${{ secrets.RANCHER_TOKEN }}

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/.gitignore

@@ -163,4 +163,6 @@ tests/nmdcdb/
 neon_cache.sqlite
 
 # output of changesheet generation
-nmdc_runtime/site/changesheets/changesheets_output/
+nmdc_runtime/site/changesheets/changesheets_output/
+
+mdb-nmdc-dev.tar.gz

nmdc_runtime-1.0.10/.gitpod.Dockerfile

@@ -0,0 +1,7 @@
+FROM gitpod/workspace-full
+
+RUN wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-ubuntu2204-x86_64-100.9.3.tgz
+
+RUN tar -zxvf mongodb-database-tools-*.tgz
+
+RUN sudo cp mongodb-database-tools-*/bin/* /usr/local/bin/

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/.gitpod.yml

@@ -3,9 +3,10 @@
 # and commit this file to your remote git repository to share the goodness with others.
 
 # Learn more from ready-to-use templates: https://www.gitpod.io/docs/introduction/getting-started/quickstart
-
+image:
+    file: .gitpod.Dockerfile
 tasks:
   - name: Start Dev on Fresh Gitpod
     before: cp .env.example .env
-    init: make up-dev
-    command: docker-compose logs -f fastapi
+    init: docker compose up mongo --detach && make mongorestore-nmdc-dev
+    command: make up-dev && docker-compose logs -f fastapi

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/Makefile

@@ -57,15 +57,9 @@ down-test:
 follow-fastapi:
 	docker-compose logs fastapi -f
 
-fastapi-docker:
-	./docker-build.sh microbiomedata/nmdc-runtime-fastapi nmdc_runtime/fastapi.Dockerfile
-
 fastapi-deploy-spin:
 	rancher kubectl rollout restart deployment/runtime-fastapi --namespace=nmdc-dev
 
-dagster-docker:
-	./docker-build.sh microbiomedata/nmdc-runtime-dagster nmdc_runtime/dagster.Dockerfile
-
 dagster-deploy-spin:
 	rancher kubectl rollout restart deployment/dagit --namespace=nmdc-dev
 	rancher kubectl rollout restart deployment/dagster-daemon --namespace=nmdc-dev
@@ -77,23 +71,26 @@ docs-dev:
 	mkdocs serve -a localhost:8080
 
 nersc-sshproxy:
-	bash ~/nersc-sshproxy.sh # https://docs.nersc.gov/connect/mfa/#sshproxy
+	bash ./nersc-sshproxy.sh -u ${NERSC_USERNAME} # https://docs.nersc.gov/connect/mfa/#sshproxy
 
 nersc-mongo-tunnels:
 	ssh -L27072:mongo-loadbalancer.nmdc.production.svc.spin.nersc.org:27017 \
 		-L28082:mongo-loadbalancer.nmdc-dev.development.svc.spin.nersc.org:27017 \
 		-L27092:mongo-loadbalancer.nmdc-dev.production.svc.spin.nersc.org:27017 \
 		-o ServerAliveInterval=60 \
-		dtn02.nersc.gov
+		${NERSC_USERNAME}@dtn02.nersc.gov
 
-mongorestore-nmdcdb-lite-archive:
-	wget https://portal.nersc.gov/project/m3408/meta/mongodumps/nmdcdb.lite.archive.gz
-	mongorestore --host localhost:27018 --username admin --password root --drop --gzip \
-		--archive=nmdcdb.lite.archive.gz
-	rm nmdcdb.lite.archive.gz
+mongorestore-nmdc-dev:
+	wget https://portal.nersc.gov/cfs/m3408/meta/mongodumps/mdb-nmdc-dev.tar.gz
+	tar zxvf mdb-nmdc-dev.tar.gz
+	rm -rf nmdc/._* # Not sure why ._* files are extracted.
+	mongorestore -h localhost:27018 -u admin -p root --authenticationDatabase=admin \
+		--drop --gzip -d nmdc nmdc
+	rm -rf nmdc/
+	rm mdb-nmdc-dev.tar.gz
 
 quick-blade:
 	python -c "from nmdc_runtime.api.core.idgen import generate_id; print(f'nmdc:nt-11-{generate_id(length=8, split_every=0)}')"
 
 .PHONY: init update-deps update up-dev down-dev follow-fastapi \
-	fastapi-docker dagster-docker publish docs
+	publish docs

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nmdc_runtime
-Version: 1.0.9
+Version: 1.0.10
 Summary: A runtime system for NMDC data management and orchestration
 Home-page: https://github.com/microbiomedata/nmdc-runtime
 Author: Donny Winston

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/docs/howto-guides/release-process.md

@@ -15,16 +15,20 @@ released? Here's how.
      [our python-app.yml GitHub
      action](https://github.com/microbiomedata/nmdc-runtime/blob/main/.github/workflows/python-app.yml),
      which is triggered automatically when a change to any Python file in the repository is pushed to the
-     `main` branch, or to a Pull Request. You can monitor the status of Github Actions
+     `main` branch, or to a Pull Request. You can monitor the status of GitHub Actions
      [here](https://github.com/microbiomedata/nmdc-runtime/actions).
 
-2. Add a summary for the release to `RELEASES.md`. You can make an edit and push to the `main`
-   branch [via GitHub](https://github.com/microbiomedata/nmdc-runtime/blob/main/RELEASES.md). This will
-   trigger two GitHub actions in sequence to
+2. Create a new [GitHub Release](https://github.com/microbiomedata/nmdc-runtime/releases). When creating the new release:
+   - Use the "Choose a tag" dropdown to **create a new tag** by typing in a tag name which does not exist yet.
+   - The new tag name should start with `v` and be followed by a [semantic version number](https://semver.org/), for example `v3.0.2`.
+   - If the last published version is `vX.Y.Z`, the next release number should be `vX.Y.{Z+1}` for a patch release (bux fixes and refactoring), `vX.{Y+1}.0` for a minor release (new functionality that is backwards-compatible), or `v{X+1}.0.0` for a major release (new functionality that may be backwards-incompatible).
+   - You may leave the "Release title" input blank. If it is blank the release title will be populated with the tag name.
 
-   - [build and push updated Docker images](https://github.com/microbiomedata/nmdc-runtime/blob/main/.github/workflows/build-and-push-docker-images.yml) for the API server and for the NMDC Runtime site's Dagster daemon and Dagit dashboard, and
+    Once the GitHub Release has been created, two GitHub Actions will be triggered which:
 
-   - [deploy the new images to Spin](https://github.com/microbiomedata/nmdc-runtime/blob/main/.github/workflows/release-to-spin.yml).
+   - [build Docker images and deploy them to Spin](https://github.com/microbiomedata/nmdc-runtime/blob/main/.github/workflows/build-and-release-to-spin.yml) for the API server and for the NMDC Runtime site's Dagster daemon and Dagit dashboard.
+
+   - [build a Python package and publish it to PyPI](https://github.com/microbiomedata/nmdc-runtime/blob/main/.github/workflows/release-to-pypi.yml).
 
 
 ## Data Releases

nmdc_runtime-1.0.10/nersc-sshproxy.sh

@@ -0,0 +1,353 @@
+#!/bin/bash
+
+
+# SSH Proxy (sshproxy), Copyright (c) 2019, The Regents of the University of California,
+# through Lawrence Berkeley National Laboratory (subject to receipt of any required
+# approvals from the U.S. Dept. of Energy).  All rights reserved.
+# 
+# If you have questions about your rights to use or distribute this software,
+# please contact Berkeley Lab's Intellectual Property Office at  IPO@lbl.gov.
+# 
+# NOTICE.  This Software was developed under funding from the U.S. Department of Energy
+# and the U.S. Government consequently retains certain rights. As such, the U.S.
+# Government has been granted for itself and others acting on its behalf a paid-up,
+# nonexclusive, irrevocable, worldwide license in the Software to reproduce, distribute
+# copies to the public, prepare derivative works, and perform publicly and display
+# publicly, and to permit other to do so.
+# 
+# See LICENSE for full text.
+
+progname=$(basename $0)
+version="1.1.0"
+
+# Save tty state for trap function below
+original_tty_state=$(stty -g)
+
+tmpkey=''
+tmpcert=''
+tmppub=''
+pw=''
+
+# Default values
+id=nersc			# Name of key file
+user=$USER			# Username
+sshdir=~/.ssh			# SSH directory
+scope="default"			# Default scope
+url="https://sshproxy.nersc.gov"	# hostname for reaching proxy
+
+#############
+# Functions
+#############
+
+# Error(error string, ...)
+# 
+# prints out error string.  Joins multiple arguments with ": "
+
+
+Error () {
+
+	# Slightly complicated print statement so that output consists of
+	# arguments joined with ": " 
+
+	printf "$progname: %s" "$1" 1>&2
+	shift
+	printf ': %s' "$@" 1>&2
+	printf "\n" 1>&2
+}
+
+# Bail(exit code, error string, ...)
+# 
+# prints out error string and exits with given exit code
+
+Bail () {
+	# get exit code
+	exitcode=$1
+	shift
+
+	Error "$@"
+
+	# restore terminal to original state, in case we're interrupted
+	# while reading password
+
+	stty $original_tty_state
+
+	# Go bye-bye
+	exit $exitcode
+}
+
+
+# Cleanup()
+#
+# Cleans up temp files on exit
+
+Cleanup () {
+	for f in "$tmpkey" "$tmpcert" "$tmppub"
+	do
+		if [[ "$f" != "" && -e "$f" ]]; then
+			/bin/rm -f "$f"
+		fi
+	done
+}
+
+# Abort ()
+#
+# Trap on errors otherwise unhandled, does cleanup and exit(1)
+
+Abort () {
+	Bail 255 "Exited on interrupt/error"
+}
+
+
+Usage () {
+
+	if [[ $# -ne 0 ]]; then
+		printf "$progname: %s\n\n", "$*"
+	fi
+	printf "Usage: $progname [-u <user>] [-o <filename>] [-s <scope>] [-c <account>] [-p] [-a] [-x <proxy-url>] [-U <server URL>] [-v] [-h]\n"
+
+	printf "\n"
+	printf "\t -u <user>\tSpecify remote (NERSC) username\n"
+	printf "\t\t\t(default: $user)\n"
+	printf "\t -o <filename>\tSpecify pathname for private key\n"
+	printf "\t\t\t(default: $sshdir/$id)\n"
+	printf "\t -s <scope>\tSpecify scope (default: '$scope')\n"
+	printf "\t -p\t\tGet keys in PuTTY compatible (ppk) format\n"
+	printf "\t -a\t\tAdd key to ssh-agent (with expiration)\n"
+	printf "\t -c <account>\tSpecify a collaboration account (no default)\n"
+	printf "\t -x <URL>\tUse socks proxy to connect to sshproxy server.\n"
+	printf "\t\t\t(format: <protocol>://<host>[:port], see curl manpage\n"
+	printf "\t\t\tsection on "--proxy" for details)\n"
+	printf "\t -U <URL>\tSpecify alternate URL for sshproxy server\n"
+	printf "\t\t\t(generally only used for testing purposes)\n"
+	printf "\t -v \t\tPrint version number and exit\n"
+	printf "\t -h \t\tPrint this usage message and exit\n"
+	printf "\n"
+	
+	exit 0
+}
+
+#############
+# Actual code starts here...
+#############
+
+# Make sure we cleanup on exit
+
+trap Cleanup exit
+trap Abort int kill term hup pipe abrt
+
+
+# for command-line arguments.  In reality, not all of these get used,
+# but here for completeness
+opt_scope=''	# -s
+opt_url=''	# -U
+opt_user=''	# -u
+opt_out=''	# -o
+opt_agent=0	# -a
+opt_version=''	# -v
+opt_putty=''     # -p
+opt_socks=''     # -x
+
+# Process getopts.  See Usage() above for description of arguments
+
+while getopts "aphvs:k:U:u:o:x:c:" opt; do
+	case ${opt} in
+
+		h )
+			Usage
+		;;
+ 		v )
+ 			printf "$progname v$version\n"
+ 			exit 0
+ 		;;
+
+		s )
+			opt_scope=$OPTARG
+			scope=$opt_scope
+		;;
+
+		U )
+			url=$OPTARG
+		;;
+
+		u )
+			user=$OPTARG
+		;;
+
+		o )
+			opt_out=$OPTARG
+		;;
+		a )
+			opt_agent=1
+		;;
+		p )
+			opt_putty="?putty"
+		;;
+
+		x )
+			opt_socks="--proxy $OPTARG"
+		;;
+		c )
+			opt_collab=$OPTARG
+		;;
+
+		\? )
+			Usage "Unknown argument"
+		;;
+
+		: )
+			Usage "Invalid option: $OPTARG requires an argument"
+		;;
+
+	esac
+done
+
+# If user has specified a keyfile, then use that.
+# Otherwise, if user has specified a scope, use that for the keyfile name
+# And if it's the default, then use the "id" defined above ("nersc")
+data=''
+if [[ "$opt_collab" != "" ]] ; then
+    if [[ "$opt_scope" == "" ]] ; then
+        scope="collab"
+        opt_scope=$opt_collab
+    fi
+    data='{"target_user": "'$opt_collab'"}'
+fi
+
+if [[ $opt_out != "" ]]; then
+	idfile=$opt_out
+elif [[ "$opt_scope" != "" ]]; then
+	idfile="$sshdir/$opt_scope"
+else
+	idfile="$sshdir/$id"
+fi
+
+certfile="$idfile-cert.pub"
+pubfile="$idfile.pub"
+
+# Have user enter password+OTP.  Curl can do this, but does not
+# provide any control over the prompt
+#
+# N.B. INPWPROMPT variable is used in Bail() above for when password
+# prompt is interrupted by ctrl-c.  Otherwise terminal gets left in
+# a weird state.
+
+read -r -p "Enter the password+OTP for ${user}: " -s pw
+
+# read -p doesn't output a newline after entry
+printf "\n"
+
+# Make temp files.  We want them in the same target directory as the
+# final keys
+
+tmpdir=$(dirname $idfile)
+tmpdir="$tmpdir"
+tmpkey="$(mktemp $tmpdir/key.XXXXXX)"
+tmpcert="$(mktemp $tmpdir/cert.XXXXXX)"
+tmppub="$(mktemp $tmpdir/pub.XXXXXX)"
+
+# And get the key/cert
+curl -s -S -X POST $opt_socks $url/create_pair/$scope/$opt_putty \
+	-d "$data" -o $tmpkey -K - <<< "-u \"${user}:${pw}\""
+
+# Check for error
+err=$?
+if [[ $err -ne 0 ]] ; then
+	Bail 1 "Failed." "Curl returned" $err
+fi
+
+# Get the first line of the file to check for errors from the
+# server
+
+read x < $tmpkey
+
+# Check whether password failed
+
+if [[ "$x" =~ "Authentication failed. Failed login" ]]; then
+	Error "The sshproxy server said: $x"
+	Bail 2 "This usually means you did not enter the correct password or OTP"
+fi
+
+# Check whether the file appears to contain a valid key
+
+if [[ "$x" == "PuTTY-User-Key-File-2: ssh-rsa" ]]; then
+	mv $tmpkey $idfile.ppk
+	printf "Successfully obtained PuTTY Key file %s\n" "$idfile.ppk"
+	exit
+fi
+
+if [[ "$x" != "-----BEGIN RSA PRIVATE KEY-----" ]]; then
+	Error "Did not get in a proper ssh private key. Output was:"
+	cat $tmpkey 1>&2
+	Bail 3 "Hopefully that's informative"
+fi
+
+# The private key and certificate are all in one file.
+# Extract the cert into its own file, and move into place
+
+grep ssh-rsa $tmpkey > $tmpcert \
+	&& ssh-keygen -y -f $tmpkey > $tmppub \
+	&& chmod 600 $tmpkey* \
+	&& /bin/mv $tmpkey $idfile \
+	&& /bin/mv $tmppub $pubfile \
+	&& /bin/mv $tmpcert $certfile
+
+if [[ $? -ne 0 ]]; then
+	Bail 4 "An error occured after successfully downloading keys (!!!)"
+fi
+
+# A few shenanigans to clean up line formatting
+valid=$(ssh-keygen -L -f $certfile | grep Valid)
+shopt -s extglob
+valid=${valid/+( )/}
+valid=${valid/Valid/valid}
+
+
+if [[ $opt_agent -ne 0 ]]; then
+	# extract expiration date from "valid" line (above)
+	expiry=${valid/valid*to /}
+
+	distro=$(uname)
+	dateError=0
+
+	case $distro in
+		Darwin|*BSD*)
+			# Convert the date to epoch
+			expepoch=$(date -j -f '%FT%T' $expiry +%s)
+
+			# get current epoch time
+			epoch=$(date -j +%s)
+		;;
+
+		Linux|GNU|CYGWIN*)
+			# Convert the date to epoch
+			expepoch=$(date -d $expiry +%s)
+
+			# get current epoch time
+			epoch=$(date +%s)
+		;;
+
+		*)
+			Error "Unrecognized OS; I don't know how to convert a date to epoch time"
+			dateError=1
+		;;
+	esac
+
+	# compute the interval between expiration and now
+	# (minus one second just to be sure)
+	interval=$(( $expepoch - $epoch - 1 ))
+
+	# add the interval to ssh-agent
+	if [[ $dateError -gt 0 ]]; then
+		Error "Can't add key to ssh-agent."
+	elif [[ $interval -gt 0 ]]; then
+		ssh-add -t $interval $idfile
+	else
+		Error "cert $certfile is either expired or otherwise invalid. Expiration read as: $expiry"
+	fi
+
+fi
+
+# And give the user some feedback
+printf "Successfully obtained ssh key %s\n" "$idfile"
+
+printf "Key $idfile is %s\n" "$valid"

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/nmdc_runtime/api/core/metadata.py

@@ -20,7 +20,7 @@ from starlette import status
 from toolz.dicttoolz import dissoc, assoc_in, get_in
 
 from nmdc_runtime.api.models.metadata import ChangesheetIn
-from nmdc_runtime.util import get_nmdc_jsonschema_dict, collection_name_to_class_name
+from nmdc_runtime.util import get_nmdc_jsonschema_dict, collection_name_to_class_names
 
 # custom named tuple to hold path property information
 SchemaPathProperties = namedtuple(
@@ -169,7 +169,14 @@ def load_changesheet(
             class_name = data["type"].split(":")[-1]
             class_name = class_name_dict[class_name]
         else:
-            class_name = class_name_dict[collection_name_to_class_name[collection_name]]
+            class_names = collection_name_to_class_names[collection_name]
+            if len(class_names) > 1:
+                raise ValueError(
+                    "cannot unambiguously infer class of document"
+                    f" with `id` {id_} in collection {collection_name}."
+                    " Please ensure explicit `type` is present in document."
+                )
+            class_name = class_name_dict[class_names[0]]
 
         # set class name for id
         df["linkml_class"] = class_name
@@ -733,6 +740,11 @@ def _validate_changesheet(df_change: pd.DataFrame, mdb: MongoDatabase):
     for result in results_of_updates:
         if len(result.get("validation_errors", [])) > 0:
             validation_errors.append(result["validation_errors"])
+        if (
+            len(write_errors := result.get("update_info", {}).get("writeErrors", {}))
+            > 0
+        ):
+            validation_errors.append(write_errors)
 
     if validation_errors:
         raise HTTPException(

{nmdc_runtime-1.0.9 → nmdc_runtime-1.0.10}/nmdc_runtime/api/core/util.py

@@ -28,10 +28,13 @@ def hash_from_str(s: str, algo="sha256") -> str:
     return getattr(hashlib, algo)(s.encode("utf-8")).hexdigest()
 
 
-def sha256hash_from_file(file_path: str):
+def sha256hash_from_file(file_path: str, timestamp: str):
     # https://stackoverflow.com/a/55542529
     h = hashlib.sha256()
 
+    timestamp_bytes = timestamp.encode("utf-8")
+    h.update(timestamp_bytes)
+
     with open(file_path, "rb") as file:
         while True:
             # Reading is buffered, so we can read smaller chunks.