nmdc-runtime 2.10.0__tar.gz → 2.11.0__tar.gz

{nmdc_runtime-2.10.0 → nmdc_runtime-2.11.0}/.github/workflows/build-and-release-to-spin.yml

@@ -11,7 +11,8 @@ on:
       - 'Makefile'
       - '**/Dockerfile'
       - '**.py'
-      - 'requirements/main.txt'
+      - 'pyproject.toml'
+      - 'uv.lock'
 
 env:
   # We don't want to do certain steps if this is running in a fork

{nmdc_runtime-2.10.0 → nmdc_runtime-2.11.0}/.github/workflows/lint.yml

@@ -16,13 +16,18 @@ jobs:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.ref }}
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Install uv
+      uses: astral-sh/setup-uv@v6
       with:
-        python-version: '3.10'
+        # Note: Caching (of the packages that `uv` builds while resolving dependencies) is "enabled by default on GitHub-hosted runners."
+        #       Reference: https://github.com/astral-sh/setup-uv/blob/main/README.md#enable-caching
+        #       Here, we make it so the cache gets invalidated whenever the `uv.lock` file changes.
+        #       Reference: https://github.com/astral-sh/setup-uv/blob/main/README.md#cache-dependency-glob
+        cache-dependency-glob: "uv.lock"
+    - name: Set up Python virtual environment
+      run: uv sync
     - name: Lint with flake8 and Reformat with black
       run: |
-        make init-lint-and-black
         make lint
         make black
     - name: commit and push if reformatted

{nmdc_runtime-2.10.0 → nmdc_runtime-2.11.0}/.github/workflows/mkdocs.yml

@@ -11,13 +11,17 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event.repository.fork == false
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: 3.x
-      - name: Install Python package dependencies
-        run: pip install mkdocs-material mkdocs-mermaid2-plugin mkdocs-jupyter
-
+          # Note: Caching (of the packages that `uv` builds while resolving dependencies) is "enabled by default on GitHub-hosted runners."
+          #       Reference: https://github.com/astral-sh/setup-uv/blob/main/README.md#enable-caching
+          #       Here, we make it so the cache gets invalidated whenever the `uv.lock` file changes.
+          #       Reference: https://github.com/astral-sh/setup-uv/blob/main/README.md#cache-dependency-glob
+          cache-dependency-glob: "uv.lock"
+      - name: Set up Python virtual environment
+        run: uv sync --group docs
       # Build the MkDocs site (but don't deploy it).
       #
       # Note: Until January 17, 2025, we would build and deploy the MkDocs website to GitHub Pages.
@@ -39,4 +43,4 @@ jobs:
       #       file, or due to one of the dependencies becoming unavailable), which can prompt remedial action.
       #
       - name: Build MkDocs site
-        run: mkdocs build -f docs/mkdocs.yml
+        run: uv run mkdocs build -f docs/mkdocs.yml

{nmdc_runtime-2.10.0 → nmdc_runtime-2.11.0}/.github/workflows/python-app.yml

@@ -19,7 +19,8 @@ on:
       - 'Makefile'
       - '**/Dockerfile'
       - '**.py'
-      - 'requirements/main.txt'
+      - 'pyproject.toml'
+      - 'uv.lock'
       - '!util/load_testing/**'
   pull_request:
     paths:
@@ -27,7 +28,8 @@ on:
       - 'Makefile'
       - '**/Dockerfile'
       - '**.py'
-      - 'requirements/main.txt'
+      - 'pyproject.toml'
+      - 'uv.lock'
       - '!util/load_testing/**'
   # Allow developers to trigger this workflow manually via the "Actions" page on GitHub.
   # Reference: https://docs.github.com/en/actions/managing-workflow-runs-and-deployments/managing-workflow-runs/manually-running-a-workflow
@@ -73,16 +75,22 @@ jobs:
           sh -c 'cp /mongoKeyFile /out/mongoKeyFile && chmod 600 /out/mongoKeyFile && chown 999:999 /out/mongoKeyFile'
         mv _tmp/mongoKeyFile ./.docker/mongoKeyFile
         rmdir _tmp
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.10'
-# deprecated: Consider merging python.app.yml and lint.yml
-#    - name: Lint with flake8
-#      run: |
-#        pip install flake8
-#        make lint
     - name: Build and start containers upon which test runner depends
       run: make up-test
+    # When running this GHA workflow job with the "Enable debug logging" checkbox marked
+    # in the GitHub web UI; wait a few seconds, then dump the `fastapi` container logs.
+    #
+    # Note: This is in an attempt to facilitate debugging. The failure of that container to
+    #       start up has been a relatively common source of GitHub Actions workflow failures
+    #       (and we think seeing the container's logs will make debugging easier).
+    #
+    - name: Sleep
+      if: runner.debug == '1'
+      uses: juliangruber/sleep-action@v2.0.0
+      with:
+        time: 20s
+    - name: Dump FastAPI container logs (for debugging)
+      if: runner.debug == '1'
+      run: docker compose --file docker-compose.test.yml logs fastapi
     - name: Run tests
       run: make run-test

nmdc_runtime-2.11.0/.github/workflows/release-to-pypi.yml

@@ -0,0 +1,37 @@
+name: Publish Python Package
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build-n-publish:
+    name: Build and publish Python 🐍 distributions 📦 to PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          # Note: Caching (of the packages that `uv` builds while resolving dependencies) is "enabled by default on GitHub-hosted runners."
+          #       Reference: https://github.com/astral-sh/setup-uv/blob/main/README.md#enable-caching
+          #       Here, we make it so the cache gets invalidated whenever the `uv.lock` file changes.
+          #       Reference: https://github.com/astral-sh/setup-uv/blob/main/README.md#cache-dependency-glob
+          cache-dependency-glob: "uv.lock"
+      - name: Set up Python virtual environment
+        run: uv sync
+
+      - name: Preview derived version number
+        run: uvx uv-dynamic-versioning
+      # Here, we build the package using the build settings specified in `pyproject.toml`,
+      # one of which is to derive the package's version number from the Git repository.
+      - name: Build source and wheel archives
+        run: uv build
+
+      - name: Publish distribution 📦 to PyPI
+        if: github.repository == 'microbiomedata/nmdc-runtime'
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_PASSWORD }}

{nmdc_runtime-2.10.0 → nmdc_runtime-2.11.0}/Makefile

@@ -1,43 +1,8 @@
-init:
-	pip install -r requirements/main.txt
-	pip install -r requirements/dev.txt
-	pip install --editable .[dev]
-
-# Updates Python dependencies based upon the contents of the `requirements/main.in` and `requirements/dev.in` files.
-#
-# Note: To omit the `--upgrade` option (included by default) from the constituent `pip-compile` commands:
-#       ```
-#       $ make update-deps UPDATE_DEPS_MAIN_UPGRADE_OPT='' UPDATE_DEPS_DEV_UPGRADE_OPT=''
-#       ```
-#
-# Note: You can run the following command on your host machine to have `$ make update-deps` run within a container:
-#       ```sh
-#       $ docker compose run --rm --no-deps fastapi sh -c 'make update-deps'
-#       ```
-#       To omit the `--upgrade` option (included by default) from the constituent `pip-compile` commands:
-#       ```sh
-#       $ docker compose run --rm --no-deps fastapi sh -c 'make update-deps UPDATE_DEPS_MAIN_UPGRADE_OPT="" UPDATE_DEPS_DEV_UPGRADE_OPT=""'
-#       ```
-#
-UPDATE_DEPS_MAIN_UPGRADE_OPT ?= --upgrade
-UPDATE_DEPS_DEV_UPGRADE_OPT ?= --upgrade
-update-deps:
-    # --allow-unsafe pins packages considered unsafe: distribute, pip, setuptools.
-	pip install --upgrade pip-tools pip setuptools
-	pip-compile $(UPDATE_DEPS_MAIN_UPGRADE_OPT) --build-isolation \
-		--allow-unsafe --resolver=backtracking --strip-extras \
-		--output-file requirements/main.txt \
-		requirements/main.in
-	pip-compile --allow-unsafe $(UPDATE_DEPS_DEV_UPGRADE_OPT) --build-isolation \
-		--allow-unsafe --resolver=backtracking --strip-extras \
-		--output-file requirements/dev.txt \
-		requirements/dev.in
-
-update: update-deps init
-
+# Spin up the development stack.
 up-dev:
 	docker compose up --build --force-recreate --detach --remove-orphans
 
+# Spin down the development stack.
 down-dev:
 	docker compose down
 
@@ -51,7 +16,7 @@ up-test:
 	docker compose --file docker-compose.test.yml \
 		up --build --force-recreate --detach --remove-orphans
 
-# Tears down the `test` stack, including removing data volumes such as that for the test MongoDB database.
+# Tears down the `test` stack, including removing data volumes such as that containing the test MongoDB database.
 down-test:
 	docker compose --file docker-compose.test.yml down --volumes
 
@@ -80,40 +45,38 @@ reset-db-test:
 run-test:
 	docker compose --file docker-compose.test.yml exec -it test \
 		./.docker/wait-for-it.sh fastapi:8000 --strict --timeout=300 -- \
-			pytest --cov=nmdc_runtime \
-			       --doctest-modules \
-			       --ignore=util/load_testing \
-			       $(ARGS)
+			uv run --active \
+				pytest --cov=nmdc_runtime \
+				       --doctest-modules \
+				       --ignore=util/load_testing \
+				       $(ARGS)
 
 # Uses Docker Compose to
-# 1. Ensure the `test` stack is torn down, including data volumes such as that of the test MongoDB database.
+# 1. Ensure the `test` stack is torn down, including data volumes such as that containing the test MongoDB database.
 # 2. Build and spin up the stack upon which the `test` container (i.e. the test runner) depends.
 # 3. Run tests on the `test` container, passing `ARGS` to `pytest` (see Tip in comment above for `run-test` target).
 test: down-test up-test run-test
 
+# Format Python code using `black`.
+# TODO: Migrate from `black` to `ruff`.
 black:
-	black nmdc_runtime
+	uv run --active black nmdc_runtime
 
+# Lint Python code using `flake8`.
+# TODO: Migrate from `flake8` to `ruff`.
 lint:
 	# Python syntax errors or undefined names
-	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F722
+	uv run --active flake8 --count --select=E9,F63,F7,F82 --show-source --statistics --extend-ignore=F722 \
+		./nmdc_runtime ./tests
 	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 \
-		--statistics --extend-exclude="./build/" --extend-ignore=F722
-
-PIP_PINNED_FLAKE8 := $(shell grep 'flake8==' requirements/dev.txt)
-PIP_PINNED_BLACK := $(shell grep 'black==' requirements/dev.txt)
-
-init-lint-and-black:
-	pip install $(PIP_PINNED_FLAKE8)
-	pip install $(PIP_PINNED_BLACK)
-
-follow-fastapi:
-	docker compose logs fastapi -f
+	uv run --active flake8 --count --exit-zero --max-complexity=10 --max-line-length=127 \
+		--statistics --extend-exclude="./build/" --extend-ignore=F722 \
+		./nmdc_runtime ./tests
 
 # Build the MkDocs documentation website and serve it at http://localhost:8080.
 # Docs: https://www.mkdocs.org/user-guide/cli/#mkdocs-serve
 docs-dev:
+	uv sync --group docs
 	mkdocs serve --config-file docs/mkdocs.yml --dev-addr localhost:8080
 
 # 🙋 Prerequisites:
@@ -134,6 +97,8 @@ nersc-mongo-tunnels:
 		-o ServerAliveInterval=60 \
 		${NERSC_USERNAME}@dtn02.nersc.gov
 
+DEV_STACK_HOST_MACHINE_PORT_MONGO:=$(shell cat .env | grep DEV_STACK_HOST_MACHINE_PORT_MONGO= | cut -d= -f2)
+
 mongorestore-nmdc-db:
 	mkdir -p /tmp/remote-mongodump/nmdc
 	# Optionally, manually update MONGO_REMOTE_DUMP_DIR env var:
@@ -153,15 +118,19 @@ mongorestore-nmdc-db:
 		${NERSC_USERNAME}@dtn01.nersc.gov:${MONGO_REMOTE_DUMP_DIR}/nmdc/ \
 		/tmp/remote-mongodump/nmdc
 	# Restore from `rsync`ed local directory:
-	mongorestore -v -h localhost:59639 -u admin -p root --authenticationDatabase=admin \
+	mongorestore -v -h localhost:$(DEV_STACK_HOST_MACHINE_PORT_MONGO) -u admin -p root --authenticationDatabase=admin \
 		--drop --nsInclude='nmdc.*' --gzip --dir /tmp/remote-mongodump
 
 quick-blade:
-	python -c "from nmdc_runtime.api.core.idgen import generate_id; print(f'nmdc:nt-11-{generate_id(length=8, split_every=0)}')"
+	uv run --active python -c "from nmdc_runtime.api.core.idgen import generate_id; print(f'nmdc:nt-11-{generate_id(length=8, split_every=0)}')"
 
-.PHONY: init update-deps update up-dev down-dev follow-fastapi \
-	publish docs
+# List of Make targets that do not represent files being created.
+# Note: I think _most_ of the targets in this Makefile meet that criterion,
+#       despite them not being listed here.
+.PHONY: up-dev down-dev publish docs quick-blade
 
+# Note: The leading hyphen in each command below tells `make` to ignore whether
+#       the command fails (i.e. returns a non-zero exit code).
 docker-clean:
 	# Down the dev stack with volumes
 	-docker compose down --volumes --remove-orphans

nmdc_runtime-2.11.0/PKG-INFO

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: nmdc-runtime
+Version: 2.11.0
+Summary: A runtime system for NMDC data management and orchestration
+Project-URL: Changelog, https://github.com/microbiomedata/nmdc-runtime/releases
+Project-URL: Documentation, https://docs.microbiomedata.org/runtime
+Project-URL: Issues, https://github.com/microbiomedata/nmdc-runtime/issues
+Project-URL: Repository, https://github.com/microbiomedata/nmdc-runtime
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: base32-lib
+Requires-Dist: boto3
+Requires-Dist: click
+Requires-Dist: dagit
+Requires-Dist: dagster
+Requires-Dist: dagster-graphql
+Requires-Dist: dagster-postgres
+Requires-Dist: fastapi>=0.115.0
+Requires-Dist: fastjsonschema
+Requires-Dist: frozendict
+Requires-Dist: git-root
+Requires-Dist: jq
+Requires-Dist: jsonasobj2
+Requires-Dist: linkml
+Requires-Dist: linkml-runtime
+Requires-Dist: lxml
+Requires-Dist: nmdc-schema==11.11.1
+Requires-Dist: ontology-loader==0.2.2
+Requires-Dist: pandas
+Requires-Dist: passlib[bcrypt]
+Requires-Dist: pydantic[email]>=1.10.0
+Requires-Dist: pyinstrument
+Requires-Dist: pymongo
+Requires-Dist: python-dotenv
+Requires-Dist: python-jose[cryptography]
+Requires-Dist: python-multipart>=0.0.18
+Requires-Dist: pyyaml
+Requires-Dist: refscan==0.3.2
+Requires-Dist: requests
+Requires-Dist: requests-cache
+Requires-Dist: scalar-fastapi<2.0.0,>=1.4.1
+Requires-Dist: tenacity
+Requires-Dist: toolz
+Requires-Dist: tqdm
+Requires-Dist: unidecode
+Requires-Dist: uvicorn[standard]

nmdc_runtime-2.10.0/nmdc_runtime.egg-info/PKG-INFO → nmdc_runtime-2.11.0/README.md

@@ -1,26 +1,3 @@
-Metadata-Version: 2.4
-Name: nmdc_runtime
-Version: 2.10.0
-Summary: A runtime system for NMDC data management and orchestration
-Home-page: https://github.com/microbiomedata/nmdc-runtime
-Author: Donny Winston
-Author-email: donny@polyneme.xyz
-Classifier: Development Status :: 3 - Alpha
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-python
-Dynamic: summary
-
 A runtime system for NMDC data management and orchestration.
 
 ## Service Status
@@ -150,15 +127,36 @@ The Dagit web server is viewable at http://127.0.0.1:3000/.
 The FastAPI service is viewable at http://127.0.0.1:8000/ -- e.g., rendered documentation at
 http://127.0.0.1:8000/redoc/.
 
+### Dependency management
 
-*  NOTE: Any time you add or change requirements in requirements/main.in or requirements/dev.in, you must run:
-```bash
-pip-compile --build-isolation --allow-unsafe --resolver=backtracking --strip-extras --output-file requirements/[main|dev].txt requirements/[main|dev].in
+We use [`uv`](https://docs.astral.sh/uv/) to manage dependencies of the application. Here's how you can use `uv` both on your host machine and within a container in the Docker Compose stack.
+
+#### On the host
+
+Although we typically run the application within a container, some developers prefer that application's dependencies be installed locally also (so that their code editors will provide auto-completion, type checking, etc.).
+
+Here's how you can install the application's dependencies locally:
+
+```sh
+uv sync
+```
+
+That will...
+1. (If you made changes to `pyproject.toml`) **Update the lock file** (at `uv.lock`) to reflect those changes
+2. (If a Python virtual environment doesn't exist at `.venv/` yet) **Create a Python virtual environment** at `.venv/`
+3. (If the Python virtual environment and `uv.lock` files are out of sync) **Synchronize the Python virtual environment** with `uv.lock` (by installing and uninstalling packages)
+
+> Note: Long term, we may implement a [devcontainer](https://containers.dev/) for this project, which will streamline the process of setting up a local development environment.
+
+#### In the Docker Compose stack
+
+In the Docker Compose stack, the Python virtual environment is located at the path specified by the `VIRTUAL_ENV` environment variable (which is defined in the `Dockerfile`) instead of at `.venv/`. That helps with containerization, but it deviates from `uv`'s default behavior, which is to use the Python virtual environment at `.venv/`. So, when running `uv` commands within the Docker Compose stack, we always include the [`--active`](https://docs.astral.sh/uv/reference/cli/#uv-sync--active) flag (which tells `uv` to use the Python virtual environment at the path specified by `VIRTUAL_ENV`).
+
+Here's how you can install the application's dependencies within a container in the Docker Compose stack:
+
+```sh
+uv sync --active
 ```
-to generate main.txt and dev.txt files respectively. main.in is kind of like a poetry dependency stanza, dev.in is kind 
-of like poetry dev.dependencies stanza. main.txt and dev.txt are kind of like poetry.lock files to specify the exact 
-versions of dependencies to use. main.txt and dev.txt are combined in the docker compose build process to create the 
-final requirements.txt file and import the dependencies into the Docker image.
 
 ## Local Testing
 
@@ -247,19 +245,11 @@ found **12 GB** to be sufficient for running the tests.
 
 This repository contains a GitHub Actions workflow that publishes a Python package to [PyPI](https://pypi.org/project/nmdc-runtime/).
 
-You can also _manually_ publish the Python package to PyPI by issuing the following commands in the root directory of the repository:
-
-```
-rm -rf dist
-python -m build
-twine upload dist/*
-```
-
 ## Links
 
 Here are links related to this repository:
 
 - Production API server: https://api.microbiomedata.org
 - PyPI package: https://pypi.org/project/nmdc-runtime
-- DockerHub image (API server): https://hub.docker.com/r/microbiomedata/nmdc-runtime-fastapi
-- DockerHub image (Dagster): https://hub.docker.com/r/microbiomedata/nmdc-runtime-dagster
+- Container image (API server): https://github.com/microbiomedata/nmdc-runtime/pkgs/container/nmdc-runtime-fastapi
+- Container image (Dagster): https://github.com/microbiomedata/nmdc-runtime/pkgs/container/nmdc-runtime-dagster