nmdc-runtime 2.10.0__py3-none-any.whl → 2.11.1__py3-none-any.whl

nmdc_runtime/Dockerfile

@@ -0,0 +1,177 @@
+# Note: Most of the steps for the `base` image were copied verbatim from either `fastapi.Dockerfile`,
+#       `dagster.Dockerfile`, or `test.Dockerfile` (indeed, most of the steps were present in all three files).
+#       Reference: https://docs.docker.com/get-started/docker-concepts/building-images/multi-stage-builds/
+#
+# Base this image upon a variant of the official Python 3.10 image that is, in turn,
+# based upon a minimal (slim) variant of the Debian 11 (bullseye) image.
+# Reference: https://hub.docker.com/_/python
+# ────────────────────────────────────────────────────────────────────────────┐
+FROM python:3.10-slim-bullseye AS base
+# ────────────────────────────────────────────────────────────────────────────┘
+
+# Install and upgrade system-level software in a non-interactive way, then delete temporary files.
+# Note: Setting `DEBIAN_FRONTEND=noninteractive` and passing `-y` to `apt-get` makes things non-interactive.
+RUN export DEBIAN_FRONTEND=noninteractive && \
+  apt-get update && \
+  apt-get -y upgrade && \
+  apt-get install -y --no-install-recommends \
+    tini \
+    procps \
+    net-tools \
+    build-essential \
+    git \
+    make \
+    zip \
+    curl \
+    wget \
+    gnupg && \
+  apt-get -y clean && \
+  rm -rf /var/lib/apt/lists/*
+
+# Enable Python's "fault handler" feature, so, when low-level errors occur (e.g. segfaults), Python prints lots of info.
+# Reference: https://docs.python.org/3/using/cmdline.html#envvar-PYTHONFAULTHANDLER
+ENV PYTHONFAULTHANDLER=1
+
+# Configure Git to consider the `/code` directory to be "safe", so that, when a Git repository
+# created outside of the container gets mounted at that path within the container, the
+# `uv-dynamic-versioning` tool running within the container does not fail with the error:
+# > "Detected Git repository, but failed because of dubious ownership"
+# Reference: https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory
+RUN git config --global --add safe.directory /code
+
+# Install `uv`.
+# Reference: https://docs.astral.sh/uv/guides/integration/docker/#installing-uv
+ADD https://astral.sh/uv/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && \
+    rm /uv-installer.sh
+ENV PATH="/root/.local/bin/:$PATH"
+
+# Install Python dependencies (production dependencies only).
+#
+# Note: We copy only the files that `uv` needs in order to install dependencies. That way,
+#       we minimize the number of files whose changes would invalidate cached image layers
+#
+# Note: We use the `VIRTUAL_ENV` environment variable to specify the path to the Python virtual
+#       environment that we want the `uv` program inside the container to create and use.
+#
+#       Q: Why don't we use `./.venv` in the repository file tree?
+#       A: If we were to do that, then, whenever a developer would mount (via our Docker Compose file)
+#          the repository file tree from their host machine (which may include a `.venv/` directory
+#          created by their host machine) into the container, it would overwrite the Python virtual
+#          environment that the `uv` program inside the container is using.
+#
+#       Q: What is special about the `VIRTUAL_ENV` environment variable?
+#       A: When using `uv`'s `--active` option (as we do in later stages of this Dockerfile),
+#          `uv` determines which virtual environment is active by looking at `VIRTUAL_ENV'. This
+#          is the case, even though the documentation of the `venv` module (in Python's standard
+#          library) specifically says: "`VIRTUAL_ENV` cannot be relied upon to determine whether
+#          a virtual environment is being used."
+#
+#       References:
+#       - https://docs.astral.sh/uv/pip/environments/#using-arbitrary-python-environments (RE: `VIRTUAL_ENV`)
+#       - https://docs.astral.sh/uv/reference/environment/#virtual_env (RE: `VIRTUAL_ENV`, from uv's perspective)
+#       - https://docs.python.org/3/library/venv.html#how-venvs-work (RE: `VIRTUAL_ENV`, from venv's perspective)
+#       - https://docs.astral.sh/uv/concepts/projects/sync/#partial-installations (RE: `--no-install-project`)
+#
+# Note: In the `RUN` command, we use a "cache mount" (a feature of Docker) to cache production dependencies
+#       across builds. This is a performance optimization technique shown in the `uv` docs.
+#       Reference:
+#       - https://docs.astral.sh/uv/guides/integration/docker/#caching (RE: the technique)
+#       - https://docs.docker.com/build/cache/optimize/#use-cache-mounts (RE: the feature)
+#       - https://docs.astral.sh/uv/reference/settings/#link-mode (RE: `UV_LINK_MODE`)
+#       - https://docs.astral.sh/uv/reference/cli/#uv-sync--no-install-project (RE: `--no-install-project`)
+#
+# Note: We use `--compile-bytecode` so that Python compiles `.py` files to `.pyc` files now,
+#       instead of when the container is running. By default, `uv` defers this compilation
+#       to "import time," whereas `pip` (by default) performs it at "install time" (like this).
+#
+# Note: We use `--locked` so that `uv sync` exits with an error if the `uv.lock` file isn't _already_
+#       up to date. By default, `uv sync` would automatically update the lock file if necessary.
+#       Reference: https://docs.astral.sh/uv/reference/cli/#uv-sync--locked
+#
+ENV VIRTUAL_ENV="/venv"
+RUN mkdir -p "${VIRTUAL_ENV}"
+COPY ./pyproject.toml /code/pyproject.toml
+COPY ./uv.lock        /code/uv.lock
+RUN --mount=type=cache,target=/root/.cache/uv \
+    cd /code && \
+    UV_LINK_MODE=copy uv sync --active --no-dev --no-install-project --compile-bytecode --locked
+
+# ────────────────────────────────────────────────────────────────────────────┐
+FROM base AS fastapi
+# ────────────────────────────────────────────────────────────────────────────┘
+
+# Copy repository contents into image.
+COPY . /code
+
+# Install the project in editable mode.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    cd /code && \
+    uv sync --active --no-dev --compile-bytecode --locked
+
+# Use Uvicorn to serve the FastAPI app on port 8000.
+#
+# Note: We include the `--no-sync` option to prevent `uv run` from automatically syncing dependencies.
+#       If it were to sync dependencies at this point, it would install development dependencies, since
+#       we exclude them above, but they are listed in uv's `default-groups` configuration by default.
+#       This is explained at: https://github.com/astral-sh/uv/issues/12558#issuecomment-2764611918
+#
+EXPOSE 8000
+WORKDIR /code
+CMD ["uv", "run", "--active", "--no-sync", "uvicorn", "nmdc_runtime.api.main:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "8000"]
+
+# ────────────────────────────────────────────────────────────────────────────┐
+FROM base AS dagster
+# ────────────────────────────────────────────────────────────────────────────┘
+
+# Copy repository contents into image.
+#
+# Note: This path (i.e. "/opt/dagster/lib/") is hard-coded in a few places in `nmdc_runtime/site/ops.py`. That's why
+#       this image does not store the repository contents in `/code`, unlike the other images in this Dockerfile.
+#
+COPY . /opt/dagster/lib
+
+# Install the project in editable mode.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    cd /opt/dagster/lib && \
+    uv sync --active --no-dev --compile-bytecode --locked
+
+# Move Dagster configuration files to the place Dagster expects.
+ENV DAGSTER_HOME="/opt/dagster/dagster_home/"
+RUN mkdir -p                                             "${DAGSTER_HOME}" && \
+    cp /opt/dagster/lib/nmdc_runtime/site/dagster.yaml   "${DAGSTER_HOME}" && \
+    cp /opt/dagster/lib/nmdc_runtime/site/workspace.yaml "${DAGSTER_HOME}"
+
+# Use Tini to run Dagit.
+#
+# Notes:
+# - The port number (i.e. "3000") is hard-coded in `nmdc_runtime/site/entrypoint-dagit.sh`.
+# - Dagster daemon (versus Dagit) can be launched by overriding the `ENTRYPOINT` defined here.
+#
+# Reference: https://github.com/krallin/tini
+#
+EXPOSE 3000
+WORKDIR /opt/dagster/dagster_home/
+ENTRYPOINT ["tini", "--", "../lib/nmdc_runtime/site/entrypoint-dagit.sh"]
+
+# ────────────────────────────────────────────────────────────────────────────┐
+FROM base AS test
+# ────────────────────────────────────────────────────────────────────────────┘
+
+# Copy all repository contents into image.
+COPY . /code
+
+# Install the project in editable mode, and install development dependencies.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    cd /code && \
+    uv sync --active --compile-bytecode --locked
+
+# Make `wait-for-it.sh` executable.
+RUN chmod +x /code/.docker/wait-for-it.sh
+
+WORKDIR /code
+
+# Ensure started container does not exit, so that a subsequent `docker exec` command can run tests.
+# For an example `docker exec` command, see `Makefile`'s `run-test` target.
+# Such a command should use `wait-for-it.sh` to run `pytest` no earlier than when the FastAPI server is accessible.
+ENTRYPOINT ["tail", "-f", "/dev/null"]

nmdc_runtime/api/analytics.py

@@ -16,25 +16,42 @@ from toolz import merge
 
 from nmdc_runtime.api.db.mongo import get_mongo_db
 
+# This is a queue of the "request descriptors" that we will eventually insert into the database.
 _requests = []
 _last_posted = datetime.now()
 
 
 def _post_requests(collection: str, requests_data: List[Dict], source: str):
+    """Inserts the specified request descriptors into the specified MongoDB collection."""
     mdb = get_mongo_db()
     mdb[collection].insert_many([merge(d, {"source": source}) for d in requests_data])
 
 
 def log_request(collection: str, request_data: Dict, source: str = "FastAPI"):
+    """Flushes the queue of request descriptors to the database if enough time has passed since the previous time."""
     global _requests, _last_posted
     _requests.append(request_data)
     now = datetime.now()
     # flush queue every minute at most
     if (now - _last_posted).total_seconds() > 60.0:
+        # Note: This use of threading is an attempt to avoid blocking the current thread
+        #       while performing the insertion(s).
+        #
+        # TODO: Is there is a race condition here? If multiple requests arrive at approximately
+        #       the same time, is it possible that each one causes a different thread to be
+        #       started, each with a different (and possibly overlapping) set of requests to
+        #       insert?
+        #
+        # TODO: If the insertion fails, will the requests be lost?
+        #
+        # Note: The author of this function said it may have been a "standard" solution copied
+        #       from some documentation. Indeed, the comment at the top of this module contains
+        #       a link to code on which it was based.
+        #
         threading.Thread(
             target=_post_requests, args=(collection, _requests, source)
         ).start()
-        _requests = []
+        _requests = []  # empties the queue
         _last_posted = now
 
 
@@ -49,6 +66,9 @@ class Analytics(BaseHTTPMiddleware):
         start = time()
         response = await call_next(request)
 
+        # Use a fallback IP address value (currently an empty string) if we can't derive one from the request.
+        ip_address: str = "" if request.client is None else request.client.host
+
         # Build a dictionary that describes the incoming request.
         #
         # Note: `request.headers` is an instance of `MultiDict`. References:
@@ -57,7 +77,7 @@ class Analytics(BaseHTTPMiddleware):
         #
         request_data = {
             "hostname": request.url.hostname,
-            "ip_address": request.client.host,
+            "ip_address": ip_address,
             "path": request.url.path,
             "user_agent": request.headers.get("user-agent"),
             "method": request.method,

nmdc_runtime/api/core/idgen.py

@@ -89,7 +89,35 @@ def generate_ids(
     shoulder: str = "fk4",
 ) -> List[str]:
     r"""
-    TODO: Document this function.
+    Generate the specified number of identifiers, storing them in a MongoDB collection
+    whose name is derived from the specified Name-Assigning Authority (NAA) and Shoulder.
+
+    :param mdb: Handle to a MongoDB database
+    :param owner: String that will go in the "__ao" field of the identifier record.
+                  Callers will oftentimes set this to the name of a Runtime "site"
+                  (as in, a "site client" site, not a "Dagster" site).
+    :param populator: String that will go in the "who" field of the identifier record.
+                      Indicates "who generated this ID." Callers will oftentimes set
+                      this to the name of a Runtime "site" (as in, a "site client" site,
+                      not a "Dagster" site).
+    :param ns: Namespace (see Minter docs); e.g. "changesheets"
+    :param naa: Name-Assigning Authority (see Minter docs); e.g. "nmdc"
+    :param shoulder: String that will go in the "how" field (see Minter docs); e.g. "sys0"
+
+    This function was written the way it was in an attempt to mirror the ARK spec:
+    https://www.ietf.org/archive/id/draft-kunze-ark-41.html (found via: https://arks.org/specs/)
+
+    Deviations from the ARK spec include:
+    1. The inclusion of a typecode.
+       The inclusion of a typecode came out of discussions with team members,
+       who wanted identifiers to include some non-opaque substring that could be used
+       to determine what type of resource a given identifier refers to.
+    2. Making hyphens mandatory.
+       We decided to make the hyphens mandatory, whereas the spec says they are optional.
+       > "Hyphens are considered to be insignificant and are always ignored in ARKs."
+       > Reference: https://www.ietf.org/archive/id/draft-kunze-ark-41.html#name-character-repertoires
+       In our case, we require that users include an identifier's hyphens whenever
+       they are using that identifier.
     """
     collection = mdb.get_collection(collection_name(naa, shoulder))
     estimated_document_count = collection.estimated_document_count()
@@ -119,7 +147,9 @@ def generate_ids(
         if not_taken:
             # All attribute names beginning with "__a" are reserved...
             # https://github.com/jkunze/n2t-eggnog/blob/0f0f4c490e6dece507dba710d3557e29b8f6627e/egg#L1882
-            # XXX mongo is a pain with '.'s in field names, so not using e.g. "_.e" names.
+            # The author of this function opted to refrain from using property names beginning with "_.e",
+            # because he thought it would complicate MongoDB queries involving those properties, given that
+            # the "." is used as a field delimiter in MongoDB syntax (e.g. "foo.bar.baz").
             docs = [
                 {
                     "@context": "https://n2t.net/e/n2t_apidoc.html#identifier-metadata",
@@ -145,9 +175,9 @@ def generate_ids(
 
 
 def generate_one_id(
-    mdb: MongoDatabase = None,
+    mdb: MongoDatabase,
     ns: str = "",
-    shoulder: str = "sys0",
+    shoulder: str = "sys0",  # "sys0" represents the Runtime
 ) -> str:
     """Generate unique Crockford Base32-encoded ID for mdb repository.
 
@@ -156,8 +186,8 @@ def generate_one_id(
     """
     return generate_ids(
         mdb,
-        owner="_system",
-        populator="_system",
+        owner="_system",  # "_system" represents the Runtime
+        populator="_system",  # "_system" represents the Runtime
         number=1,
         ns=ns,
         naa="nmdc",

nmdc_runtime/api/db/mongo.py

@@ -10,7 +10,6 @@ import bson
 from jsonschema import Draft7Validator
 from nmdc_schema.nmdc import Database as NMDCDatabase
 from pymongo.errors import AutoReconnect, OperationFailure
-from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase
 from refscan.lib.Finder import Finder
 from refscan.scanner import scan_outgoing_references
 from tenacity import wait_random_exponential, retry, retry_if_exception_type
@@ -83,17 +82,6 @@ def get_session_bound_mongo_db(session=None) -> MongoDatabase:
     return SessionBoundDatabase(mdb, session) if session is not None else mdb
 
 
-@lru_cache
-def get_async_mongo_db() -> AsyncIOMotorDatabase:
-    _client = AsyncIOMotorClient(
-        host=os.getenv("MONGO_HOST"),
-        username=os.getenv("MONGO_USERNAME"),
-        password=os.getenv("MONGO_PASSWORD"),
-        directConnection=True,
-    )
-    return _client[os.getenv("MONGO_DBNAME")]
-
-
 def get_nonempty_nmdc_schema_collection_names(mdb: MongoDatabase) -> Set[str]:
     """
     Returns the names of the collections that (a) exist in the database,