nmdc-runtime 2.10.0__py3-none-any.whl → 2.11.1__py3-none-any.whl

nmdc_runtime/site/drsobjects/ingest.py

@@ -1,93 +0,0 @@
-import json
-from datetime import datetime, timezone
-
-from toolz import dissoc
-
-from nmdc_runtime.api.models.job import JobOperationMetadata
-from nmdc_runtime.api.models.operation import Operation
-from nmdc_runtime.api.models.operation import UpdateOperationRequest
-from nmdc_runtime.api.models.util import ListRequest
-from nmdc_runtime.api.models.util import ResultT
-
-
-def load_local_json(url, prefixes_url_to_local=None):
-    """Useful for large files cached on local filesystem.
-
-    You may, for example, `cp --parents ` many files on a remote filesystem to a staging
-    folder on that remote filesystem, gzip that folder, scp it to your local machine, and then
-    extract to your local machine.
-
-    Example:
-    prefixes_url_to_local = {
-        "https://data.microbiomedata.org/data/": "/Users/dwinston/nmdc_files/2021-09-scanon-meta/ficus/pipeline_products/",
-        "https://portal.nersc.gov/project/m3408/": "/Users/dwinston/nmdc_files/2021-09-scanon-meta/www/",
-    }
-    """
-    path = url
-    for before, after in prefixes_url_to_local.items():
-        path = path.replace(before, after)
-    with open(path) as f:
-        return json.load(f)
-
-
-def claim_metadata_ingest_jobs(
-    client, drs_object_ids_to_ingest, wf_id, max_page_size=1000
-):
-    lr = ListRequest(
-        filter=json.dumps(
-            {
-                "workflow.id": wf_id,
-                "config.object_id": {"$in": drs_object_ids_to_ingest},
-            }
-        ),
-        max_page_size=max_page_size,
-    )
-    jobs = []
-    while True:
-        rv = client.list_jobs(lr.model_dump()).json()
-        jobs.extend(rv["resources"])
-        if "next_page_token" not in rv:
-            break
-        else:
-            lr.page_token = rv["next_page_token"]
-
-        # safety escape
-        if len(jobs) == len(drs_object_ids_to_ingest):
-            break
-
-    job_claim_responses = [client.claim_job(j["id"]) for j in jobs]
-
-    return job_claim_responses
-
-
-def mongo_add_docs_result_as_dict(rv):
-    return {
-        collection_name: dissoc(bulk_write_result.bulk_api_result, "upserted")
-        for collection_name, bulk_write_result in rv.items()
-    }
-
-
-def get_metadata_ingest_job_ops(mongo, wf_id, drs_object_ids_to_ingest):
-    return list(
-        mongo.db.operations.find(
-            {
-                "metadata.job.workflow.id": wf_id,
-                "metadata.job.config.object_id": {"$in": drs_object_ids_to_ingest},
-                "done": False,
-            }
-        )
-    )
-
-
-def do_metadata_ingest_job(client, mongo, job_op_doc):
-    op = Operation[ResultT, JobOperationMetadata](**job_op_doc)
-    object_info = client.get_object_info(op.metadata.job.config["object_id"]).json()
-    url = object_info["access_methods"][0]["access_url"]["url"]
-    docs = load_local_json(url)
-    op_result = mongo.add_docs(docs, validate=False, replace=False)
-    op_patch = UpdateOperationRequest(
-        done=True,
-        result=mongo_add_docs_result_as_dict(op_result),
-        metadata={"done_at": datetime.now(timezone.utc).isoformat(timespec="seconds")},
-    )
-    return client.update_operation(op.id, op_patch)

nmdc_runtime/site/drsobjects/registration.py

@@ -1,131 +0,0 @@
-import json
-import os
-import re
-from datetime import datetime, timezone, timedelta
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-import requests
-from bs4 import BeautifulSoup
-
-from nmdc_runtime.api.models.object import DrsObjectIn
-from nmdc_runtime.util import (
-    drs_metadata_for,
-    nmdc_jsonschema_validator,
-    specialize_activity_set_docs,
-)
-
-pattern = re.compile(r"https?://(?P<domain>[^/]+)/(?P<path>.+)")
-
-
-def url_to_name(url):
-    m = pattern.match(url)
-    return (
-        f"{'.'.join(reversed(m.group('domain').split('.')))}"
-        f"__{m.group('path').replace('/', '.')}"
-    )
-
-
-def fetch_url(url, timeout=30):
-    return requests.get(url, timeout=timeout)
-
-
-class HttpResponseNotOk(Exception):
-    pass
-
-
-class HttpResponseNotJson(Exception):
-    pass
-
-
-def response_to_json(response):
-    if response.status_code != 200:
-        raise HttpResponseNotOk()
-    try:
-        json_data = response.json()
-    except ValueError:
-        raise HttpResponseNotJson()
-    return json_data
-
-
-def json_data_from_url_to_file(json_data, url, save_dir):
-    filepath = os.path.join(save_dir, url_to_name(url))
-    with open(filepath, "w") as f:
-        json.dump(json_data, f)
-    return filepath
-
-
-def json_clean(d, model, exclude_unset=False):
-    return json.loads(model(**d).json(exclude_unset=exclude_unset))
-
-
-def drs_object_in_for(url):
-    with TemporaryDirectory() as save_dir:
-        response = fetch_url(url)
-        try:
-            json_data = response_to_json(response)
-        except HttpResponseNotOk:
-            return {"error": "HttpResponseNotOk"}
-
-        except HttpResponseNotJson:
-            return {"error": "HttpResponseNotJson"}
-
-        filepath = json_data_from_url_to_file(json_data, url, save_dir)
-        drs_object_in = DrsObjectIn(
-            **drs_metadata_for(
-                filepath,
-                {
-                    "access_methods": [{"access_url": {"url": url}}],
-                    "name": Path(filepath).name.replace(":", "-"),
-                },
-            )
-        )
-        return {"result": drs_object_in}
-
-
-def create_drs_object_for(url, drs_object_in, client):
-    rv = client.create_object(json.loads(drs_object_in.json(exclude_unset=True)))
-    return {"url": url, "response": rv}
-
-
-def validate_as_metadata_and_ensure_tags_for(
-    drs_id, client, tags=("schema#/definitions/Database", "metadata-in")
-):
-    docs = client.get_object_bytes(drs_id).json()
-    docs, _ = specialize_activity_set_docs(docs)
-    _ = nmdc_jsonschema_validator(docs)
-    return {tag: client.ensure_object_tag(drs_id, tag) for tag in tags}
-
-
-def recent_metadata_urls(
-    urlpath="https://portal.nersc.gov/project/m3408/meta/anno2/",
-    urlpath_extra="?C=M;O=D",
-    since="2021-09",
-):
-    """Scrapes recent URLs from Apache/2.4.38 (Debian) Server listing.
-
-    Designed with urlpath.startwsith("https://portal.nersc.gov/project/m3408/") in mind.
-    """
-    if since is None:
-        now = datetime.now(timezone.utc)
-        recent_enuf = now - timedelta(days=30)
-        since = f"{recent_enuf.year}-{recent_enuf.month}"
-
-    rv = requests.get(f"{urlpath}{urlpath_extra}")
-
-    soup = BeautifulSoup(rv.text, "html.parser")
-
-    urls = []
-
-    for tr in soup.find_all("tr"):
-        tds = tr.find_all("td")
-        if len(tds) != 5:
-            continue
-
-        _, td_name, td_last_modified, td_size, _ = tds
-        if td_last_modified.text.startswith(since):
-            name = td_name.a.text
-            if name.endswith(".json"):
-                urls.append(f"{urlpath}{name}")
-
-    return urls

nmdc_runtime-2.10.0.dist-info/METADATA

@@ -1,265 +0,0 @@
-Metadata-Version: 2.4
-Name: nmdc_runtime
-Version: 2.10.0
-Summary: A runtime system for NMDC data management and orchestration
-Home-page: https://github.com/microbiomedata/nmdc-runtime
-Author: Donny Winston
-Author-email: donny@polyneme.xyz
-Classifier: Development Status :: 3 - Alpha
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: requires-python
-Dynamic: summary
-
-A runtime system for NMDC data management and orchestration.
-
-## Service Status
-
-http://nmdcstatus.polyneme.xyz/
-
-## How It Fits In
-
-* [issues](https://github.com/microbiomedata/issues)  
-tracks issues related to NMDC, which may necessitate work across multiple repos.
-  
-* [nmdc-schema](https://github.com/microbiomedata/nmdc-schema/)
-houses the LinkML schema specification, as well as generated artifacts (e.g. JSON Schema).
-
-* [nmdc-server](https://github.com/microbiomedata/nmdc-server)
-houses code specific to the data portal -- its database, back-end API, and front-end application.
-
-* Workflows — documented in the [workflows](https://docs.microbiomedata.org/workflows/) section of the NMDC documentation website — take source data and produce computed data.
-
-* This repo (nmdc-runtime)
-   * houses code that takes source data and computed data, and transforms it
-     to broadly accommodate downstream applications such as the data portal
-   * manages execution of the above (i.e., lightweight data transformations) and also
-     of computationally- and data-intensive workflows performed at other sites,
-     ensuring that claimed jobs have access to needed configuration and data resources.
-
-## Data exports
-
-The NMDC metadata as of 2021-10 is available here:
-
-https://drs.microbiomedata.org/ga4gh/drs/v1/objects/sys086d541
-
-The link returns a [GA4GH DRS API bundle object record](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.0.0/docs/#_drs_datatypes), with the NMDC metadata collections (study_set, biosample_set, etc.) as contents, each a DRS API blob object.
-
-For example the blob for the study_set collection export, named "study_set.jsonl.gz", is listed with DRS API ID "sys0xsry70". Thus, it is retrievable via
-
-https://drs.microbiomedata.org/ga4gh/drs/v1/objects/sys0xsry70
-
-The returned blob object record lists https://nmdc-runtime.files.polyneme.xyz/nmdcdb-mongoexport/2021-10-14/study_set.jsonl.gz as the url for an access method.
-
-The 2021-10 exports are currently all accessible at `https://nmdc-runtime.files.polyneme.xyz/nmdcdb-mongoexport/2021-10-14/${COLLECTION_NAME}.jsonl.gz`, but the DRS API indirection allows these links to change in the future, for mirroring via other URLs, etc. So, the DRS API links should be the links you share.
-
-## Overview
-
-The runtime features:
-
-1. [Dagster](https://docs.dagster.io/concepts) orchestration:
-    - dagit - a web UI to monitor and manage the running system.
-    - dagster-daemon - a service that triggers pipeline runs based on time or external state.
-    - PostgresSQL database - for storing run history, event logs, and scheduler state.
-    - workspace code
-      - Code to run is loaded into a Dagster `workspace`. This code is loaded from
-         one or more dagster `repositories`. Each Dagster `repository` may be run with a different
-         Python virtual environment if need be, and may be loaded from a local Python file or
-         `pip install`ed from an external source. In our case, each Dagster `repository` is simply
-         loaded from a Python file local to the nmdc-runtime GitHub repository, and all code is
-         run in the same Python environment.
-      - A Dagster repository consists of `solids` and `pipelines`,
-         and optionally `schedules` and `sensors`.
-         - `solids` represent individual units of computation
-         - `pipelines` are built up from solids
-         - `schedules` trigger recurring pipeline runs based on time
-         - `sensors` trigger pipeline runs based on external state
-      - Each `pipeline` can declare dependencies on any runtime `resources` or additional
-         configuration. There are MongoDB `resources` defined, as well as `preset`
-         configuration definitions for both "dev" and "prod" `modes`. The `preset`s tell Dagster to
-         look to a set of known environment variables to load resources configurations, depending on
-         the `mode`.
-   
-2. A MongoDB database supporting write-once, high-throughput internal
-data storage by the nmdc-runtime FastAPI instance.
-   
-3. A [FastAPI](https://fastapi.tiangolo.com/) service to interface with the orchestrator and
-database, as a hub for data management and workflow automation.
-
-## Local Development
-
-Ensure Docker (and Docker Compose) are installed; and the Docker engine is running.
-
-```shell
-docker --version
-docker compose version
-docker info
-```
-
-Ensure the permissions of `./.docker/mongoKeyFile` are such that only the file's owner can read or write the file.
-
-```shell
-chmod 600 ./.docker/mongoKeyFile
-```
-
-Ensure you have a `.env` file for the Docker services to source from. You may copy `.env.example` to
-`.env` (which is gitignore'd) to get started.
-
-```shell
-cp .env.example .env
-```
-
-Create environment variables in your shell session, based upon the contents of the `.env` file.
-
-```shell
-set -a # automatically export all variables
-source .env
-set +a
-```
-
-If you are connecting to resources that require an SSH tunnel—for example, a MongoDB server that is only accessible on 
-the NERSC network—set up the SSH tunnel.
-
-The following command could be useful to you, either directly or as a template (see `Makefile`).
-
-```shell
-make nersc-mongo-tunnels
-```
-
-Finally, spin up the Docker Compose stack.
-
-```bash
-make up-dev
-```
-
-Docker Compose is used to start local MongoDB and PostgresSQL (used by Dagster) instances, as well
-as a Dagster web server (dagit) and daemon (dagster-daemon).
-
-The Dagit web server is viewable at http://127.0.0.1:3000/. 
-
-The FastAPI service is viewable at http://127.0.0.1:8000/ -- e.g., rendered documentation at
-http://127.0.0.1:8000/redoc/.
-
-
-*  NOTE: Any time you add or change requirements in requirements/main.in or requirements/dev.in, you must run:
-```bash
-pip-compile --build-isolation --allow-unsafe --resolver=backtracking --strip-extras --output-file requirements/[main|dev].txt requirements/[main|dev].in
-```
-to generate main.txt and dev.txt files respectively. main.in is kind of like a poetry dependency stanza, dev.in is kind 
-of like poetry dev.dependencies stanza. main.txt and dev.txt are kind of like poetry.lock files to specify the exact 
-versions of dependencies to use. main.txt and dev.txt are combined in the docker compose build process to create the 
-final requirements.txt file and import the dependencies into the Docker image.
-
-## Local Testing
-
-Tests can be found in `tests` and are run with the following commands:
-
-```bash
-make up-test
-make test
-
-# Run a Specific test file eg. tests/test_api/test_endpoints.py
-make test ARGS="tests/test_api/test_endpoints.py"
-
-docker compose --file docker-compose.test.yml run test
-```
-
-As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
-desired and does not break over time.
-
-[For hints on how to write tests for solids and pipelines in Dagster, see their documentation
-tutorial on Testing](https://docs.dagster.io/guides/test/unit-testing-assets-and-ops).
-
-### Performance profiling
-
-We use a tool called [Pyinstrument](https://pyinstrument.readthedocs.io) to profile the performance of the Runtime API while processing an individual HTTP request.
-
-Here's how you can do that:
-
-1. In your `.env` file, set `IS_PROFILING_ENABLED` to `true`
-2. Start/restart your development stack: `$ make up-dev`
-3. Ensure the endpoint function whose performance you want to profile is defined using `async def` (as opposed to just `def`) ([reference](https://github.com/joerick/pyinstrument/issues/257))
-
-Then—with all of that done—submit an HTTP request that includes the URL query parameter: `profile=true`. Instructions for doing that are in the sections below.
-
-<details>
-<summary>Show/hide instructions for <code>GET</code> requests only (involves web browser)</summary>
-
-1. In your web browser, visit the endpoint's URL, but add the `profile=true` query parameter to the URL. Examples:
-   ```diff
-   A. If the URL doesn't already have query parameters, append `?profile=true`.
-   - http://127.0.0.1:8000/nmdcschema/biosample_set
-   + http://127.0.0.1:8000/nmdcschema/biosample_set?profile=true
-
-   B. If the URL already has query parameters, append `&profile=true`.
-   - http://127.0.0.1:8000/nmdcschema/biosample_set?filter={}
-   + http://127.0.0.1:8000/nmdcschema/biosample_set?filter={}&profile=true
-   ```
-2. Your web browser will display a performance profiling report.
-   > Note: The Runtime API will have responded with a performance profiling report web page, instead of its normal response (which the Runtime discards).
-
-That'll only work for `GET` requests, though, since you're limited to specifying the request via the address bar.
-
-</details>
-
-<details>
-<summary>Show/hide instructions for <strong>all</strong> kinds of requests (involves <code>curl</code> + web browser)</summary>
-
-1. At your terminal, type or paste the `curl` command you want to run (you can copy/paste one from Swagger UI).
-2. Append the `profile=true` query parameter to the URL in the command, and use the `-o` option to save the response to a file whose name ends with `.html`. For example:
-   ```diff
-     curl -X 'POST' \
-   -   'http://127.0.0.1:8000/metadata/json:validate' \
-   +   'http://127.0.0.1:8000/metadata/json:validate?profile=true' \
-   +    -o /tmp/profile.html
-        -H 'accept: application/json' \
-        -H 'Content-Type: application/json' \
-        -d '{"biosample_set": []}'
-   ```
-3. Run the command.
-   > Note: The Runtime API will respond with a performance profiling report web page, instead of its normal response (which the Runtime discards). The performance profiling report web page will be saved to the `.html` file to which you redirected the command output.
-4. Double-click on the `.html` file to view it in your web browser.
-   1. Alternatively, open your web browser and navigate to the `.html` file; e.g., enter `file:///tmp/profile.html` into the address bar.
-
-</details>
-
-### RAM usage
-
-The `dagster-daemon` and `dagster-dagit` containers can consume a lot of RAM. If tests are failing and the console of
-the `test` container shows "Error 137," here is something you can try as a workaround: In Docker Desktop, go to 
-"Settings > Resources > Advanced," and increase the memory limit. One of our team members has
-found **12 GB** to be sufficient for running the tests.
-
-> Dedicating 12 GB of RAM to Docker may be prohibitive for some prospective developers.
-> There is an open [issue](https://github.com/microbiomedata/nmdc-runtime/issues/928) about the memory requirement.
-
-## Publish to PyPI
-
-This repository contains a GitHub Actions workflow that publishes a Python package to [PyPI](https://pypi.org/project/nmdc-runtime/).
-
-You can also _manually_ publish the Python package to PyPI by issuing the following commands in the root directory of the repository:
-
-```
-rm -rf dist
-python -m build
-twine upload dist/*
-```
-
-## Links
-
-Here are links related to this repository:
-
-- Production API server: https://api.microbiomedata.org
-- PyPI package: https://pypi.org/project/nmdc-runtime
-- DockerHub image (API server): https://hub.docker.com/r/microbiomedata/nmdc-runtime-fastapi
-- DockerHub image (Dagster): https://hub.docker.com/r/microbiomedata/nmdc-runtime-dagster

nmdc_runtime-2.10.0.dist-info/top_level.txt

@@ -1 +0,0 @@
-nmdc_runtime