nmdc-runtime 2.9.0__py3-none-any.whl → 2.11.0__py3-none-any.whl

nmdc_runtime/site/drsobjects/ingest.py

@@ -1,93 +0,0 @@
-import json
-from datetime import datetime, timezone
-
-from toolz import dissoc
-
-from nmdc_runtime.api.models.job import JobOperationMetadata
-from nmdc_runtime.api.models.operation import Operation
-from nmdc_runtime.api.models.operation import UpdateOperationRequest
-from nmdc_runtime.api.models.util import ListRequest
-from nmdc_runtime.api.models.util import ResultT
-
-
-def load_local_json(url, prefixes_url_to_local=None):
-    """Useful for large files cached on local filesystem.
-
-    You may, for example, `cp --parents ` many files on a remote filesystem to a staging
-    folder on that remote filesystem, gzip that folder, scp it to your local machine, and then
-    extract to your local machine.
-
-    Example:
-    prefixes_url_to_local = {
-        "https://data.microbiomedata.org/data/": "/Users/dwinston/nmdc_files/2021-09-scanon-meta/ficus/pipeline_products/",
-        "https://portal.nersc.gov/project/m3408/": "/Users/dwinston/nmdc_files/2021-09-scanon-meta/www/",
-    }
-    """
-    path = url
-    for before, after in prefixes_url_to_local.items():
-        path = path.replace(before, after)
-    with open(path) as f:
-        return json.load(f)
-
-
-def claim_metadata_ingest_jobs(
-    client, drs_object_ids_to_ingest, wf_id, max_page_size=1000
-):
-    lr = ListRequest(
-        filter=json.dumps(
-            {
-                "workflow.id": wf_id,
-                "config.object_id": {"$in": drs_object_ids_to_ingest},
-            }
-        ),
-        max_page_size=max_page_size,
-    )
-    jobs = []
-    while True:
-        rv = client.list_jobs(lr.model_dump()).json()
-        jobs.extend(rv["resources"])
-        if "next_page_token" not in rv:
-            break
-        else:
-            lr.page_token = rv["next_page_token"]
-
-        # safety escape
-        if len(jobs) == len(drs_object_ids_to_ingest):
-            break
-
-    job_claim_responses = [client.claim_job(j["id"]) for j in jobs]
-
-    return job_claim_responses
-
-
-def mongo_add_docs_result_as_dict(rv):
-    return {
-        collection_name: dissoc(bulk_write_result.bulk_api_result, "upserted")
-        for collection_name, bulk_write_result in rv.items()
-    }
-
-
-def get_metadata_ingest_job_ops(mongo, wf_id, drs_object_ids_to_ingest):
-    return list(
-        mongo.db.operations.find(
-            {
-                "metadata.job.workflow.id": wf_id,
-                "metadata.job.config.object_id": {"$in": drs_object_ids_to_ingest},
-                "done": False,
-            }
-        )
-    )
-
-
-def do_metadata_ingest_job(client, mongo, job_op_doc):
-    op = Operation[ResultT, JobOperationMetadata](**job_op_doc)
-    object_info = client.get_object_info(op.metadata.job.config["object_id"]).json()
-    url = object_info["access_methods"][0]["access_url"]["url"]
-    docs = load_local_json(url)
-    op_result = mongo.add_docs(docs, validate=False, replace=False)
-    op_patch = UpdateOperationRequest(
-        done=True,
-        result=mongo_add_docs_result_as_dict(op_result),
-        metadata={"done_at": datetime.now(timezone.utc).isoformat(timespec="seconds")},
-    )
-    return client.update_operation(op.id, op_patch)

nmdc_runtime/site/drsobjects/registration.py

@@ -1,131 +0,0 @@
-import json
-import os
-import re
-from datetime import datetime, timezone, timedelta
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-import requests
-from bs4 import BeautifulSoup
-
-from nmdc_runtime.api.models.object import DrsObjectIn
-from nmdc_runtime.util import (
-    drs_metadata_for,
-    nmdc_jsonschema_validator,
-    specialize_activity_set_docs,
-)
-
-pattern = re.compile(r"https?://(?P<domain>[^/]+)/(?P<path>.+)")
-
-
-def url_to_name(url):
-    m = pattern.match(url)
-    return (
-        f"{'.'.join(reversed(m.group('domain').split('.')))}"
-        f"__{m.group('path').replace('/', '.')}"
-    )
-
-
-def fetch_url(url, timeout=30):
-    return requests.get(url, timeout=timeout)
-
-
-class HttpResponseNotOk(Exception):
-    pass
-
-
-class HttpResponseNotJson(Exception):
-    pass
-
-
-def response_to_json(response):
-    if response.status_code != 200:
-        raise HttpResponseNotOk()
-    try:
-        json_data = response.json()
-    except ValueError:
-        raise HttpResponseNotJson()
-    return json_data
-
-
-def json_data_from_url_to_file(json_data, url, save_dir):
-    filepath = os.path.join(save_dir, url_to_name(url))
-    with open(filepath, "w") as f:
-        json.dump(json_data, f)
-    return filepath
-
-
-def json_clean(d, model, exclude_unset=False):
-    return json.loads(model(**d).json(exclude_unset=exclude_unset))
-
-
-def drs_object_in_for(url):
-    with TemporaryDirectory() as save_dir:
-        response = fetch_url(url)
-        try:
-            json_data = response_to_json(response)
-        except HttpResponseNotOk:
-            return {"error": "HttpResponseNotOk"}
-
-        except HttpResponseNotJson:
-            return {"error": "HttpResponseNotJson"}
-
-        filepath = json_data_from_url_to_file(json_data, url, save_dir)
-        drs_object_in = DrsObjectIn(
-            **drs_metadata_for(
-                filepath,
-                {
-                    "access_methods": [{"access_url": {"url": url}}],
-                    "name": Path(filepath).name.replace(":", "-"),
-                },
-            )
-        )
-        return {"result": drs_object_in}
-
-
-def create_drs_object_for(url, drs_object_in, client):
-    rv = client.create_object(json.loads(drs_object_in.json(exclude_unset=True)))
-    return {"url": url, "response": rv}
-
-
-def validate_as_metadata_and_ensure_tags_for(
-    drs_id, client, tags=("schema#/definitions/Database", "metadata-in")
-):
-    docs = client.get_object_bytes(drs_id).json()
-    docs, _ = specialize_activity_set_docs(docs)
-    _ = nmdc_jsonschema_validator(docs)
-    return {tag: client.ensure_object_tag(drs_id, tag) for tag in tags}
-
-
-def recent_metadata_urls(
-    urlpath="https://portal.nersc.gov/project/m3408/meta/anno2/",
-    urlpath_extra="?C=M;O=D",
-    since="2021-09",
-):
-    """Scrapes recent URLs from Apache/2.4.38 (Debian) Server listing.
-
-    Designed with urlpath.startwsith("https://portal.nersc.gov/project/m3408/") in mind.
-    """
-    if since is None:
-        now = datetime.now(timezone.utc)
-        recent_enuf = now - timedelta(days=30)
-        since = f"{recent_enuf.year}-{recent_enuf.month}"
-
-    rv = requests.get(f"{urlpath}{urlpath_extra}")
-
-    soup = BeautifulSoup(rv.text, "html.parser")
-
-    urls = []
-
-    for tr in soup.find_all("tr"):
-        tds = tr.find_all("td")
-        if len(tds) != 5:
-            continue
-
-        _, td_name, td_last_modified, td_size, _ = tds
-        if td_last_modified.text.startswith(since):
-            name = td_name.a.text
-            if name.endswith(".json"):
-                urls.append(f"{urlpath}{name}")
-
-    return urls

nmdc_runtime/site/translation/emsl.py

@@ -1,43 +0,0 @@
-"""
-Translates EMSL data into JSON conformant with the NMDC JSON schema
-"""
-
-from dagster import op, graph
-
-from nmdc_runtime.lib.nmdc_etl_class import NMDC_ETL
-from nmdc_runtime.site.translation.util import (
-    load_nmdc_etl_class,
-    load_mongo_collection,
-    preset_prod,
-    preset_test,
-    schema_validate,
-)
-
-
-@op
-def transform_emsl_omics_processing(_context, nmdc_etl: NMDC_ETL) -> tuple:
-    return ("emsl.omics_processing_set", nmdc_etl.transform_emsl_omics_processing())
-
-
-@op
-def transform_emsl_data_object(_context, nmdc_etl: NMDC_ETL) -> tuple:
-    return ("emsl.data_object_set", nmdc_etl.transform_emsl_data_object())
-
-
-@graph
-def emsl():
-    # load_merged_data_source()
-    nmdc_etl = load_nmdc_etl_class()
-    emsl_omics_processing = transform_emsl_omics_processing(nmdc_etl)
-    emsl_omics_processing_validated = schema_validate(emsl_omics_processing)
-
-    emsl_data_object = transform_emsl_data_object(nmdc_etl)
-    emsl_data_object_validated = schema_validate(emsl_data_object)
-
-    # load data into mongo
-    load_mongo_collection(emsl_omics_processing_validated)
-    load_mongo_collection(emsl_data_object_validated)
-
-
-emsl_job = emsl.to_job(**preset_prod)
-test_emsl_job = emsl.to_job(name="test_emsl", **preset_test)

nmdc_runtime/site/translation/gold.py

@@ -1,53 +0,0 @@
-"""
-Translate an export of the JGI GOLD [1] study, project, and biosample data into JSON conformant with the NMDC JSON schema.
-[1] Genomes OnLine Database (GOLD) <https://gold.jgi.doe.gov/>.
-"""
-
-from dagster import op, graph
-
-from nmdc_runtime.lib.nmdc_etl_class import NMDC_ETL
-from nmdc_runtime.site.translation.util import (
-    load_nmdc_etl_class,
-    load_mongo_collection,
-    preset_prod,
-    preset_test,
-    schema_validate,
-)
-
-
-@op
-def transform_study(_context, nmdc_etl: NMDC_ETL) -> tuple:
-    # return {"study_set": nmdc_etl.transform_study()}
-    return ("gold.study_set", nmdc_etl.transform_study())
-
-
-@op
-def transform_gold_omics_processing(_context, nmdc_etl: NMDC_ETL) -> tuple:
-    return ("gold.omics_processing_set", nmdc_etl.transform_omics_processing())
-
-
-@op
-def transform_biosample(_context, nmdc_etl: NMDC_ETL) -> tuple:
-    return ("gold.biosample_set", nmdc_etl.transform_biosample())
-
-
-@graph
-def gold():
-    nmdc_etl = load_nmdc_etl_class()
-    gold_study = transform_study(nmdc_etl)
-    gold_study_validated = schema_validate(gold_study)
-
-    gold_omics_processing = transform_gold_omics_processing(nmdc_etl)
-    gold_omics_processing_validated = schema_validate(gold_omics_processing)
-
-    gold_biosample = transform_biosample(nmdc_etl)
-    gold_biosample_validated = schema_validate(gold_biosample)
-
-    # load data into mongo
-    load_mongo_collection(gold_study_validated)
-    load_mongo_collection(gold_omics_processing_validated)
-    load_mongo_collection(gold_biosample_validated)
-
-
-gold_job = gold.to_job(**preset_prod)
-test_gold_job = gold.to_job(name="test_gold", **preset_test)

nmdc_runtime/site/translation/jgi.py

@@ -1,32 +0,0 @@
-"""
-Translates EMSL data into JSON conformant with the NMDC JSON schema
-"""
-
-from dagster import op, graph
-
-from nmdc_runtime.lib.nmdc_etl_class import NMDC_ETL
-from nmdc_runtime.site.translation.util import (
-    load_nmdc_etl_class,
-    load_mongo_collection,
-    preset_prod,
-    preset_test,
-    schema_validate,
-)
-
-
-@op
-def transform_jgi_data_object(_context, nmdc_etl: NMDC_ETL) -> tuple:
-    # return "jgi.data_object_set", [{"foo": "bar"}]  # used for testing failure
-    return "jgi.data_object_set", nmdc_etl.transform_jgi_data_object()
-
-
-@graph
-def jgi():
-    nmdc_etl = load_nmdc_etl_class()
-    jgi_data_object = transform_jgi_data_object(nmdc_etl)
-    jgi_data_object_validated = schema_validate(jgi_data_object)
-    load_mongo_collection(jgi_data_object_validated)
-
-
-jgi_job = jgi.to_job(**preset_prod)
-test_jgi_job = jgi.to_job(name="test_jgi", **preset_test)

nmdc_runtime/site/translation/util.py

@@ -1,132 +0,0 @@
-from pathlib import Path
-
-from dagster import op, Failure, AssetMaterialization
-from dagster.core.definitions.events import AssetKey, Output
-from fastjsonschema import JsonSchemaValueException
-
-from nmdc_runtime.lib.nmdc_etl_class import NMDC_ETL
-from nmdc_runtime.site.resources import mongo_resource
-from nmdc_runtime.util import nmdc_jsonschema_validator
-
-mode_prod = {"resource_defs": {"mongo": mongo_resource}}
-mode_dev = {
-    "resource_defs": {"mongo": mongo_resource}
-}  # Connect to a real MongoDB instance for development.
-mode_test = {
-    "resource_defs": {"mongo": mongo_resource}
-}  # Connect to a real MongoDB instance for testing.
-
-config_prod = {
-    "resources": {
-        "mongo": {
-            "config": {
-                "host": {"env": "MONGO_HOST"},
-                "username": {"env": "MONGO_USERNAME"},
-                "password": {"env": "MONGO_PASSWORD"},
-                "dbname": "nmdc_etl_staging",
-            },
-        }
-    },
-    "ops": {
-        "load_nmdc_etl_class": {
-            "config": {
-                "data_file": str(
-                    Path(__file__).parent.parent.parent.parent.joinpath(
-                        "metadata-translation/src/data/nmdc_merged_data.tsv.zip"
-                    )
-                ),
-                "sssom_map_file": "",
-                "spec_file": str(
-                    Path(__file__).parent.parent.parent.parent.joinpath(
-                        "nmdc_runtime/lib/nmdc_data_source.yaml"
-                    )
-                ),
-            }
-        }
-    },
-}
-
-config_test = {
-    "resources": {
-        "mongo": {
-            "config": {
-                # local docker container via docker-compose.yml
-                "host": "mongo",
-                "username": "admin",
-                "password": "root",
-                "dbname": "nmdc_etl_staging",
-            },
-        }
-    },
-    "ops": {
-        "load_nmdc_etl_class": {
-            "config": {
-                "data_file": str(
-                    Path(__file__).parent.parent.parent.parent.joinpath(
-                        "metadata-translation/src/data/nmdc_merged_data.tsv.zip"
-                    )
-                ),
-                "sssom_map_file": "",
-                "spec_file": str(
-                    Path(__file__).parent.parent.parent.parent.joinpath(
-                        "nmdc_runtime/lib/nmdc_data_source.yaml"
-                    )
-                ),
-            }
-        }
-    },
-}
-
-preset_prod = dict(**mode_prod, config=config_prod)
-preset_test = dict(**mode_test, config=config_test)
-
-
-@op
-def load_nmdc_etl_class(context) -> NMDC_ETL:
-    # build instance of NMDC_ETL class
-    etl = NMDC_ETL(
-        merged_data_file=context.op_config["data_file"],
-        data_source_spec_file=context.op_config["spec_file"],
-        sssom_file="",
-    )
-    return etl
-
-
-@op(required_resource_keys={"mongo"})
-def load_mongo_collection(context, data: tuple):
-    mongo_db = context.resources.mongo.db
-    collection_name, documents = data
-    collection = mongo_db[collection_name]  # get mongo collection
-
-    # drop collection if exists
-    collection.drop()
-
-    # insert data
-    collection.insert(documents)
-    context.log.info(f"inserted {len(documents)} documents into {collection.name}")
-    return collection_name
-
-
-@op()
-def schema_validate(context, data: tuple):
-    def schema_validate_asset(collection_name, status, errors):
-        return AssetMaterialization(
-            asset_key=AssetKey(["translation", f"{collection_name}_translation"]),
-            description=f"{collection_name} translation validation",
-            metadata={"status": status, "errors": errors},
-        )
-
-    collection_name, documents = data
-    _, schema_collection_name = collection_name.split(".")
-    try:
-        nmdc_jsonschema_validator({schema_collection_name: documents})
-        context.log.info(f"data for {collection_name} is valid")
-        yield schema_validate_asset(collection_name, "valid", "none")
-        return data  # do I need a return statement and an Output?
-    except JsonSchemaValueException as e:
-        context.log.error(f"validation failed for {schema_collection_name} " + str(e))
-        context.log.error(f"documents: {documents}")
-        yield schema_validate_asset(collection_name, "not valid", str(e))
-        raise Failure(str(e))
-    finally:
-        yield Output(data)

nmdc_runtime/site/validation/jgi.py

@@ -1,43 +0,0 @@
-"""
-Validates data in the JGI collection in the nmdc_etl_staging database.
-"""
-
-from dagster import op, graph
-
-from nmdc_runtime.site.ops import local_file_to_api_object
-from nmdc_runtime.site.validation.util import (
-    preset_prod,
-    preset_test,
-    validate_mongo_collection,
-    write_to_local_file,
-    announce_validation_report,
-)
-
-
-@op
-def jgi_data_object_set_collection_name():
-    return "jgi.data_object_set"
-
-
-@graph()
-def jgi():
-    report = validate_mongo_collection(jgi_data_object_set_collection_name())
-    # the below could also be a @graph and loaded as a "subgraph" by e.g. the jgi graph job.
-    local_path = write_to_local_file(report)
-    obj = local_file_to_api_object(local_path)
-    announce_validation_report(report, obj)
-
-
-# passing the collecton name via the config
-# problem: not sure if this best when multiple sets need to be validated
-# from toolz import assoc_in
-# config_ops = {
-#     "validate_mongo_collection": {"config": {"collection_name": "jgi.data_object_set"}}
-# }
-# validate_jgi_job = jgi.to_job(**assoc_in(preset_prod, ["config", "ops"], config_ops))
-# test_validate_jgi_job = jgi.to_job(
-#     **assoc_in(preset_test, ["config", "ops"], config_ops)
-# )
-
-validate_jgi_job = jgi.to_job(**preset_prod)
-test_validate_jgi_job = jgi.to_job(**preset_test)