nmdc-runtime 1.3.1__py3-none-any.whl → 2.12.0__py3-none-any.whl

nmdc_runtime/site/drsobjects/ingest.py

@@ -1,93 +0,0 @@
-import json
-from datetime import datetime, timezone
-
-from toolz import dissoc
-
-from nmdc_runtime.api.models.job import JobOperationMetadata
-from nmdc_runtime.api.models.operation import Operation
-from nmdc_runtime.api.models.operation import UpdateOperationRequest
-from nmdc_runtime.api.models.util import ListRequest
-from nmdc_runtime.api.models.util import ResultT
-
-
-def load_local_json(url, prefixes_url_to_local=None):
-    """Useful for large files cached on local filesystem.
-
-    You may, for example, `cp --parents ` many files on a remote filesystem to a staging
-    folder on that remote filesystem, gzip that folder, scp it to your local machine, and then
-    extract to your local machine.
-
-    Example:
-    prefixes_url_to_local = {
-        "https://data.microbiomedata.org/data/": "/Users/dwinston/nmdc_files/2021-09-scanon-meta/ficus/pipeline_products/",
-        "https://portal.nersc.gov/project/m3408/": "/Users/dwinston/nmdc_files/2021-09-scanon-meta/www/",
-    }
-    """
-    path = url
-    for before, after in prefixes_url_to_local.items():
-        path = path.replace(before, after)
-    with open(path) as f:
-        return json.load(f)
-
-
-def claim_metadata_ingest_jobs(
-    client, drs_object_ids_to_ingest, wf_id, max_page_size=1000
-):
-    lr = ListRequest(
-        filter=json.dumps(
-            {
-                "workflow.id": wf_id,
-                "config.object_id": {"$in": drs_object_ids_to_ingest},
-            }
-        ),
-        max_page_size=max_page_size,
-    )
-    jobs = []
-    while True:
-        rv = client.list_jobs(lr.model_dump()).json()
-        jobs.extend(rv["resources"])
-        if "next_page_token" not in rv:
-            break
-        else:
-            lr.page_token = rv["next_page_token"]
-
-        # safety escape
-        if len(jobs) == len(drs_object_ids_to_ingest):
-            break
-
-    job_claim_responses = [client.claim_job(j["id"]) for j in jobs]
-
-    return job_claim_responses
-
-
-def mongo_add_docs_result_as_dict(rv):
-    return {
-        collection_name: dissoc(bulk_write_result.bulk_api_result, "upserted")
-        for collection_name, bulk_write_result in rv.items()
-    }
-
-
-def get_metadata_ingest_job_ops(mongo, wf_id, drs_object_ids_to_ingest):
-    return list(
-        mongo.db.operations.find(
-            {
-                "metadata.job.workflow.id": wf_id,
-                "metadata.job.config.object_id": {"$in": drs_object_ids_to_ingest},
-                "done": False,
-            }
-        )
-    )
-
-
-def do_metadata_ingest_job(client, mongo, job_op_doc):
-    op = Operation[ResultT, JobOperationMetadata](**job_op_doc)
-    object_info = client.get_object_info(op.metadata.job.config["object_id"]).json()
-    url = object_info["access_methods"][0]["access_url"]["url"]
-    docs = load_local_json(url)
-    op_result = mongo.add_docs(docs, validate=False, replace=False)
-    op_patch = UpdateOperationRequest(
-        done=True,
-        result=mongo_add_docs_result_as_dict(op_result),
-        metadata={"done_at": datetime.now(timezone.utc).isoformat(timespec="seconds")},
-    )
-    return client.update_operation(op.id, op_patch)

nmdc_runtime/site/drsobjects/registration.py

@@ -1,131 +0,0 @@
-import json
-import os
-import re
-from datetime import datetime, timezone, timedelta
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-import requests
-from bs4 import BeautifulSoup
-
-from nmdc_runtime.api.models.object import DrsObjectIn
-from nmdc_runtime.util import (
-    drs_metadata_for,
-    nmdc_jsonschema_validator,
-    specialize_activity_set_docs,
-)
-
-pattern = re.compile(r"https?://(?P<domain>[^/]+)/(?P<path>.+)")
-
-
-def url_to_name(url):
-    m = pattern.match(url)
-    return (
-        f"{'.'.join(reversed(m.group('domain').split('.')))}"
-        f"__{m.group('path').replace('/', '.')}"
-    )
-
-
-def fetch_url(url, timeout=30):
-    return requests.get(url, timeout=timeout)
-
-
-class HttpResponseNotOk(Exception):
-    pass
-
-
-class HttpResponseNotJson(Exception):
-    pass
-
-
-def response_to_json(response):
-    if response.status_code != 200:
-        raise HttpResponseNotOk()
-    try:
-        json_data = response.json()
-    except ValueError:
-        raise HttpResponseNotJson()
-    return json_data
-
-
-def json_data_from_url_to_file(json_data, url, save_dir):
-    filepath = os.path.join(save_dir, url_to_name(url))
-    with open(filepath, "w") as f:
-        json.dump(json_data, f)
-    return filepath
-
-
-def json_clean(d, model, exclude_unset=False):
-    return json.loads(model(**d).json(exclude_unset=exclude_unset))
-
-
-def drs_object_in_for(url):
-    with TemporaryDirectory() as save_dir:
-        response = fetch_url(url)
-        try:
-            json_data = response_to_json(response)
-        except HttpResponseNotOk:
-            return {"error": "HttpResponseNotOk"}
-
-        except HttpResponseNotJson:
-            return {"error": "HttpResponseNotJson"}
-
-        filepath = json_data_from_url_to_file(json_data, url, save_dir)
-        drs_object_in = DrsObjectIn(
-            **drs_metadata_for(
-                filepath,
-                {
-                    "access_methods": [{"access_url": {"url": url}}],
-                    "name": Path(filepath).name.replace(":", "-"),
-                },
-            )
-        )
-        return {"result": drs_object_in}
-
-
-def create_drs_object_for(url, drs_object_in, client):
-    rv = client.create_object(json.loads(drs_object_in.json(exclude_unset=True)))
-    return {"url": url, "response": rv}
-
-
-def validate_as_metadata_and_ensure_tags_for(
-    drs_id, client, tags=("schema#/definitions/Database", "metadata-in")
-):
-    docs = client.get_object_bytes(drs_id).json()
-    docs, _ = specialize_activity_set_docs(docs)
-    _ = nmdc_jsonschema_validator(docs)
-    return {tag: client.ensure_object_tag(drs_id, tag) for tag in tags}
-
-
-def recent_metadata_urls(
-    urlpath="https://portal.nersc.gov/project/m3408/meta/anno2/",
-    urlpath_extra="?C=M;O=D",
-    since="2021-09",
-):
-    """Scrapes recent URLs from Apache/2.4.38 (Debian) Server listing.
-
-    Designed with urlpath.startwsith("https://portal.nersc.gov/project/m3408/") in mind.
-    """
-    if since is None:
-        now = datetime.now(timezone.utc)
-        recent_enuf = now - timedelta(days=30)
-        since = f"{recent_enuf.year}-{recent_enuf.month}"
-
-    rv = requests.get(f"{urlpath}{urlpath_extra}")
-
-    soup = BeautifulSoup(rv.text, "html.parser")
-
-    urls = []
-
-    for tr in soup.find_all("tr"):
-        tds = tr.find_all("td")
-        if len(tds) != 5:
-            continue
-
-        _, td_name, td_last_modified, td_size, _ = tds
-        if td_last_modified.text.startswith(since):
-            name = td_name.a.text
-            if name.endswith(".json"):
-                urls.append(f"{urlpath}{name}")
-
-    return urls

nmdc_runtime/site/terminusdb/generate.py

@@ -1,198 +0,0 @@
-"""
-Example usage:
-$ schemagen-terminusdb ../nmdc-schema/src/schema/nmdc.yaml \
-    > nmdc_runtime/site/terminusdb/nmdc.schema.terminusdb.json
-"""
-
-import json
-import os
-from typing import Union, TextIO, List
-
-import click
-from linkml.utils.generator import Generator, shared_arguments
-from linkml_runtime.linkml_model.meta import (
-    SchemaDefinition,
-    ClassDefinition,
-    SlotDefinition,
-)
-from linkml_runtime.utils.formatutils import camelcase, be, underscore
-
-# http://books.xmlschemata.org/relaxng/relax-CHP-19.html
-XSD_Ok = {
-    "xsd:anyURI",
-    "xsd:base64Binary",
-    "xsd:boolean",
-    "xsd:byte",
-    "xsd:date",
-    "xsd:dateTime",
-    "xsd:decimal",
-    "xsd:double",
-    "xsd:duration",
-    "xsd:ENTITIES",
-    "xsd:ENTITY",
-    "xsd:float",
-    "xsd:gDay",
-    "xsd:gMonth",
-    "xsd:gMonthDay",
-    "xsd:gYear",
-    "xsd:gYearMonth",
-    "xsd:hexBinary",
-    "xsd:ID",
-    "xsd:IDREF",
-    "xsd:IDREFS",
-    "xsd:int",
-    "xsd:integer",
-    "xsd:language",
-    "xsd:long",
-    "xsd:Name",
-    "xsd:NCName",
-    "xsd:negativeInteger",
-    "xsd:NMTOKEN",
-    "xsd:NMTOKENS",
-    "xsd:nonNegativeInteger",
-    "xsd:nonPositiveInteger",
-    "xsd:normalizedString",
-    "xsd:NOTATION",
-    "xsd:positiveInteger",
-    "xsd:short",
-    "xsd:string",
-    "xsd:time",
-    "xsd:token",
-    "xsd:unsignedByte",
-    "xsd:unsignedInt",
-    "xsd:unsignedLong",
-    "xsd:unsignedShort",
-}
-
-
-def as_list(thing) -> list:
-    return thing if isinstance(thing, list) else [thing]
-
-
-def has_field(graph: List[dict], cls: dict, field: str) -> bool:
-    if field in cls:
-        return True
-    for parent_id in as_list(cls.get("@inherits", [])):
-        parent_cls = next(
-            graph_cls for graph_cls in graph if graph_cls.get("@id") == parent_id
-        )
-        if parent_cls and has_field(graph, parent_cls, field):
-            return True
-    return False
-
-
-class TerminusdbGenerator(Generator):
-    """Generates JSON file to pass to WOQLClient.insert_document(..., graph_type="schema")`."""
-
-    generatorname = os.path.basename(__file__)
-    generatorversion = "0.1.0"
-    valid_formats = ["json"]
-    visit_all_class_slots = True
-
-    def __init__(self, schema: Union[str, TextIO, SchemaDefinition], **kwargs) -> None:
-        super().__init__(schema, **kwargs)
-        self.graph = []
-        self.cls_json = {}
-
-    def visit_schema(self, inline: bool = False, **kwargs) -> None:
-        self.graph.append(
-            {
-                "@type": "@context",
-                "@base": "https://api.microbiomedata.org/nmdcschema/ids/",
-                "@schema": "https://w3id.org/nmdc/",
-            }
-        )
-
-    def end_schema(self, **_) -> None:
-        for cls in self.graph:
-            if has_field(self.graph, cls, "id"):
-                cls["@key"] = {"@type": "Lexical", "@fields": ["id"]}
-        print(json.dumps(self.graph, indent=2))
-
-    def visit_class(self, cls: ClassDefinition) -> bool:
-        self.cls_json = {
-            "@type": "Class",
-            "@id": camelcase(cls.name),
-            "@documentation": {
-                "@comment": be(cls.description),
-                "@properties": {},
-            },
-        }
-        if cls.is_a:
-            self.cls_json["@inherits"] = camelcase(cls.is_a)
-        if cls.abstract:
-            self.cls_json["@abstract"] = []
-        return True
-
-    def end_class(self, cls: ClassDefinition) -> None:
-        self.cls_json["@id"] = cls.definition_uri.split(":")[-1].rpartition("/")[-1]
-        self.graph.append(self.cls_json)
-
-    # sounding board as solist
-    # safe space to ask questions. more of a whatsapp group.
-    # both re: business, how to structure proposals, etc.
-    # And also technical content suggestions. R data pipeline / copy/paste in Figma
-    #  - how far do you go in automation in delivery
-
-    def visit_class_slot(
-        self, cls: ClassDefinition, aliased_slot_name: str, slot: SlotDefinition
-    ) -> None:
-        if slot not in self.own_slots(cls):
-            return
-        if slot.is_usage_slot:
-            # TerminusDB does not support calling different things the same name.
-            # So, ignore usage overrides.
-            slot = self.schema.slots[aliased_slot_name]
-
-        if slot.range in self.schema.classes:
-            rng = camelcase(slot.range)
-        elif slot.range in self.schema.types:
-            # XXX Why does `linkml.utils.metamodelcore.Identifier` subclass `str`??
-            rng = str(self.schema.types[slot.range].uri)
-        else:
-            rng = "xsd:string"
-
-        # name = (
-        #     f"{cls.name} {aliased_slot_name}"
-        #     if slot.is_usage_slot
-        #     else aliased_slot_name
-        # )
-        name = slot.name
-        # TODO fork nmdc schema and make any slots NOT required in parent class
-        #  also NOT required in child classes. Can have opt-in entity validation logic in code.
-
-        # XXX MAG bin -> bin name goes to "mAGBin__bin_name", etc. Weird.
-
-        # # translate to terminusdb xsd builtins:
-        # if rng == "xsd:int":
-        #     rng = "xsd:integer"
-        # elif rng == "xsd:float":
-        #     rng = "xsd:double"
-        # elif rng == "xsd:language":
-        #     rng = "xsd:string"
-
-        if rng not in XSD_Ok and slot.range not in self.schema.classes:
-            raise Exception(
-                f"slot range for {name} must be schema class or supported xsd type. "
-                f"Range {rng} is of type {type(rng)}."
-            )
-
-        self.cls_json[underscore(name)] = rng
-        self.cls_json["@documentation"]["@properties"][
-            underscore(name)
-        ] = slot.description
-        if not slot.required:
-            self.cls_json[underscore(name)] = {"@type": "Optional", "@class": rng}
-        if slot.multivalued:  # XXX what about an required multivalued field?
-            self.cls_json[underscore(name)] = {"@type": "Set", "@class": rng}
-
-
-@shared_arguments(TerminusdbGenerator)
-@click.command()
-def cli(yamlfile, **args):
-    """Generate graphql representation of a biolink model"""
-    print(TerminusdbGenerator(yamlfile, **args).serialize(**args))
-
-
-if __name__ == "__main__":
-    cli()

nmdc_runtime/site/terminusdb/ingest.py

@@ -1,44 +0,0 @@
-import json
-from pathlib import Path
-
-from terminusdb_client import WOQLClient
-
-team = "admin"
-client = WOQLClient(f"http://localhost:6364/")
-# make sure you have put the token in environment variable
-# https://docs.terminusdb.com/v10.0/#/terminusx/get-your-api-key
-client.connect(user=team, team=team, key="root")
-
-dbid = "nmdc"
-label = "NMDC"
-description = "."
-prefixes = {
-    "@base": "terminusdb:///data/",
-    "@schema": "terminusdb:///schema#",
-    "gold": "https://gold.jgi.doe.gov/biosample?id=",
-}
-
-
-def import_schema(client):
-    # sd = get_nmdc_schema_definition()
-    # sd.source_file = f"{REPO_ROOT_DIR.parent}/nmdc-schema/src/schema/nmdc.yaml"
-    # print(sd.source_file)
-    with open(Path(__file__).parent.joinpath("nmdc.schema.terminusdb.json")) as f:
-        schema_objects = json.load(f)
-
-    client.message = "Adding NMDC Schema"
-    results = client.insert_document(schema_objects, graph_type="schema")
-    print(f"Added schema: {results}")
-
-
-if __name__ == "__main__":
-    exists = client.get_database(dbid)
-
-    if exists:
-        client.delete_database(dbid, team=team, force=True)
-
-    client.create_database(
-        dbid, team, label=label, description=description, prefixes=prefixes
-    )
-
-    import_schema(client)