nmdc-runtime 2.8.0__py3-none-any.whl → 2.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of nmdc-runtime might be problematic. Click here for more details.
- nmdc_runtime/api/__init__.py +0 -0
- nmdc_runtime/api/analytics.py +70 -0
- nmdc_runtime/api/boot/__init__.py +0 -0
- nmdc_runtime/api/boot/capabilities.py +9 -0
- nmdc_runtime/api/boot/object_types.py +126 -0
- nmdc_runtime/api/boot/triggers.py +84 -0
- nmdc_runtime/api/boot/workflows.py +116 -0
- nmdc_runtime/api/core/__init__.py +0 -0
- nmdc_runtime/api/core/auth.py +208 -0
- nmdc_runtime/api/core/idgen.py +170 -0
- nmdc_runtime/api/core/metadata.py +788 -0
- nmdc_runtime/api/core/util.py +109 -0
- nmdc_runtime/api/db/__init__.py +0 -0
- nmdc_runtime/api/db/mongo.py +447 -0
- nmdc_runtime/api/db/s3.py +37 -0
- nmdc_runtime/api/endpoints/__init__.py +0 -0
- nmdc_runtime/api/endpoints/capabilities.py +25 -0
- nmdc_runtime/api/endpoints/find.py +794 -0
- nmdc_runtime/api/endpoints/ids.py +192 -0
- nmdc_runtime/api/endpoints/jobs.py +143 -0
- nmdc_runtime/api/endpoints/lib/__init__.py +0 -0
- nmdc_runtime/api/endpoints/lib/helpers.py +274 -0
- nmdc_runtime/api/endpoints/lib/path_segments.py +165 -0
- nmdc_runtime/api/endpoints/metadata.py +260 -0
- nmdc_runtime/api/endpoints/nmdcschema.py +581 -0
- nmdc_runtime/api/endpoints/object_types.py +38 -0
- nmdc_runtime/api/endpoints/objects.py +277 -0
- nmdc_runtime/api/endpoints/operations.py +105 -0
- nmdc_runtime/api/endpoints/queries.py +679 -0
- nmdc_runtime/api/endpoints/runs.py +98 -0
- nmdc_runtime/api/endpoints/search.py +38 -0
- nmdc_runtime/api/endpoints/sites.py +229 -0
- nmdc_runtime/api/endpoints/triggers.py +25 -0
- nmdc_runtime/api/endpoints/users.py +214 -0
- nmdc_runtime/api/endpoints/util.py +774 -0
- nmdc_runtime/api/endpoints/workflows.py +353 -0
- nmdc_runtime/api/main.py +401 -0
- nmdc_runtime/api/middleware.py +43 -0
- nmdc_runtime/api/models/__init__.py +0 -0
- nmdc_runtime/api/models/capability.py +14 -0
- nmdc_runtime/api/models/id.py +92 -0
- nmdc_runtime/api/models/job.py +37 -0
- nmdc_runtime/api/models/lib/__init__.py +0 -0
- nmdc_runtime/api/models/lib/helpers.py +78 -0
- nmdc_runtime/api/models/metadata.py +11 -0
- nmdc_runtime/api/models/minter.py +0 -0
- nmdc_runtime/api/models/nmdc_schema.py +146 -0
- nmdc_runtime/api/models/object.py +180 -0
- nmdc_runtime/api/models/object_type.py +20 -0
- nmdc_runtime/api/models/operation.py +66 -0
- nmdc_runtime/api/models/query.py +246 -0
- nmdc_runtime/api/models/query_continuation.py +111 -0
- nmdc_runtime/api/models/run.py +161 -0
- nmdc_runtime/api/models/site.py +87 -0
- nmdc_runtime/api/models/trigger.py +13 -0
- nmdc_runtime/api/models/user.py +140 -0
- nmdc_runtime/api/models/util.py +253 -0
- nmdc_runtime/api/models/workflow.py +15 -0
- nmdc_runtime/api/openapi.py +242 -0
- nmdc_runtime/config.py +55 -4
- nmdc_runtime/core/db/Database.py +1 -3
- nmdc_runtime/infrastructure/database/models/user.py +0 -9
- nmdc_runtime/lib/extract_nmdc_data.py +0 -8
- nmdc_runtime/lib/nmdc_dataframes.py +3 -7
- nmdc_runtime/lib/nmdc_etl_class.py +1 -7
- nmdc_runtime/minter/adapters/repository.py +1 -2
- nmdc_runtime/minter/config.py +2 -0
- nmdc_runtime/minter/domain/model.py +35 -1
- nmdc_runtime/minter/entrypoints/fastapi_app.py +1 -1
- nmdc_runtime/mongo_util.py +1 -2
- nmdc_runtime/site/backup/nmdcdb_mongodump.py +1 -1
- nmdc_runtime/site/backup/nmdcdb_mongoexport.py +1 -3
- nmdc_runtime/site/export/ncbi_xml.py +1 -2
- nmdc_runtime/site/export/ncbi_xml_utils.py +1 -1
- nmdc_runtime/site/graphs.py +33 -28
- nmdc_runtime/site/ops.py +97 -237
- nmdc_runtime/site/repair/database_updater.py +8 -0
- nmdc_runtime/site/repository.py +7 -117
- nmdc_runtime/site/resources.py +4 -4
- nmdc_runtime/site/translation/gold_translator.py +22 -21
- nmdc_runtime/site/translation/neon_benthic_translator.py +0 -1
- nmdc_runtime/site/translation/neon_soil_translator.py +4 -5
- nmdc_runtime/site/translation/neon_surface_water_translator.py +0 -2
- nmdc_runtime/site/translation/submission_portal_translator.py +64 -54
- nmdc_runtime/site/translation/translator.py +63 -1
- nmdc_runtime/site/util.py +8 -3
- nmdc_runtime/site/validation/util.py +10 -5
- nmdc_runtime/util.py +9 -321
- {nmdc_runtime-2.8.0.dist-info → nmdc_runtime-2.10.0.dist-info}/METADATA +57 -6
- nmdc_runtime-2.10.0.dist-info/RECORD +138 -0
- nmdc_runtime/site/translation/emsl.py +0 -43
- nmdc_runtime/site/translation/gold.py +0 -53
- nmdc_runtime/site/translation/jgi.py +0 -32
- nmdc_runtime/site/translation/util.py +0 -132
- nmdc_runtime/site/validation/jgi.py +0 -43
- nmdc_runtime-2.8.0.dist-info/RECORD +0 -84
- {nmdc_runtime-2.8.0.dist-info → nmdc_runtime-2.10.0.dist-info}/WHEEL +0 -0
- {nmdc_runtime-2.8.0.dist-info → nmdc_runtime-2.10.0.dist-info}/entry_points.txt +0 -0
- {nmdc_runtime-2.8.0.dist-info → nmdc_runtime-2.10.0.dist-info}/licenses/LICENSE +0 -0
- {nmdc_runtime-2.8.0.dist-info → nmdc_runtime-2.10.0.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from dagster import op, AssetMaterialization, AssetKey,
|
|
1
|
+
from dagster import op, AssetMaterialization, AssetKey, MetadataValue
|
|
2
2
|
from jsonschema import Draft7Validator
|
|
3
3
|
from nmdc_runtime.util import get_nmdc_jsonschema_dict
|
|
4
4
|
from toolz import dissoc
|
|
@@ -92,10 +92,15 @@ def announce_validation_report(context, report, api_object):
|
|
|
92
92
|
asset_key=AssetKey(["validation", f"{collection_name}_validation"]),
|
|
93
93
|
description=f"{collection_name} translation validation",
|
|
94
94
|
metadata={
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
|
|
98
|
-
|
|
95
|
+
# Note: When this code was originally written, it used Dagster's `EventMetadata` class,
|
|
96
|
+
# which has since been replaced by Dagster's `MetadataValue` class.
|
|
97
|
+
#
|
|
98
|
+
# Reference:
|
|
99
|
+
# - https://docs.dagster.io/api/dagster/ops#dagster.MetadataValue
|
|
100
|
+
# - https://docs.dagster.io/api/dagster/metadata#dagster.MetadataValue
|
|
101
|
+
#
|
|
102
|
+
"n_errors": MetadataValue.int(len(report["errors"])),
|
|
103
|
+
"object_id": MetadataValue.text(api_object["id"]),
|
|
99
104
|
},
|
|
100
105
|
)
|
|
101
106
|
|
nmdc_runtime/util.py
CHANGED
|
@@ -3,78 +3,26 @@ import mimetypes
|
|
|
3
3
|
import os
|
|
4
4
|
import pkgutil
|
|
5
5
|
from collections.abc import Iterable
|
|
6
|
-
from contextlib import AbstractContextManager
|
|
7
6
|
from copy import deepcopy
|
|
8
7
|
from datetime import datetime, timezone
|
|
9
8
|
from functools import lru_cache
|
|
10
9
|
from io import BytesIO
|
|
11
10
|
from itertools import chain
|
|
12
11
|
from pathlib import Path
|
|
13
|
-
from uuid import uuid4
|
|
14
12
|
from typing import Callable, List, Optional, Set, Dict
|
|
15
13
|
|
|
16
14
|
import fastjsonschema
|
|
17
15
|
import requests
|
|
18
16
|
from frozendict import frozendict
|
|
19
|
-
from jsonschema.validators import Draft7Validator
|
|
20
|
-
from linkml_runtime import linkml_model
|
|
21
|
-
from linkml_runtime.utils.schemaview import SchemaView
|
|
22
|
-
from nmdc_schema.nmdc import Database as NMDCDatabase
|
|
23
17
|
from nmdc_schema.get_nmdc_view import ViewGetter
|
|
24
|
-
from pydantic import Field, BaseModel
|
|
25
18
|
from pymongo.database import Database as MongoDatabase
|
|
26
19
|
from pymongo.errors import OperationFailure
|
|
27
20
|
from refscan.lib.helpers import identify_references
|
|
28
|
-
from refscan.lib.Finder import Finder
|
|
29
21
|
from refscan.lib.ReferenceList import ReferenceList
|
|
30
|
-
from
|
|
31
|
-
from toolz import merge, unique
|
|
22
|
+
from toolz import merge
|
|
32
23
|
|
|
33
24
|
from nmdc_runtime.api.core.util import sha256hash_from_file
|
|
34
25
|
from nmdc_runtime.api.models.object import DrsObjectIn
|
|
35
|
-
from typing_extensions import Annotated
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
def get_names_of_classes_in_effective_range_of_slot(
|
|
39
|
-
schema_view: SchemaView, slot_definition: linkml_model.SlotDefinition
|
|
40
|
-
) -> List[str]:
|
|
41
|
-
r"""
|
|
42
|
-
Determine the slot's "effective" range, by taking into account its `any_of` constraints (if defined).
|
|
43
|
-
|
|
44
|
-
Note: The `any_of` constraints constrain the slot's "effective" range beyond that described by the
|
|
45
|
-
induced slot definition's `range` attribute. `SchemaView` does not seem to provide the result
|
|
46
|
-
of applying those additional constraints, so we do it manually here (if any are defined).
|
|
47
|
-
Reference: https://github.com/orgs/linkml/discussions/2101#discussion-6625646
|
|
48
|
-
|
|
49
|
-
Reference: https://linkml.io/linkml-model/latest/docs/any_of/
|
|
50
|
-
"""
|
|
51
|
-
|
|
52
|
-
# Initialize the list to be empty.
|
|
53
|
-
names_of_eligible_target_classes = []
|
|
54
|
-
|
|
55
|
-
# If the `any_of` constraint is defined on this slot, use that instead of the `range`.
|
|
56
|
-
if "any_of" in slot_definition and len(slot_definition.any_of) > 0:
|
|
57
|
-
for slot_expression in slot_definition.any_of:
|
|
58
|
-
# Use the slot expression's `range` to get the specified eligible class name
|
|
59
|
-
# and the names of all classes that inherit from that eligible class.
|
|
60
|
-
if slot_expression.range in schema_view.all_classes():
|
|
61
|
-
own_and_descendant_class_names = schema_view.class_descendants(
|
|
62
|
-
slot_expression.range
|
|
63
|
-
)
|
|
64
|
-
names_of_eligible_target_classes.extend(own_and_descendant_class_names)
|
|
65
|
-
else:
|
|
66
|
-
# Use the slot's `range` to get the specified eligible class name
|
|
67
|
-
# and the names of all classes that inherit from that eligible class.
|
|
68
|
-
if slot_definition.range in schema_view.all_classes():
|
|
69
|
-
own_and_descendant_class_names = schema_view.class_descendants(
|
|
70
|
-
slot_definition.range
|
|
71
|
-
)
|
|
72
|
-
names_of_eligible_target_classes.extend(own_and_descendant_class_names)
|
|
73
|
-
|
|
74
|
-
# Remove duplicate class names.
|
|
75
|
-
names_of_eligible_target_classes = list(set(names_of_eligible_target_classes))
|
|
76
|
-
|
|
77
|
-
return names_of_eligible_target_classes
|
|
78
26
|
|
|
79
27
|
|
|
80
28
|
def get_class_names_from_collection_spec(
|
|
@@ -332,9 +280,9 @@ def find_one(k_v: dict, entities: Iterable[dict]):
|
|
|
332
280
|
"""Find the first entity with key-value pair k_v, if any?
|
|
333
281
|
|
|
334
282
|
>>> find_one({"id": "foo"}, [{"id": "foo"}])
|
|
283
|
+
{'id': 'foo'}
|
|
284
|
+
>>> find_one({"id": "foo"}, [{"id": "bar"}]) is None
|
|
335
285
|
True
|
|
336
|
-
>>> find_one({"id": "foo"}, [{"id": "bar"}])
|
|
337
|
-
False
|
|
338
286
|
"""
|
|
339
287
|
if len(k_v) > 1:
|
|
340
288
|
raise Exception("Supports only one key-value pair")
|
|
@@ -378,7 +326,7 @@ def nmdc_database_collection_names():
|
|
|
378
326
|
TODO: Document this function.
|
|
379
327
|
|
|
380
328
|
TODO: Assuming this function was designed to return a list of names of all Database slots that represents database
|
|
381
|
-
collections, use the function named `get_collection_names_from_schema`
|
|
329
|
+
collections, import/use the function named `get_collection_names_from_schema` from `refscan.lib.helpers`
|
|
382
330
|
instead, since (a) it includes documentation and (b) it performs the additional checks the lead schema
|
|
383
331
|
maintainer expects (e.g. checking whether a slot is `multivalued` and `inlined_as_list`).
|
|
384
332
|
"""
|
|
@@ -499,6 +447,11 @@ def populated_schema_collection_names_with_id_field(mdb: MongoDatabase) -> List[
|
|
|
499
447
|
|
|
500
448
|
def ensure_unique_id_indexes(mdb: MongoDatabase):
|
|
501
449
|
"""Ensure that any collections with an "id" field have an index on "id"."""
|
|
450
|
+
|
|
451
|
+
# Note: The pipe (i.e. `|`) operator performs a union of the two sets. In this case,
|
|
452
|
+
# it creates a set (i.e. `candidate_names`) consisting of the names of both
|
|
453
|
+
# (a) all collections in the real database, and (b) all collections that
|
|
454
|
+
# the NMDC schema says can contain instances of classes that have an "id" slot.
|
|
502
455
|
candidate_names = (
|
|
503
456
|
set(mdb.list_collection_names()) | schema_collection_names_with_id_field()
|
|
504
457
|
)
|
|
@@ -533,271 +486,6 @@ def ensure_unique_id_indexes(mdb: MongoDatabase):
|
|
|
533
486
|
raise
|
|
534
487
|
|
|
535
488
|
|
|
536
|
-
class UpdateStatement(BaseModel):
|
|
537
|
-
q: dict
|
|
538
|
-
u: dict
|
|
539
|
-
upsert: bool = False
|
|
540
|
-
multi: bool = False
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
class DeleteStatement(BaseModel):
|
|
544
|
-
q: dict
|
|
545
|
-
limit: Annotated[int, Field(ge=0, le=1)] = 1
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
class OverlayDBError(Exception):
|
|
549
|
-
pass
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
class OverlayDB(AbstractContextManager):
|
|
553
|
-
"""Provides a context whereby a base Database is overlaid with a temporary one.
|
|
554
|
-
|
|
555
|
-
If you need to run basic simulations of updates to a base database,
|
|
556
|
-
you don't want to actually commit transactions to the base database.
|
|
557
|
-
|
|
558
|
-
For example, to insert or replace (matching on "id") many documents into a collection in order
|
|
559
|
-
to then validate the resulting total set of collection documents, an OverlayDB writes to
|
|
560
|
-
an overlay collection that "shadows" the base collection during a "find" query
|
|
561
|
-
(the "merge_find" method of an OverlayDB object): if a document with `id0` is found in the
|
|
562
|
-
overlay collection, that id is marked as "seen" and will not also be returned when
|
|
563
|
-
subsequently scanning the (unmodified) base-database collection.
|
|
564
|
-
|
|
565
|
-
Note: The OverlayDB object does not provide a means to perform arbitrary MongoDB queries on the virtual "merged"
|
|
566
|
-
database. Callers can access the real database via `overlay_db._bottom_db` and the overlaying database via
|
|
567
|
-
`overlay_db._top_db` and perform arbitrary MongoDB queries on the individual databases that way. Access to
|
|
568
|
-
the virtual "merged" database is limited to the methods of the `OverlayDB` class, which simulates the
|
|
569
|
-
"merging" just-in-time to process the method invocation. You can see an example of this in the implementation
|
|
570
|
-
of the `merge_find` method, which internally accesses both the real database and the overlaying database.
|
|
571
|
-
|
|
572
|
-
Mongo "update" commands (as the "apply_updates" method) are simulated by first copying affected
|
|
573
|
-
documents from a base collection to the overlay, and then applying the updates to the overlay,
|
|
574
|
-
so that again, base collections are unmodified, and a "merge_find" call will produce a result
|
|
575
|
-
*as if* the base collection(s) were modified.
|
|
576
|
-
|
|
577
|
-
Mongo deletions (as the "delete" method) also copy affected documents from the base collection
|
|
578
|
-
to the overlay collection, and flag them using the "_deleted" field. In this way, a `merge_find`
|
|
579
|
-
call will match a relevant document given a suitable filter, and will mark the document's id
|
|
580
|
-
as "seen" *without* returning the document. Thus, the result is as if the document were deleted.
|
|
581
|
-
|
|
582
|
-
Usage:
|
|
583
|
-
````
|
|
584
|
-
with OverlayDB(mdb) as odb:
|
|
585
|
-
# do stuff, e.g. `odb.replace_or_insert_many(...)`
|
|
586
|
-
```
|
|
587
|
-
"""
|
|
588
|
-
|
|
589
|
-
def __init__(self, mdb: MongoDatabase):
|
|
590
|
-
self._bottom_db = mdb
|
|
591
|
-
self._top_db = self._bottom_db.client.get_database(f"overlay-{uuid4()}")
|
|
592
|
-
ensure_unique_id_indexes(self._top_db)
|
|
593
|
-
|
|
594
|
-
def __enter__(self):
|
|
595
|
-
return self
|
|
596
|
-
|
|
597
|
-
def __exit__(self, exc_type, exc_value, traceback):
|
|
598
|
-
self._bottom_db.client.drop_database(self._top_db.name)
|
|
599
|
-
|
|
600
|
-
def replace_or_insert_many(self, coll_name, documents: list):
|
|
601
|
-
try:
|
|
602
|
-
self._top_db[coll_name].insert_many(documents)
|
|
603
|
-
except OperationFailure as e:
|
|
604
|
-
raise OverlayDBError(str(e.details))
|
|
605
|
-
|
|
606
|
-
def apply_updates(self, coll_name, updates: list):
|
|
607
|
-
"""prepare overlay db and apply updates to it."""
|
|
608
|
-
assert all(UpdateStatement(**us) for us in updates)
|
|
609
|
-
for update_spec in updates:
|
|
610
|
-
for bottom_doc in self._bottom_db[coll_name].find(update_spec["q"]):
|
|
611
|
-
self._top_db[coll_name].insert_one(bottom_doc)
|
|
612
|
-
try:
|
|
613
|
-
self._top_db.command({"update": coll_name, "updates": updates})
|
|
614
|
-
except OperationFailure as e:
|
|
615
|
-
raise OverlayDBError(str(e.details))
|
|
616
|
-
|
|
617
|
-
def delete(self, coll_name, deletes: list):
|
|
618
|
-
""" "apply" delete command by flagging docs in overlay database"""
|
|
619
|
-
assert all(DeleteStatement(**us) for us in deletes)
|
|
620
|
-
for delete_spec in deletes:
|
|
621
|
-
for bottom_doc in self._bottom_db[coll_name].find(
|
|
622
|
-
delete_spec["q"], limit=delete_spec.get("limit", 1)
|
|
623
|
-
):
|
|
624
|
-
bottom_doc["_deleted"] = True
|
|
625
|
-
self._top_db[coll_name].insert_one(bottom_doc)
|
|
626
|
-
|
|
627
|
-
def merge_find(self, coll_name, find_spec: dict):
|
|
628
|
-
"""Yield docs first from overlay and then from base db, minding deletion flags."""
|
|
629
|
-
# ensure projection of "id" and "_deleted"
|
|
630
|
-
if "projection" in find_spec:
|
|
631
|
-
proj = find_spec["projection"]
|
|
632
|
-
if isinstance(proj, dict):
|
|
633
|
-
proj = merge(proj, {"id": 1, "_deleted": 1})
|
|
634
|
-
elif isinstance(proj, list):
|
|
635
|
-
proj = list(unique(proj + ["id", "_deleted"]))
|
|
636
|
-
|
|
637
|
-
top_docs = self._top_db[coll_name].find(**find_spec)
|
|
638
|
-
bottom_docs = self._bottom_db[coll_name].find(**find_spec)
|
|
639
|
-
top_seen_ids = set()
|
|
640
|
-
for doc in top_docs:
|
|
641
|
-
if not doc.get("_deleted"):
|
|
642
|
-
yield doc
|
|
643
|
-
top_seen_ids.add(doc["id"])
|
|
644
|
-
|
|
645
|
-
for doc in bottom_docs:
|
|
646
|
-
if doc["id"] not in top_seen_ids:
|
|
647
|
-
yield doc
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
def validate_json(
|
|
651
|
-
in_docs: dict, mdb: MongoDatabase, check_inter_document_references: bool = False
|
|
652
|
-
):
|
|
653
|
-
r"""
|
|
654
|
-
Checks whether the specified dictionary represents a valid instance of the `Database` class
|
|
655
|
-
defined in the NMDC Schema. Referential integrity checking is performed on an opt-in basis.
|
|
656
|
-
|
|
657
|
-
Example dictionary:
|
|
658
|
-
{
|
|
659
|
-
"biosample_set": [
|
|
660
|
-
{"id": "nmdc:bsm-00-000001", ...},
|
|
661
|
-
{"id": "nmdc:bsm-00-000002", ...}
|
|
662
|
-
],
|
|
663
|
-
"study_set": [
|
|
664
|
-
{"id": "nmdc:sty-00-000001", ...},
|
|
665
|
-
{"id": "nmdc:sty-00-000002", ...}
|
|
666
|
-
]
|
|
667
|
-
}
|
|
668
|
-
|
|
669
|
-
:param in_docs: The dictionary you want to validate
|
|
670
|
-
:param mdb: A reference to a MongoDB database
|
|
671
|
-
:param check_inter_document_references: Whether you want this function to check whether every document that
|
|
672
|
-
is referenced by any of the documents passed in would, indeed, exist
|
|
673
|
-
in the database, if the documents passed in were to be inserted into
|
|
674
|
-
the database. In other words, set this to `True` if you want this
|
|
675
|
-
function to perform referential integrity checks.
|
|
676
|
-
"""
|
|
677
|
-
validator = Draft7Validator(get_nmdc_jsonschema_dict())
|
|
678
|
-
docs = deepcopy(in_docs)
|
|
679
|
-
validation_errors = {}
|
|
680
|
-
|
|
681
|
-
known_coll_names = set(nmdc_database_collection_names())
|
|
682
|
-
for coll_name, coll_docs in docs.items():
|
|
683
|
-
if coll_name not in known_coll_names:
|
|
684
|
-
# FIXME: Document what `@type` is (conceptually; e.g., why this function accepts it as a collection name).
|
|
685
|
-
# See: https://github.com/microbiomedata/nmdc-runtime/discussions/858
|
|
686
|
-
if coll_name == "@type" and coll_docs in ("Database", "nmdc:Database"):
|
|
687
|
-
continue
|
|
688
|
-
else:
|
|
689
|
-
validation_errors[coll_name] = [
|
|
690
|
-
f"'{coll_name}' is not a known schema collection name"
|
|
691
|
-
]
|
|
692
|
-
continue
|
|
693
|
-
|
|
694
|
-
errors = list(validator.iter_errors({coll_name: coll_docs}))
|
|
695
|
-
validation_errors[coll_name] = [e.message for e in errors]
|
|
696
|
-
if coll_docs:
|
|
697
|
-
if not isinstance(coll_docs, list):
|
|
698
|
-
validation_errors[coll_name].append("value must be a list")
|
|
699
|
-
elif not all(isinstance(d, dict) for d in coll_docs):
|
|
700
|
-
validation_errors[coll_name].append(
|
|
701
|
-
"all elements of list must be dicts"
|
|
702
|
-
)
|
|
703
|
-
if not validation_errors[coll_name]:
|
|
704
|
-
try:
|
|
705
|
-
with OverlayDB(mdb) as odb:
|
|
706
|
-
odb.replace_or_insert_many(coll_name, coll_docs)
|
|
707
|
-
except OverlayDBError as e:
|
|
708
|
-
validation_errors[coll_name].append(str(e))
|
|
709
|
-
|
|
710
|
-
if all(len(v) == 0 for v in validation_errors.values()):
|
|
711
|
-
# Second pass. Try instantiating linkml-sourced dataclass
|
|
712
|
-
in_docs.pop("@type", None)
|
|
713
|
-
try:
|
|
714
|
-
NMDCDatabase(**in_docs)
|
|
715
|
-
except Exception as e:
|
|
716
|
-
return {"result": "errors", "detail": str(e)}
|
|
717
|
-
|
|
718
|
-
# Third pass (if enabled): Check inter-document references.
|
|
719
|
-
if check_inter_document_references is True:
|
|
720
|
-
# Prepare to use `refscan`.
|
|
721
|
-
#
|
|
722
|
-
# Note: We check the inter-document references in two stages, which are:
|
|
723
|
-
# 1. For each document in the JSON payload, check whether each document it references already exists
|
|
724
|
-
# (in the collections the schema says it can exist in) in the database. We use the
|
|
725
|
-
# `refscan` package to do this, which returns violation details we'll use in the second stage.
|
|
726
|
-
# 2. For each violation found in the first stage (i.e. each reference to a not-found document), we
|
|
727
|
-
# check whether that document exists (in the collections the schema says it can exist in) in the
|
|
728
|
-
# JSON payload. If it does, then we "waive" (i.e. discard) that violation.
|
|
729
|
-
# The violations that remain after those two stages are the ones we return to the caller.
|
|
730
|
-
#
|
|
731
|
-
# Note: The reason we do not insert documents into an `OverlayDB` and scan _that_, is that the `OverlayDB`
|
|
732
|
-
# does not provide a means to perform arbitrary queries against its virtual "merged" database. It
|
|
733
|
-
# is not a drop-in replacement for a pymongo's `Database` class, which is the only thing that
|
|
734
|
-
# `refscan`'s `Finder` class accepts.
|
|
735
|
-
#
|
|
736
|
-
finder = Finder(database=mdb)
|
|
737
|
-
references = get_allowed_references()
|
|
738
|
-
reference_field_names_by_source_class_name = (
|
|
739
|
-
references.get_reference_field_names_by_source_class_name()
|
|
740
|
-
)
|
|
741
|
-
|
|
742
|
-
# Iterate over the collections in the JSON payload.
|
|
743
|
-
for source_collection_name, documents in in_docs.items():
|
|
744
|
-
for document in documents:
|
|
745
|
-
# Add an `_id` field to the document, since `refscan` requires the document to have one.
|
|
746
|
-
source_document = dict(document, _id=None)
|
|
747
|
-
violations = scan_outgoing_references(
|
|
748
|
-
document=source_document,
|
|
749
|
-
schema_view=nmdc_schema_view(),
|
|
750
|
-
reference_field_names_by_source_class_name=reference_field_names_by_source_class_name,
|
|
751
|
-
references=references,
|
|
752
|
-
finder=finder,
|
|
753
|
-
collection_names=nmdc_database_collection_names(),
|
|
754
|
-
source_collection_name=source_collection_name,
|
|
755
|
-
user_wants_to_locate_misplaced_documents=False,
|
|
756
|
-
)
|
|
757
|
-
|
|
758
|
-
# For each violation, check whether the misplaced document is in the JSON payload, itself.
|
|
759
|
-
for violation in violations:
|
|
760
|
-
can_waive_violation = False
|
|
761
|
-
# Determine which collections can contain the referenced document, based upon
|
|
762
|
-
# the schema class of which this source document is an instance.
|
|
763
|
-
target_collection_names = (
|
|
764
|
-
references.get_target_collection_names(
|
|
765
|
-
source_class_name=violation.source_class_name,
|
|
766
|
-
source_field_name=violation.source_field_name,
|
|
767
|
-
)
|
|
768
|
-
)
|
|
769
|
-
# Check whether the referenced document exists in any of those collections in the JSON payload.
|
|
770
|
-
for json_coll_name, json_coll_docs in in_docs.items():
|
|
771
|
-
if json_coll_name in target_collection_names:
|
|
772
|
-
for json_coll_doc in json_coll_docs:
|
|
773
|
-
if json_coll_doc["id"] == violation.target_id:
|
|
774
|
-
can_waive_violation = True
|
|
775
|
-
break # stop checking
|
|
776
|
-
if can_waive_violation:
|
|
777
|
-
break # stop checking
|
|
778
|
-
if not can_waive_violation:
|
|
779
|
-
violation_as_str = (
|
|
780
|
-
f"Document '{violation.source_document_id}' "
|
|
781
|
-
f"in collection '{violation.source_collection_name}' "
|
|
782
|
-
f"has a field '{violation.source_field_name}' that "
|
|
783
|
-
f"references a document having id "
|
|
784
|
-
f"'{violation.target_id}', but the latter document "
|
|
785
|
-
f"does not exist in any of the collections the "
|
|
786
|
-
f"NMDC Schema says it can exist in."
|
|
787
|
-
)
|
|
788
|
-
validation_errors[source_collection_name].append(
|
|
789
|
-
violation_as_str
|
|
790
|
-
)
|
|
791
|
-
|
|
792
|
-
# If any collection's error list is not empty, return an error response.
|
|
793
|
-
if any(len(v) > 0 for v in validation_errors.values()):
|
|
794
|
-
return {"result": "errors", "detail": validation_errors}
|
|
795
|
-
|
|
796
|
-
return {"result": "All Okay!"}
|
|
797
|
-
else:
|
|
798
|
-
return {"result": "errors", "detail": validation_errors}
|
|
799
|
-
|
|
800
|
-
|
|
801
489
|
def decorate_if(condition: bool = False) -> Callable:
|
|
802
490
|
r"""
|
|
803
491
|
Decorator that applies another decorator only when `condition` is `True`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nmdc_runtime
|
|
3
|
-
Version: 2.
|
|
3
|
+
Version: 2.10.0
|
|
4
4
|
Summary: A runtime system for NMDC data management and orchestration
|
|
5
5
|
Home-page: https://github.com/microbiomedata/nmdc-runtime
|
|
6
6
|
Author: Donny Winston
|
|
@@ -152,7 +152,7 @@ http://127.0.0.1:8000/redoc/.
|
|
|
152
152
|
|
|
153
153
|
|
|
154
154
|
* NOTE: Any time you add or change requirements in requirements/main.in or requirements/dev.in, you must run:
|
|
155
|
-
```
|
|
155
|
+
```bash
|
|
156
156
|
pip-compile --build-isolation --allow-unsafe --resolver=backtracking --strip-extras --output-file requirements/[main|dev].txt requirements/[main|dev].in
|
|
157
157
|
```
|
|
158
158
|
to generate main.txt and dev.txt files respectively. main.in is kind of like a poetry dependency stanza, dev.in is kind
|
|
@@ -160,9 +160,6 @@ of like poetry dev.dependencies stanza. main.txt and dev.txt are kind of like po
|
|
|
160
160
|
versions of dependencies to use. main.txt and dev.txt are combined in the docker compose build process to create the
|
|
161
161
|
final requirements.txt file and import the dependencies into the Docker image.
|
|
162
162
|
|
|
163
|
-
|
|
164
|
-
```bash
|
|
165
|
-
|
|
166
163
|
## Local Testing
|
|
167
164
|
|
|
168
165
|
Tests can be found in `tests` and are run with the following commands:
|
|
@@ -173,8 +170,9 @@ make test
|
|
|
173
170
|
|
|
174
171
|
# Run a Specific test file eg. tests/test_api/test_endpoints.py
|
|
175
172
|
make test ARGS="tests/test_api/test_endpoints.py"
|
|
176
|
-
|
|
173
|
+
|
|
177
174
|
docker compose --file docker-compose.test.yml run test
|
|
175
|
+
```
|
|
178
176
|
|
|
179
177
|
As you create Dagster solids and pipelines, add tests in `tests/` to check that your code behaves as
|
|
180
178
|
desired and does not break over time.
|
|
@@ -182,6 +180,59 @@ desired and does not break over time.
|
|
|
182
180
|
[For hints on how to write tests for solids and pipelines in Dagster, see their documentation
|
|
183
181
|
tutorial on Testing](https://docs.dagster.io/guides/test/unit-testing-assets-and-ops).
|
|
184
182
|
|
|
183
|
+
### Performance profiling
|
|
184
|
+
|
|
185
|
+
We use a tool called [Pyinstrument](https://pyinstrument.readthedocs.io) to profile the performance of the Runtime API while processing an individual HTTP request.
|
|
186
|
+
|
|
187
|
+
Here's how you can do that:
|
|
188
|
+
|
|
189
|
+
1. In your `.env` file, set `IS_PROFILING_ENABLED` to `true`
|
|
190
|
+
2. Start/restart your development stack: `$ make up-dev`
|
|
191
|
+
3. Ensure the endpoint function whose performance you want to profile is defined using `async def` (as opposed to just `def`) ([reference](https://github.com/joerick/pyinstrument/issues/257))
|
|
192
|
+
|
|
193
|
+
Then—with all of that done—submit an HTTP request that includes the URL query parameter: `profile=true`. Instructions for doing that are in the sections below.
|
|
194
|
+
|
|
195
|
+
<details>
|
|
196
|
+
<summary>Show/hide instructions for <code>GET</code> requests only (involves web browser)</summary>
|
|
197
|
+
|
|
198
|
+
1. In your web browser, visit the endpoint's URL, but add the `profile=true` query parameter to the URL. Examples:
|
|
199
|
+
```diff
|
|
200
|
+
A. If the URL doesn't already have query parameters, append `?profile=true`.
|
|
201
|
+
- http://127.0.0.1:8000/nmdcschema/biosample_set
|
|
202
|
+
+ http://127.0.0.1:8000/nmdcschema/biosample_set?profile=true
|
|
203
|
+
|
|
204
|
+
B. If the URL already has query parameters, append `&profile=true`.
|
|
205
|
+
- http://127.0.0.1:8000/nmdcschema/biosample_set?filter={}
|
|
206
|
+
+ http://127.0.0.1:8000/nmdcschema/biosample_set?filter={}&profile=true
|
|
207
|
+
```
|
|
208
|
+
2. Your web browser will display a performance profiling report.
|
|
209
|
+
> Note: The Runtime API will have responded with a performance profiling report web page, instead of its normal response (which the Runtime discards).
|
|
210
|
+
|
|
211
|
+
That'll only work for `GET` requests, though, since you're limited to specifying the request via the address bar.
|
|
212
|
+
|
|
213
|
+
</details>
|
|
214
|
+
|
|
215
|
+
<details>
|
|
216
|
+
<summary>Show/hide instructions for <strong>all</strong> kinds of requests (involves <code>curl</code> + web browser)</summary>
|
|
217
|
+
|
|
218
|
+
1. At your terminal, type or paste the `curl` command you want to run (you can copy/paste one from Swagger UI).
|
|
219
|
+
2. Append the `profile=true` query parameter to the URL in the command, and use the `-o` option to save the response to a file whose name ends with `.html`. For example:
|
|
220
|
+
```diff
|
|
221
|
+
curl -X 'POST' \
|
|
222
|
+
- 'http://127.0.0.1:8000/metadata/json:validate' \
|
|
223
|
+
+ 'http://127.0.0.1:8000/metadata/json:validate?profile=true' \
|
|
224
|
+
+ -o /tmp/profile.html
|
|
225
|
+
-H 'accept: application/json' \
|
|
226
|
+
-H 'Content-Type: application/json' \
|
|
227
|
+
-d '{"biosample_set": []}'
|
|
228
|
+
```
|
|
229
|
+
3. Run the command.
|
|
230
|
+
> Note: The Runtime API will respond with a performance profiling report web page, instead of its normal response (which the Runtime discards). The performance profiling report web page will be saved to the `.html` file to which you redirected the command output.
|
|
231
|
+
4. Double-click on the `.html` file to view it in your web browser.
|
|
232
|
+
1. Alternatively, open your web browser and navigate to the `.html` file; e.g., enter `file:///tmp/profile.html` into the address bar.
|
|
233
|
+
|
|
234
|
+
</details>
|
|
235
|
+
|
|
185
236
|
### RAM usage
|
|
186
237
|
|
|
187
238
|
The `dagster-daemon` and `dagster-dagit` containers can consume a lot of RAM. If tests are failing and the console of
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
nmdc_runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
nmdc_runtime/config.py,sha256=fVxcqdXv13Fa9CSRPnFIsfmvmlos8o4SFUZcmsXfX_8,2020
|
|
3
|
+
nmdc_runtime/containers.py,sha256=8m_S1wiFu8VOWvY7tyqzf-02X9gXY83YGc8FgjWzLGA,418
|
|
4
|
+
nmdc_runtime/main.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
5
|
+
nmdc_runtime/mongo_util.py,sha256=L6UxK_6f0wQw2NTKCUVKCp-QLhBudQczDLUdF5odbP8,2943
|
|
6
|
+
nmdc_runtime/util.py,sha256=4-LndkMcQox9fKDfqYkNMU1aKrRVkxhk_7U9L2Kdfno,17644
|
|
7
|
+
nmdc_runtime/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
nmdc_runtime/api/analytics.py,sha256=DyxHYCLlpG-IyOvlcLoyurqWgTT2MoiliC06JkZ3aAY,2416
|
|
9
|
+
nmdc_runtime/api/main.py,sha256=KN8sjesJumLn3fRtyOlLrTFknPgQf8XPerO0FuVFPKo,14734
|
|
10
|
+
nmdc_runtime/api/middleware.py,sha256=GUVN26Ym9H87gaxrBs0NAMpOoA7qQfv-7UnIJOkcQkI,1703
|
|
11
|
+
nmdc_runtime/api/openapi.py,sha256=ABjwttrVOwEShYHI6zEkJEVyzKo9D040wpbaKLBubaM,9753
|
|
12
|
+
nmdc_runtime/api/boot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
nmdc_runtime/api/boot/capabilities.py,sha256=9Cp_yULUeEXN7Nz-WC5XJXTaB7fhOWOCGp8mx050qgg,291
|
|
14
|
+
nmdc_runtime/api/boot/object_types.py,sha256=JL6OZw34lKkbKJJXDIiswfLmn1tkOng4ZKF6ypqWKhs,4382
|
|
15
|
+
nmdc_runtime/api/boot/triggers.py,sha256=fLM588CBYft_no1ENN13XSO6Cj4DB90ZKJl-1UgfsYw,2723
|
|
16
|
+
nmdc_runtime/api/boot/workflows.py,sha256=UpOAMjVKagat-PCKPx7HuriLTnCbhj0EVgpk7UuLpQQ,3826
|
|
17
|
+
nmdc_runtime/api/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
+
nmdc_runtime/api/core/auth.py,sha256=c38EjN3Ckd3uuR70qqTiAKF3EedZCihoVvTKBmBSRMQ,7453
|
|
19
|
+
nmdc_runtime/api/core/idgen.py,sha256=jedXPPRVMq4MR3hXK67JkLE6O_BYjOPrP-mv2Fx81CI,5898
|
|
20
|
+
nmdc_runtime/api/core/metadata.py,sha256=AMX-EWfyOU_xobIlDf98PTQLuNCbZuyCPCIELKAoMIk,27943
|
|
21
|
+
nmdc_runtime/api/core/util.py,sha256=OGE9kCui-wEfkfdUeUOBNeqmHJ5fNpJepBo0ivYkMnE,3232
|
|
22
|
+
nmdc_runtime/api/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
23
|
+
nmdc_runtime/api/db/mongo.py,sha256=jnig92qWDuiiY0de_l-khg7GMkPYvPVBaC_bISSUH4U,20371
|
|
24
|
+
nmdc_runtime/api/db/s3.py,sha256=tRFEjjVXHMiUdZtRiq1ImvLza2s86TkgubDw3kchDOA,1046
|
|
25
|
+
nmdc_runtime/api/endpoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
26
|
+
nmdc_runtime/api/endpoints/capabilities.py,sha256=AkSh8wcsVSOkkjH0kzcM2TcO2o8AuKyXGt1km_VgwVs,717
|
|
27
|
+
nmdc_runtime/api/endpoints/find.py,sha256=kSRmJNDqV-iKV9b4UANSUAcr1CFNU06vHZ7nBRFjgOc,34297
|
|
28
|
+
nmdc_runtime/api/endpoints/ids.py,sha256=fuWNP9x_oVPh-VrrrkFT_wtPo-rBTDRr7dOqr1dxAt4,6713
|
|
29
|
+
nmdc_runtime/api/endpoints/jobs.py,sha256=waQk3A7gGIqcr2GNNX988Bxbvx5bxq-kGKmWXeNBrpY,5010
|
|
30
|
+
nmdc_runtime/api/endpoints/metadata.py,sha256=Msj7k5DxXUzXCZWcvy91T7AIFj1chhn_53Dsc44_qmc,9886
|
|
31
|
+
nmdc_runtime/api/endpoints/nmdcschema.py,sha256=yXbp5jgqvJR_tYtXka_sTgrYgPWFHt8bOukaj1Aay5E,24009
|
|
32
|
+
nmdc_runtime/api/endpoints/object_types.py,sha256=dTbl3A9j9lyk186arA7cszTEKOY7vXWJO_aKYfFAV8s,1179
|
|
33
|
+
nmdc_runtime/api/endpoints/objects.py,sha256=jWydxuVkJKImvuh2omXMPp6Ip2s-d8HBnJCy2JVFi08,10357
|
|
34
|
+
nmdc_runtime/api/endpoints/operations.py,sha256=mza7yaOQnjb41ZfoPK3TTBnTr4tfzMX43MKxACXP-y8,3130
|
|
35
|
+
nmdc_runtime/api/endpoints/queries.py,sha256=AXGhfST6FzkH2uH4EwhJ_Y4ldNXRL-Hptt9NKTmjkdg,29241
|
|
36
|
+
nmdc_runtime/api/endpoints/runs.py,sha256=PWoEKCFQhWgpWklmRUjTc0UZCvDAa1i59gnPhAcZirA,3326
|
|
37
|
+
nmdc_runtime/api/endpoints/search.py,sha256=_h30mu8_Xndjggg3IllMDn5h8k92BX0ubxqRO85R0Ss,1187
|
|
38
|
+
nmdc_runtime/api/endpoints/sites.py,sha256=wQ0uTDHfKg1nMArE6KNNZHCUljGM6uSlW6nsIGVKDfg,7138
|
|
39
|
+
nmdc_runtime/api/endpoints/triggers.py,sha256=1DG2oEOV7zu5bT2qoeHrLNajY6K6sEGi7O7po3Bcmbk,673
|
|
40
|
+
nmdc_runtime/api/endpoints/users.py,sha256=syu_Tz05k-OAsDMhe125ofqeiCkmDMjdbRlfvRZ5_rI,7816
|
|
41
|
+
nmdc_runtime/api/endpoints/util.py,sha256=5ATuq4k6FmxG-wvK7_-GBWRYR99mJrq8aVPj0rUwGDo,26894
|
|
42
|
+
nmdc_runtime/api/endpoints/workflows.py,sha256=HWTnt-yrHp2DcrQT8BS_-SRQWNibkKNPVOpwqjS9QCA,14383
|
|
43
|
+
nmdc_runtime/api/endpoints/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
44
|
+
nmdc_runtime/api/endpoints/lib/helpers.py,sha256=E6pH0NtzKiSGBgIfoeukH5qeHKtxjLtCvw4F8LOacTQ,13874
|
|
45
|
+
nmdc_runtime/api/endpoints/lib/path_segments.py,sha256=4nIy_KrYvTc80Np3ELnT94VCk2QfR-2055fMlcbBSPw,5724
|
|
46
|
+
nmdc_runtime/api/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
47
|
+
nmdc_runtime/api/models/capability.py,sha256=4__rqgLS4MCMjbaCM3e_ncR9KW001Klm34p2t_bp65k,262
|
|
48
|
+
nmdc_runtime/api/models/id.py,sha256=D8kDS-f3-OIxaNKrkhrdvyxu90ac4SDeFpVHboycDac,2724
|
|
49
|
+
nmdc_runtime/api/models/job.py,sha256=GrmGDlw7qJS8FCz4Z5PvC2U5LcwC_DZXvTYAmEOSI4g,755
|
|
50
|
+
nmdc_runtime/api/models/metadata.py,sha256=mVCC0KODtKzNEYABk70jaKVoOTyZP87eCtzf96lgysw,196
|
|
51
|
+
nmdc_runtime/api/models/minter.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
|
+
nmdc_runtime/api/models/nmdc_schema.py,sha256=C-Jb_eHr4dSVoYP-r7ldBUW-JHhCqSMtxSMT9HWKZ70,5768
|
|
53
|
+
nmdc_runtime/api/models/object.py,sha256=A-U5RkWfmEsb50x9ai5xdpHo9xy-O9mZj2gKCoBR87E,5004
|
|
54
|
+
nmdc_runtime/api/models/object_type.py,sha256=2Ejn5iCbqwqVEaOcYo4mvUJgBTDMhlmw8cLD92bWwSE,399
|
|
55
|
+
nmdc_runtime/api/models/operation.py,sha256=Nb_Ys8P_vdxL-5fcKTeNTmB9CongxK3-JWs0vhgkNq8,1606
|
|
56
|
+
nmdc_runtime/api/models/query.py,sha256=899fPzA55xyskflLXQKlIADQATsnpl8-Pu5rhrE7MvA,6739
|
|
57
|
+
nmdc_runtime/api/models/query_continuation.py,sha256=bWVX7ijk34kOCedxAWFGgSTNgpy98yp0IZni31oD3-Y,4150
|
|
58
|
+
nmdc_runtime/api/models/run.py,sha256=OhWBc6lQeEM-GSgUpvJCE2xbZQzv8kdPsEkWlL0xolM,4406
|
|
59
|
+
nmdc_runtime/api/models/site.py,sha256=KLLgln2KJrinUDp6ixxci1JFmcLAL4O4vEtFTZKc82U,2310
|
|
60
|
+
nmdc_runtime/api/models/trigger.py,sha256=TviQMk9-2HMZgCiaXYAF0WFnjD295jxnJLJCWsmtem4,201
|
|
61
|
+
nmdc_runtime/api/models/user.py,sha256=R_K7f2Zaajo8-gCwKwu4Ytfg063v31Yb5xQSIH_iYxw,4083
|
|
62
|
+
nmdc_runtime/api/models/util.py,sha256=M9flQgkrnJIaV5udKLHs2e8ImcO6R9TUuRLiw7aS4gY,11026
|
|
63
|
+
nmdc_runtime/api/models/workflow.py,sha256=etPFP_L9DcRoIAFwvMYzLLT2WlwRG6T68-7tzNzXnQ0,326
|
|
64
|
+
nmdc_runtime/api/models/lib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
65
|
+
nmdc_runtime/api/models/lib/helpers.py,sha256=k6AihKIiQ0kg2kk_qY_VNWTb96LGkazuztARhgjHr8M,2410
|
|
66
|
+
nmdc_runtime/client/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
67
|
+
nmdc_runtime/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
68
|
+
nmdc_runtime/core/db/Database.py,sha256=95FFFJAfPABdQXhbTYLNhW6kvr8Cj6RNImhpLhQrlXY,310
|
|
69
|
+
nmdc_runtime/core/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
70
|
+
nmdc_runtime/core/exceptions/__init__.py,sha256=s486odD0uhUuk9K7M5_NISOgRrUE5RNnDJSypA2Qe_I,520
|
|
71
|
+
nmdc_runtime/core/exceptions/base.py,sha256=G5azYv0FJvbjrpQtK90BkM-KK2f534szdwrHj9N-SNo,1343
|
|
72
|
+
nmdc_runtime/core/exceptions/token.py,sha256=7iTdfRQjfijDExd6-KJBjN7t0BGI_Kc1F6Lc-d0AsE8,293
|
|
73
|
+
nmdc_runtime/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
74
|
+
nmdc_runtime/domain/users/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
75
|
+
nmdc_runtime/domain/users/queriesInterface.py,sha256=0DjOehnsA5oKADmRKh8NTool2zoQZaejFigXHuUGoOg,476
|
|
76
|
+
nmdc_runtime/domain/users/userSchema.py,sha256=eVpsB5aSbT89XjPh2_m7ao8XyyinEC94hpZQIouV4uk,758
|
|
77
|
+
nmdc_runtime/domain/users/userService.py,sha256=b-HD7N-wWQyAux_iZsXMBFrz5_j9ygRc3qsJlm-vQGI,428
|
|
78
|
+
nmdc_runtime/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
79
|
+
nmdc_runtime/infrastructure/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
80
|
+
nmdc_runtime/infrastructure/database/db.py,sha256=djdqVxXvvJWtJUj4yariINcOuYOkQ_OiAYI_jGqOtM8,32
|
|
81
|
+
nmdc_runtime/infrastructure/database/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
82
|
+
nmdc_runtime/infrastructure/database/models/user.py,sha256=U4S_2y3zgLZVfMenHRaJFBW8yqh2mUBuI291LGQVOJ8,35
|
|
83
|
+
nmdc_runtime/lib/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
84
|
+
nmdc_runtime/lib/extract_nmdc_data.py,sha256=zodxHn1OILRYAZUcKoej5uFfzB0esA1UZVubNlzv0sg,987
|
|
85
|
+
nmdc_runtime/lib/load_nmdc_data.py,sha256=KO2cIqkY3cBCVcFIwsGokZNOKntOejZVG8ecq43NjFM,3934
|
|
86
|
+
nmdc_runtime/lib/nmdc_dataframes.py,sha256=AMtL8IoVuvT2SIac_yx49UK_EP6fiProImjjeugaOOU,28721
|
|
87
|
+
nmdc_runtime/lib/nmdc_etl_class.py,sha256=EfLm6TfXEg-wurCJe-jTJg85j9TkMdCQe1hRtc8ancg,13379
|
|
88
|
+
nmdc_runtime/lib/transform_nmdc_data.py,sha256=hij4lR3IMQRJQdL-rsP_I-m_WyFPsBMchV2MNFUkh0M,39906
|
|
89
|
+
nmdc_runtime/minter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
90
|
+
nmdc_runtime/minter/bootstrap.py,sha256=5Ej6pJVBRryRIi0ZwEloY78Zky7iE2okF6tPwRI2axM,822
|
|
91
|
+
nmdc_runtime/minter/config.py,sha256=-E1kQXTDraabrN4CENuVCHcNJafjVdiHWeUrucxBzMQ,2741
|
|
92
|
+
nmdc_runtime/minter/adapters/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
93
|
+
nmdc_runtime/minter/adapters/repository.py,sha256=XxUVD_csKR0tM7yJT2psXc7WSj1t-tM50JSMfYA9Euo,8281
|
|
94
|
+
nmdc_runtime/minter/domain/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
95
|
+
nmdc_runtime/minter/domain/model.py,sha256=Ex1ADUZCOXpDoTWb6THRjlZxeDcvvsfsFqkkbiJpUsc,2823
|
|
96
|
+
nmdc_runtime/minter/entrypoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
97
|
+
nmdc_runtime/minter/entrypoints/fastapi_app.py,sha256=I_lgExs6g1MRpMQdpedrnYdA1L7r_TBi4RiiD8ogrkM,4015
|
|
98
|
+
nmdc_runtime/site/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
99
|
+
nmdc_runtime/site/graphs.py,sha256=ngDeTTZZWCHUuId3rqjMUF7RpeFNIx2F2Wp_fuy0DE0,17676
|
|
100
|
+
nmdc_runtime/site/ops.py,sha256=dPlOQg9hNTYWDsVK7UKVkRazZkph06k7uyLRSn5l3BY,60870
|
|
101
|
+
nmdc_runtime/site/repository.py,sha256=BCZjaYdI2zyc28OlQm4vkz1w13D34eXtNMqINPMjMAk,44025
|
|
102
|
+
nmdc_runtime/site/resources.py,sha256=dLNtNa4FfSKN_6b21eItn-i8e0ZHyveoBsexl2I6zmo,20144
|
|
103
|
+
nmdc_runtime/site/util.py,sha256=4h0X_fhjf3HdX6XDR8GvHgkrpxQY4OnZVtaOeXJVxJQ,1935
|
|
104
|
+
nmdc_runtime/site/backup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
105
|
+
nmdc_runtime/site/backup/nmdcdb_mongodump.py,sha256=My2ORKVIk_Z9wzfnIuamDe3_hv4viid9ToSJDC5J4mY,2689
|
|
106
|
+
nmdc_runtime/site/backup/nmdcdb_mongoexport.py,sha256=y1x3B4-qxF5_itXOKYaix99OvDhW_PYxhLoLc4Y5E1M,4028
|
|
107
|
+
nmdc_runtime/site/backup/nmdcdb_mongoimport.py,sha256=k6w5yscMNYoMBVkaAA9soWS0Dj2CB0FRBSFlifRO3Ro,1739
|
|
108
|
+
nmdc_runtime/site/changesheets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
109
|
+
nmdc_runtime/site/changesheets/base.py,sha256=lZT6WCsEBl-FsTr7Ki8_ploT93uMiVyIWWKM36aOrRk,3171
|
|
110
|
+
nmdc_runtime/site/drsobjects/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
111
|
+
nmdc_runtime/site/drsobjects/ingest.py,sha256=pcMP69WSzFHFqHB9JIL55ePFhilnCLRc2XHCQ97w1Ik,3107
|
|
112
|
+
nmdc_runtime/site/drsobjects/registration.py,sha256=D1T3QUuxEOxqKZIvB5rkb_6ZxFZiA-U9SMPajyeWC2Y,3572
|
|
113
|
+
nmdc_runtime/site/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
114
|
+
nmdc_runtime/site/export/ncbi_xml.py,sha256=W686Fufs0UynnvzcQf3qHzgCa4ToynTqo5ZnRU0p514,29690
|
|
115
|
+
nmdc_runtime/site/export/ncbi_xml_utils.py,sha256=XaJZWxxgE-x1t5NUUGpuijryQifzjIs185nw4BmpF7Y,10693
|
|
116
|
+
nmdc_runtime/site/export/study_metadata.py,sha256=yR5pXL6JG8d7cAtqcF-60Hp7bLD3dJ0Rut4AtYc0tXA,4844
|
|
117
|
+
nmdc_runtime/site/normalization/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
118
|
+
nmdc_runtime/site/normalization/gold.py,sha256=iISDD4qs4d6uLhv631WYNeQVOzY5DO201ZpPtxHdkVk,1311
|
|
119
|
+
nmdc_runtime/site/repair/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
120
|
+
nmdc_runtime/site/repair/database_updater.py,sha256=a6POYZcLEl0JvnuWxPjaOJtwZjkJhhvvUg1ABhnBiP8,21268
|
|
121
|
+
nmdc_runtime/site/translation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
122
|
+
nmdc_runtime/site/translation/gold_translator.py,sha256=4AFgbJtHaVwme3a57Y6Foi-uzI8oBHUlOt3Ns7_a5_o,32879
|
|
123
|
+
nmdc_runtime/site/translation/neon_benthic_translator.py,sha256=CMoC56ymA0DKPkzqdMR4m5yYV6EcyH3tOvOiA3P6goE,23762
|
|
124
|
+
nmdc_runtime/site/translation/neon_soil_translator.py,sha256=MMntFXwK62PdPNGpurTq5L3-pct7xAmUymRE2QqMPso,38572
|
|
125
|
+
nmdc_runtime/site/translation/neon_surface_water_translator.py,sha256=_-KDZzC30dQ1y57lXEKWXE6ZfGozNHxGFvbGaj4f0Lg,30536
|
|
126
|
+
nmdc_runtime/site/translation/neon_utils.py,sha256=d00o7duKKugpLHmsEifNbp4WjeC4GOqcgw0b5qlCg4I,5549
|
|
127
|
+
nmdc_runtime/site/translation/submission_portal_translator.py,sha256=x9iUxPxZeWi2ajISBAY09bwzNugqqxkKmuOPw7887v0,41822
|
|
128
|
+
nmdc_runtime/site/translation/translator.py,sha256=kVmDIVfxQ0ry6obAc_SyvlEcPwPCupYeKenRylZWWbI,3643
|
|
129
|
+
nmdc_runtime/site/validation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
130
|
+
nmdc_runtime/site/validation/emsl.py,sha256=OG20mv_3E2rkQqTQtYO0_SVRqFb-Z_zKCiAVbty6Wl0,671
|
|
131
|
+
nmdc_runtime/site/validation/gold.py,sha256=Z5ZzYdjERbrJ2Tu06d0TDTBSfwaFdL1Z23Rl-YkZ2Ow,803
|
|
132
|
+
nmdc_runtime/site/validation/util.py,sha256=Fz3c7dOEKOiYXssitYb50uUI1yRx3wJzFBl0s95Za7s,3632
|
|
133
|
+
nmdc_runtime-2.10.0.dist-info/licenses/LICENSE,sha256=VWiv65r7gHGjgtr3jMJYVmQny5GRpQ6H-W9sScb1x70,2408
|
|
134
|
+
nmdc_runtime-2.10.0.dist-info/METADATA,sha256=y7q8IQWm8basdnSM6ZZL3BraYycLH54QdUT80Cfoupw,11885
|
|
135
|
+
nmdc_runtime-2.10.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
136
|
+
nmdc_runtime-2.10.0.dist-info/entry_points.txt,sha256=JxdvOnvxHK_8046cwlvE30s_fV0-k-eTpQtkKYA69nQ,224
|
|
137
|
+
nmdc_runtime-2.10.0.dist-info/top_level.txt,sha256=b0K1s09L_iHH49ueBKaLrB5-lh6cyrSv9vL6x4Qvyz8,13
|
|
138
|
+
nmdc_runtime-2.10.0.dist-info/RECORD,,
|