nmdc-runtime 1.3.1__py3-none-any.whl → 2.12.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- nmdc_runtime/Dockerfile +177 -0
- nmdc_runtime/api/analytics.py +90 -0
- nmdc_runtime/api/boot/capabilities.py +9 -0
- nmdc_runtime/api/boot/object_types.py +126 -0
- nmdc_runtime/api/boot/triggers.py +84 -0
- nmdc_runtime/api/boot/workflows.py +116 -0
- nmdc_runtime/api/core/auth.py +212 -0
- nmdc_runtime/api/core/idgen.py +200 -0
- nmdc_runtime/api/core/metadata.py +777 -0
- nmdc_runtime/api/core/util.py +114 -0
- nmdc_runtime/api/db/mongo.py +436 -0
- nmdc_runtime/api/db/s3.py +37 -0
- nmdc_runtime/api/endpoints/capabilities.py +25 -0
- nmdc_runtime/api/endpoints/find.py +634 -0
- nmdc_runtime/api/endpoints/jobs.py +206 -0
- nmdc_runtime/api/endpoints/lib/helpers.py +274 -0
- nmdc_runtime/api/endpoints/lib/linked_instances.py +193 -0
- nmdc_runtime/api/endpoints/lib/path_segments.py +165 -0
- nmdc_runtime/api/endpoints/metadata.py +260 -0
- nmdc_runtime/api/endpoints/nmdcschema.py +515 -0
- nmdc_runtime/api/endpoints/object_types.py +38 -0
- nmdc_runtime/api/endpoints/objects.py +277 -0
- nmdc_runtime/api/endpoints/operations.py +78 -0
- nmdc_runtime/api/endpoints/queries.py +701 -0
- nmdc_runtime/api/endpoints/runs.py +98 -0
- nmdc_runtime/api/endpoints/search.py +38 -0
- nmdc_runtime/api/endpoints/sites.py +205 -0
- nmdc_runtime/api/endpoints/triggers.py +25 -0
- nmdc_runtime/api/endpoints/users.py +214 -0
- nmdc_runtime/api/endpoints/util.py +817 -0
- nmdc_runtime/api/endpoints/wf_file_staging.py +307 -0
- nmdc_runtime/api/endpoints/workflows.py +353 -0
- nmdc_runtime/api/entrypoint.sh +7 -0
- nmdc_runtime/api/main.py +495 -0
- nmdc_runtime/api/middleware.py +43 -0
- nmdc_runtime/api/models/capability.py +14 -0
- nmdc_runtime/api/models/id.py +92 -0
- nmdc_runtime/api/models/job.py +57 -0
- nmdc_runtime/api/models/lib/helpers.py +78 -0
- nmdc_runtime/api/models/metadata.py +11 -0
- nmdc_runtime/api/models/nmdc_schema.py +146 -0
- nmdc_runtime/api/models/object.py +180 -0
- nmdc_runtime/api/models/object_type.py +20 -0
- nmdc_runtime/api/models/operation.py +66 -0
- nmdc_runtime/api/models/query.py +246 -0
- nmdc_runtime/api/models/query_continuation.py +111 -0
- nmdc_runtime/api/models/run.py +161 -0
- nmdc_runtime/api/models/site.py +87 -0
- nmdc_runtime/api/models/trigger.py +13 -0
- nmdc_runtime/api/models/user.py +207 -0
- nmdc_runtime/api/models/util.py +260 -0
- nmdc_runtime/api/models/wfe_file_stages.py +122 -0
- nmdc_runtime/api/models/workflow.py +15 -0
- nmdc_runtime/api/openapi.py +178 -0
- nmdc_runtime/api/swagger_ui/assets/EllipsesButton.js +146 -0
- nmdc_runtime/api/swagger_ui/assets/EndpointSearchWidget.js +369 -0
- nmdc_runtime/api/swagger_ui/assets/script.js +252 -0
- nmdc_runtime/api/swagger_ui/assets/style.css +155 -0
- nmdc_runtime/api/swagger_ui/swagger_ui.py +34 -0
- nmdc_runtime/config.py +56 -0
- nmdc_runtime/minter/adapters/repository.py +22 -2
- nmdc_runtime/minter/config.py +30 -4
- nmdc_runtime/minter/domain/model.py +55 -1
- nmdc_runtime/minter/entrypoints/fastapi_app.py +1 -1
- nmdc_runtime/mongo_util.py +89 -0
- nmdc_runtime/site/backup/nmdcdb_mongodump.py +1 -1
- nmdc_runtime/site/backup/nmdcdb_mongoexport.py +1 -3
- nmdc_runtime/site/changesheets/data/OmicsProcessing-to-catted-Biosamples.tsv +1561 -0
- nmdc_runtime/site/changesheets/scripts/missing_neon_soils_ecosystem_data.py +311 -0
- nmdc_runtime/site/changesheets/scripts/neon_soils_add_ncbi_ids.py +210 -0
- nmdc_runtime/site/dagster.yaml +53 -0
- nmdc_runtime/site/entrypoint-daemon.sh +29 -0
- nmdc_runtime/site/entrypoint-dagit-readonly.sh +26 -0
- nmdc_runtime/site/entrypoint-dagit.sh +29 -0
- nmdc_runtime/site/export/ncbi_xml.py +1331 -0
- nmdc_runtime/site/export/ncbi_xml_utils.py +405 -0
- nmdc_runtime/site/export/study_metadata.py +27 -4
- nmdc_runtime/site/graphs.py +294 -45
- nmdc_runtime/site/ops.py +1008 -230
- nmdc_runtime/site/repair/database_updater.py +451 -0
- nmdc_runtime/site/repository.py +368 -133
- nmdc_runtime/site/resources.py +154 -80
- nmdc_runtime/site/translation/gold_translator.py +235 -83
- nmdc_runtime/site/translation/neon_benthic_translator.py +212 -188
- nmdc_runtime/site/translation/neon_soil_translator.py +82 -58
- nmdc_runtime/site/translation/neon_surface_water_translator.py +698 -0
- nmdc_runtime/site/translation/neon_utils.py +24 -7
- nmdc_runtime/site/translation/submission_portal_translator.py +616 -162
- nmdc_runtime/site/translation/translator.py +73 -3
- nmdc_runtime/site/util.py +26 -7
- nmdc_runtime/site/validation/emsl.py +1 -0
- nmdc_runtime/site/validation/gold.py +1 -0
- nmdc_runtime/site/validation/util.py +16 -12
- nmdc_runtime/site/workspace.yaml +13 -0
- nmdc_runtime/static/NMDC_logo.svg +1073 -0
- nmdc_runtime/static/ORCID-iD_icon_vector.svg +4 -0
- nmdc_runtime/static/README.md +5 -0
- nmdc_runtime/static/favicon.ico +0 -0
- nmdc_runtime/util.py +236 -192
- nmdc_runtime-2.12.0.dist-info/METADATA +45 -0
- nmdc_runtime-2.12.0.dist-info/RECORD +131 -0
- {nmdc_runtime-1.3.1.dist-info → nmdc_runtime-2.12.0.dist-info}/WHEEL +1 -2
- {nmdc_runtime-1.3.1.dist-info → nmdc_runtime-2.12.0.dist-info}/entry_points.txt +0 -1
- nmdc_runtime/containers.py +0 -14
- nmdc_runtime/core/db/Database.py +0 -15
- nmdc_runtime/core/exceptions/__init__.py +0 -23
- nmdc_runtime/core/exceptions/base.py +0 -47
- nmdc_runtime/core/exceptions/token.py +0 -13
- nmdc_runtime/domain/users/queriesInterface.py +0 -18
- nmdc_runtime/domain/users/userSchema.py +0 -37
- nmdc_runtime/domain/users/userService.py +0 -14
- nmdc_runtime/infrastructure/database/db.py +0 -3
- nmdc_runtime/infrastructure/database/models/user.py +0 -10
- nmdc_runtime/lib/__init__.py +0 -1
- nmdc_runtime/lib/extract_nmdc_data.py +0 -41
- nmdc_runtime/lib/load_nmdc_data.py +0 -121
- nmdc_runtime/lib/nmdc_dataframes.py +0 -829
- nmdc_runtime/lib/nmdc_etl_class.py +0 -402
- nmdc_runtime/lib/transform_nmdc_data.py +0 -1117
- nmdc_runtime/site/drsobjects/ingest.py +0 -93
- nmdc_runtime/site/drsobjects/registration.py +0 -131
- nmdc_runtime/site/terminusdb/generate.py +0 -198
- nmdc_runtime/site/terminusdb/ingest.py +0 -44
- nmdc_runtime/site/terminusdb/schema.py +0 -1671
- nmdc_runtime/site/translation/emsl.py +0 -42
- nmdc_runtime/site/translation/gold.py +0 -53
- nmdc_runtime/site/translation/jgi.py +0 -31
- nmdc_runtime/site/translation/util.py +0 -132
- nmdc_runtime/site/validation/jgi.py +0 -42
- nmdc_runtime-1.3.1.dist-info/METADATA +0 -181
- nmdc_runtime-1.3.1.dist-info/RECORD +0 -81
- nmdc_runtime-1.3.1.dist-info/top_level.txt +0 -1
- /nmdc_runtime/{client → api}/__init__.py +0 -0
- /nmdc_runtime/{core → api/boot}/__init__.py +0 -0
- /nmdc_runtime/{core/db → api/core}/__init__.py +0 -0
- /nmdc_runtime/{domain → api/db}/__init__.py +0 -0
- /nmdc_runtime/{domain/users → api/endpoints}/__init__.py +0 -0
- /nmdc_runtime/{infrastructure → api/endpoints/lib}/__init__.py +0 -0
- /nmdc_runtime/{infrastructure/database → api/models}/__init__.py +0 -0
- /nmdc_runtime/{infrastructure/database/models → api/models/lib}/__init__.py +0 -0
- /nmdc_runtime/{site/drsobjects/__init__.py → api/models/minter.py} +0 -0
- /nmdc_runtime/site/{terminusdb → repair}/__init__.py +0 -0
- {nmdc_runtime-1.3.1.dist-info → nmdc_runtime-2.12.0.dist-info/licenses}/LICENSE +0 -0
nmdc_runtime/api/main.py
ADDED
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from contextlib import asynccontextmanager
|
|
3
|
+
from html import escape
|
|
4
|
+
from importlib import import_module
|
|
5
|
+
from importlib.metadata import version
|
|
6
|
+
from typing import Annotated
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import fastapi
|
|
10
|
+
import requests
|
|
11
|
+
import uvicorn
|
|
12
|
+
from fastapi import APIRouter, FastAPI, Cookie
|
|
13
|
+
from fastapi.middleware.cors import CORSMiddleware
|
|
14
|
+
from fastapi.openapi.docs import get_swagger_ui_html
|
|
15
|
+
from fastapi.staticfiles import StaticFiles
|
|
16
|
+
from starlette import status
|
|
17
|
+
from starlette.responses import RedirectResponse, HTMLResponse, FileResponse
|
|
18
|
+
from refscan.lib.helpers import get_collection_names_from_schema
|
|
19
|
+
from scalar_fastapi import get_scalar_api_reference
|
|
20
|
+
from nmdc_runtime.api.models.wfe_file_stages import WorkflowFileStagingCollectionName
|
|
21
|
+
from nmdc_runtime import config
|
|
22
|
+
from nmdc_runtime.api.analytics import Analytics
|
|
23
|
+
from nmdc_runtime.api.middleware import PyinstrumentMiddleware
|
|
24
|
+
from nmdc_runtime.config import IS_SCALAR_ENABLED
|
|
25
|
+
from nmdc_runtime.util import (
|
|
26
|
+
decorate_if,
|
|
27
|
+
get_allowed_references,
|
|
28
|
+
ensure_unique_id_indexes,
|
|
29
|
+
REPO_ROOT_DIR,
|
|
30
|
+
nmdc_schema_view,
|
|
31
|
+
)
|
|
32
|
+
from nmdc_runtime.api.core.auth import (
|
|
33
|
+
get_password_hash,
|
|
34
|
+
ORCID_NMDC_CLIENT_ID,
|
|
35
|
+
ORCID_BASE_URL,
|
|
36
|
+
)
|
|
37
|
+
from nmdc_runtime.api.db.mongo import get_mongo_db
|
|
38
|
+
from nmdc_runtime.api.endpoints import (
|
|
39
|
+
capabilities,
|
|
40
|
+
find,
|
|
41
|
+
jobs,
|
|
42
|
+
metadata,
|
|
43
|
+
nmdcschema,
|
|
44
|
+
object_types,
|
|
45
|
+
objects,
|
|
46
|
+
operations,
|
|
47
|
+
queries,
|
|
48
|
+
runs,
|
|
49
|
+
sites,
|
|
50
|
+
triggers,
|
|
51
|
+
users,
|
|
52
|
+
workflows,
|
|
53
|
+
wf_file_staging,
|
|
54
|
+
)
|
|
55
|
+
from nmdc_runtime.api.endpoints.util import BASE_URL_EXTERNAL
|
|
56
|
+
from nmdc_runtime.api.models.site import SiteClientInDB, SiteInDB
|
|
57
|
+
from nmdc_runtime.api.models.user import UserInDB
|
|
58
|
+
from nmdc_runtime.api.models.util import entity_attributes_to_index
|
|
59
|
+
from nmdc_runtime.api.openapi import (
|
|
60
|
+
OpenAPITag,
|
|
61
|
+
ordered_tag_descriptors,
|
|
62
|
+
make_api_description,
|
|
63
|
+
)
|
|
64
|
+
from nmdc_runtime.api.swagger_ui.swagger_ui import base_swagger_ui_parameters
|
|
65
|
+
from nmdc_runtime.minter.bootstrap import bootstrap as minter_bootstrap
|
|
66
|
+
from nmdc_runtime.minter.entrypoints.fastapi_app import router as minter_router
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
api_router = APIRouter()
|
|
70
|
+
api_router.include_router(find.router, tags=[OpenAPITag.METADATA_ACCESS.value])
|
|
71
|
+
api_router.include_router(nmdcschema.router, tags=[OpenAPITag.METADATA_ACCESS.value])
|
|
72
|
+
api_router.include_router(queries.router, tags=[OpenAPITag.METADATA_ACCESS.value])
|
|
73
|
+
api_router.include_router(metadata.router, tags=[OpenAPITag.METADATA_ACCESS.value])
|
|
74
|
+
api_router.include_router(sites.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
75
|
+
api_router.include_router(workflows.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
76
|
+
api_router.include_router(capabilities.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
77
|
+
api_router.include_router(object_types.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
78
|
+
api_router.include_router(triggers.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
79
|
+
api_router.include_router(jobs.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
80
|
+
api_router.include_router(objects.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
81
|
+
api_router.include_router(operations.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
82
|
+
api_router.include_router(runs.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
83
|
+
api_router.include_router(minter_router, prefix="/pids", tags=[OpenAPITag.MINTER.value])
|
|
84
|
+
api_router.include_router(users.router, tags=[OpenAPITag.USERS.value])
|
|
85
|
+
api_router.include_router(wf_file_staging.router, tags=[OpenAPITag.WORKFLOWS.value])
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def ensure_initial_resources_on_boot():
|
|
89
|
+
"""ensure these resources are loaded when (re-)booting the system."""
|
|
90
|
+
mdb = get_mongo_db()
|
|
91
|
+
|
|
92
|
+
collections = ["workflows", "capabilities", "object_types", "triggers"]
|
|
93
|
+
for collection_name in collections:
|
|
94
|
+
mdb[collection_name].create_index("id", unique=True)
|
|
95
|
+
collection_boot = import_module(f"nmdc_runtime.api.boot.{collection_name}")
|
|
96
|
+
|
|
97
|
+
for model in collection_boot.construct():
|
|
98
|
+
doc = model.model_dump()
|
|
99
|
+
mdb[collection_name].replace_one({"id": doc["id"]}, doc, upsert=True)
|
|
100
|
+
|
|
101
|
+
username = os.getenv("API_ADMIN_USER")
|
|
102
|
+
admin_ok = mdb.users.count_documents(({"username": username})) > 0
|
|
103
|
+
if not admin_ok:
|
|
104
|
+
mdb.users.replace_one(
|
|
105
|
+
{"username": username},
|
|
106
|
+
UserInDB(
|
|
107
|
+
username=username,
|
|
108
|
+
hashed_password=get_password_hash(os.getenv("API_ADMIN_PASS")),
|
|
109
|
+
site_admin=[os.getenv("API_SITE_ID")],
|
|
110
|
+
).model_dump(exclude_unset=True),
|
|
111
|
+
upsert=True,
|
|
112
|
+
)
|
|
113
|
+
mdb.users.create_index("username", unique=True)
|
|
114
|
+
|
|
115
|
+
site_id = os.getenv("API_SITE_ID")
|
|
116
|
+
runtime_site_ok = mdb.sites.count_documents(({"id": site_id})) > 0
|
|
117
|
+
if not runtime_site_ok:
|
|
118
|
+
client_id = os.getenv("API_SITE_CLIENT_ID")
|
|
119
|
+
mdb.sites.replace_one(
|
|
120
|
+
{"id": site_id},
|
|
121
|
+
SiteInDB(
|
|
122
|
+
id=site_id,
|
|
123
|
+
clients=[
|
|
124
|
+
SiteClientInDB(
|
|
125
|
+
id=client_id,
|
|
126
|
+
hashed_secret=get_password_hash(
|
|
127
|
+
os.getenv("API_SITE_CLIENT_SECRET")
|
|
128
|
+
),
|
|
129
|
+
)
|
|
130
|
+
],
|
|
131
|
+
).model_dump(),
|
|
132
|
+
upsert=True,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
ensure_unique_id_indexes(mdb)
|
|
136
|
+
|
|
137
|
+
# No two object documents can have the same checksum of the same type.
|
|
138
|
+
mdb.objects.create_index(
|
|
139
|
+
[("checksums.type", 1), ("checksums.checksum", 1)], unique=True
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
# Minting resources
|
|
143
|
+
minter_bootstrap()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def ensure_type_field_is_indexed():
|
|
147
|
+
r"""
|
|
148
|
+
Ensures that each schema-described collection has an index on its `type` field.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
mdb = get_mongo_db()
|
|
152
|
+
schema_view = nmdc_schema_view()
|
|
153
|
+
for collection_name in get_collection_names_from_schema(schema_view):
|
|
154
|
+
mdb.get_collection(collection_name).create_index("type", background=True)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def ensure_attribute_indexes():
|
|
158
|
+
r"""
|
|
159
|
+
Ensures that the MongoDB collection identified by each key (i.e. collection name) in the
|
|
160
|
+
`entity_attributes_to_index` dictionary, has an index on each field identified by the value
|
|
161
|
+
(i.e. set of field names) associated with that key.
|
|
162
|
+
|
|
163
|
+
Example dictionary (notice each item's value is a _set_, not a _dict_):
|
|
164
|
+
```
|
|
165
|
+
{
|
|
166
|
+
"coll_name_1": {"field_name_1"},
|
|
167
|
+
"coll_name_2": {"field_name_1", "field_name_2"},
|
|
168
|
+
}
|
|
169
|
+
```
|
|
170
|
+
"""
|
|
171
|
+
|
|
172
|
+
mdb = get_mongo_db()
|
|
173
|
+
for collection_name, index_specs in entity_attributes_to_index.items():
|
|
174
|
+
for spec in index_specs:
|
|
175
|
+
if not isinstance(spec, str):
|
|
176
|
+
raise ValueError(
|
|
177
|
+
"only supports basic single-key ascending index specs at this time."
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
mdb[collection_name].create_index([(spec, 1)], name=spec, background=True)
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def ensure_globus_tasks_id_is_indexed():
|
|
184
|
+
"""
|
|
185
|
+
Ensures that the `wf_file_staging.globus_tasks` collection has an index on its `task_id` field and that the index is unique.
|
|
186
|
+
"""
|
|
187
|
+
|
|
188
|
+
mdb = get_mongo_db()
|
|
189
|
+
mdb["wf_file_staging.globus_tasks"].create_index(
|
|
190
|
+
"task_id", background=True, unique=True
|
|
191
|
+
)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def ensure_jgi_samples_id_is_indexed():
|
|
195
|
+
"""
|
|
196
|
+
Ensures that the `wf_file_staging.jgi_samples` collection has an index on its `jdp_file_id` field and that the index is unique.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
mdb = get_mongo_db()
|
|
200
|
+
mdb["wf_file_staging.jgi_samples"].create_index(
|
|
201
|
+
"jdp_file_id", background=True, unique=True
|
|
202
|
+
)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def ensure_sequencing_project_name_is_indexed():
|
|
206
|
+
"""
|
|
207
|
+
Ensures that the `wf_file_staging.sequencing_projects` collection has an index on its `sequencing_project_name` field and that the index is unique.
|
|
208
|
+
"""
|
|
209
|
+
|
|
210
|
+
mdb = get_mongo_db()
|
|
211
|
+
mdb[WorkflowFileStagingCollectionName.JGI_SEQUENCING_PROJECTS.value].create_index(
|
|
212
|
+
"sequencing_project_name", background=True, unique=True
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def ensure_default_api_perms():
|
|
217
|
+
"""
|
|
218
|
+
Ensures that specific users (currently only "admin") are allowed to perform
|
|
219
|
+
specific actions, and creates MongoDB indexes to speed up allowance queries.
|
|
220
|
+
|
|
221
|
+
Note: If a MongoDB index already exists, the call to `create_index` does nothing.
|
|
222
|
+
"""
|
|
223
|
+
|
|
224
|
+
db = get_mongo_db()
|
|
225
|
+
if db["_runtime.api.allow"].count_documents({}):
|
|
226
|
+
return
|
|
227
|
+
|
|
228
|
+
allowances = {
|
|
229
|
+
"/metadata/changesheets:submit": [
|
|
230
|
+
"admin",
|
|
231
|
+
],
|
|
232
|
+
"/queries:run(query_cmd:DeleteCommand)": [
|
|
233
|
+
"admin",
|
|
234
|
+
],
|
|
235
|
+
"/queries:run(query_cmd:AggregateCommand)": [
|
|
236
|
+
"admin",
|
|
237
|
+
],
|
|
238
|
+
"/metadata/json:submit": [
|
|
239
|
+
"admin",
|
|
240
|
+
],
|
|
241
|
+
"/wf_file_staging": [
|
|
242
|
+
"admin",
|
|
243
|
+
],
|
|
244
|
+
}
|
|
245
|
+
for doc in [
|
|
246
|
+
{"username": username, "action": action}
|
|
247
|
+
for action, usernames in allowances.items()
|
|
248
|
+
for username in usernames
|
|
249
|
+
]:
|
|
250
|
+
db["_runtime.api.allow"].replace_one(doc, doc, upsert=True)
|
|
251
|
+
db["_runtime.api.allow"].create_index("username")
|
|
252
|
+
db["_runtime.api.allow"].create_index("action")
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
@asynccontextmanager
|
|
256
|
+
async def lifespan(app: FastAPI):
|
|
257
|
+
r"""
|
|
258
|
+
Prepares the application to receive requests.
|
|
259
|
+
|
|
260
|
+
From the [FastAPI documentation](https://fastapi.tiangolo.com/advanced/events/#lifespan-function):
|
|
261
|
+
> You can define logic (code) that should be executed before the application starts up. This means that
|
|
262
|
+
> this code will be executed once, before the application starts receiving requests.
|
|
263
|
+
"""
|
|
264
|
+
ensure_initial_resources_on_boot()
|
|
265
|
+
ensure_attribute_indexes()
|
|
266
|
+
ensure_type_field_is_indexed()
|
|
267
|
+
ensure_default_api_perms()
|
|
268
|
+
ensure_globus_tasks_id_is_indexed()
|
|
269
|
+
ensure_sequencing_project_name_is_indexed()
|
|
270
|
+
ensure_jgi_samples_id_is_indexed()
|
|
271
|
+
# Invoke a function—thereby priming its memoization cache—in order to speed up all future invocations.
|
|
272
|
+
get_allowed_references() # we ignore the return value here
|
|
273
|
+
|
|
274
|
+
yield
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
@api_router.get("/", include_in_schema=False)
|
|
278
|
+
async def root():
|
|
279
|
+
return RedirectResponse(
|
|
280
|
+
BASE_URL_EXTERNAL + "/docs",
|
|
281
|
+
status_code=status.HTTP_303_SEE_OTHER,
|
|
282
|
+
)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
@api_router.get("/version", tags=[OpenAPITag.SYSTEM_ADMINISTRATION.value])
|
|
286
|
+
async def get_versions():
|
|
287
|
+
return {
|
|
288
|
+
"nmdc-runtime": version("nmdc_runtime"),
|
|
289
|
+
"fastapi": fastapi.__version__,
|
|
290
|
+
"nmdc-schema": version("nmdc_schema"),
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# Build an ORCID Login URL for the Swagger UI page, based upon some environment variables.
|
|
295
|
+
orcid_login_url = f"{ORCID_BASE_URL}/oauth/authorize?client_id={ORCID_NMDC_CLIENT_ID}&response_type=code&scope=openid&redirect_uri={BASE_URL_EXTERNAL}/orcid_code"
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
app = FastAPI(
|
|
299
|
+
title="NMDC Runtime API",
|
|
300
|
+
version=version("nmdc_runtime"),
|
|
301
|
+
description=make_api_description(
|
|
302
|
+
api_version=version("nmdc_runtime"), schema_version=version("nmdc_schema")
|
|
303
|
+
),
|
|
304
|
+
openapi_tags=ordered_tag_descriptors,
|
|
305
|
+
lifespan=lifespan,
|
|
306
|
+
docs_url=None,
|
|
307
|
+
)
|
|
308
|
+
app.include_router(api_router)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
app.add_middleware(
|
|
312
|
+
CORSMiddleware,
|
|
313
|
+
# Allow requests from client-side web apps hosted in local development environments, on microbiomedata.org, and on GitHub Pages.
|
|
314
|
+
allow_origin_regex=r"(http://localhost:\d+)|(https://.+?\.microbiomedata\.org)|(https://microbiomedata\.github\.io)",
|
|
315
|
+
allow_credentials=True,
|
|
316
|
+
allow_methods=["*"],
|
|
317
|
+
allow_headers=["*"],
|
|
318
|
+
)
|
|
319
|
+
app.add_middleware(Analytics)
|
|
320
|
+
|
|
321
|
+
if config.IS_PROFILING_ENABLED:
|
|
322
|
+
app.add_middleware(PyinstrumentMiddleware)
|
|
323
|
+
|
|
324
|
+
# Note: Here, we are mounting a `StaticFiles` instance (which is bound to the directory that
|
|
325
|
+
# contains static files) as a "sub-application" of the main FastAPI application. This
|
|
326
|
+
# makes the contents of that directory be accessible under the `/static` URL path.
|
|
327
|
+
# Reference: https://fastapi.tiangolo.com/tutorial/static-files/
|
|
328
|
+
static_files_path: Path = REPO_ROOT_DIR.joinpath("nmdc_runtime/static/")
|
|
329
|
+
app.mount("/static", StaticFiles(directory=static_files_path), name="static")
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
@app.get("/favicon.ico", include_in_schema=False)
|
|
333
|
+
async def favicon():
|
|
334
|
+
r"""Returns the application's favicon."""
|
|
335
|
+
favicon_path = static_files_path / "favicon.ico"
|
|
336
|
+
return FileResponse(favicon_path)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
@decorate_if(condition=IS_SCALAR_ENABLED)(app.get("/scalar", include_in_schema=False))
|
|
340
|
+
async def get_scalar_html():
|
|
341
|
+
r"""
|
|
342
|
+
Returns the HTML markup for an interactive API docs web page
|
|
343
|
+
(alternative to Swagger UI) powered by Scalar.
|
|
344
|
+
"""
|
|
345
|
+
return get_scalar_api_reference(
|
|
346
|
+
openapi_url=app.openapi_url,
|
|
347
|
+
title="NMDC Runtime API",
|
|
348
|
+
)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
@app.get("/docs", include_in_schema=False)
|
|
352
|
+
def custom_swagger_ui_html(
|
|
353
|
+
user_id_token: Annotated[str | None, Cookie()] = None,
|
|
354
|
+
):
|
|
355
|
+
r"""Returns the HTML markup for an interactive API docs web page powered by Swagger UI.
|
|
356
|
+
|
|
357
|
+
If the `user_id_token` cookie is present and not empty, this function will send its value to
|
|
358
|
+
the `/token` endpoint in an attempt to get an access token. If it gets one, this function will
|
|
359
|
+
inject that access token into the web page so Swagger UI will consider the user to be logged in.
|
|
360
|
+
|
|
361
|
+
Reference: https://fastapi.tiangolo.com/tutorial/cookie-params/
|
|
362
|
+
"""
|
|
363
|
+
access_token = None
|
|
364
|
+
if user_id_token:
|
|
365
|
+
# get bearer token
|
|
366
|
+
rv = requests.post(
|
|
367
|
+
url=f"{BASE_URL_EXTERNAL}/token",
|
|
368
|
+
data={
|
|
369
|
+
"client_id": user_id_token,
|
|
370
|
+
"client_secret": "",
|
|
371
|
+
"grant_type": "client_credentials",
|
|
372
|
+
},
|
|
373
|
+
headers={
|
|
374
|
+
"Content-type": "application/x-www-form-urlencoded",
|
|
375
|
+
"Accept": "application/json",
|
|
376
|
+
},
|
|
377
|
+
)
|
|
378
|
+
if rv.status_code != 200:
|
|
379
|
+
rv.reason = rv.text
|
|
380
|
+
rv.raise_for_status()
|
|
381
|
+
access_token = rv.json()["access_token"]
|
|
382
|
+
|
|
383
|
+
onComplete = ""
|
|
384
|
+
if access_token is not None:
|
|
385
|
+
onComplete += f"ui.preauthorizeApiKey('bearerAuth', '{access_token}');"
|
|
386
|
+
if os.getenv("INFO_BANNER_INNERHTML"):
|
|
387
|
+
info_banner_innerhtml = os.getenv("INFO_BANNER_INNERHTML")
|
|
388
|
+
onComplete += f"""
|
|
389
|
+
banner = document.createElement('section');
|
|
390
|
+
banner.classList.add('nmdc-info', 'nmdc-info-banner', 'block', 'col-12');
|
|
391
|
+
banner.innerHTML = `{info_banner_innerhtml.replace('"', '<double-quote>')}`;
|
|
392
|
+
document.querySelector('.information-container').prepend(banner);
|
|
393
|
+
""".replace(
|
|
394
|
+
"\n", " "
|
|
395
|
+
)
|
|
396
|
+
swagger_ui_parameters = base_swagger_ui_parameters.copy()
|
|
397
|
+
# Note: The `nmdcInit` JavaScript event is a custom event we use to trigger anything that is listening for it.
|
|
398
|
+
# Reference: https://developer.mozilla.org/en-US/docs/Web/Events/Creating_and_triggering_events
|
|
399
|
+
swagger_ui_parameters.update(
|
|
400
|
+
{
|
|
401
|
+
"onComplete": f"""<unquote-safe>() => {{ {onComplete}; dispatchEvent(new Event('nmdcInit')); }}</unquote-safe>""",
|
|
402
|
+
}
|
|
403
|
+
)
|
|
404
|
+
# Pin the Swagger UI version to avoid breaking changes (to our own JavaScript code that depends on Swagger UI internals).
|
|
405
|
+
# Note: version `5.29.4` was released on October 10, 2025.
|
|
406
|
+
pinned_swagger_ui_version = "5.29.4"
|
|
407
|
+
swagger_ui_css_url = f"https://cdn.jsdelivr.net/npm/swagger-ui-dist@{pinned_swagger_ui_version}/swagger-ui.css"
|
|
408
|
+
swagger_ui_js_url = f"https://cdn.jsdelivr.net/npm/swagger-ui-dist@{pinned_swagger_ui_version}/swagger-ui-bundle.js"
|
|
409
|
+
response = get_swagger_ui_html(
|
|
410
|
+
swagger_css_url=swagger_ui_css_url,
|
|
411
|
+
swagger_js_url=swagger_ui_js_url,
|
|
412
|
+
openapi_url=app.openapi_url,
|
|
413
|
+
title=app.title,
|
|
414
|
+
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
|
|
415
|
+
swagger_favicon_url="/static/favicon.ico",
|
|
416
|
+
swagger_ui_parameters=swagger_ui_parameters,
|
|
417
|
+
)
|
|
418
|
+
assets_dir_path = Path(__file__).parent / "swagger_ui" / "assets"
|
|
419
|
+
style_css: str = Path(assets_dir_path / "style.css").read_text()
|
|
420
|
+
script_js: str = Path(assets_dir_path / "script.js").read_text()
|
|
421
|
+
ellipses_button_js: str = Path(assets_dir_path / "EllipsesButton.js").read_text()
|
|
422
|
+
endpoint_search_widget_js: str = Path(
|
|
423
|
+
assets_dir_path / "EndpointSearchWidget.js"
|
|
424
|
+
).read_text()
|
|
425
|
+
content = (
|
|
426
|
+
response.body.decode()
|
|
427
|
+
.replace('"<unquote-safe>', "")
|
|
428
|
+
.replace('</unquote-safe>"', "")
|
|
429
|
+
.replace("<double-quote>", '"')
|
|
430
|
+
.replace("</double-quote>", '"')
|
|
431
|
+
# TODO: Consider using a "custom layout" implemented as a React component.
|
|
432
|
+
# Reference: https://github.com/swagger-api/swagger-ui/blob/master/docs/customization/custom-layout.md
|
|
433
|
+
#
|
|
434
|
+
# Note: Custom layouts are specified via the Swagger UI parameter named `layout`, whose value identifies
|
|
435
|
+
# a component that is specified via the Swagger UI parameter named `plugins`. The Swagger UI
|
|
436
|
+
# JavaScript code expects each item in the `plugins` array to be a JavaScript function,
|
|
437
|
+
# but FastAPI's `get_swagger_ui_html` function serializes each parameter's value into JSON,
|
|
438
|
+
# preventing us from specifying a JavaScript function as a value in the `plugins` array.
|
|
439
|
+
#
|
|
440
|
+
# As a workaround, we could use the string `replace`-ment technique shown below to put the literal
|
|
441
|
+
# JavaScript characters into place in the final HTML document. Using that approach, I _have_ been
|
|
442
|
+
# able to display a custom layout (a custom React component), but I have _not_ been able to get
|
|
443
|
+
# that custom layout to display Swagger UI's `BaseLayout` component (which includes the core
|
|
444
|
+
# Swagger UI functionality). That's a deal breaker.
|
|
445
|
+
#
|
|
446
|
+
.replace(r'"{{ NMDC_SWAGGER_UI_PARAMETERS_PLUGINS_PLACEHOLDER }}"', r"[]")
|
|
447
|
+
# Inject HTML elements containing data that can be read via JavaScript (e.g., `swagger_ui/assets/script.js`).
|
|
448
|
+
# Note: We escape the values here so they can be safely used as HTML attribute values.
|
|
449
|
+
.replace(
|
|
450
|
+
"</head>",
|
|
451
|
+
f"""
|
|
452
|
+
</head>
|
|
453
|
+
<div
|
|
454
|
+
id="nmdc-access-token"
|
|
455
|
+
data-token="{escape(access_token if access_token is not None else '')}"
|
|
456
|
+
style="display: none"
|
|
457
|
+
></div>
|
|
458
|
+
<div
|
|
459
|
+
id="nmdc-orcid-login-url"
|
|
460
|
+
data-url="{escape(orcid_login_url)}"
|
|
461
|
+
style="display: none"
|
|
462
|
+
></div>
|
|
463
|
+
""",
|
|
464
|
+
1,
|
|
465
|
+
)
|
|
466
|
+
# Inject a custom CSS stylesheet immediately before the closing `</head>` tag.
|
|
467
|
+
.replace(
|
|
468
|
+
"</head>",
|
|
469
|
+
f"""
|
|
470
|
+
<style>
|
|
471
|
+
{style_css}
|
|
472
|
+
</style>
|
|
473
|
+
</head>
|
|
474
|
+
""",
|
|
475
|
+
1,
|
|
476
|
+
)
|
|
477
|
+
# Inject custom JavaScript scripts immediately before the closing `</body>` tag.
|
|
478
|
+
.replace(
|
|
479
|
+
"</body>",
|
|
480
|
+
f"""
|
|
481
|
+
<script>
|
|
482
|
+
{ellipses_button_js}
|
|
483
|
+
{endpoint_search_widget_js}
|
|
484
|
+
{script_js}
|
|
485
|
+
</script>
|
|
486
|
+
</body>
|
|
487
|
+
""",
|
|
488
|
+
1,
|
|
489
|
+
)
|
|
490
|
+
)
|
|
491
|
+
return HTMLResponse(content=content)
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
if __name__ == "__main__":
|
|
495
|
+
uvicorn.run(app, host="0.0.0.0", port=8000)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
from fastapi import Request
|
|
2
|
+
from fastapi.responses import HTMLResponse
|
|
3
|
+
from pyinstrument import Profiler
|
|
4
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PyinstrumentMiddleware(BaseHTTPMiddleware):
|
|
8
|
+
r"""
|
|
9
|
+
FastAPI middleware that uses Pyinstrument to do performance profiling.
|
|
10
|
+
|
|
11
|
+
If the requested URL includes the query parameter, `profile=true`, this middleware
|
|
12
|
+
will profile the performance of the application for the duration of the HTTP request,
|
|
13
|
+
and then override the HTTP response to consist of a performance report.
|
|
14
|
+
|
|
15
|
+
References:
|
|
16
|
+
- https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-web-request-in-fastapi
|
|
17
|
+
- https://stackoverflow.com/a/71526036
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
async def dispatch(self, request: Request, call_next):
|
|
21
|
+
# Get the `profile` query parameter and check whether its value is "true" (case insensitive).
|
|
22
|
+
profile_param = request.query_params.get("profile", None)
|
|
23
|
+
is_profiling = (
|
|
24
|
+
isinstance(profile_param, str) and profile_param.lower() == "true"
|
|
25
|
+
)
|
|
26
|
+
|
|
27
|
+
# If profiling is enabled for this request, profile the request processing.
|
|
28
|
+
if is_profiling:
|
|
29
|
+
# Start the profiler.
|
|
30
|
+
profiler = Profiler()
|
|
31
|
+
profiler.start()
|
|
32
|
+
|
|
33
|
+
# Allow the request to be processed as usual, and discard the normal response.
|
|
34
|
+
_ = await call_next(request)
|
|
35
|
+
|
|
36
|
+
# Stop the profiler.
|
|
37
|
+
profiler.stop()
|
|
38
|
+
|
|
39
|
+
# Override the normal response with the profiling report.
|
|
40
|
+
return HTMLResponse(profiler.output_html())
|
|
41
|
+
else:
|
|
42
|
+
# Allow the request to be processed as usual.
|
|
43
|
+
return await call_next(request)
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import datetime
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class CapabilityBase(BaseModel):
|
|
8
|
+
name: Optional[str] = None
|
|
9
|
+
description: Optional[str] = None
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Capability(CapabilityBase):
|
|
13
|
+
id: str
|
|
14
|
+
created_at: datetime.datetime
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from enum import Enum
|
|
3
|
+
from typing import Union, Any, Optional, Literal
|
|
4
|
+
|
|
5
|
+
from pydantic import model_validator, StringConstraints, BaseModel, PositiveInt
|
|
6
|
+
from typing_extensions import Annotated
|
|
7
|
+
|
|
8
|
+
# NO i, l, o or u.
|
|
9
|
+
base32_letters = "abcdefghjkmnpqrstvwxyz"
|
|
10
|
+
|
|
11
|
+
NAA_VALUES = ["nmdc"]
|
|
12
|
+
_fake_shoulders = [f"fk{n}" for n in range(10)]
|
|
13
|
+
SHOULDER_VALUES = _fake_shoulders + ["mga0", "mta0", "mba0", "mpa0", "oma0"]
|
|
14
|
+
|
|
15
|
+
_naa = rf"(?P<naa>({'|'.join(NAA_VALUES)}))"
|
|
16
|
+
pattern_naa = re.compile(_naa)
|
|
17
|
+
_shoulder = rf"(?P<shoulder>({'|'.join(SHOULDER_VALUES)}))"
|
|
18
|
+
pattern_shoulder = re.compile(_shoulder)
|
|
19
|
+
_blade = rf"(?P<blade>[0-9{base32_letters}]+)"
|
|
20
|
+
pattern_blade = re.compile(_blade)
|
|
21
|
+
_assigned_base_name = f"{_shoulder}{_blade}"
|
|
22
|
+
pattern_assigned_base_name = re.compile(_assigned_base_name)
|
|
23
|
+
_base_object_name = f"{_naa}:{_shoulder}{_blade}"
|
|
24
|
+
pattern_base_object_name = re.compile(_base_object_name)
|
|
25
|
+
|
|
26
|
+
Naa = Annotated[str, StringConstraints(pattern=_naa)]
|
|
27
|
+
Shoulder = Annotated[str, StringConstraints(pattern=rf"^{_shoulder}$", min_length=2)]
|
|
28
|
+
Blade = Annotated[str, StringConstraints(pattern=_blade, min_length=4)]
|
|
29
|
+
AssignedBaseName = Annotated[str, StringConstraints(pattern=_assigned_base_name)]
|
|
30
|
+
BaseObjectName = Annotated[str, StringConstraints(pattern=_base_object_name)]
|
|
31
|
+
|
|
32
|
+
NameAssigningAuthority = Literal[tuple(NAA_VALUES)]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class MintRequest(BaseModel):
|
|
36
|
+
populator: str = ""
|
|
37
|
+
naa: NameAssigningAuthority = "nmdc"
|
|
38
|
+
shoulder: Shoulder = "fk0"
|
|
39
|
+
number: PositiveInt = 1
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class IdThreeParts(BaseModel):
|
|
43
|
+
naa: Naa
|
|
44
|
+
shoulder: Shoulder
|
|
45
|
+
blade: Blade
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class IdTwoParts(BaseModel):
|
|
49
|
+
naa: Naa
|
|
50
|
+
assigned_base_name: AssignedBaseName
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class IdWhole(BaseModel):
|
|
54
|
+
base_object_name: BaseObjectName
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class Id(BaseModel):
|
|
58
|
+
id: Union[IdWhole, IdTwoParts, IdThreeParts]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class IdBindings(BaseModel):
|
|
62
|
+
where: BaseObjectName
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class IdBindingOp(str, Enum):
|
|
66
|
+
set = "set"
|
|
67
|
+
addToSet = "addToSet"
|
|
68
|
+
rm = "rm"
|
|
69
|
+
purge = "purge"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class IdBindingRequest(BaseModel):
|
|
73
|
+
i: BaseObjectName
|
|
74
|
+
o: IdBindingOp = IdBindingOp.set
|
|
75
|
+
a: Optional[str] = None
|
|
76
|
+
v: Any = None
|
|
77
|
+
|
|
78
|
+
@model_validator(mode="before")
|
|
79
|
+
def set_or_add_needs_value(cls, values):
|
|
80
|
+
op = values.get("o")
|
|
81
|
+
if op in (IdBindingOp.set, IdBindingOp.addToSet):
|
|
82
|
+
if "v" not in values:
|
|
83
|
+
raise ValueError("{'set','add'} operations needs value 'v'.")
|
|
84
|
+
return values
|
|
85
|
+
|
|
86
|
+
@model_validator(mode="before")
|
|
87
|
+
def set_or_add_or_rm_needs_attribute(cls, values):
|
|
88
|
+
op = values.get("o")
|
|
89
|
+
if op in (IdBindingOp.set, IdBindingOp.addToSet, IdBindingOp.rm):
|
|
90
|
+
if not values.get("a"):
|
|
91
|
+
raise ValueError("{'set','add','rm'} operations need attribute 'a'.")
|
|
92
|
+
return values
|