dcicutils 8.14.0__py3-none-any.whl → 8.14.0.1b3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dcicutils/scripts/update_portal_object.py +30 -3
- dcicutils/scripts/view_portal_object.py +33 -5
- {dcicutils-8.14.0.dist-info → dcicutils-8.14.0.1b3.dist-info}/METADATA +1 -1
- {dcicutils-8.14.0.dist-info → dcicutils-8.14.0.1b3.dist-info}/RECORD +7 -7
- {dcicutils-8.14.0.dist-info → dcicutils-8.14.0.1b3.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.14.0.dist-info → dcicutils-8.14.0.1b3.dist-info}/WHEEL +0 -0
- {dcicutils-8.14.0.dist-info → dcicutils-8.14.0.1b3.dist-info}/entry_points.txt +0 -0
@@ -19,7 +19,7 @@ from typing import Callable, List, Optional, Tuple, Union
|
|
19
19
|
from dcicutils.command_utils import yes_or_no
|
20
20
|
from dcicutils.common import ORCHESTRATED_APPS, APP_SMAHT
|
21
21
|
from dcicutils.ff_utils import delete_metadata, purge_metadata
|
22
|
-
from dcicutils.misc_utils import get_error_message, PRINT
|
22
|
+
from dcicutils.misc_utils import get_error_message, ignored, PRINT
|
23
23
|
from dcicutils.portal_utils import Portal as PortalFromUtils
|
24
24
|
|
25
25
|
|
@@ -40,7 +40,7 @@ _DEFAULT_APP = "smaht"
|
|
40
40
|
_SMAHT_ENV_ENVIRON_NAME = "SMAHT_ENV"
|
41
41
|
|
42
42
|
# Schema properties to ignore (by default) for the view schema usage.
|
43
|
-
|
43
|
+
_IGNORE_PROPERTIES_ON_UPDATE = [
|
44
44
|
"date_created",
|
45
45
|
"last_modified",
|
46
46
|
"principals_allowed",
|
@@ -122,6 +122,9 @@ def main():
|
|
122
122
|
parser.add_argument("--upsert", type=str, required=False, default=None, help="Upsert data.")
|
123
123
|
parser.add_argument("--delete", type=str, required=False, default=None, help="Delete data.")
|
124
124
|
parser.add_argument("--purge", type=str, required=False, default=None, help="Purge data.")
|
125
|
+
parser.add_argument("--noignore", action="store_true", required=False, default=False,
|
126
|
+
help="Do not ignore standard fields on update(s).")
|
127
|
+
parser.add_argument("--ignore", nargs="+", help="Ignore these additional fields.")
|
125
128
|
parser.add_argument("--confirm", action="store_true", required=False, default=False, help="Confirm before action.")
|
126
129
|
parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
|
127
130
|
parser.add_argument("--quiet", action="store_true", required=False, default=False, help="Quiet output.")
|
@@ -156,6 +159,7 @@ def main():
|
|
156
159
|
explicit_schema_name=explicit_schema_name,
|
157
160
|
update_function=post_data,
|
158
161
|
update_action_name="POST",
|
162
|
+
noignore=args.noignore, ignore=args.ignore,
|
159
163
|
confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
|
160
164
|
if args.patch:
|
161
165
|
_post_or_patch_or_upsert(portal=portal,
|
@@ -164,6 +168,7 @@ def main():
|
|
164
168
|
update_function=patch_data,
|
165
169
|
update_action_name="PATCH",
|
166
170
|
patch_delete_fields=args.delete,
|
171
|
+
noignore=args.noignore, ignore=args.ignore,
|
167
172
|
confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
|
168
173
|
args.delete = None
|
169
174
|
if args.upsert:
|
@@ -173,6 +178,7 @@ def main():
|
|
173
178
|
update_function=upsert_data,
|
174
179
|
update_action_name="UPSERT",
|
175
180
|
patch_delete_fields=args.delete,
|
181
|
+
noignore=args.noignore, ignore=args.ignore,
|
176
182
|
confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
|
177
183
|
args.delete = None
|
178
184
|
|
@@ -196,6 +202,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
196
202
|
explicit_schema_name: str,
|
197
203
|
update_function: Callable, update_action_name: str,
|
198
204
|
patch_delete_fields: Optional[str] = None,
|
205
|
+
noignore: bool = False, ignore: Optional[List[str]] = None,
|
199
206
|
confirm: bool = False, verbose: bool = False,
|
200
207
|
quiet: bool = False, debug: bool = False) -> None:
|
201
208
|
|
@@ -222,6 +229,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
222
229
|
_print(f"DEBUG: File ({file}) contains an object of type: {schema_name}")
|
223
230
|
update_function(portal, data, schema_name, file=file,
|
224
231
|
patch_delete_fields=patch_delete_fields,
|
232
|
+
noignore=noignore, ignore=ignore,
|
225
233
|
confirm=confirm, verbose=verbose, debug=debug)
|
226
234
|
elif is_schema_name_list(portal, list(data.keys())):
|
227
235
|
if debug:
|
@@ -233,6 +241,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
233
241
|
for index, item in enumerate(schema_data):
|
234
242
|
update_function(portal, item, schema_name, file=file, index=index,
|
235
243
|
patch_delete_fields=patch_delete_fields,
|
244
|
+
noignore=noignore, ignore=ignore,
|
236
245
|
confirm=confirm, verbose=verbose, debug=debug)
|
237
246
|
else:
|
238
247
|
_print(f"WARNING: File ({file}) contains schema item which is not a list: {schema_name}")
|
@@ -244,6 +253,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
244
253
|
for index, item in enumerate(data):
|
245
254
|
update_function(portal, item, schema_name, file=file, index=index,
|
246
255
|
patch_delete_fields=patch_delete_fields,
|
256
|
+
noignore=noignore, ignore=ignore,
|
247
257
|
confirm=confirm, verbose=verbose, debug=debug)
|
248
258
|
if debug:
|
249
259
|
_print(f"DEBUG: Processing {update_action_name} file done: {file}")
|
@@ -278,8 +288,10 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
|
|
278
288
|
|
279
289
|
def post_data(portal: Portal, data: dict, schema_name: str,
|
280
290
|
file: Optional[str] = None, index: int = 0,
|
281
|
-
patch_delete_fields: Optional[str] = None,
|
291
|
+
patch_delete_fields: Optional[str] = None,
|
292
|
+
noignore: bool = False, ignore: Optional[List[str]] = None,
|
282
293
|
confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
|
294
|
+
ignored(patch_delete_fields)
|
283
295
|
if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
|
284
296
|
if isinstance(file, str) and isinstance(index, int):
|
285
297
|
_print(f"ERROR: Item for POST has no identifying property: {file} (#{index + 1})")
|
@@ -294,6 +306,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
|
|
294
306
|
if verbose:
|
295
307
|
_print(f"POST {schema_name} item: {identifying_path}")
|
296
308
|
try:
|
309
|
+
data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
|
297
310
|
portal.post_metadata(schema_name, data)
|
298
311
|
if debug:
|
299
312
|
_print(f"DEBUG: POST {schema_name} item done: {identifying_path}")
|
@@ -306,6 +319,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
|
|
306
319
|
def patch_data(portal: Portal, data: dict, schema_name: str,
|
307
320
|
file: Optional[str] = None, index: int = 0,
|
308
321
|
patch_delete_fields: Optional[str] = None,
|
322
|
+
noignore: bool = False, ignore: Optional[List[str]] = None,
|
309
323
|
confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
|
310
324
|
if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
|
311
325
|
if isinstance(file, str) and isinstance(index, int):
|
@@ -323,6 +337,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
|
|
323
337
|
try:
|
324
338
|
if delete_fields := _parse_delete_fields(patch_delete_fields):
|
325
339
|
identifying_path += f"?delete_fields={delete_fields}"
|
340
|
+
data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
|
326
341
|
portal.patch_metadata(identifying_path, data)
|
327
342
|
if debug:
|
328
343
|
_print(f"DEBUG: PATCH {schema_name} item OK: {identifying_path}")
|
@@ -335,6 +350,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
|
|
335
350
|
def upsert_data(portal: Portal, data: dict, schema_name: str,
|
336
351
|
file: Optional[str] = None, index: int = 0,
|
337
352
|
patch_delete_fields: Optional[str] = None,
|
353
|
+
noignore: bool = False, ignore: Optional[List[str]] = None,
|
338
354
|
confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
|
339
355
|
if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
|
340
356
|
if isinstance(file, str) and isinstance(index, int):
|
@@ -349,10 +365,12 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
|
|
349
365
|
_print(f"{'PATCH' if exists else 'POST'} {schema_name} item: {identifying_path}")
|
350
366
|
try:
|
351
367
|
if not exists:
|
368
|
+
data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
|
352
369
|
portal.post_metadata(schema_name, data)
|
353
370
|
else:
|
354
371
|
if delete_fields := _parse_delete_fields(patch_delete_fields):
|
355
372
|
identifying_path += f"?delete_fields={delete_fields}"
|
373
|
+
data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
|
356
374
|
portal.patch_metadata(identifying_path, data)
|
357
375
|
if debug:
|
358
376
|
_print(f"DEBUG: UPSERT {schema_name} item OK: {identifying_path}")
|
@@ -362,6 +380,15 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
|
|
362
380
|
return
|
363
381
|
|
364
382
|
|
383
|
+
def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
|
384
|
+
ignore_these_properties = [] if noignore is True else _IGNORE_PROPERTIES_ON_UPDATE
|
385
|
+
if isinstance(ignore, list):
|
386
|
+
ignore_these_properties = ignore_these_properties + ignore
|
387
|
+
if not ignore_these_properties:
|
388
|
+
return data
|
389
|
+
return {key: value for key, value in data.items() if key not in ignore_these_properties}
|
390
|
+
|
391
|
+
|
365
392
|
def _create_portal(env: Optional[str] = None, app: Optional[str] = None,
|
366
393
|
verbose: bool = False, debug: bool = False) -> Optional[Portal]:
|
367
394
|
|
@@ -104,6 +104,7 @@ def main():
|
|
104
104
|
parser.add_argument("--raw", action="store_true", required=False, default=False, help="Raw output.")
|
105
105
|
parser.add_argument("--inserts", action="store_true", required=False, default=False,
|
106
106
|
help="Format output for subsequent inserts.")
|
107
|
+
parser.add_argument("--ignore", nargs="+", help="Ignore these fields for --inserts.")
|
107
108
|
parser.add_argument("--tree", action="store_true", required=False, default=False, help="Tree output for schemas.")
|
108
109
|
parser.add_argument("--database", action="store_true", required=False, default=False,
|
109
110
|
help="Read from database output.")
|
@@ -191,7 +192,7 @@ def main():
|
|
191
192
|
return
|
192
193
|
|
193
194
|
data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw, inserts=args.inserts,
|
194
|
-
database=args.database, check=args.bool, verbose=args.verbose)
|
195
|
+
ignore=args.ignore, database=args.database, check=args.bool, verbose=args.verbose)
|
195
196
|
if args.bool:
|
196
197
|
if data:
|
197
198
|
_print(f"{args.uuid}: found")
|
@@ -242,7 +243,15 @@ def _create_portal(ini: str, env: Optional[str] = None,
|
|
242
243
|
|
243
244
|
def _get_portal_object(portal: Portal, uuid: str,
|
244
245
|
raw: bool = False, inserts: bool = False, database: bool = False,
|
246
|
+
ignore: Optional[List[str]] = None,
|
245
247
|
check: bool = False, verbose: bool = False) -> dict:
|
248
|
+
|
249
|
+
def prune_data(data: dict) -> dict:
|
250
|
+
nonlocal ignore
|
251
|
+
if not isinstance(ignore, list) or not ignore:
|
252
|
+
return data
|
253
|
+
return {key: value for key, value in data.items() if key not in ignore}
|
254
|
+
|
246
255
|
response = None
|
247
256
|
try:
|
248
257
|
if not uuid.startswith("/"):
|
@@ -279,13 +288,32 @@ def _get_portal_object(portal: Portal, uuid: str,
|
|
279
288
|
if not ((supertypes := portal.get_schemas_super_type_map()) and (subtypes := supertypes.get(results_type))):
|
280
289
|
subtypes = None
|
281
290
|
response = {}
|
291
|
+
results_index = 0
|
292
|
+
results_total = len(results)
|
293
|
+
def get_metadata_for_individual_result_type(uuid: str) -> Optional[dict]: # noqa
|
294
|
+
# There can be a lot of individual results for which we may need to get the actual type,
|
295
|
+
# so do this in a function we were can give verbose output feedback.
|
296
|
+
nonlocal portal, results_index, results_total, verbose
|
297
|
+
if verbose:
|
298
|
+
_print(f"Getting actual type for {results_type} result:"
|
299
|
+
f" {uuid} [{results_index} of {results_total}]", end="")
|
300
|
+
result = portal.get_metadata(uuid, raise_exception=False)
|
301
|
+
if (isinstance(result_types := result.get("@type"), list) and
|
302
|
+
result_types and (result_type := result_types[0])): # noqa
|
303
|
+
if verbose:
|
304
|
+
_print(f" -> {result_type}")
|
305
|
+
return result_type
|
306
|
+
if verbose:
|
307
|
+
_print()
|
308
|
+
return None
|
282
309
|
for result in results:
|
310
|
+
results_index += 1
|
283
311
|
result.pop("schema_version", None)
|
312
|
+
result = prune_data(result)
|
284
313
|
if (subtypes and
|
285
314
|
(result_uuid := result.get("uuid")) and
|
286
|
-
(
|
287
|
-
|
288
|
-
result_type = result_type[0]
|
315
|
+
(individual_result_type := get_metadata_for_individual_result_type(result_uuid))): # noqa
|
316
|
+
result_type = individual_result_type
|
289
317
|
else:
|
290
318
|
result_type = results_type
|
291
319
|
if response.get(result_type):
|
@@ -295,7 +323,7 @@ def _get_portal_object(portal: Portal, uuid: str,
|
|
295
323
|
# Get the result as non-raw so we can get its type.
|
296
324
|
elif ((response_cooked := portal.get(path, database=database)) and
|
297
325
|
(isinstance(response_type := response_cooked.json().get("@type"), list) and response_type)):
|
298
|
-
response = {f"{response_type[0]}": [response]}
|
326
|
+
response = {f"{response_type[0]}": [prune_data(response)]}
|
299
327
|
elif raw:
|
300
328
|
response.pop("schema_version", None)
|
301
329
|
return response
|
@@ -60,8 +60,8 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
|
|
60
60
|
dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
|
61
61
|
dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
|
62
62
|
dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
|
63
|
-
dcicutils/scripts/update_portal_object.py,sha256=
|
64
|
-
dcicutils/scripts/view_portal_object.py,sha256=
|
63
|
+
dcicutils/scripts/update_portal_object.py,sha256=qo_TGji-naaK2UXa-VbCfTSMsvRSEduqbQDuVXX3y4g,22188
|
64
|
+
dcicutils/scripts/view_portal_object.py,sha256=6day_AXtFf9y4oYNvZC_fGa-h9xQKbNwUBuoluVcuiI,33691
|
65
65
|
dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
|
66
66
|
dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
|
67
67
|
dcicutils/snapshot_utils.py,sha256=YDeI3vD-MhAtHwKDzfEm2q-n3l-da2yRpRR3xp0Ah1M,23021
|
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
|
75
75
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
76
76
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
77
77
|
dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
|
78
|
-
dcicutils-8.14.0.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
79
|
-
dcicutils-8.14.0.dist-info/METADATA,sha256=
|
80
|
-
dcicutils-8.14.0.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
81
|
-
dcicutils-8.14.0.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
|
82
|
-
dcicutils-8.14.0.dist-info/RECORD,,
|
78
|
+
dcicutils-8.14.0.1b3.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
79
|
+
dcicutils-8.14.0.1b3.dist-info/METADATA,sha256=iuq31-VVzXhHa2-CxoblCc7iu0CgVP7hT4hXWU75szQ,3439
|
80
|
+
dcicutils-8.14.0.1b3.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
81
|
+
dcicutils-8.14.0.1b3.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
|
82
|
+
dcicutils-8.14.0.1b3.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|