dcicutils 8.14.0__py3-none-any.whl → 8.14.0.1b3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -19,7 +19,7 @@ from typing import Callable, List, Optional, Tuple, Union
19
19
  from dcicutils.command_utils import yes_or_no
20
20
  from dcicutils.common import ORCHESTRATED_APPS, APP_SMAHT
21
21
  from dcicutils.ff_utils import delete_metadata, purge_metadata
22
- from dcicutils.misc_utils import get_error_message, PRINT
22
+ from dcicutils.misc_utils import get_error_message, ignored, PRINT
23
23
  from dcicutils.portal_utils import Portal as PortalFromUtils
24
24
 
25
25
 
@@ -40,7 +40,7 @@ _DEFAULT_APP = "smaht"
40
40
  _SMAHT_ENV_ENVIRON_NAME = "SMAHT_ENV"
41
41
 
42
42
  # Schema properties to ignore (by default) for the view schema usage.
43
- _SCHEMAS_IGNORE_PROPERTIES = [
43
+ _IGNORE_PROPERTIES_ON_UPDATE = [
44
44
  "date_created",
45
45
  "last_modified",
46
46
  "principals_allowed",
@@ -122,6 +122,9 @@ def main():
122
122
  parser.add_argument("--upsert", type=str, required=False, default=None, help="Upsert data.")
123
123
  parser.add_argument("--delete", type=str, required=False, default=None, help="Delete data.")
124
124
  parser.add_argument("--purge", type=str, required=False, default=None, help="Purge data.")
125
+ parser.add_argument("--noignore", action="store_true", required=False, default=False,
126
+ help="Do not ignore standard fields on update(s).")
127
+ parser.add_argument("--ignore", nargs="+", help="Ignore these additional fields.")
125
128
  parser.add_argument("--confirm", action="store_true", required=False, default=False, help="Confirm before action.")
126
129
  parser.add_argument("--verbose", action="store_true", required=False, default=False, help="Verbose output.")
127
130
  parser.add_argument("--quiet", action="store_true", required=False, default=False, help="Quiet output.")
@@ -156,6 +159,7 @@ def main():
156
159
  explicit_schema_name=explicit_schema_name,
157
160
  update_function=post_data,
158
161
  update_action_name="POST",
162
+ noignore=args.noignore, ignore=args.ignore,
159
163
  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
160
164
  if args.patch:
161
165
  _post_or_patch_or_upsert(portal=portal,
@@ -164,6 +168,7 @@ def main():
164
168
  update_function=patch_data,
165
169
  update_action_name="PATCH",
166
170
  patch_delete_fields=args.delete,
171
+ noignore=args.noignore, ignore=args.ignore,
167
172
  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
168
173
  args.delete = None
169
174
  if args.upsert:
@@ -173,6 +178,7 @@ def main():
173
178
  update_function=upsert_data,
174
179
  update_action_name="UPSERT",
175
180
  patch_delete_fields=args.delete,
181
+ noignore=args.noignore, ignore=args.ignore,
176
182
  confirm=args.confirm, verbose=args.verbose, quiet=args.quiet, debug=args.debug)
177
183
  args.delete = None
178
184
 
@@ -196,6 +202,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
196
202
  explicit_schema_name: str,
197
203
  update_function: Callable, update_action_name: str,
198
204
  patch_delete_fields: Optional[str] = None,
205
+ noignore: bool = False, ignore: Optional[List[str]] = None,
199
206
  confirm: bool = False, verbose: bool = False,
200
207
  quiet: bool = False, debug: bool = False) -> None:
201
208
 
@@ -222,6 +229,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
222
229
  _print(f"DEBUG: File ({file}) contains an object of type: {schema_name}")
223
230
  update_function(portal, data, schema_name, file=file,
224
231
  patch_delete_fields=patch_delete_fields,
232
+ noignore=noignore, ignore=ignore,
225
233
  confirm=confirm, verbose=verbose, debug=debug)
226
234
  elif is_schema_name_list(portal, list(data.keys())):
227
235
  if debug:
@@ -233,6 +241,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
233
241
  for index, item in enumerate(schema_data):
234
242
  update_function(portal, item, schema_name, file=file, index=index,
235
243
  patch_delete_fields=patch_delete_fields,
244
+ noignore=noignore, ignore=ignore,
236
245
  confirm=confirm, verbose=verbose, debug=debug)
237
246
  else:
238
247
  _print(f"WARNING: File ({file}) contains schema item which is not a list: {schema_name}")
@@ -244,6 +253,7 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
244
253
  for index, item in enumerate(data):
245
254
  update_function(portal, item, schema_name, file=file, index=index,
246
255
  patch_delete_fields=patch_delete_fields,
256
+ noignore=noignore, ignore=ignore,
247
257
  confirm=confirm, verbose=verbose, debug=debug)
248
258
  if debug:
249
259
  _print(f"DEBUG: Processing {update_action_name} file done: {file}")
@@ -278,8 +288,10 @@ def _post_or_patch_or_upsert(portal: Portal, file_or_directory: str,
278
288
 
279
289
  def post_data(portal: Portal, data: dict, schema_name: str,
280
290
  file: Optional[str] = None, index: int = 0,
281
- patch_delete_fields: Optional[str] = None, # unused here
291
+ patch_delete_fields: Optional[str] = None,
292
+ noignore: bool = False, ignore: Optional[List[str]] = None,
282
293
  confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
294
+ ignored(patch_delete_fields)
283
295
  if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
284
296
  if isinstance(file, str) and isinstance(index, int):
285
297
  _print(f"ERROR: Item for POST has no identifying property: {file} (#{index + 1})")
@@ -294,6 +306,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
294
306
  if verbose:
295
307
  _print(f"POST {schema_name} item: {identifying_path}")
296
308
  try:
309
+ data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
297
310
  portal.post_metadata(schema_name, data)
298
311
  if debug:
299
312
  _print(f"DEBUG: POST {schema_name} item done: {identifying_path}")
@@ -306,6 +319,7 @@ def post_data(portal: Portal, data: dict, schema_name: str,
306
319
  def patch_data(portal: Portal, data: dict, schema_name: str,
307
320
  file: Optional[str] = None, index: int = 0,
308
321
  patch_delete_fields: Optional[str] = None,
322
+ noignore: bool = False, ignore: Optional[List[str]] = None,
309
323
  confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
310
324
  if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
311
325
  if isinstance(file, str) and isinstance(index, int):
@@ -323,6 +337,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
323
337
  try:
324
338
  if delete_fields := _parse_delete_fields(patch_delete_fields):
325
339
  identifying_path += f"?delete_fields={delete_fields}"
340
+ data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
326
341
  portal.patch_metadata(identifying_path, data)
327
342
  if debug:
328
343
  _print(f"DEBUG: PATCH {schema_name} item OK: {identifying_path}")
@@ -335,6 +350,7 @@ def patch_data(portal: Portal, data: dict, schema_name: str,
335
350
  def upsert_data(portal: Portal, data: dict, schema_name: str,
336
351
  file: Optional[str] = None, index: int = 0,
337
352
  patch_delete_fields: Optional[str] = None,
353
+ noignore: bool = False, ignore: Optional[List[str]] = None,
338
354
  confirm: bool = False, verbose: bool = False, debug: bool = False) -> None:
339
355
  if not (identifying_path := portal.get_identifying_path(data, portal_type=schema_name)):
340
356
  if isinstance(file, str) and isinstance(index, int):
@@ -349,10 +365,12 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
349
365
  _print(f"{'PATCH' if exists else 'POST'} {schema_name} item: {identifying_path}")
350
366
  try:
351
367
  if not exists:
368
+ data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
352
369
  portal.post_metadata(schema_name, data)
353
370
  else:
354
371
  if delete_fields := _parse_delete_fields(patch_delete_fields):
355
372
  identifying_path += f"?delete_fields={delete_fields}"
373
+ data = _prune_data_for_update(data, noignore=noignore, ignore=ignore)
356
374
  portal.patch_metadata(identifying_path, data)
357
375
  if debug:
358
376
  _print(f"DEBUG: UPSERT {schema_name} item OK: {identifying_path}")
@@ -362,6 +380,15 @@ def upsert_data(portal: Portal, data: dict, schema_name: str,
362
380
  return
363
381
 
364
382
 
383
+ def _prune_data_for_update(data: dict, noignore: bool = False, ignore: Optional[List[str]] = None) -> dict:
384
+ ignore_these_properties = [] if noignore is True else _IGNORE_PROPERTIES_ON_UPDATE
385
+ if isinstance(ignore, list):
386
+ ignore_these_properties = ignore_these_properties + ignore
387
+ if not ignore_these_properties:
388
+ return data
389
+ return {key: value for key, value in data.items() if key not in ignore_these_properties}
390
+
391
+
365
392
  def _create_portal(env: Optional[str] = None, app: Optional[str] = None,
366
393
  verbose: bool = False, debug: bool = False) -> Optional[Portal]:
367
394
 
@@ -104,6 +104,7 @@ def main():
104
104
  parser.add_argument("--raw", action="store_true", required=False, default=False, help="Raw output.")
105
105
  parser.add_argument("--inserts", action="store_true", required=False, default=False,
106
106
  help="Format output for subsequent inserts.")
107
+ parser.add_argument("--ignore", nargs="+", help="Ignore these fields for --inserts.")
107
108
  parser.add_argument("--tree", action="store_true", required=False, default=False, help="Tree output for schemas.")
108
109
  parser.add_argument("--database", action="store_true", required=False, default=False,
109
110
  help="Read from database output.")
@@ -191,7 +192,7 @@ def main():
191
192
  return
192
193
 
193
194
  data = _get_portal_object(portal=portal, uuid=args.uuid, raw=args.raw, inserts=args.inserts,
194
- database=args.database, check=args.bool, verbose=args.verbose)
195
+ ignore=args.ignore, database=args.database, check=args.bool, verbose=args.verbose)
195
196
  if args.bool:
196
197
  if data:
197
198
  _print(f"{args.uuid}: found")
@@ -242,7 +243,15 @@ def _create_portal(ini: str, env: Optional[str] = None,
242
243
 
243
244
  def _get_portal_object(portal: Portal, uuid: str,
244
245
  raw: bool = False, inserts: bool = False, database: bool = False,
246
+ ignore: Optional[List[str]] = None,
245
247
  check: bool = False, verbose: bool = False) -> dict:
248
+
249
+ def prune_data(data: dict) -> dict:
250
+ nonlocal ignore
251
+ if not isinstance(ignore, list) or not ignore:
252
+ return data
253
+ return {key: value for key, value in data.items() if key not in ignore}
254
+
246
255
  response = None
247
256
  try:
248
257
  if not uuid.startswith("/"):
@@ -279,13 +288,32 @@ def _get_portal_object(portal: Portal, uuid: str,
279
288
  if not ((supertypes := portal.get_schemas_super_type_map()) and (subtypes := supertypes.get(results_type))):
280
289
  subtypes = None
281
290
  response = {}
291
+ results_index = 0
292
+ results_total = len(results)
293
+ def get_metadata_for_individual_result_type(uuid: str) -> Optional[dict]: # noqa
294
+ # There can be a lot of individual results for which we may need to get the actual type,
295
+ # so do this in a function we were can give verbose output feedback.
296
+ nonlocal portal, results_index, results_total, verbose
297
+ if verbose:
298
+ _print(f"Getting actual type for {results_type} result:"
299
+ f" {uuid} [{results_index} of {results_total}]", end="")
300
+ result = portal.get_metadata(uuid, raise_exception=False)
301
+ if (isinstance(result_types := result.get("@type"), list) and
302
+ result_types and (result_type := result_types[0])): # noqa
303
+ if verbose:
304
+ _print(f" -> {result_type}")
305
+ return result_type
306
+ if verbose:
307
+ _print()
308
+ return None
282
309
  for result in results:
310
+ results_index += 1
283
311
  result.pop("schema_version", None)
312
+ result = prune_data(result)
284
313
  if (subtypes and
285
314
  (result_uuid := result.get("uuid")) and
286
- (individual_result := portal.get_metadata(result_uuid, raise_exception=False)) and
287
- isinstance(result_type:= individual_result.get("@type"), list) and result_type and result_type[0]): # noqa
288
- result_type = result_type[0]
315
+ (individual_result_type := get_metadata_for_individual_result_type(result_uuid))): # noqa
316
+ result_type = individual_result_type
289
317
  else:
290
318
  result_type = results_type
291
319
  if response.get(result_type):
@@ -295,7 +323,7 @@ def _get_portal_object(portal: Portal, uuid: str,
295
323
  # Get the result as non-raw so we can get its type.
296
324
  elif ((response_cooked := portal.get(path, database=database)) and
297
325
  (isinstance(response_type := response_cooked.json().get("@type"), list) and response_type)):
298
- response = {f"{response_type[0]}": [response]}
326
+ response = {f"{response_type[0]}": [prune_data(response)]}
299
327
  elif raw:
300
328
  response.pop("schema_version", None)
301
329
  return response
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.14.0
3
+ Version: 8.14.0.1b3
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -60,8 +60,8 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
60
60
  dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
61
61
  dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
62
62
  dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
63
- dcicutils/scripts/update_portal_object.py,sha256=Lrgj01JuxmQSoWikukYgwm5LKi9CgSzjAogoMIBsQC4,20373
64
- dcicutils/scripts/view_portal_object.py,sha256=h8COy0lcLNWF9b5spjrlQ28wfqyTTMqAeC_xpFXutus,32262
63
+ dcicutils/scripts/update_portal_object.py,sha256=qo_TGji-naaK2UXa-VbCfTSMsvRSEduqbQDuVXX3y4g,22188
64
+ dcicutils/scripts/view_portal_object.py,sha256=6day_AXtFf9y4oYNvZC_fGa-h9xQKbNwUBuoluVcuiI,33691
65
65
  dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
66
66
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
67
67
  dcicutils/snapshot_utils.py,sha256=YDeI3vD-MhAtHwKDzfEm2q-n3l-da2yRpRR3xp0Ah1M,23021
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
75
75
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
76
76
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
77
77
  dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
78
- dcicutils-8.14.0.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
79
- dcicutils-8.14.0.dist-info/METADATA,sha256=hgCt8yg5wJE5esuZCuj9uCXfJNS2uqdR_w4ge8xR8CQ,3435
80
- dcicutils-8.14.0.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
81
- dcicutils-8.14.0.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
82
- dcicutils-8.14.0.dist-info/RECORD,,
78
+ dcicutils-8.14.0.1b3.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
79
+ dcicutils-8.14.0.1b3.dist-info/METADATA,sha256=iuq31-VVzXhHa2-CxoblCc7iu0CgVP7hT4hXWU75szQ,3439
80
+ dcicutils-8.14.0.1b3.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
81
+ dcicutils-8.14.0.1b3.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
82
+ dcicutils-8.14.0.1b3.dist-info/RECORD,,