dcicutils 8.14.0.1b21__py3-none-any.whl → 8.14.0.1b22__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,9 +2,16 @@
2
2
  # Command-line utility to update (post, patch, upsert) portal objects for SMaHT/CGAP/Fourfront.
3
3
  # ------------------------------------------------------------------------------------------------------
4
4
  # Example commands:
5
- # update-portal-object --post file_format.json
6
- # update-portal-object --upsert directory-with-schema-named-dot-json-files
7
- # update-portal-object --patch file-not-named-for-schema-name.json --schema UnalignedReads
5
+ #
6
+ # update-portal-object --load {json-file | directory-with-json-files}
7
+ # update-portal-object --post {json-file | directory-with-json-files}
8
+ # update-portal-object --upsert {json-file | directory-with-json-files}
9
+ # update-portal-object --patch {json-file | directory-with-json-files}
10
+ #
11
+ # The specified json-file or file withing directory-with-jaon-files must be JSON containing either
12
+ # a list of objects, which which case the file name for the target schema name, or if not, then
13
+ # the --schema option must be used to specified the target schema; or the JSON must be a dictionary
14
+ # of schema names, where the value of each is a list of objects for that schema.
8
15
  # --------------------------------------------------------------------------------------------------
9
16
 
10
17
  import argparse
@@ -464,8 +471,7 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
464
471
  # view_name: \\\'22813a02-906b-4b60-b2b2-4afaea24aa28\\\', subpath: (), traversed: (), root:
465
472
  # <encoded.root.SMAHTRoot object at 0x136d41460>, vroot: <encoded.root.SMAHTRoot object at 0x136d41460>, vroot_path: ()"}\' # noqa
466
473
  #
467
- if ((item_type := re.search(r"https?://.*/(.*)\?skip_indexing=.*", item)) and
468
- (len(item_type.groups()) == 1)): # noqa
474
+ if (item_type := re.search(r"https?://.*/(.*)\?skip_indexing=.*", item)) and (len(item_type.groups()) == 1): # noqa
469
475
  item_type = to_snake_case(item_type.group(1))
470
476
  identifying_value = f"/{to_camel_case(item_type)}{identifying_value}"
471
477
  unresolved_link_error_message_prefix = "Unable to resolve link:"
@@ -628,7 +634,7 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
628
634
  _print(f"Total items loaded: {loadxl_total_item_count // 2}" # TODO: straightend out this arithmetic
629
635
  f"{f' (errors: {loadxl_total_error_count})' if loadxl_total_error_count else ''}")
630
636
  for item in sorted(loadxl_summary.keys()):
631
- _print(f"▷ {to_camel_case(item)}: {loadxl_summary[item]}")
637
+ _print(f"▷ {to_camel_case(item)}: {loadxl_summary[item] // 2}") # TODO: straightend out this arithmetic
632
638
  if loadxl_unresolved:
633
639
  _print("✗ Unresolved references:")
634
640
  for item in loadxl_unresolved:
@@ -290,6 +290,23 @@ def _get_portal_object(portal: Portal, uuid: str,
290
290
  _print()
291
291
  return None
292
292
 
293
+ def get_metadata_types(path: str) -> Optional[dict]:
294
+ nonlocal portal, debug
295
+ metadata_types = {}
296
+ try:
297
+ if debug:
298
+ _print(f"Executing separted query to get actual metadata types for raw/inserts query.")
299
+ if ((response := portal.get(path)) and (response.status_code in [200, 307]) and
300
+ (response := response.json()) and (results := response.get("@graph"))): # noqa
301
+ for result in results:
302
+ if (result_type := result.get("@type")) and (result_uuid := result.get("uuid")):
303
+ if ((isinstance(result_type, list) and (result_type := result_type[0])) or
304
+ isinstance(result_type, str)): # noqa
305
+ metadata_types[result_uuid] = result_type
306
+ except Exception:
307
+ return None
308
+ return metadata_types
309
+
293
310
  response = None
294
311
  try:
295
312
  if not uuid.startswith("/"):
@@ -312,6 +329,7 @@ def _get_portal_object(portal: Portal, uuid: str,
312
329
  if not response.json:
313
330
  _exit(f"Invalid JSON getting Portal object: {uuid}")
314
331
  response = response.json()
332
+ response_types = {}
315
333
  if inserts:
316
334
  # Format results as suitable for inserts (e.g. via update-portal-object).
317
335
  response.pop("schema_version", None)
@@ -319,10 +337,12 @@ def _get_portal_object(portal: Portal, uuid: str,
319
337
  (isinstance(results_type := response.get("@type"), list) and results_type) and
320
338
  (isinstance(results_type := results_type[0], str) and results_type.endswith("SearchResults")) and
321
339
  (results_type := results_type[0:-len("SearchResults")])): # noqa
322
- # For search results, the type (from XyzSearchResults, above) may not be precisely correct for
323
- # each of the results; it may be the supertype (e.g. QualityMetric vs QualityMetricWorkflowRun);
340
+ # For (raw frame) search results, the type (from XyzSearchResults, above) may not be precisely correct
341
+ # for each of the results; it may be the supertype (e.g. QualityMetric vs QualityMetricWorkflowRun);
324
342
  # so for types which are supertypes (gotten via Portal.get_schemas_super_type_map) we actually
325
- # lookup each result individually to determine its actual precise type.
343
+ # lookup each result individually to determine its actual precise type. Although, if we have
344
+ # more than (say) 5 results to do this for, then do a separate query (get_metadata_types)
345
+ # to get the result types all at once.
326
346
  if not ((supertypes := portal.get_schemas_super_type_map()) and (subtypes := supertypes.get(results_type))):
327
347
  subtypes = None
328
348
  response = {}
@@ -335,9 +355,16 @@ def _get_portal_object(portal: Portal, uuid: str,
335
355
  result.pop("schema_version", None)
336
356
  result = prune_data(result)
337
357
  if (subtypes and one_or_more_objects_of_types_exists(portal, subtypes, debug=debug) and
338
- (result_uuid := result.get("uuid")) and
339
- (individual_result_type := get_metadata_for_individual_result_type(result_uuid))): # noqa
340
- result_type = individual_result_type
358
+ (result_uuid := result.get("uuid"))): # noqa
359
+ # If we have more than (say) 5 results for which we need to determine that actual result type,
360
+ # then get them all at once via separate query (get_metadata_types)) which is not the raw frame.
361
+ if (results_total > 5) and (not response_types):
362
+ response_types = get_metadata_types(path)
363
+ if not (response_types and (result_type := response_types.get(result_uuid))):
364
+ if individual_result_type := get_metadata_for_individual_result_type(result_uuid):
365
+ result_type = individual_result_type
366
+ else:
367
+ result_type = results_type
341
368
  else:
342
369
  result_type = results_type
343
370
  if response.get(result_type):
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: dcicutils
3
- Version: 8.14.0.1b21
3
+ Version: 8.14.0.1b22
4
4
  Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
5
5
  Home-page: https://github.com/4dn-dcic/utils
6
6
  License: MIT
@@ -60,8 +60,8 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
60
60
  dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
61
61
  dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
62
62
  dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
63
- dcicutils/scripts/update_portal_object.py,sha256=_Wk9UYcT4XQKQQoqyTmiNEEh0ttY3TG1vMATsHr8W2Y,39661
64
- dcicutils/scripts/view_portal_object.py,sha256=lcgXWH9ooVf7tJDIRnoFGOgT0wYLGhiJlJW3a9w6A_c,36983
63
+ dcicutils/scripts/update_portal_object.py,sha256=pRTzbfT4Ul64KwlBJa6cqfNRDIqPh2Uyi9BmXeAjoeU,40135
64
+ dcicutils/scripts/view_portal_object.py,sha256=7I7wEBLFoVLpP8VZNGQuxh6N1PALJ38IKZTJ6WSNRRM,38654
65
65
  dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
66
66
  dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
67
67
  dcicutils/snapshot_utils.py,sha256=YDeI3vD-MhAtHwKDzfEm2q-n3l-da2yRpRR3xp0Ah1M,23021
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
75
75
  dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
76
76
  dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
77
77
  dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
78
- dcicutils-8.14.0.1b21.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
79
- dcicutils-8.14.0.1b21.dist-info/METADATA,sha256=dBs3CAErEqR7h0pLE7K9CZKVJw5FOcSk1JCuZo7ZkZg,3440
80
- dcicutils-8.14.0.1b21.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
81
- dcicutils-8.14.0.1b21.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
82
- dcicutils-8.14.0.1b21.dist-info/RECORD,,
78
+ dcicutils-8.14.0.1b22.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
79
+ dcicutils-8.14.0.1b22.dist-info/METADATA,sha256=ACXqTzYQxjKfS18CdckhbfPZZF2AeIQ88Ta25nkDMPM,3440
80
+ dcicutils-8.14.0.1b22.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
81
+ dcicutils-8.14.0.1b22.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
82
+ dcicutils-8.14.0.1b22.dist-info/RECORD,,