dcicutils 8.14.0.1b17__py3-none-any.whl → 8.14.0.1b19__py3-none-any.whl

dcicutils/scripts/update_portal_object.py

@@ -433,15 +433,19 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
         current_item_total = 0
 
         for item in LoadGenWrapper(load_all_gen(testapp=portal.vapp, inserts=inserts_directory,
-                                                docsdir=None, overwrite=True, verbose=True)):
+                                                docsdir=None, overwrite=True, verbose=True,
+                                                continue_on_exception=True)):
             loadxl_total_item_count += 1
             item = decode_bytes(item)
             match = LOADXL_RESPONSE_PATTERN.match(item)
             if not match or match.re.groups < 3:
                 continue
             if (action := LOADXL_ACTION_NAME[match.group(1).upper()]) == "Error":
+                loadxl_total_error_count += 1
                 identifying_value = match.group(2)
-                # Example message for unresolved link:
+                #
+                # Example message for unresolved link ...
+                #
                 # ERROR: /22813a02-906b-4b60-b2b2-4afaea24aa28 Bad response: 422 Unprocessable Entity
                 # (not 200 OK or 3xx redirect for http://localhost/file_set?skip_indexing=true)b\'{"@type":
                 # ["ValidationFailure", "Error"], "status": "error", "code": # 422, "title": "Unprocessable Entity",
@@ -449,21 +453,33 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
                 # "description": "Unable to resolve link: /Library/a4e8f79f-4d47-4e85-9707-c343c940a315"},
                 # {"location": "body", "name": "Schema: libraries.0",
                 # "description": "\\\'a4e8f79f-4d47-4e85-9707-c343c940a315\\\' not found"}]}\'
+                #
+                # OR ...
+                #
+                # ERROR: /22813a02-906b-4b60-b2b2-4afaea24aa28 Bad response: 404 Not Found (not 200 OK or 3xx
+                # redirect for http://localhost/22813a02-906b-4b60-b2b2-4afaea24aa28)b\'{"@type": ["HTTPNotFound",
+                # "Error"], "status": "error", "code": 404, "title": "Not Found", "description": "The resource
+                # could not be found.", "detail": "debug_notfound of url http://localhost/22813a02-906b-4b60-b2b2-4afaea24aa28; # noqa
+                # path_info: \\\'/22813a02-906b-4b60-b2b2-4afaea24aa28\\\', context: <encoded.root.SMAHTRoot object at 0x136d41460>, # noqa
+                # view_name: \\\'22813a02-906b-4b60-b2b2-4afaea24aa28\\\', subpath: (), traversed: (), root:
+                # <encoded.root.SMAHTRoot object at 0x136d41460>, vroot: <encoded.root.SMAHTRoot object at 0x136d41460>, vroot_path: ()"}\' # noqa
+                #
+                if (item_type := re.search(r"https?://.*/(.*)\?skip_indexing=.*", item)) and (len(item_type.groups()) == 1):  # noqa
+                    item_type = to_snake_case(item_type.group(1))
+                    identifying_value = f"/{to_camel_case(item_type)}{identifying_value}"
                 unresolved_link_error_message_prefix = "Unable to resolve link:"
                 if (i := item.find(unresolved_link_error_message_prefix)) > 0:
                     unresolved_link = item[i + len(unresolved_link_error_message_prefix):].strip()
                     if (i := unresolved_link.find("\"")) > 0:
                         if (unresolved_link := unresolved_link[0:i]):
-                            if ((error_type := re.search(r"https?://.*/(.*)\?skip_indexing=.*", item)) and
-                                (len(error_type.groups()) == 1)):  # noqa
-                                error_type = to_camel_case(error_type.group(1))
-                                identifying_value = f"/{error_type}{identifying_value}"
-                                if not loadxl_unresolved.get(identifying_value):
-                                    loadxl_unresolved[identifying_value] = []
-                                loadxl_unresolved[identifying_value].append(unresolved_link)
-                loadxl_total_error_count += 1
-                continue
-            item_type = match.group(3)
+                            if not loadxl_unresolved.get(unresolved_link):
+                                loadxl_unresolved[unresolved_link] = []
+                            if identifying_value not in loadxl_unresolved[unresolved_link]:
+                                loadxl_unresolved[unresolved_link].append(identifying_value)
+                if not item_type:
+                    continue
+            else:
+                item_type = match.group(3)
             if current_item_type != item_type:
                 if noprogress and debug and current_item_type is not None:
                     _print()
@@ -482,6 +498,7 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
                 _print(f"{current_item_type}: {current_item_count} or {current_item_total} ({action})")
         if progress_bar:
             progress_bar.set_description("▶ Load Complete")
+            progress_bar.set_progress(progress_total)
             if loadxl_total_item_count > loadxl_total_error_count:
                 _print()
 
@@ -607,16 +624,16 @@ def _load_data(portal: Portal, load: str, ini_file: str, explicit_schema_name: O
             _print(f"Done loading data into Portal (via snovault.loadxl) from file: {_single_insert_file}")
         else:
             _print(f"Done loading data into Portal (via snovault.loadxl) from directory: {inserts_directory}")
-        _print(f"Total items loaded: {loadxl_total_item_count}"
+        _print(f"Total items loaded: {loadxl_total_item_count // 2}"  # TODO: straightend out this arithmetic
                f"{f' (errors: {loadxl_total_error_count})' if loadxl_total_error_count else ''}")
         for item in sorted(loadxl_summary.keys()):
             _print(f"▷ {to_camel_case(item)}: {loadxl_summary[item]}")
     if loadxl_unresolved:
         _print("✗ Unresolved references:")
         for item in loadxl_unresolved:
-            _print(f"  ▶ {item}: {len(loadxl_unresolved[item])}")
+            _print(f"  ✗ {item}: {len(loadxl_unresolved[item])}")
             for subitem in loadxl_unresolved[item]:
-                _print(f"    ▷ {subitem}")
+                _print(f"     ▶ {subitem}")
     if debug and loadxl_output:
         _print("✗ Output from loadxl:")
         for item in loadxl_output:

{dcicutils-8.14.0.1b17.dist-info → dcicutils-8.14.0.1b19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.14.0.1b17
+Version: 8.14.0.1b19
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.14.0.1b17.dist-info → dcicutils-8.14.0.1b19.dist-info}/RECORD

@@ -60,7 +60,7 @@ dcicutils/s3_utils.py,sha256=h2B9ftOo-kxqfiKth5ZDC_cAUFy1Pbu7BrVanFnE5Iw,28839
 dcicutils/schema_utils.py,sha256=GmRm-XqZKJ6qine16SQF1txcby9WougDav_sYmKNs9E,12400
 dcicutils/scripts/publish_to_pypi.py,sha256=sMd4WASQGlxlh7uLrt2eGkFRXYgONVmvIg8mClMS5RQ,13903
 dcicutils/scripts/run_license_checker.py,sha256=z2keYnRDZsHQbTeo1XORAXSXNJK5axVzL5LjiNqZ7jE,4184
-dcicutils/scripts/update_portal_object.py,sha256=MsNpF6eHGVxr7dUg8QrwZo9XfIjPjAavOWqyI_RrfpA,38397
+dcicutils/scripts/update_portal_object.py,sha256=Xp1Jho_LC9gkaR6pV3Jvukxx27LW2N18ucDgxz6gc9w,39639
 dcicutils/scripts/view_portal_object.py,sha256=lcgXWH9ooVf7tJDIRnoFGOgT0wYLGhiJlJW3a9w6A_c,36983
 dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19745
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
@@ -75,8 +75,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=_Y9EmL3D2dUZhxucxHvrtmmlbZmK4FpSsHEb7rGSJLU,3265
-dcicutils-8.14.0.1b17.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.14.0.1b17.dist-info/METADATA,sha256=WxgzreGqLqRZT1QRIzME9quTpxihdyduDQKkxX9kdS0,3440
-dcicutils-8.14.0.1b17.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.14.0.1b17.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
-dcicutils-8.14.0.1b17.dist-info/RECORD,,
+dcicutils-8.14.0.1b19.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.14.0.1b19.dist-info/METADATA,sha256=fNFLeE-npH786qVeBJgJemerLOwaIGYkbOT1jZWKqKs,3440
+dcicutils-8.14.0.1b19.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.14.0.1b19.dist-info/entry_points.txt,sha256=W6kEWdUJk9tQ4myAgpehPdebcwvCAZ7UgB-wyPgDUMg,335
+dcicutils-8.14.0.1b19.dist-info/RECORD,,