dcicutils 8.8.3.1b5__py3-none-any.whl → 8.8.3.1b8__py3-none-any.whl

dcicutils/progress_bar.py

@@ -61,6 +61,8 @@ class ProgressBar:
         self._bar = None
         self._disabled = False
         self._done = False
+        self._really_done = False
+        self.foo = time.time()
         self._tidy_output_hack = (tidy_output_hack is True)
         self._started = time.time()
         self._stop_requested = False
@@ -104,11 +106,9 @@ class ProgressBar:
             return True
         return False
 
-    def set_total(self, value: int, reset_eta: bool = False) -> None:
+    def set_total(self, value: int) -> None:
         if value == self._total:
             # If the total has not changed since last set then do nothing.
-            if reset_eta and self._bar is not None:
-                self._bar.reset()
             return
         if isinstance(value, int) and value > 0:
             self._total = value
@@ -119,16 +119,6 @@ class ProgressBar:
                 self._bar.total = value
                 self._bar.refresh()
 
-    def reset_eta(self) -> None:
-        # Since set_total does nothing if total is the same, provide
-        # a way to reset the ETA if starting over with the same total.
-        if self._bar is not None:
-            progress = self._bar.n
-            self._bar.reset()
-            self._bar.total = self._total
-            self._bar.n = progress
-            self._bar.refresh()
-
     def set_progress(self, value: int) -> None:
         if isinstance(value, int) and value >= 0:
             if (self._bar is not None) or self._initialize():
@@ -141,17 +131,29 @@ class ProgressBar:
                 self._bar.update(value)
                 self._bar.refresh()
 
-    def set_description(self, value: str) -> None:
-        self._description = self._format_description(value)
+    def reset_eta(self) -> None:
+        # Since set_total does nothing if total is the same, provide
+        # a way to reset the ETA if starting over with the same total.
+        # But NOTE that resetting ETA will ALSO reset the ELAPSED time.
         if self._bar is not None:
-            self._bar.set_description(self._description)
+            progress = self._bar.n
+            self._bar.reset()
+            self._bar.total = self._total
+            self._bar.n = progress
+            self._bar.refresh()
 
-    def done(self) -> None:
+    def set_description(self, value: str) -> None:
+        if isinstance(value, str):
+            self._description = self._format_description(value)
+            if self._bar is not None:
+                self._bar.set_description(self._description)
+
+    def done(self, description: Optional[str] = None) -> None:
         if self._done or self._bar is None:
             return
         self._ended = time.time()
         self.set_progress(self.total)
-        self._bar.set_description(self._description)
+        self.set_description(description)
         self._bar.refresh()
         # FYI: Do NOT do a bar.disable = True before a bar.close() or it messes up output
         # on multiple calls; found out the hard way; a couple hours will never get back :-/
@@ -202,6 +204,12 @@ class ProgressBar:
     def captured_output_for_testing(self) -> Optional[List[str]]:
         return self._captured_output_for_testing
 
+    @staticmethod
+    def format_captured_output_for_testing(description: str, total: int, progress: int) -> str:
+        percent = round((progress / total) * 100.0)
+        separator = "✓" if percent == 100 else "|"
+        return f"{description} {separator} {percent:>3}% ◀|### | {progress}/{total} | 0.0/s | 00:00 | ETA: 00:00"
+
     def _format_description(self, value: str) -> str:
         if not isinstance(value, str):
             value = ""
@@ -261,22 +269,19 @@ class ProgressBar:
         # string in the display string where the progress bar should actually go,
         # which we do in _format_description. Other minor things too; see below.
         sys_stdout_write = sys.stdout.write
-        last_total = None ; last_progress = None ; last_text = None  # noqa
+        last_text = None ; last_captured_output_text = None  # noqa
         def tidy_stdout_write(text: str) -> None:  # noqa
             nonlocal self, sys_stdout_write, sentinel_internal, spina, spini, spinn
-            nonlocal last_total, last_progress, last_text
+            nonlocal last_text, last_captured_output_text
             def replace_first(value: str, match: str, replacement: str) -> str:  # noqa
                 return value[:i] + replacement + value[i + len(match):] if (i := value.find(match)) >= 0 else value
             def remove_extra_trailing_spaces(text: str) -> str:  # noqa
                 while text.endswith("  "):
                     text = text[:-1]
                 return text
-            if not text:
+            if (not text) or (last_text == text) or self._really_done:
                 return
-            if self._bar:
-                if ((self._bar.total == last_total) and (self._bar.n == last_progress) and (last_text == text)):
-                    return
-                last_total = self._bar.total ; last_progress = self._bar.n ; last_text = text  # noqa
+            last_text = text
             if (self._disabled or self._done) and sentinel_internal in text:
                 # Another hack to really disable output on interrupt; in this case we set
                 # tqdm.disable to True, but output can still dribble out, so if the output
@@ -293,6 +298,9 @@ class ProgressBar:
                 text = replace_first(text, "s/ ", "/s ")
             sys_stdout_write(text)
             sys.stdout.flush()
+            if self._done:
+                self._really_done = True
+                return
             if self._captured_output_for_testing is not None:
                 # For testing only we replace vacilliting values in the out like rate,
                 # time elapsed, and ETA with static values; so that something like this:
@@ -300,7 +308,7 @@ class ProgressBar:
                 # becomes something more static like this after calling this function:
                 # > Working |  20% ◀|### | 1/5 | 0.0/s | 00:00 | ETA: 00:00
                 # This function obviously has intimate knowledge of the output; better here than in tests.
-                def replace_vacillating_values_with_static(text: str) -> str:
+                def replace_time_dependent_values_with_static(text: str) -> str:
                     blocks = "\u2587|\u2588|\u2589|\u258a|\u258b|\u258c|\u258d|\u258e|\u258f"
                     if (n := find_nth_from_end(text, "|", 5)) >= 8:
                         pattern = re.compile(
@@ -311,7 +319,11 @@ class ProgressBar:
                             return (text[0:n - 6].replace("\r", "") +
                                     match.expand(rf"\g<1>\g<2>### \g<3>\g<4>0.0\g<5>00:00\g<6>00:00"))
                     return text
-                self._captured_output_for_testing.append(replace_vacillating_values_with_static(text))
+                if text != "\n":
+                    captured_output_text = replace_time_dependent_values_with_static(text)
+                    if captured_output_text != last_captured_output_text:
+                        self._captured_output_for_testing.append(captured_output_text)
+                        last_captured_output_text = captured_output_text
         def restore_stdout_write() -> None:  # noqa
             nonlocal sys_stdout_write
             if sys_stdout_write is not None:

dcicutils/structured_data.py

@@ -154,7 +154,32 @@ class StructuredDataSet:
 
     @property
     def resolved_refs_with_uuids(self) -> List[str]:
-        return list([{"path": resolved_ref[0], "uuid": resolved_ref[1]} for resolved_ref in self._resolved_refs])
+        return list([{"path": resolved_ref[0],
+                      "uuid": resolved_ref[1] if len(resolved_ref) >= 2 else None}
+                     for resolved_ref in self._resolved_refs])
+
+    @property
+    def unchecked_refs(self) -> List[str]:
+        """
+        Returns list of unchecked (for existence) references, grouped by reference path;
+        each object in the list has a path property and a srcs property which is a list of
+        src objects containing the type, column and row of the reference to the reference.
+        Note that this is only populated if the norefs option is specified.
+        """
+        def load_json(value: str) -> Optional[dict]:
+            try:
+                return json.loads(value)
+            except Exception:
+                return None
+        result = []
+        if self._norefs:
+            for ref in self._resolved_refs:
+                if len(ref) >= 3 and (ref_path := ref[0]) and (ref_src := load_json(ref[2])):
+                    if existing_ref := [item for item in result if item.get("path") == ref_path]:
+                        existing_ref[0]["srcs"].append(ref_src)
+                    else:
+                        result.append({"path": ref_path, "srcs": [ref_src]})
+        return result
 
     @property
     def upload_files(self) -> List[str]:
@@ -653,10 +678,14 @@ class Schema(SchemaBase):
         def map_ref(value: str, link_to: str, portal: Optional[Portal], src: Optional[str]) -> Any:
             nonlocal self, typeinfo
             if self._norefs:
+                # Here the caller has specified the (StructuredDataSet) norefs option
+                # which means we do not check for the existence of references at all.
                 if value:
-                    # Dump the src because this cannot add dictionary to a set; note that
-                    # this is ONLY used for smaht-submitr/submit-metadata-bundle --info --refs.
-                    self._resolved_refs.add((f"/{link_to}/{value}", json.dumps(src) if isinstance(src, dict) else None))
+                    # Dump the src as a JSON string because a dictionary cannot be added to a set; note
+                    # that this is ONLY used for smaht-submitr/submit-metadata-bundle --info --refs.
+                    # This info can be gotten at using StructureDataSet.unchecked_refs.
+                    self._resolved_refs.add((f"/{link_to}/{value}", None,
+                                             json.dumps(src) if isinstance(src, dict) else None))
                 return value
             if not value:
                 if (column := typeinfo.get("column")) and column in self.data.get("required", []):

{dcicutils-8.8.3.1b5.dist-info → dcicutils-8.8.3.1b8.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.3.1b5
+Version: 8.8.3.1b8
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.3.1b5.dist-info → dcicutils-8.8.3.1b8.dist-info}/RECORD

@@ -48,7 +48,7 @@ dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmj
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
 dcicutils/portal_utils.py,sha256=Xm0IqL2dA9C2gx98cPEbvlo81V76bEmpUpxb_8S3VqM,30480
-dcicutils/progress_bar.py,sha256=BWd-a1Og7SL_FNtIQv8gAkGYMyp6VgwIwTlLClsl-8g,15988
+dcicutils/progress_bar.py,sha256=j6dZdh5EeZo52GWIBu2JL5254-V6eOhKHHB2eCkgevI,16569
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -63,7 +63,7 @@ dcicutils/secrets_utils.py,sha256=8dppXAsiHhJzI6NmOcvJV5ldvKkQZzh3Fl-cb8Wm7MI,19
 dcicutils/sheet_utils.py,sha256=VlmzteONW5VF_Q4vo0yA5vesz1ViUah1MZ_yA1rwZ0M,33629
 dcicutils/snapshot_utils.py,sha256=ymP7PXH6-yEiXAt75w0ldQFciGNqWBClNxC5gfX2FnY,22961
 dcicutils/ssl_certificate_utils.py,sha256=F0ifz_wnRRN9dfrfsz7aCp4UDLgHEY8LaK7PjnNvrAQ,9707
-dcicutils/structured_data.py,sha256=YCLGtjsCMXCm5ZM96aiTRQ9OPUuj0lK8O9vSkLx1gv8,58450
+dcicutils/structured_data.py,sha256=Ho97KAqPgfn9QVfIMpjp0gS1FXLAnEJu6usdVi2lGHc,59927
 dcicutils/submitr/progress_constants.py,sha256=5bxyX77ql8qEJearfHEvsvXl7D0GuUODW0T65mbRmnE,2895
 dcicutils/submitr/ref_lookup_strategy.py,sha256=Js2cVznTmgjciLWBPLCvMiwLIHXjDn3jww-gJPjYuFw,3467
 dcicutils/task_utils.py,sha256=MF8ujmTD6-O2AC2gRGPHyGdUrVKgtr8epT5XU8WtNjk,8082
@@ -72,8 +72,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.3.1b5.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.3.1b5.dist-info/METADATA,sha256=Z9Not6bJDezXyZb3afrGADA6mRDuOd2QbETxSbht6DM,3356
-dcicutils-8.8.3.1b5.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.3.1b5.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.3.1b5.dist-info/RECORD,,
+dcicutils-8.8.3.1b8.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.3.1b8.dist-info/METADATA,sha256=saFsKdu4mbE4RBPh2331yy2nC-ZUcE5n3fT740wn9iY,3356
+dcicutils-8.8.3.1b8.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.3.1b8.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.3.1b8.dist-info/RECORD,,