dcicutils 8.8.3.1b4__py3-none-any.whl → 8.8.3.1b7__py3-none-any.whl

dcicutils/misc_utils.py

@@ -2548,6 +2548,29 @@ def normalize_spaces(value: str) -> str:
     return re.sub(r"\s+", " ", value).strip()
 
 
+def find_nth_from_end(string: str, substring: str, nth: int) -> int:
+    """
+    Returns the index of the nth occurrence of the given substring within
+    the given string from the END of the given string; or -1 if not found.
+    """
+    index = -1
+    string = string[::-1]
+    for i in range(0, nth):
+        index = string.find(substring, index + 1)
+    return len(string) - index - 1 if index >= 0 else -1
+
+
+def set_nth(string: str, nth: int, replacement: str) -> str:
+    """
+    Sets the nth character of the given string to the given replacement string.
+    """
+    if not isinstance(string, str) or not isinstance(nth, int) or not isinstance(replacement, str):
+        return string
+    if nth < 0:
+        nth += len(string)
+    return string[:nth] + replacement + string[nth + 1:] if 0 <= nth < len(string) else string
+
+
 class JsonLinesReader:
 
     def __init__(self, fp, padded=False, padding=None):

dcicutils/progress_bar.py

@@ -1,4 +1,5 @@
 from collections import namedtuple
+import re
 from signal import signal, SIGINT
 import sys
 import threading
@@ -8,6 +9,7 @@ from types import FrameType as frame
 from typing import Callable, List, Optional, Union
 from contextlib import contextmanager
 from dcicutils.command_utils import yes_or_no
+from dcicutils.misc_utils import find_nth_from_end, set_nth
 
 
 class TQDM(tqdm):
@@ -59,6 +61,8 @@ class ProgressBar:
         self._bar = None
         self._disabled = False
         self._done = False
+        self._really_done = False
+        self.foo = time.time()
         self._tidy_output_hack = (tidy_output_hack is True)
         self._started = time.time()
         self._stop_requested = False
@@ -102,11 +106,9 @@ class ProgressBar:
             return True
         return False
 
-    def set_total(self, value: int, reset_eta: bool = False) -> None:
+    def set_total(self, value: int) -> None:
         if value == self._total:
             # If the total has not changed since last set then do nothing.
-            if reset_eta and self._bar is not None:
-                self._bar.reset()
             return
         if isinstance(value, int) and value > 0:
             self._total = value
@@ -117,16 +119,6 @@ class ProgressBar:
                 self._bar.total = value
                 self._bar.refresh()
 
-    def reset_eta(self) -> None:
-        # Since set_total does nothing if total is the same, provide
-        # a way to reset the ETA if starting over with the same total.
-        if self._bar is not None:
-            progress = self._bar.n
-            self._bar.reset()
-            self._bar.total = self._total
-            self._bar.n = progress
-            self._bar.refresh()
-
     def set_progress(self, value: int) -> None:
         if isinstance(value, int) and value >= 0:
             if (self._bar is not None) or self._initialize():
@@ -139,17 +131,29 @@ class ProgressBar:
                 self._bar.update(value)
                 self._bar.refresh()
 
-    def set_description(self, value: str) -> None:
-        self._description = self._format_description(value)
+    def reset_eta(self) -> None:
+        # Since set_total does nothing if total is the same, provide
+        # a way to reset the ETA if starting over with the same total.
+        # But NOTE that resetting ETA will ALSO reset the ELAPSED time.
         if self._bar is not None:
-            self._bar.set_description(self._description)
+            progress = self._bar.n
+            self._bar.reset()
+            self._bar.total = self._total
+            self._bar.n = progress
+            self._bar.refresh()
+
+    def set_description(self, value: str) -> None:
+        if isinstance(value, str):
+            self._description = self._format_description(value)
+            if self._bar is not None:
+                self._bar.set_description(self._description)
 
-    def done(self) -> None:
+    def done(self, description: Optional[str] = None) -> None:
         if self._done or self._bar is None:
             return
         self._ended = time.time()
         self.set_progress(self.total)
-        self._bar.set_description(self._description)
+        self.set_description(description)
         self._bar.refresh()
         # FYI: Do NOT do a bar.disable = True before a bar.close() or it messes up output
         # on multiple calls; found out the hard way; a couple hours will never get back :-/
@@ -200,6 +204,12 @@ class ProgressBar:
     def captured_output_for_testing(self) -> Optional[List[str]]:
         return self._captured_output_for_testing
 
+    @staticmethod
+    def format_captured_output_for_testing(description: str, total: int, progress: int) -> str:
+        percent = round((progress / total) * 100.0)
+        separator = "✓" if percent == 100 else "|"
+        return f"{description} {separator} {percent:>3}% ◀|### | {progress}/{total} | 0.0/s | 00:00 | ETA: 00:00"
+
     def _format_description(self, value: str) -> str:
         if not isinstance(value, str):
             value = ""
@@ -259,29 +269,26 @@ class ProgressBar:
         # string in the display string where the progress bar should actually go,
         # which we do in _format_description. Other minor things too; see below.
         sys_stdout_write = sys.stdout.write
-        last_total = None ; last_progress = None ; last_text = None  # noqa
+        last_text = None ; last_captured_output_text = None  # noqa
         def tidy_stdout_write(text: str) -> None:  # noqa
             nonlocal self, sys_stdout_write, sentinel_internal, spina, spini, spinn
-            nonlocal last_total, last_progress, last_text
+            nonlocal last_text, last_captured_output_text
             def replace_first(value: str, match: str, replacement: str) -> str:  # noqa
                 return value[:i] + replacement + value[i + len(match):] if (i := value.find(match)) >= 0 else value
             def remove_extra_trailing_spaces(text: str) -> str:  # noqa
                 while text.endswith("  "):
                     text = text[:-1]
                 return text
-            if not text:
+            if (not text) or (last_text == text) or self._really_done:
                 return
-            if self._bar:
-                if ((self._bar.total == last_total) and (self._bar.n == last_progress) and (last_text == text)):
-                    return
-                last_total = self._bar.total ; last_progress = self._bar.n ; last_text = text  # noqa
+            last_text = text
             if (self._disabled or self._done) and sentinel_internal in text:
                 # Another hack to really disable output on interrupt; in this case we set
                 # tqdm.disable to True, but output can still dribble out, so if the output
                 # looks like it is from tqdm and we are disabled/done then do no output.
                 return
             if sentinel_internal in text:
-                spinc = spina[spini % spinn] if not ("100%|" in text) else "| ✓" ; spini += 1  # noqa
+                spinc = spina[spini % spinn] if not ("100%|" in text) else "✓" ; spini += 1  # noqa
                 text = replace_first(text, sentinel_internal, f" {spinc}")
                 text = replace_first(text, "%|", "% ◀|")
                 text = remove_extra_trailing_spaces(text)
@@ -290,9 +297,33 @@ class ProgressBar:
                 # the unit we gave, which is empty; idunno; just replace it here.
                 text = replace_first(text, "s/ ", "/s ")
             sys_stdout_write(text)
-            if self._captured_output_for_testing is not None:
-                self._captured_output_for_testing.append(text)
             sys.stdout.flush()
+            if self._done:
+                self._really_done = True
+                return
+            if self._captured_output_for_testing is not None:
+                # For testing only we replace vacilliting values in the out like rate,
+                # time elapsed, and ETA with static values; so that something like this:
+                # > Working /  20% ◀|█████████▌  | 1/5 | 536.00/s | 00:01 | ETA: 00:02
+                # becomes something more static like this after calling this function:
+                # > Working |  20% ◀|### | 1/5 | 0.0/s | 00:00 | ETA: 00:00
+                # This function obviously has intimate knowledge of the output; better here than in tests.
+                def replace_time_dependent_values_with_static(text: str) -> str:
+                    blocks = "\u2587|\u2588|\u2589|\u258a|\u258b|\u258c|\u258d|\u258e|\u258f"
+                    if (n := find_nth_from_end(text, "|", 5)) >= 8:
+                        pattern = re.compile(
+                            rf"(\s*)(\d*%? ◀\|)(?:\s*{blocks}|#)*\s*(\|\s*\d+/\d+)?(\s*\|\s*)"
+                            rf"(?:\d+\.?\d*|\?)(\/s\s*\|\s*)(?:\d+:\d+)?(\s*\|\s*ETA:\s*)(?:\d+:\d+|\?)?")
+                        if match := pattern.match(text[n - 6:]):
+                            if text[n - 8:n - 7] != "✓": text = set_nth(text, n - 8, "|")  # noqa
+                            return (text[0:n - 6].replace("\r", "") +
+                                    match.expand(rf"\g<1>\g<2>### \g<3>\g<4>0.0\g<5>00:00\g<6>00:00"))
+                    return text
+                if text != "\n":
+                    captured_output_text = replace_time_dependent_values_with_static(text)
+                    if captured_output_text != last_captured_output_text:
+                        self._captured_output_for_testing.append(captured_output_text)
+                        last_captured_output_text = captured_output_text
         def restore_stdout_write() -> None:  # noqa
             nonlocal sys_stdout_write
             if sys_stdout_write is not None:

{dcicutils-8.8.3.1b4.dist-info → dcicutils-8.8.3.1b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.3.1b4
+Version: 8.8.3.1b7
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.3.1b4.dist-info → dcicutils-8.8.3.1b7.dist-info}/RECORD

@@ -43,12 +43,12 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
 dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
 dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=zVc4urdVGgnWjQ4UQlrGH-URAzr2l_PwZWI3u_GJdFE,102210
+dcicutils/misc_utils.py,sha256=a_grjJdiYgEMctwnXy7uDoFtDvCfAv1gjrLxdrOkptM,103041
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
 dcicutils/portal_utils.py,sha256=Xm0IqL2dA9C2gx98cPEbvlo81V76bEmpUpxb_8S3VqM,30480
-dcicutils/progress_bar.py,sha256=ZuWJJ7FAkv8ndU1vwcg-0TAG76Tm05m3-d97G4b7YzE,14463
+dcicutils/progress_bar.py,sha256=j6dZdh5EeZo52GWIBu2JL5254-V6eOhKHHB2eCkgevI,16569
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -72,8 +72,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.3.1b4.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.3.1b4.dist-info/METADATA,sha256=SU3-GNJsxb2pu-dBBV8wXLAigORCxsI0-oYH6uE2gdc,3356
-dcicutils-8.8.3.1b4.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.3.1b4.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.3.1b4.dist-info/RECORD,,
+dcicutils-8.8.3.1b7.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.3.1b7.dist-info/METADATA,sha256=v8fZzOT3EWHNMI-R_bT-jbA1LTjKnGd1LOGD_oLn1Yg,3356
+dcicutils-8.8.3.1b7.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.3.1b7.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.3.1b7.dist-info/RECORD,,