dcicutils 8.8.3.1b9__tar.gz → 8.8.3.1b12__tar.gz

{dcicutils-8.8.3.1b9 → dcicutils-8.8.3.1b12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.3.1b9
+Version: 8.8.3.1b12
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.3.1b9 → dcicutils-8.8.3.1b12}/dcicutils/misc_utils.py

@@ -2571,6 +2571,48 @@ def set_nth(string: str, nth: int, replacement: str) -> str:
     return string[:nth] + replacement + string[nth + 1:] if 0 <= nth < len(string) else string
 
 
+def format_size(nbytes: Union[int, float], precision: int = 2, nospace: bool = False, terse: bool = False) -> str:
+    if isinstance(nbytes, str) and nbytes.isdigit():
+        nbytes = int(nbytes)
+    elif not isinstance(nbytes, (int, float)):
+        return ""
+    UNITS = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+    UNITS_TERSE = ['b', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+    MAX_UNITS_INDEX = len(UNITS) - 1
+    ONE_K = 1024
+    index = 0
+    if (precision := max(precision, 0)) and (nbytes <= ONE_K):
+        precision -= 1
+    while abs(nbytes) >= ONE_K and index < MAX_UNITS_INDEX:
+        nbytes /= ONE_K
+        index += 1
+    if index == 0:
+        nbytes = int(nbytes)
+        return f"{nbytes} byte{'s' if nbytes != 1 else ''}"
+    unit = (UNITS_TERSE if terse else UNITS)[index]
+    return f"{nbytes:.{precision}f}{'' if nospace else ' '}{unit}"
+
+
+def format_duration(seconds: Union[int, float]) -> str:
+    seconds_actual = seconds
+    seconds = round(max(seconds, 0))
+    durations = [("year", 31536000), ("day", 86400), ("hour", 3600), ("minute", 60), ("second", 1)]
+    parts = []
+    for name, duration in durations:
+        if seconds >= duration:
+            count = seconds // duration
+            seconds %= duration
+            if count != 1:
+                name += "s"
+            parts.append(f"{count} {name}")
+    if len(parts) == 0:
+        return f"{seconds_actual:.1f} seconds"
+    elif len(parts) == 1:
+        return f"{seconds_actual:.1f} seconds"
+    else:
+        return " ".join(parts[:-1]) + " " + parts[-1]
+
+
 class JsonLinesReader:
 
     def __init__(self, fp, padded=False, padding=None):

{dcicutils-8.8.3.1b9 → dcicutils-8.8.3.1b12}/dcicutils/portal_utils.py

@@ -282,15 +282,17 @@ class Portal:
             except Exception:
                 return None
 
-    def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
+    def patch_metadata(self, object_id: str, data: dict, check_only: bool = False) -> Optional[dict]:
         if self.key:
-            return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
-        return self.patch(f"/{object_id}", data).json()
+            return patch_metadata(obj_id=object_id, patch_item=data, key=self.key,
+                                  add_on="check_only=True" if check_only else "")
+        return self.patch(f"/{object_id}{'?check_only=True' if check_only else ''}", data).json()
 
-    def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
+    def post_metadata(self, object_type: str, data: dict, check_only: bool = False) -> Optional[dict]:
         if self.key:
-            return post_metadata(schema_name=object_type, post_item=data, key=self.key)
-        return self.post(f"/{object_type}", data).json()
+            return post_metadata(schema_name=object_type, post_item=data, key=self.key,
+                                 add_on="check_only=True" if check_only else "")
+        return self.post(f"/{object_type}{'?check_only=True' if check_only else ''}", data).json()
 
     def get_health(self) -> OptionalResponse:
         return self.get("/health")

{dcicutils-8.8.3.1b9 → dcicutils-8.8.3.1b12}/dcicutils/progress_bar.py

@@ -9,7 +9,7 @@ from types import FrameType as frame
 from typing import Callable, List, Optional, Union
 from contextlib import contextmanager
 from dcicutils.command_utils import yes_or_no
-from dcicutils.misc_utils import find_nth_from_end, set_nth
+from dcicutils.misc_utils import find_nth_from_end, format_size, set_nth
 
 
 class TQDM(tqdm):
@@ -49,6 +49,7 @@ class ProgressBar:
 
     def __init__(self, total: Optional[int] = None,
                  description: Optional[str] = None,
+                 use_byte_size_for_rate: bool = False,
                  catch_interrupt: bool = True,
                  interrupt: Optional[Callable] = None,
                  interrupt_continue: Optional[Callable] = None,
@@ -59,13 +60,12 @@ class ProgressBar:
                  tidy_output_hack: bool = True,
                  capture_output_for_testing: bool = False) -> None:
         self._bar = None
+        self._started = 0
         self._disabled = False
         self._done = False
-        self._really_done = False
-        self.foo = time.time()
         self._tidy_output_hack = (tidy_output_hack is True)
-        self._started = time.time()
         self._stop_requested = False
+        self._use_byte_size_for_rate = (use_byte_size_for_rate is True and self._tidy_output_hack)
         # Interrupt handling. We do not do the actual (signal) interrupt setup
         # in self._initialize as that could be called from a (sub) thread; and in
         # Python we can only set a signal (SIGINT in our case) on the main thread.
@@ -98,9 +98,13 @@ class ProgressBar:
     def _initialize(self) -> bool:
         # Do not actually create the tqdm object unless/until we have a positive total.
         if (self._bar is None) and (self._total > 0):
-            bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | {rate_fmt} | {elapsed}{postfix} | ETA: {remaining} "
+            if self._use_byte_size_for_rate:
+                bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | [rate] | {elapsed}{postfix} | ETA: {remaining} "
+            else:
+                bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | {rate_fmt} | {elapsed}{postfix} | ETA: {remaining} "
             self._bar = TQDM(total=self._total, desc=self._description,
                              dynamic_ncols=True, bar_format=bar_format, unit="", file=sys.stdout)
+            self._started = time.time()
             if self._disabled:
                 self._bar.disable = True
             return True
@@ -155,6 +159,7 @@ class ProgressBar:
         self.set_total(total, _norefresh=True)
         self.set_progress(progress, _norefresh=True)
         self.set_description(description)
+        self._started = time.time()
 
     def done(self, description: Optional[str] = None) -> None:
         if self._done or self._bar is None:
@@ -200,14 +205,6 @@ class ProgressBar:
     def stop_requested(self) -> bool:
         return self._stop_requested
 
-    @property
-    def started(self) -> None:
-        return self._started
-
-    @property
-    def duration(self) -> None:
-        return time.time() - self._started
-
     @property
     def captured_output_for_testing(self) -> Optional[List[str]]:
         return self._captured_output_for_testing
@@ -287,7 +284,7 @@ class ProgressBar:
                 while text.endswith("  "):
                     text = text[:-1]
                 return text
-            if (not text) or (last_text == text) or self._really_done:
+            if (not text) or (last_text == text):
                 return
             last_text = text
             if (self._disabled or self._done) and sentinel_internal in text:
@@ -304,11 +301,11 @@ class ProgressBar:
                 # something like "1.54s/" rather than "1.54/s"; something to do with
                 # the unit we gave, which is empty; idunno; just replace it here.
                 text = replace_first(text, "s/ ", "/s ")
+            if self._use_byte_size_for_rate and self._bar:
+                rate = self._bar.n / (time.time() - self._started)
+                text = text.replace("[rate]", f"{format_size(rate)}/s")
             sys_stdout_write(text)
             sys.stdout.flush()
-            if self._done:
-                self._really_done = True
-                return
             if self._captured_output_for_testing is not None:
                 # For testing only we replace vacilliting values in the out like rate,
                 # time elapsed, and ETA with static values; so that something like this:

{dcicutils-8.8.3.1b9 → dcicutils-8.8.3.1b12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "dcicutils"
-version = "8.8.3.1b9"  # TODO: To become 8.8.4
+version = "8.8.3.1b12"  # TODO: To become 8.8.4
 description = "Utility package for interacting with the 4DN Data Portal and other 4DN resources"
 authors = ["4DN-DCIC Team <support@4dnucleome.org>"]
 license = "MIT"