dcicutils 8.8.3.1b11__py3-none-any.whl → 8.8.3.1b13__py3-none-any.whl

dcicutils/data_readers.py

@@ -66,6 +66,13 @@ class RowReader(abc.ABC):
         else:
             return value
 
+    @property
+    def nrows(self) -> int:
+        nrows = 0
+        for row in self:
+            nrows += 1
+        return nrows
+
     def open(self) -> None:
         pass
 
@@ -192,6 +199,10 @@ class Excel:
                 return True
         return False
 
+    @property
+    def nsheets(self) -> int:
+        return len(self.sheet_names)
+
     def __del__(self) -> None:
         if (workbook := self._workbook) is not None:
             self._workbook = None

dcicutils/datetime_utils.py

@@ -1,6 +1,7 @@
 from dcicutils.misc_utils import normalize_spaces
 from datetime import datetime, timedelta, timezone
-from typing import Optional, Tuple
+from dateutil import parser as datetime_parser
+from typing import Optional, Tuple, Union
 
 
 def parse_datetime_string(value: str) -> Optional[datetime]:
@@ -82,6 +83,32 @@ def normalize_date_string(value: str) -> Optional[str]:
     return d.strftime("%Y-%m-%d") if d else None
 
 
+def get_timezone(hours: int, minutes: Optional[int] = None) -> timezone:
+    try:
+        return timezone(timedelta(hours=hours, minutes=minutes or 0))
+    except Exception:
+        return timezone.utc
+
+
+def get_timezone_hours_minutes(tz: timezone) -> Tuple[int, int]:
+    """
+    Returns a tuple with the integer hours and minutes offset for the given timezone.
+    """
+    tz_minutes = datetime.now(tz).utcoffset().total_seconds() / 60
+    return int(tz_minutes // 60), int(abs(tz_minutes % 60))
+
+
+def get_utc_timezone() -> timezone:
+    return timezone.utc
+
+
+def get_local_timezone() -> timezone:
+    """
+    Returns current/local timezone as a datetime.timezone object.
+    """
+    return datetime.now().astimezone().tzinfo
+
+
 def get_local_timezone_string() -> str:
     """
     Returns current/local timezone in format like: "-05:00".
@@ -96,3 +123,85 @@ def get_local_timezone_hours_minutes() -> Tuple[int, int]:
     """
     tz_minutes = datetime.now(timezone.utc).astimezone().utcoffset().total_seconds() / 60
     return int(tz_minutes // 60), int(abs(tz_minutes % 60))
+
+
+def parse_datetime(value: str, utc: bool = False, tz: Optional[timezone] = None) -> Optional[datetime]:
+    """
+    Parses the given string into a datetime, if possible, and returns that value,
+    or None if not able to parse. The timezone of the returned datetime will be the
+    local timezone; or if the given utc argument is True then it will be UTC; or if the
+    given tz argument is a datetime.timezone then return datetime will be in that timezone.
+    """
+    if isinstance(value, datetime):
+        return value
+    elif not isinstance(value, str):
+        return None
+    try:
+        # This dateutil.parser handles quite a wide variety of formats and suits our needs.
+        value = datetime_parser.parse(value)
+        if utc is True:
+            # If the given utc argument is True then it trumps any tz argument if given.
+            tz = timezone.utc
+        if value.tzinfo is not None:
+            # The given value had an explicit timezone specified.
+            if isinstance(tz, timezone):
+                return value.astimezone(tz)
+            return value
+        return value.replace(tzinfo=tz if isinstance(tz, timezone) else get_local_timezone())
+    except Exception:
+        return None
+
+
+def format_datetime(value: datetime,
+                    utc: bool = False,
+                    iso: bool = False,
+                    ms: bool = False,
+                    tz: Optional[Union[timezone, bool]] = None,
+                    notz: bool = False,
+                    noseconds: bool = False,
+                    verbose: bool = False,
+                    noseparator: bool = False,
+                    noday: bool = False) -> Optional[str]:
+    """
+    Returns the given datetime as a string in "YYYY:MM:DD hh:mm:ss tz" format, for
+    example "2024-04-17 15:42:26 EDT". If the given notz argument is True then omits
+    the timezone; if the noseconds argument is given the omits the seconds. If the given
+    verbose argument is True then returns a really verbose version of the datetime, for
+    example "Wednesday, April 17, 2024 | 15:42:26 EDT"; if the noseparator argument is
+    True then omits the "|" separator; if the noday argument is True then omits the day
+    of week part. The timezone of the returned datetime string will default to the local
+    one; if the given utc argument is True then it will be UTC; or if the given tz
+    argument is a datetime.timezone it will be in that timezone.
+    """
+    if not isinstance(value, datetime):
+        if not isinstance(value, str) or not (value := parse_datetime(value)):
+            return None
+    try:
+        if utc is True:
+            tz = timezone.utc
+        elif not isinstance(tz, timezone):
+            tz = get_local_timezone()
+            if tz is True:
+                notz = False
+            elif tz is False:
+                notz = True
+        value = value.astimezone(tz)
+        if iso:
+            if notz is True:
+                value = value.replace(tzinfo=None)
+            if not (ms is True):
+                value = value.replace(microsecond=0)
+            return value.isoformat()
+        if noseconds is True:
+            ms = False
+        if verbose:
+            return value.strftime(
+                f"{'' if noday is True else '%A, '}%B %-d, %Y{'' if noseparator is True else ' |'}"
+                f" %-I:%M{'' if noseconds is True else ':%S'}"
+                f"{f'.%f' if ms is True else ''} %p{'' if notz is True else ' %Z'}")
+        else:
+            return value.strftime(
+                f"%Y-%m-%d %H:%M{'' if noseconds is True else ':%S'}"
+                f"{f'.%f' if ms is True else ''}{'' if notz is True else ' %Z'}")
+    except Exception:
+        return None

dcicutils/misc_utils.py

@@ -2571,6 +2571,48 @@ def set_nth(string: str, nth: int, replacement: str) -> str:
     return string[:nth] + replacement + string[nth + 1:] if 0 <= nth < len(string) else string
 
 
+def format_size(nbytes: Union[int, float], precision: int = 2, nospace: bool = False, terse: bool = False) -> str:
+    if isinstance(nbytes, str) and nbytes.isdigit():
+        nbytes = int(nbytes)
+    elif not isinstance(nbytes, (int, float)):
+        return ""
+    UNITS = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
+    UNITS_TERSE = ['b', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
+    MAX_UNITS_INDEX = len(UNITS) - 1
+    ONE_K = 1024
+    index = 0
+    if (precision := max(precision, 0)) and (nbytes <= ONE_K):
+        precision -= 1
+    while abs(nbytes) >= ONE_K and index < MAX_UNITS_INDEX:
+        nbytes /= ONE_K
+        index += 1
+    if index == 0:
+        nbytes = int(nbytes)
+        return f"{nbytes} byte{'s' if nbytes != 1 else ''}"
+    unit = (UNITS_TERSE if terse else UNITS)[index]
+    return f"{nbytes:.{precision}f}{'' if nospace else ' '}{unit}"
+
+
+def format_duration(seconds: Union[int, float]) -> str:
+    seconds_actual = seconds
+    seconds = round(max(seconds, 0))
+    durations = [("year", 31536000), ("day", 86400), ("hour", 3600), ("minute", 60), ("second", 1)]
+    parts = []
+    for name, duration in durations:
+        if seconds >= duration:
+            count = seconds // duration
+            seconds %= duration
+            if count != 1:
+                name += "s"
+            parts.append(f"{count} {name}")
+    if len(parts) == 0:
+        return f"{seconds_actual:.1f} seconds"
+    elif len(parts) == 1:
+        return f"{seconds_actual:.1f} seconds"
+    else:
+        return " ".join(parts[:-1]) + " " + parts[-1]
+
+
 class JsonLinesReader:
 
     def __init__(self, fp, padded=False, padding=None):

dcicutils/portal_utils.py

@@ -282,15 +282,17 @@ class Portal:
             except Exception:
                 return None
 
-    def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
+    def patch_metadata(self, object_id: str, data: dict, check_only: bool = False) -> Optional[dict]:
         if self.key:
-            return patch_metadata(obj_id=object_id, patch_item=data, key=self.key)
-        return self.patch(f"/{object_id}", data).json()
+            return patch_metadata(obj_id=object_id, patch_item=data, key=self.key,
+                                  add_on="check_only=True" if check_only else "")
+        return self.patch(f"/{object_id}{'?check_only=True' if check_only else ''}", data).json()
 
-    def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
+    def post_metadata(self, object_type: str, data: dict, check_only: bool = False) -> Optional[dict]:
         if self.key:
-            return post_metadata(schema_name=object_type, post_item=data, key=self.key)
-        return self.post(f"/{object_type}", data).json()
+            return post_metadata(schema_name=object_type, post_item=data, key=self.key,
+                                 add_on="check_only=True" if check_only else "")
+        return self.post(f"/{object_type}{'?check_only=True' if check_only else ''}", data).json()
 
     def get_health(self) -> OptionalResponse:
         return self.get("/health")

dcicutils/progress_bar.py

@@ -9,7 +9,7 @@ from types import FrameType as frame
 from typing import Callable, List, Optional, Union
 from contextlib import contextmanager
 from dcicutils.command_utils import yes_or_no
-from dcicutils.misc_utils import find_nth_from_end, set_nth
+from dcicutils.misc_utils import find_nth_from_end, format_size, set_nth
 
 
 class TQDM(tqdm):
@@ -49,6 +49,8 @@ class ProgressBar:
 
     def __init__(self, total: Optional[int] = None,
                  description: Optional[str] = None,
+                 use_byte_size_for_rate: bool = False,
+                 use_ascii: bool = False,
                  catch_interrupt: bool = True,
                  interrupt: Optional[Callable] = None,
                  interrupt_continue: Optional[Callable] = None,
@@ -59,11 +61,13 @@ class ProgressBar:
                  tidy_output_hack: bool = True,
                  capture_output_for_testing: bool = False) -> None:
         self._bar = None
+        self._started = 0
         self._disabled = False
         self._done = False
         self._tidy_output_hack = (tidy_output_hack is True)
-        self._started = time.time()
         self._stop_requested = False
+        self._use_byte_size_for_rate = (use_byte_size_for_rate is True and self._tidy_output_hack)
+        self._use_ascii = (use_ascii is True)
         # Interrupt handling. We do not do the actual (signal) interrupt setup
         # in self._initialize as that could be called from a (sub) thread; and in
         # Python we can only set a signal (SIGINT in our case) on the main thread.
@@ -96,9 +100,13 @@ class ProgressBar:
     def _initialize(self) -> bool:
         # Do not actually create the tqdm object unless/until we have a positive total.
         if (self._bar is None) and (self._total > 0):
-            bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | {rate_fmt} | {elapsed}{postfix} | ETA: {remaining} "
+            if self._use_byte_size_for_rate:
+                bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | [rate] | {elapsed}{postfix} | ETA: {remaining} "
+            else:
+                bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | {rate_fmt} | {elapsed}{postfix} | ETA: {remaining} "
             self._bar = TQDM(total=self._total, desc=self._description,
-                             dynamic_ncols=True, bar_format=bar_format, unit="", file=sys.stdout)
+                             dynamic_ncols=True, bar_format=bar_format, unit="", file=sys.stdout, ascii=self._use_ascii)
+            self._started = time.time()
             if self._disabled:
                 self._bar.disable = True
             return True
@@ -153,6 +161,7 @@ class ProgressBar:
         self.set_total(total, _norefresh=True)
         self.set_progress(progress, _norefresh=True)
         self.set_description(description)
+        self._started = time.time()
 
     def done(self, description: Optional[str] = None) -> None:
         if self._done or self._bar is None:
@@ -198,14 +207,6 @@ class ProgressBar:
     def stop_requested(self) -> bool:
         return self._stop_requested
 
-    @property
-    def started(self) -> None:
-        return self._started
-
-    @property
-    def duration(self) -> None:
-        return time.time() - self._started
-
     @property
     def captured_output_for_testing(self) -> Optional[List[str]]:
         return self._captured_output_for_testing
@@ -302,6 +303,9 @@ class ProgressBar:
                 # something like "1.54s/" rather than "1.54/s"; something to do with
                 # the unit we gave, which is empty; idunno; just replace it here.
                 text = replace_first(text, "s/ ", "/s ")
+            if self._use_byte_size_for_rate and self._bar:
+                rate = self._bar.n / (time.time() - self._started)
+                text = text.replace("[rate]", f"{format_size(rate)}/s")
             sys_stdout_write(text)
             sys.stdout.flush()
             if self._captured_output_for_testing is not None:

{dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcicutils
-Version: 8.8.3.1b11
+Version: 8.8.3.1b13
 Summary: Utility package for interacting with the 4DN Data Portal and other 4DN resources
 Home-page: https://github.com/4dn-dcic/utils
 License: MIT

{dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b13.dist-info}/RECORD

@@ -10,9 +10,9 @@ dcicutils/common.py,sha256=YE8Mt5-vaZWWz4uaChSVhqGFbFtW5QKtnIyOr4zG4vM,3955
 dcicutils/contribution_scripts.py,sha256=0k5Gw1TumcD5SAcXVkDd6-yvuMEw-jUp5Kfb7FJH6XQ,2015
 dcicutils/contribution_utils.py,sha256=vYLS1JUB3sKd24BUxZ29qUBqYeQBLK9cwo8x3k64uPg,25653
 dcicutils/creds_utils.py,sha256=xrLekD49Ex0GOpL9n7LlJA4gvNcY7txTVFOSYD7LvEU,11113
-dcicutils/data_readers.py,sha256=WWH_VDz2KnNv_FoTjfFwrg6zh9asl8Q-uEV2V3XuyUg,7414
+dcicutils/data_readers.py,sha256=6EMrY7TjDE8H7bA_TCWtpLQP7slJ0YTL77_dNh6e7sg,7626
 dcicutils/data_utils.py,sha256=k2OxOlsx7AJ6jF-YNlMyGus_JqSUBe4_n1s65Mv1gQQ,3098
-dcicutils/datetime_utils.py,sha256=EODDGAngp1yh2ZlDIuI7tB74JBJucw2DljqfPknzK0Y,4666
+dcicutils/datetime_utils.py,sha256=OXdNpOKjNL3osTwhmLoJeL3DXoMksp0egExDT1hif_Y,9152
 dcicutils/deployment_utils.py,sha256=sKv8Jb-_Zw2aH3OAywRlsre-Cqm3a7fEGG3_1PT-r-s,69908
 dcicutils/diff_utils.py,sha256=sQx-yz56DHAcQWOChYbAG3clXu7TbiZKlw-GggeveO0,8118
 dcicutils/docker_utils.py,sha256=30gUiqz7X9rJwSPXTPn4ewjQibUgoSJqhP9o9vn5X-A,1747
@@ -43,12 +43,12 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
 dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
 dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
 dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
-dcicutils/misc_utils.py,sha256=a_grjJdiYgEMctwnXy7uDoFtDvCfAv1gjrLxdrOkptM,103041
+dcicutils/misc_utils.py,sha256=YH_TTmv6ABWeMERwVvA2-rIfdS-CoPYLXJru9TvWxgM,104610
 dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
 dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
 dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
-dcicutils/portal_utils.py,sha256=Xm0IqL2dA9C2gx98cPEbvlo81V76bEmpUpxb_8S3VqM,30480
-dcicutils/progress_bar.py,sha256=OFgLXJL-03tQw2WM6JVDr8_YhxN3eay090iUhiAJKZ4,16830
+dcicutils/portal_utils.py,sha256=DYyE5o15GekDgzpJWas9iS7klAYbjJZUPW0G42McArk,30779
+dcicutils/progress_bar.py,sha256=th-M7Q1UfkmQWc8M36lfVoRhFAQVRT9fSpdHi5RHqNY,17389
 dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
 dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
 dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
@@ -72,8 +72,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
 dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
 dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
 dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
-dcicutils-8.8.3.1b11.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
-dcicutils-8.8.3.1b11.dist-info/METADATA,sha256=X_eKy99Z5pjG-7rAnp-JC1sFIRJscq29KBx-J-CV-n0,3357
-dcicutils-8.8.3.1b11.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
-dcicutils-8.8.3.1b11.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
-dcicutils-8.8.3.1b11.dist-info/RECORD,,
+dcicutils-8.8.3.1b13.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
+dcicutils-8.8.3.1b13.dist-info/METADATA,sha256=3mHRqaCku3ZV_Yea9_Na_QZ8LTCscx9sELeZtt1BaUc,3357
+dcicutils-8.8.3.1b13.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
+dcicutils-8.8.3.1b13.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
+dcicutils-8.8.3.1b13.dist-info/RECORD,,