dcicutils 8.8.3.1b11__py3-none-any.whl → 8.8.3.1b12__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- dcicutils/misc_utils.py +42 -0
- dcicutils/portal_utils.py +8 -6
- dcicutils/progress_bar.py +13 -11
- {dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b12.dist-info}/METADATA +1 -1
- {dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b12.dist-info}/RECORD +8 -8
- {dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b12.dist-info}/LICENSE.txt +0 -0
- {dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b12.dist-info}/WHEEL +0 -0
- {dcicutils-8.8.3.1b11.dist-info → dcicutils-8.8.3.1b12.dist-info}/entry_points.txt +0 -0
dcicutils/misc_utils.py
CHANGED
@@ -2571,6 +2571,48 @@ def set_nth(string: str, nth: int, replacement: str) -> str:
|
|
2571
2571
|
return string[:nth] + replacement + string[nth + 1:] if 0 <= nth < len(string) else string
|
2572
2572
|
|
2573
2573
|
|
2574
|
+
def format_size(nbytes: Union[int, float], precision: int = 2, nospace: bool = False, terse: bool = False) -> str:
|
2575
|
+
if isinstance(nbytes, str) and nbytes.isdigit():
|
2576
|
+
nbytes = int(nbytes)
|
2577
|
+
elif not isinstance(nbytes, (int, float)):
|
2578
|
+
return ""
|
2579
|
+
UNITS = ['bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']
|
2580
|
+
UNITS_TERSE = ['b', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y']
|
2581
|
+
MAX_UNITS_INDEX = len(UNITS) - 1
|
2582
|
+
ONE_K = 1024
|
2583
|
+
index = 0
|
2584
|
+
if (precision := max(precision, 0)) and (nbytes <= ONE_K):
|
2585
|
+
precision -= 1
|
2586
|
+
while abs(nbytes) >= ONE_K and index < MAX_UNITS_INDEX:
|
2587
|
+
nbytes /= ONE_K
|
2588
|
+
index += 1
|
2589
|
+
if index == 0:
|
2590
|
+
nbytes = int(nbytes)
|
2591
|
+
return f"{nbytes} byte{'s' if nbytes != 1 else ''}"
|
2592
|
+
unit = (UNITS_TERSE if terse else UNITS)[index]
|
2593
|
+
return f"{nbytes:.{precision}f}{'' if nospace else ' '}{unit}"
|
2594
|
+
|
2595
|
+
|
2596
|
+
def format_duration(seconds: Union[int, float]) -> str:
|
2597
|
+
seconds_actual = seconds
|
2598
|
+
seconds = round(max(seconds, 0))
|
2599
|
+
durations = [("year", 31536000), ("day", 86400), ("hour", 3600), ("minute", 60), ("second", 1)]
|
2600
|
+
parts = []
|
2601
|
+
for name, duration in durations:
|
2602
|
+
if seconds >= duration:
|
2603
|
+
count = seconds // duration
|
2604
|
+
seconds %= duration
|
2605
|
+
if count != 1:
|
2606
|
+
name += "s"
|
2607
|
+
parts.append(f"{count} {name}")
|
2608
|
+
if len(parts) == 0:
|
2609
|
+
return f"{seconds_actual:.1f} seconds"
|
2610
|
+
elif len(parts) == 1:
|
2611
|
+
return f"{seconds_actual:.1f} seconds"
|
2612
|
+
else:
|
2613
|
+
return " ".join(parts[:-1]) + " " + parts[-1]
|
2614
|
+
|
2615
|
+
|
2574
2616
|
class JsonLinesReader:
|
2575
2617
|
|
2576
2618
|
def __init__(self, fp, padded=False, padding=None):
|
dcicutils/portal_utils.py
CHANGED
@@ -282,15 +282,17 @@ class Portal:
|
|
282
282
|
except Exception:
|
283
283
|
return None
|
284
284
|
|
285
|
-
def patch_metadata(self, object_id: str, data: dict) -> Optional[dict]:
|
285
|
+
def patch_metadata(self, object_id: str, data: dict, check_only: bool = False) -> Optional[dict]:
|
286
286
|
if self.key:
|
287
|
-
return patch_metadata(obj_id=object_id, patch_item=data, key=self.key
|
288
|
-
|
287
|
+
return patch_metadata(obj_id=object_id, patch_item=data, key=self.key,
|
288
|
+
add_on="check_only=True" if check_only else "")
|
289
|
+
return self.patch(f"/{object_id}{'?check_only=True' if check_only else ''}", data).json()
|
289
290
|
|
290
|
-
def post_metadata(self, object_type: str, data: dict) -> Optional[dict]:
|
291
|
+
def post_metadata(self, object_type: str, data: dict, check_only: bool = False) -> Optional[dict]:
|
291
292
|
if self.key:
|
292
|
-
return post_metadata(schema_name=object_type, post_item=data, key=self.key
|
293
|
-
|
293
|
+
return post_metadata(schema_name=object_type, post_item=data, key=self.key,
|
294
|
+
add_on="check_only=True" if check_only else "")
|
295
|
+
return self.post(f"/{object_type}{'?check_only=True' if check_only else ''}", data).json()
|
294
296
|
|
295
297
|
def get_health(self) -> OptionalResponse:
|
296
298
|
return self.get("/health")
|
dcicutils/progress_bar.py
CHANGED
@@ -9,7 +9,7 @@ from types import FrameType as frame
|
|
9
9
|
from typing import Callable, List, Optional, Union
|
10
10
|
from contextlib import contextmanager
|
11
11
|
from dcicutils.command_utils import yes_or_no
|
12
|
-
from dcicutils.misc_utils import find_nth_from_end, set_nth
|
12
|
+
from dcicutils.misc_utils import find_nth_from_end, format_size, set_nth
|
13
13
|
|
14
14
|
|
15
15
|
class TQDM(tqdm):
|
@@ -49,6 +49,7 @@ class ProgressBar:
|
|
49
49
|
|
50
50
|
def __init__(self, total: Optional[int] = None,
|
51
51
|
description: Optional[str] = None,
|
52
|
+
use_byte_size_for_rate: bool = False,
|
52
53
|
catch_interrupt: bool = True,
|
53
54
|
interrupt: Optional[Callable] = None,
|
54
55
|
interrupt_continue: Optional[Callable] = None,
|
@@ -59,11 +60,12 @@ class ProgressBar:
|
|
59
60
|
tidy_output_hack: bool = True,
|
60
61
|
capture_output_for_testing: bool = False) -> None:
|
61
62
|
self._bar = None
|
63
|
+
self._started = 0
|
62
64
|
self._disabled = False
|
63
65
|
self._done = False
|
64
66
|
self._tidy_output_hack = (tidy_output_hack is True)
|
65
|
-
self._started = time.time()
|
66
67
|
self._stop_requested = False
|
68
|
+
self._use_byte_size_for_rate = (use_byte_size_for_rate is True and self._tidy_output_hack)
|
67
69
|
# Interrupt handling. We do not do the actual (signal) interrupt setup
|
68
70
|
# in self._initialize as that could be called from a (sub) thread; and in
|
69
71
|
# Python we can only set a signal (SIGINT in our case) on the main thread.
|
@@ -96,9 +98,13 @@ class ProgressBar:
|
|
96
98
|
def _initialize(self) -> bool:
|
97
99
|
# Do not actually create the tqdm object unless/until we have a positive total.
|
98
100
|
if (self._bar is None) and (self._total > 0):
|
99
|
-
|
101
|
+
if self._use_byte_size_for_rate:
|
102
|
+
bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | [rate] | {elapsed}{postfix} | ETA: {remaining} "
|
103
|
+
else:
|
104
|
+
bar_format = "{l_bar}{bar}| {n_fmt}/{total_fmt} | {rate_fmt} | {elapsed}{postfix} | ETA: {remaining} "
|
100
105
|
self._bar = TQDM(total=self._total, desc=self._description,
|
101
106
|
dynamic_ncols=True, bar_format=bar_format, unit="", file=sys.stdout)
|
107
|
+
self._started = time.time()
|
102
108
|
if self._disabled:
|
103
109
|
self._bar.disable = True
|
104
110
|
return True
|
@@ -153,6 +159,7 @@ class ProgressBar:
|
|
153
159
|
self.set_total(total, _norefresh=True)
|
154
160
|
self.set_progress(progress, _norefresh=True)
|
155
161
|
self.set_description(description)
|
162
|
+
self._started = time.time()
|
156
163
|
|
157
164
|
def done(self, description: Optional[str] = None) -> None:
|
158
165
|
if self._done or self._bar is None:
|
@@ -198,14 +205,6 @@ class ProgressBar:
|
|
198
205
|
def stop_requested(self) -> bool:
|
199
206
|
return self._stop_requested
|
200
207
|
|
201
|
-
@property
|
202
|
-
def started(self) -> None:
|
203
|
-
return self._started
|
204
|
-
|
205
|
-
@property
|
206
|
-
def duration(self) -> None:
|
207
|
-
return time.time() - self._started
|
208
|
-
|
209
208
|
@property
|
210
209
|
def captured_output_for_testing(self) -> Optional[List[str]]:
|
211
210
|
return self._captured_output_for_testing
|
@@ -302,6 +301,9 @@ class ProgressBar:
|
|
302
301
|
# something like "1.54s/" rather than "1.54/s"; something to do with
|
303
302
|
# the unit we gave, which is empty; idunno; just replace it here.
|
304
303
|
text = replace_first(text, "s/ ", "/s ")
|
304
|
+
if self._use_byte_size_for_rate and self._bar:
|
305
|
+
rate = self._bar.n / (time.time() - self._started)
|
306
|
+
text = text.replace("[rate]", f"{format_size(rate)}/s")
|
305
307
|
sys_stdout_write(text)
|
306
308
|
sys.stdout.flush()
|
307
309
|
if self._captured_output_for_testing is not None:
|
@@ -43,12 +43,12 @@ dcicutils/license_policies/park-lab-gpl-pipeline.jsonc,sha256=vLZkwm3Js-kjV44nug
|
|
43
43
|
dcicutils/license_policies/park-lab-pipeline.jsonc,sha256=9qlY0ASy3iUMQlr3gorVcXrSfRHnVGbLhkS427UaRy4,283
|
44
44
|
dcicutils/license_utils.py,sha256=d1cq6iwv5Ju-VjdoINi6q7CPNNL7Oz6rcJdLMY38RX0,46978
|
45
45
|
dcicutils/log_utils.py,sha256=7pWMc6vyrorUZQf-V-M3YC6zrPgNhuV_fzm9xqTPph0,10883
|
46
|
-
dcicutils/misc_utils.py,sha256=
|
46
|
+
dcicutils/misc_utils.py,sha256=YH_TTmv6ABWeMERwVvA2-rIfdS-CoPYLXJru9TvWxgM,104610
|
47
47
|
dcicutils/obfuscation_utils.py,sha256=fo2jOmDRC6xWpYX49u80bVNisqRRoPskFNX3ymFAmjw,5963
|
48
48
|
dcicutils/opensearch_utils.py,sha256=V2exmFYW8Xl2_pGFixF4I2Cc549Opwe4PhFi5twC0M8,1017
|
49
49
|
dcicutils/portal_object_utils.py,sha256=gDXRgPsRvqCFwbC8WatsuflAxNiigOnqr0Hi93k3AgE,15422
|
50
|
-
dcicutils/portal_utils.py,sha256=
|
51
|
-
dcicutils/progress_bar.py,sha256=
|
50
|
+
dcicutils/portal_utils.py,sha256=DYyE5o15GekDgzpJWas9iS7klAYbjJZUPW0G42McArk,30779
|
51
|
+
dcicutils/progress_bar.py,sha256=l9h8o-RVvjGTt0J5qnxhMq7BaatJn1SnfOpWxHC2hL0,17278
|
52
52
|
dcicutils/project_utils.py,sha256=qPdCaFmWUVBJw4rw342iUytwdQC0P-XKpK4mhyIulMM,31250
|
53
53
|
dcicutils/qa_checkers.py,sha256=cdXjeL0jCDFDLT8VR8Px78aS10hwNISOO5G_Zv2TZ6M,20534
|
54
54
|
dcicutils/qa_utils.py,sha256=TT0SiJWiuxYvbsIyhK9VO4uV_suxhB6CpuC4qPacCzQ,160208
|
@@ -72,8 +72,8 @@ dcicutils/trace_utils.py,sha256=g8kwV4ebEy5kXW6oOrEAUsurBcCROvwtZqz9fczsGRE,1769
|
|
72
72
|
dcicutils/validation_utils.py,sha256=cMZIU2cY98FYtzK52z5WUYck7urH6JcqOuz9jkXpqzg,14797
|
73
73
|
dcicutils/variant_utils.py,sha256=2H9azNx3xAj-MySg-uZ2SFqbWs4kZvf61JnK6b-h4Qw,4343
|
74
74
|
dcicutils/zip_utils.py,sha256=rnjNv_k6L9jT2SjDSgVXp4BEJYLtz9XN6Cl2Fy-tqnM,2027
|
75
|
-
dcicutils-8.8.3.
|
76
|
-
dcicutils-8.8.3.
|
77
|
-
dcicutils-8.8.3.
|
78
|
-
dcicutils-8.8.3.
|
79
|
-
dcicutils-8.8.3.
|
75
|
+
dcicutils-8.8.3.1b12.dist-info/LICENSE.txt,sha256=qnwSmfnEWMl5l78VPDEzAmEbLVrRqQvfUQiHT0ehrOo,1102
|
76
|
+
dcicutils-8.8.3.1b12.dist-info/METADATA,sha256=U0KtU7pIabtImsxBvw3OHYkEp76VaOkBy44A0NApfEw,3357
|
77
|
+
dcicutils-8.8.3.1b12.dist-info/WHEEL,sha256=7Z8_27uaHI_UZAc4Uox4PpBhQ9Y5_modZXWMxtUi4NU,88
|
78
|
+
dcicutils-8.8.3.1b12.dist-info/entry_points.txt,sha256=51Q4F_2V10L0282W7HFjP4jdzW4K8lnWDARJQVFy_hw,270
|
79
|
+
dcicutils-8.8.3.1b12.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|