pythonwrench 0.6.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pythonwrench/__init__.py +490 -0
- pythonwrench/__main__.py +7 -0
- pythonwrench/_core.py +192 -0
- pythonwrench/abc.py +30 -0
- pythonwrench/argparse/__init__.py +81 -0
- pythonwrench/argparse/dataclass_.py +284 -0
- pythonwrench/argparse/parsers.py +619 -0
- pythonwrench/cast.py +247 -0
- pythonwrench/checksum.py +427 -0
- pythonwrench/collections/__init__.py +104 -0
- pythonwrench/collections/collections.py +900 -0
- pythonwrench/collections/prop.py +104 -0
- pythonwrench/collections/reducers.py +330 -0
- pythonwrench/concurrent.py +73 -0
- pythonwrench/csv.py +12 -0
- pythonwrench/dataclasses.py +117 -0
- pythonwrench/datetime.py +17 -0
- pythonwrench/difflib.py +39 -0
- pythonwrench/disk_cache.py +615 -0
- pythonwrench/entrypoints/info.py +44 -0
- pythonwrench/entrypoints/safe_rmdir.py +98 -0
- pythonwrench/entrypoints/tree.py +113 -0
- pythonwrench/enum.py +55 -0
- pythonwrench/functools.py +234 -0
- pythonwrench/hashlib.py +95 -0
- pythonwrench/importlib.py +243 -0
- pythonwrench/inspect.py +69 -0
- pythonwrench/json.py +12 -0
- pythonwrench/jsonl.py +12 -0
- pythonwrench/logging.py +252 -0
- pythonwrench/math.py +107 -0
- pythonwrench/os.py +226 -0
- pythonwrench/pickle.py +12 -0
- pythonwrench/random.py +60 -0
- pythonwrench/re.py +139 -0
- pythonwrench/semver.py +406 -0
- pythonwrench/serialization/__init__.py +70 -0
- pythonwrench/serialization/_core.py +70 -0
- pythonwrench/serialization/csv.py +493 -0
- pythonwrench/serialization/json.py +178 -0
- pythonwrench/serialization/jsonl.py +215 -0
- pythonwrench/serialization/pickle.py +186 -0
- pythonwrench/time.py +34 -0
- pythonwrench/typing/__init__.py +125 -0
- pythonwrench/typing/checks.py +551 -0
- pythonwrench/typing/classes.py +251 -0
- pythonwrench/warnings.py +118 -0
- pythonwrench-0.6.4.dist-info/METADATA +242 -0
- pythonwrench-0.6.4.dist-info/RECORD +52 -0
- pythonwrench-0.6.4.dist-info/WHEEL +4 -0
- pythonwrench-0.6.4.dist-info/entry_points.txt +10 -0
- pythonwrench-0.6.4.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,900 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
|
|
4
|
+
import copy
|
|
5
|
+
import operator
|
|
6
|
+
import random
|
|
7
|
+
import sys
|
|
8
|
+
from typing import (
|
|
9
|
+
Any,
|
|
10
|
+
Callable,
|
|
11
|
+
Dict,
|
|
12
|
+
Generator,
|
|
13
|
+
Generic,
|
|
14
|
+
Hashable,
|
|
15
|
+
Iterable,
|
|
16
|
+
Iterator,
|
|
17
|
+
List,
|
|
18
|
+
Literal,
|
|
19
|
+
Mapping,
|
|
20
|
+
MutableSequence,
|
|
21
|
+
Optional,
|
|
22
|
+
Sequence,
|
|
23
|
+
Tuple,
|
|
24
|
+
TypeVar,
|
|
25
|
+
Union,
|
|
26
|
+
get_args,
|
|
27
|
+
overload,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
from typing_extensions import TypeGuard, TypeIs
|
|
31
|
+
|
|
32
|
+
from pythonwrench.collections.prop import all_eq
|
|
33
|
+
from pythonwrench.collections.reducers import reduce_or
|
|
34
|
+
from pythonwrench.functools import identity
|
|
35
|
+
from pythonwrench.semver import Version
|
|
36
|
+
from pythonwrench.typing.checks import is_builtin_scalar, isinstance_generic
|
|
37
|
+
from pythonwrench.typing.classes import T_BuiltinScalar
|
|
38
|
+
|
|
39
|
+
K = TypeVar("K", covariant=True, bound=Hashable)
|
|
40
|
+
|
|
41
|
+
T = TypeVar("T", covariant=True)
|
|
42
|
+
U = TypeVar("U", covariant=True)
|
|
43
|
+
V = TypeVar("V", covariant=True)
|
|
44
|
+
W = TypeVar("W", covariant=True)
|
|
45
|
+
X = TypeVar("X", covariant=True)
|
|
46
|
+
Y = TypeVar("Y", covariant=True)
|
|
47
|
+
|
|
48
|
+
KeyMode = Literal["intersect", "same", "union"]
|
|
49
|
+
Order = Literal["left", "right"]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class SizedGenerator(Generic[T]):
|
|
53
|
+
"""Wraps a generator and size to provide a sized iterable object."""
|
|
54
|
+
|
|
55
|
+
def __init__(self, generator: Generator[T, None, None], size: int) -> None:
|
|
56
|
+
"""Initialize the instance."""
|
|
57
|
+
super().__init__()
|
|
58
|
+
self._generator = generator
|
|
59
|
+
self._size = size
|
|
60
|
+
|
|
61
|
+
def __iter__(self) -> Iterator[T]:
|
|
62
|
+
"""Return an iterator over the instance."""
|
|
63
|
+
yield from self._generator
|
|
64
|
+
|
|
65
|
+
def __len__(self) -> int:
|
|
66
|
+
"""Return the number of items in the instance."""
|
|
67
|
+
return self._size
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def contained(
|
|
71
|
+
x: T,
|
|
72
|
+
include: Optional[Iterable[T]] = None,
|
|
73
|
+
exclude: Optional[Iterable[T]] = None,
|
|
74
|
+
*,
|
|
75
|
+
match_fn: Callable[[T, T], bool] = operator.eq,
|
|
76
|
+
order: Literal["left", "right"] = "right",
|
|
77
|
+
) -> bool:
|
|
78
|
+
"""Returns True if name in include set and not in exclude set."""
|
|
79
|
+
if (
|
|
80
|
+
include is not None
|
|
81
|
+
and find(x, include, match_fn=match_fn, order=order, default=-1) == -1
|
|
82
|
+
):
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
if (
|
|
86
|
+
exclude is not None
|
|
87
|
+
and find(x, exclude, match_fn=match_fn, order=order, default=-1) != -1
|
|
88
|
+
):
|
|
89
|
+
return False
|
|
90
|
+
|
|
91
|
+
return True
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
@overload
|
|
95
|
+
def dict_list_to_list_dict(
|
|
96
|
+
dic: Mapping[T, Iterable[U]],
|
|
97
|
+
key_mode: Literal["same", "intersect"] = "same",
|
|
98
|
+
default_val: Any = None,
|
|
99
|
+
) -> List[Dict[T, U]]:
|
|
100
|
+
"""Perform the dict list to list dict operation."""
|
|
101
|
+
...
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@overload
|
|
105
|
+
def dict_list_to_list_dict(
|
|
106
|
+
dic: Mapping[T, Iterable[U]],
|
|
107
|
+
key_mode: Literal["union"],
|
|
108
|
+
default_val: W = None,
|
|
109
|
+
) -> List[Dict[T, Union[U, W]]]:
|
|
110
|
+
"""Perform the dict list to list dict operation."""
|
|
111
|
+
...
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def dict_list_to_list_dict(
|
|
115
|
+
dic: Mapping[T, Iterable[U]],
|
|
116
|
+
key_mode: KeyMode = "same",
|
|
117
|
+
default_val: W = None,
|
|
118
|
+
) -> List[Dict[T, Union[U, W]]]:
|
|
119
|
+
"""Convert dict of lists with same sizes to list of dicts.
|
|
120
|
+
|
|
121
|
+
Example 1
|
|
122
|
+
---------
|
|
123
|
+
>>> dic = {"a": [1, 2], "b": [3, 4]}
|
|
124
|
+
>>> dict_list_to_list_dict(dic)
|
|
125
|
+
... [{"a": 1, "b": 3}, {"a": 2, "b": 4}]
|
|
126
|
+
|
|
127
|
+
Example 2
|
|
128
|
+
---------
|
|
129
|
+
>>> dic = {"a": [1, 2, 3], "b": [4], "c": [5, 6]}
|
|
130
|
+
>>> dict_list_to_list_dict(dic, key_mode="union", default=-1)
|
|
131
|
+
... [{"a": 1, "b": 4, "c": 5}, {"a": 2, "b": -1, "c": 6}, {"a": 3, "b": -1, "c": -1}]
|
|
132
|
+
"""
|
|
133
|
+
if len(dic) == 0:
|
|
134
|
+
return []
|
|
135
|
+
|
|
136
|
+
dic = {k: list(v) if not isinstance(v, Sequence) else v for k, v in dic.items()}
|
|
137
|
+
lengths = [len(seq) for seq in dic.values()]
|
|
138
|
+
|
|
139
|
+
if key_mode == "same":
|
|
140
|
+
if not all_eq(lengths):
|
|
141
|
+
msg = f"Invalid sequences for batch. (found different lengths in sub-lists: {set(lengths)})"
|
|
142
|
+
raise ValueError(msg)
|
|
143
|
+
length = lengths[0]
|
|
144
|
+
|
|
145
|
+
elif key_mode == "intersect":
|
|
146
|
+
length = min(lengths)
|
|
147
|
+
|
|
148
|
+
elif key_mode == "union":
|
|
149
|
+
length = max(lengths)
|
|
150
|
+
|
|
151
|
+
else:
|
|
152
|
+
msg = f"Invalid argument key_mode={key_mode}. (expected one of {get_args(KeyMode)})"
|
|
153
|
+
raise ValueError(msg)
|
|
154
|
+
|
|
155
|
+
result = [
|
|
156
|
+
{k: (v[i] if i < len(v) else default_val) for k, v in dic.items()}
|
|
157
|
+
for i in range(length)
|
|
158
|
+
]
|
|
159
|
+
return result
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def dump_dict(
|
|
163
|
+
dic: Optional[Mapping[str, T]] = None,
|
|
164
|
+
/,
|
|
165
|
+
join: str = ", ",
|
|
166
|
+
fmt: str = "{key}={value}",
|
|
167
|
+
ignore_lst: Iterable[T] = (),
|
|
168
|
+
**kwargs,
|
|
169
|
+
) -> str:
|
|
170
|
+
"""Dump dictionary of scalars to string function to customize representation.
|
|
171
|
+
|
|
172
|
+
Example 1:
|
|
173
|
+
----------
|
|
174
|
+
>>> d = {"a": 1, "b": 2}
|
|
175
|
+
>>> dump_dict(d)
|
|
176
|
+
... 'a=1, b=2'
|
|
177
|
+
"""
|
|
178
|
+
if dic is None:
|
|
179
|
+
dic = {}
|
|
180
|
+
else:
|
|
181
|
+
dic = dict(dic.items())
|
|
182
|
+
dic.update(kwargs)
|
|
183
|
+
|
|
184
|
+
ignore_lst = dict.fromkeys(ignore_lst)
|
|
185
|
+
result = join.join(
|
|
186
|
+
fmt.format(key=key, value=value)
|
|
187
|
+
for key, value in dic.items()
|
|
188
|
+
if value not in ignore_lst
|
|
189
|
+
)
|
|
190
|
+
return result
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def filter_iterable(
|
|
194
|
+
it: Iterable[T],
|
|
195
|
+
include: Optional[Iterable[T]] = None,
|
|
196
|
+
exclude: Optional[Iterable[T]] = None,
|
|
197
|
+
*,
|
|
198
|
+
match_fn: Callable[[T, T], bool] = operator.eq,
|
|
199
|
+
order: Literal["left", "right"] = "right",
|
|
200
|
+
) -> List[T]:
|
|
201
|
+
"""Perform the filter iterable operation."""
|
|
202
|
+
return [
|
|
203
|
+
item
|
|
204
|
+
for item in it
|
|
205
|
+
if contained(
|
|
206
|
+
item,
|
|
207
|
+
include=include,
|
|
208
|
+
exclude=exclude,
|
|
209
|
+
match_fn=match_fn,
|
|
210
|
+
order=order,
|
|
211
|
+
)
|
|
212
|
+
]
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@overload
|
|
216
|
+
def find(
|
|
217
|
+
target: T,
|
|
218
|
+
it: Iterable[V],
|
|
219
|
+
*,
|
|
220
|
+
match_fn: Callable[[V, T], bool] = operator.eq,
|
|
221
|
+
order: Literal["right"] = "right",
|
|
222
|
+
default: U = -1,
|
|
223
|
+
return_value: Literal[False] = False,
|
|
224
|
+
) -> Union[int, U]:
|
|
225
|
+
"""Perform the find operation."""
|
|
226
|
+
...
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
@overload
|
|
230
|
+
def find(
|
|
231
|
+
target: T,
|
|
232
|
+
it: Iterable[V],
|
|
233
|
+
*,
|
|
234
|
+
match_fn: Callable[[T, V], bool] = operator.eq,
|
|
235
|
+
order: Literal["left"],
|
|
236
|
+
default: U = -1,
|
|
237
|
+
return_value: Literal[False] = False,
|
|
238
|
+
) -> Union[int, U]:
|
|
239
|
+
"""Perform the find operation."""
|
|
240
|
+
...
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@overload
|
|
244
|
+
def find(
|
|
245
|
+
target: T,
|
|
246
|
+
it: Iterable[V],
|
|
247
|
+
*,
|
|
248
|
+
match_fn: Callable[[V, T], bool] = operator.eq,
|
|
249
|
+
order: Literal["right"] = "right",
|
|
250
|
+
default: U = -1,
|
|
251
|
+
return_value: Literal[True],
|
|
252
|
+
) -> Tuple[Union[int, U], Union[V, U]]:
|
|
253
|
+
"""Perform the find operation."""
|
|
254
|
+
...
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@overload
|
|
258
|
+
def find(
|
|
259
|
+
target: T,
|
|
260
|
+
it: Iterable[V],
|
|
261
|
+
*,
|
|
262
|
+
match_fn: Callable[[T, V], bool] = operator.eq,
|
|
263
|
+
order: Literal["left"],
|
|
264
|
+
default: U = -1,
|
|
265
|
+
return_value: Literal[True],
|
|
266
|
+
) -> Tuple[Union[int, U], Union[V, U]]:
|
|
267
|
+
"""Perform the find operation."""
|
|
268
|
+
...
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def find(
|
|
272
|
+
target: Any,
|
|
273
|
+
it: Iterable[V],
|
|
274
|
+
*,
|
|
275
|
+
match_fn: Callable[[Any, Any], bool] = operator.eq,
|
|
276
|
+
order: Order = "right",
|
|
277
|
+
default: U = -1,
|
|
278
|
+
return_value: bool = False,
|
|
279
|
+
) -> Union[int, U, Tuple[Union[int, U], Union[V, U]]]:
|
|
280
|
+
"""Perform the find operation."""
|
|
281
|
+
if not return_value:
|
|
282
|
+
result = find(
|
|
283
|
+
target,
|
|
284
|
+
it,
|
|
285
|
+
match_fn=match_fn,
|
|
286
|
+
order=order,
|
|
287
|
+
default=default,
|
|
288
|
+
return_value=True,
|
|
289
|
+
)
|
|
290
|
+
return result[0]
|
|
291
|
+
|
|
292
|
+
if order == "right":
|
|
293
|
+
pass
|
|
294
|
+
elif order == "left":
|
|
295
|
+
|
|
296
|
+
def revert(f):
|
|
297
|
+
"""Perform the revert operation."""
|
|
298
|
+
|
|
299
|
+
def reverted_f(a, b):
|
|
300
|
+
"""Perform the reverted f operation."""
|
|
301
|
+
return f(b, a)
|
|
302
|
+
|
|
303
|
+
return reverted_f
|
|
304
|
+
|
|
305
|
+
match_fn = revert(match_fn)
|
|
306
|
+
else:
|
|
307
|
+
raise ValueError(
|
|
308
|
+
f"Invalid argument {order=}. (expected one of {get_args(Order)})"
|
|
309
|
+
)
|
|
310
|
+
|
|
311
|
+
for i, xi in enumerate(it):
|
|
312
|
+
if match_fn(xi, target):
|
|
313
|
+
return i, xi
|
|
314
|
+
|
|
315
|
+
return default, default
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
@overload
|
|
319
|
+
def flatten(
|
|
320
|
+
x: T_BuiltinScalar,
|
|
321
|
+
start_dim: int = 0,
|
|
322
|
+
end_dim: Optional[int] = None,
|
|
323
|
+
) -> List[T_BuiltinScalar]:
|
|
324
|
+
"""Perform the flatten operation."""
|
|
325
|
+
...
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
@overload
|
|
329
|
+
def flatten( # type: ignore
|
|
330
|
+
x: Iterable[T_BuiltinScalar],
|
|
331
|
+
start_dim: int = 0,
|
|
332
|
+
end_dim: Optional[int] = None,
|
|
333
|
+
) -> List[T_BuiltinScalar]:
|
|
334
|
+
"""Perform the flatten operation."""
|
|
335
|
+
...
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
@overload
|
|
339
|
+
def flatten(
|
|
340
|
+
x: Any,
|
|
341
|
+
start_dim: int = 0,
|
|
342
|
+
end_dim: Optional[int] = None,
|
|
343
|
+
is_scalar_fn: Union[
|
|
344
|
+
Callable[[Any], TypeGuard[T]], Callable[[Any], TypeIs[T]]
|
|
345
|
+
] = is_builtin_scalar,
|
|
346
|
+
) -> List[Any]:
|
|
347
|
+
"""Perform the flatten operation."""
|
|
348
|
+
...
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def flatten(
|
|
352
|
+
x: Any,
|
|
353
|
+
start_dim: int = 0,
|
|
354
|
+
end_dim: Optional[int] = None,
|
|
355
|
+
is_scalar_fn: Union[
|
|
356
|
+
Callable[[Any], TypeGuard[T]], Callable[[Any], TypeIs[T]]
|
|
357
|
+
] = is_builtin_scalar,
|
|
358
|
+
) -> List[Any]:
|
|
359
|
+
"""Perform the flatten operation."""
|
|
360
|
+
if end_dim is None:
|
|
361
|
+
end_dim = sys.maxsize
|
|
362
|
+
if start_dim < 0:
|
|
363
|
+
raise ValueError(f"Invalid argument {start_dim=}. (expected positive integer)")
|
|
364
|
+
if end_dim < 0:
|
|
365
|
+
raise ValueError(f"Invalid argument {end_dim=}. (expected positive integer)")
|
|
366
|
+
if start_dim > end_dim:
|
|
367
|
+
msg = f"Invalid arguments {start_dim=} and {end_dim=}. (expected start_dim <= end_dim)"
|
|
368
|
+
raise ValueError(msg)
|
|
369
|
+
|
|
370
|
+
def flatten_impl(x: Any, start_dim: int, end_dim: int) -> List[Any]:
|
|
371
|
+
"""Perform the flatten impl operation."""
|
|
372
|
+
if is_scalar_fn(x):
|
|
373
|
+
return [x]
|
|
374
|
+
elif isinstance(x, Iterable):
|
|
375
|
+
if start_dim > 0:
|
|
376
|
+
return [flatten_impl(xi, start_dim - 1, end_dim - 1) for xi in x]
|
|
377
|
+
elif end_dim > 0:
|
|
378
|
+
return [
|
|
379
|
+
xij
|
|
380
|
+
for xi in x
|
|
381
|
+
for xij in flatten_impl(xi, start_dim - 1, end_dim - 1)
|
|
382
|
+
]
|
|
383
|
+
else:
|
|
384
|
+
return list(x)
|
|
385
|
+
else:
|
|
386
|
+
raise TypeError(f"Invalid argument type {type(x)=}.")
|
|
387
|
+
|
|
388
|
+
return flatten_impl(x, start_dim, end_dim)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def flat_dict_of_dict(
|
|
392
|
+
nested_dic: Mapping[str, Any],
|
|
393
|
+
*,
|
|
394
|
+
sep: str = ".",
|
|
395
|
+
flat_iterables: bool = False,
|
|
396
|
+
overwrite: bool = True,
|
|
397
|
+
) -> Dict[str, Any]:
|
|
398
|
+
"""Flat a nested dictionary.
|
|
399
|
+
|
|
400
|
+
Example 1
|
|
401
|
+
---------
|
|
402
|
+
>>> dic = {
|
|
403
|
+
... "a": 1,
|
|
404
|
+
... "b": {
|
|
405
|
+
... "a": 2,
|
|
406
|
+
... "b": 10,
|
|
407
|
+
... },
|
|
408
|
+
... }
|
|
409
|
+
>>> flat_dict_of_dict(dic)
|
|
410
|
+
... {"a": 1, "b.a": 2, "b.b": 10}
|
|
411
|
+
|
|
412
|
+
Example 2
|
|
413
|
+
---------
|
|
414
|
+
>>> dic = {"a": ["hello", "world"], "b": 3}
|
|
415
|
+
>>> flat_dict_of_dict(dic, flat_iterables=True)
|
|
416
|
+
... {"a.0": "hello", "a.1": "world", "b": 3}
|
|
417
|
+
|
|
418
|
+
Args:
|
|
419
|
+
nested_dic: Nested mapping containing sub-mappings or iterables.
|
|
420
|
+
sep: Separators between keys.
|
|
421
|
+
flat_iterables: If True, flat iterable and use index as key.
|
|
422
|
+
overwrite: If True, overwrite duplicated keys in output. Otherwise duplicated keys will raises a ValueError.
|
|
423
|
+
"""
|
|
424
|
+
|
|
425
|
+
def _impl(nested_dic: Mapping[str, Any]) -> Dict[str, Any]:
|
|
426
|
+
"""Perform the impl operation."""
|
|
427
|
+
output = {}
|
|
428
|
+
for k, v in nested_dic.items():
|
|
429
|
+
if isinstance_generic(v, Mapping[str, Any]):
|
|
430
|
+
v = _impl(v)
|
|
431
|
+
v = {f"{k}{sep}{kv}": vv for kv, vv in v.items()}
|
|
432
|
+
output.update(v)
|
|
433
|
+
|
|
434
|
+
elif flat_iterables and isinstance(v, Iterable) and not isinstance(v, str):
|
|
435
|
+
v = {f"{i}": vi for i, vi in enumerate(v)}
|
|
436
|
+
v = _impl(v)
|
|
437
|
+
v = {f"{k}{sep}{kv}": vv for kv, vv in v.items()}
|
|
438
|
+
output.update(v)
|
|
439
|
+
|
|
440
|
+
elif overwrite or k not in output:
|
|
441
|
+
output[k] = v
|
|
442
|
+
|
|
443
|
+
else:
|
|
444
|
+
msg = f"Ambiguous flatten dict with key '{k}'. (with value '{v}')"
|
|
445
|
+
raise ValueError(msg)
|
|
446
|
+
return output
|
|
447
|
+
|
|
448
|
+
return _impl(nested_dic)
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
@overload
|
|
452
|
+
def flat_list_of_list(
|
|
453
|
+
lst: Iterable[Sequence[T]],
|
|
454
|
+
return_sizes: Literal[True] = True,
|
|
455
|
+
) -> Tuple[List[T], List[int]]:
|
|
456
|
+
"""Perform the flat list of list operation."""
|
|
457
|
+
...
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
@overload
|
|
461
|
+
def flat_list_of_list(
|
|
462
|
+
lst: Iterable[Sequence[T]],
|
|
463
|
+
return_sizes: Literal[False],
|
|
464
|
+
) -> List[T]:
|
|
465
|
+
"""Perform the flat list of list operation."""
|
|
466
|
+
...
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def flat_list_of_list(
|
|
470
|
+
lst: Iterable[Sequence[T]],
|
|
471
|
+
return_sizes: bool = True,
|
|
472
|
+
) -> Union[Tuple[List[T], List[int]], List[T]]:
|
|
473
|
+
"""Return a flat version of the input list of sublists with each sublist size."""
|
|
474
|
+
flatten_lst = [elt for sublst in lst for elt in sublst]
|
|
475
|
+
sizes = [len(sents) for sents in lst]
|
|
476
|
+
|
|
477
|
+
if return_sizes:
|
|
478
|
+
return flatten_lst, sizes
|
|
479
|
+
else:
|
|
480
|
+
return flatten_lst
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def intersect_lists(lst_of_lst: Sequence[Iterable[T]]) -> List[T]:
|
|
484
|
+
"""Performs intersection of elements in lists (like set intersection), but keep their original order."""
|
|
485
|
+
if len(lst_of_lst) <= 0:
|
|
486
|
+
return []
|
|
487
|
+
out = list(dict.fromkeys(lst_of_lst[0]))
|
|
488
|
+
for lst_i in lst_of_lst[1:]:
|
|
489
|
+
out = [name for name in out if name in lst_i]
|
|
490
|
+
if len(out) == 0:
|
|
491
|
+
break
|
|
492
|
+
return out
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
@overload
|
|
496
|
+
def list_dict_to_dict_list(
|
|
497
|
+
lst: Iterable[Mapping[K, V]],
|
|
498
|
+
key_mode: Literal["intersect", "same"] = "same",
|
|
499
|
+
default_val: Any = None,
|
|
500
|
+
*,
|
|
501
|
+
default_val_fn: Any = None,
|
|
502
|
+
list_fn: None = None,
|
|
503
|
+
) -> Dict[K, List[V]]:
|
|
504
|
+
"""Perform the list dict to dict list operation."""
|
|
505
|
+
...
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
@overload
|
|
509
|
+
def list_dict_to_dict_list(
|
|
510
|
+
lst: Iterable[Mapping[K, V]],
|
|
511
|
+
key_mode: Literal["union"],
|
|
512
|
+
default_val: Any = None,
|
|
513
|
+
*,
|
|
514
|
+
default_val_fn: Callable[[K], X],
|
|
515
|
+
list_fn: None = None,
|
|
516
|
+
) -> Dict[K, List[Union[V, X]]]:
|
|
517
|
+
"""Perform the list dict to dict list operation."""
|
|
518
|
+
...
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
@overload
|
|
522
|
+
def list_dict_to_dict_list(
|
|
523
|
+
lst: Iterable[Mapping[K, V]],
|
|
524
|
+
key_mode: Literal["union"],
|
|
525
|
+
default_val: W = None,
|
|
526
|
+
*,
|
|
527
|
+
default_val_fn: None = None,
|
|
528
|
+
list_fn: None = None,
|
|
529
|
+
) -> Dict[K, List[Union[V, W]]]:
|
|
530
|
+
"""Perform the list dict to dict list operation."""
|
|
531
|
+
...
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
@overload
|
|
535
|
+
def list_dict_to_dict_list(
|
|
536
|
+
lst: Iterable[Mapping[K, V]],
|
|
537
|
+
key_mode: Union[KeyMode, Iterable[K]] = "same",
|
|
538
|
+
default_val: W = None,
|
|
539
|
+
*,
|
|
540
|
+
default_val_fn: Optional[Callable[[K], X]] = None,
|
|
541
|
+
list_fn: Callable[[List[Union[V, W, X]]], Y],
|
|
542
|
+
) -> Dict[K, Y]:
|
|
543
|
+
"""Perform the list dict to dict list operation."""
|
|
544
|
+
...
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def list_dict_to_dict_list(
|
|
548
|
+
lst: Iterable[Mapping[K, V]],
|
|
549
|
+
key_mode: Union[KeyMode, Iterable[K]] = "same",
|
|
550
|
+
default_val: W = None,
|
|
551
|
+
*,
|
|
552
|
+
default_val_fn: Optional[Callable[[K], X]] = None,
|
|
553
|
+
list_fn: Optional[Callable[[List[Union[V, W, X]]], Y]] = identity,
|
|
554
|
+
) -> Dict[K, Y]:
|
|
555
|
+
"""Convert list of dicts to dict of lists.
|
|
556
|
+
|
|
557
|
+
Args:
|
|
558
|
+
lst: The list of dict to merge. Cannot be a Generator.
|
|
559
|
+
key_mode: Can be "same" or "intersect". \
|
|
560
|
+
- If "same", all the dictionaries must contains the same keys otherwise a ValueError will be raised. \
|
|
561
|
+
- If "intersect", only the intersection of all keys will be used in output. \
|
|
562
|
+
- If "union", the output dict will contains the union of all keys, and the missing value will use the argument default_val. \
|
|
563
|
+
- If an iterable of elements, use them as keys for output dict.
|
|
564
|
+
default_val: Default value of an element when key_mode is "union". defaults to None.
|
|
565
|
+
default_val_fn: Function to return the default value according to a specific key. defaults to None.
|
|
566
|
+
list_fn: Optional function to build the values. defaults to identity.
|
|
567
|
+
"""
|
|
568
|
+
if isinstance(lst, Generator):
|
|
569
|
+
msg = f"Invalid argument type {type(lst)}. (expected any Iterable except Generator)"
|
|
570
|
+
raise TypeError(msg)
|
|
571
|
+
|
|
572
|
+
try:
|
|
573
|
+
item0 = next(iter(lst))
|
|
574
|
+
except StopIteration:
|
|
575
|
+
return {}
|
|
576
|
+
|
|
577
|
+
if isinstance(key_mode, str):
|
|
578
|
+
unique_keys = set(item0.keys())
|
|
579
|
+
|
|
580
|
+
if key_mode == "same":
|
|
581
|
+
invalids = [
|
|
582
|
+
list(item.keys()) for item in lst if unique_keys != set(item.keys())
|
|
583
|
+
]
|
|
584
|
+
if len(invalids) > 0:
|
|
585
|
+
msg = f"Invalid dict keys for conversion from List[dict] to Dict[list]. (with {key_mode=}, {unique_keys=} and {invalids=})"
|
|
586
|
+
raise ValueError(msg)
|
|
587
|
+
keys = list(item0.keys())
|
|
588
|
+
|
|
589
|
+
elif key_mode == "intersect":
|
|
590
|
+
keys = intersect_lists([item.keys() for item in lst])
|
|
591
|
+
|
|
592
|
+
elif key_mode == "union":
|
|
593
|
+
keys = union_lists(item.keys() for item in lst)
|
|
594
|
+
|
|
595
|
+
else:
|
|
596
|
+
msg = f"Invalid argument key_mode={key_mode}. (expected one of {get_args(KeyMode)})"
|
|
597
|
+
raise ValueError(msg)
|
|
598
|
+
else:
|
|
599
|
+
keys = list(key_mode)
|
|
600
|
+
|
|
601
|
+
if list_fn is None:
|
|
602
|
+
list_fn = identity # type: ignore
|
|
603
|
+
|
|
604
|
+
result = {
|
|
605
|
+
key: list_fn(
|
|
606
|
+
[
|
|
607
|
+
item.get(
|
|
608
|
+
key,
|
|
609
|
+
default_val_fn(key) if default_val_fn is not None else default_val,
|
|
610
|
+
)
|
|
611
|
+
for item in lst
|
|
612
|
+
]
|
|
613
|
+
) # type: ignore
|
|
614
|
+
for key in keys
|
|
615
|
+
}
|
|
616
|
+
return result # type: ignore
|
|
617
|
+
|
|
618
|
+
|
|
619
|
+
def recursive_generator(x: Any) -> Generator[Tuple[Any, int, int], None, None]:
|
|
620
|
+
"""Perform the recursive generator operation."""
|
|
621
|
+
|
|
622
|
+
def recursive_generator_impl(
|
|
623
|
+
x: Any,
|
|
624
|
+
i: int,
|
|
625
|
+
deep: int,
|
|
626
|
+
) -> Generator[Tuple[Any, int, int], None, None]:
|
|
627
|
+
"""Perform the recursive generator impl operation."""
|
|
628
|
+
if is_builtin_scalar(x):
|
|
629
|
+
yield x, i, deep
|
|
630
|
+
elif isinstance(x, Iterable):
|
|
631
|
+
for j, xj in enumerate(x):
|
|
632
|
+
if xj == x:
|
|
633
|
+
yield xj, i, deep
|
|
634
|
+
return
|
|
635
|
+
else:
|
|
636
|
+
yield from recursive_generator_impl(xj, j, deep + 1)
|
|
637
|
+
else:
|
|
638
|
+
yield x, i, deep
|
|
639
|
+
return
|
|
640
|
+
|
|
641
|
+
return recursive_generator_impl(x, 0, 0)
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
@overload
|
|
645
|
+
def sorted_dict(
|
|
646
|
+
x: Mapping[K, V],
|
|
647
|
+
/,
|
|
648
|
+
*,
|
|
649
|
+
by: Literal["key"] = "key",
|
|
650
|
+
key: Optional[Callable[[K], Any]] = None,
|
|
651
|
+
reverse: bool = False,
|
|
652
|
+
) -> Dict[K, V]:
|
|
653
|
+
"""Perform the sorted dict operation."""
|
|
654
|
+
...
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
@overload
|
|
658
|
+
def sorted_dict(
|
|
659
|
+
x: Mapping[K, V],
|
|
660
|
+
/,
|
|
661
|
+
*,
|
|
662
|
+
by: Literal["value"],
|
|
663
|
+
key: Optional[Callable[[V], Any]] = None,
|
|
664
|
+
reverse: bool = False,
|
|
665
|
+
) -> Dict[K, V]:
|
|
666
|
+
"""Perform the sorted dict operation."""
|
|
667
|
+
...
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
@overload
|
|
671
|
+
def sorted_dict(
|
|
672
|
+
x: Mapping[K, V],
|
|
673
|
+
/,
|
|
674
|
+
*,
|
|
675
|
+
by: Literal["item"],
|
|
676
|
+
key: Optional[Callable[[Tuple[K, V]], Any]] = None,
|
|
677
|
+
reverse: bool = False,
|
|
678
|
+
) -> Dict[K, V]:
|
|
679
|
+
"""Perform the sorted dict operation."""
|
|
680
|
+
...
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def sorted_dict(
|
|
684
|
+
x: Mapping[K, V],
|
|
685
|
+
/,
|
|
686
|
+
*,
|
|
687
|
+
by: Literal["key", "value", "item"] = "key",
|
|
688
|
+
key: Optional[Callable[[Any], Any]] = None,
|
|
689
|
+
reverse: bool = False,
|
|
690
|
+
) -> Dict[K, V]:
|
|
691
|
+
"""Sort a dictionnary by key, value or item."""
|
|
692
|
+
if key is None or by == "item":
|
|
693
|
+
impl_key = key
|
|
694
|
+
|
|
695
|
+
elif by == "key":
|
|
696
|
+
|
|
697
|
+
def by_key_fn(x: Tuple[K, V]) -> Any:
|
|
698
|
+
"""Perform the by key fn operation."""
|
|
699
|
+
return key(x[0])
|
|
700
|
+
|
|
701
|
+
impl_key = by_key_fn
|
|
702
|
+
|
|
703
|
+
elif by == "value":
|
|
704
|
+
|
|
705
|
+
def by_value_fn(x: Tuple[K, V]) -> Any:
|
|
706
|
+
"""Perform the by value fn operation."""
|
|
707
|
+
return key(x[1])
|
|
708
|
+
|
|
709
|
+
impl_key = by_value_fn
|
|
710
|
+
|
|
711
|
+
else:
|
|
712
|
+
msg = f"Invalid argument {by=}. (expected one of {('key', 'value', 'item')})"
|
|
713
|
+
raise ValueError(msg)
|
|
714
|
+
|
|
715
|
+
return {k: v for k, v in sorted(x.items(), key=impl_key, reverse=reverse)} # type: ignore
|
|
716
|
+
|
|
717
|
+
|
|
718
|
+
def shuffled(
|
|
719
|
+
x: MutableSequence[T],
|
|
720
|
+
*,
|
|
721
|
+
seed: Optional[int] = None,
|
|
722
|
+
deep: bool = False,
|
|
723
|
+
) -> MutableSequence[T]:
|
|
724
|
+
"""Perform the shuffled operation."""
|
|
725
|
+
if deep:
|
|
726
|
+
x = copy.deepcopy(x)
|
|
727
|
+
else:
|
|
728
|
+
x = copy.copy(x)
|
|
729
|
+
|
|
730
|
+
if seed is None:
|
|
731
|
+
random.shuffle(x)
|
|
732
|
+
return x
|
|
733
|
+
else:
|
|
734
|
+
state = random.getstate()
|
|
735
|
+
random.seed(seed)
|
|
736
|
+
random.shuffle(x)
|
|
737
|
+
state = random.setstate(state)
|
|
738
|
+
return x
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
def unflat_dict_of_dict(dic: Mapping[str, Any], *, sep: str = ".") -> Dict[str, Any]:
|
|
742
|
+
"""Unflat a dictionary.
|
|
743
|
+
|
|
744
|
+
Example 1
|
|
745
|
+
----------
|
|
746
|
+
>>> dic = {
|
|
747
|
+
"a.a": 1,
|
|
748
|
+
"b.a": 2,
|
|
749
|
+
"b.b": 3,
|
|
750
|
+
"c": 4,
|
|
751
|
+
}
|
|
752
|
+
>>> unflat_dict_of_dict(dic)
|
|
753
|
+
... {"a": {"a": 1}, "b": {"a": 2, "b": 3}, "c": 4}
|
|
754
|
+
"""
|
|
755
|
+
output = {}
|
|
756
|
+
for k, v in dic.items():
|
|
757
|
+
if sep not in k:
|
|
758
|
+
output[k] = v
|
|
759
|
+
else:
|
|
760
|
+
idx = k.index(sep)
|
|
761
|
+
k, kk = k[:idx], k[idx + 1 :]
|
|
762
|
+
if k not in output:
|
|
763
|
+
output[k] = {}
|
|
764
|
+
elif not isinstance(output[k], Mapping):
|
|
765
|
+
msg = f"Invalid dict argument. (found keys {k} and {k}{sep}{kk})"
|
|
766
|
+
raise ValueError(msg)
|
|
767
|
+
|
|
768
|
+
output[k][kk] = v
|
|
769
|
+
|
|
770
|
+
output = {
|
|
771
|
+
k: (unflat_dict_of_dict(v) if isinstance(v, Mapping) else v)
|
|
772
|
+
for k, v in output.items()
|
|
773
|
+
}
|
|
774
|
+
return output
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def unflat_list_of_list(
|
|
778
|
+
flatten_lst: Sequence[T],
|
|
779
|
+
sizes: Iterable[int],
|
|
780
|
+
) -> List[List[T]]:
|
|
781
|
+
"""Unflat a list to a list of sublists of given sizes."""
|
|
782
|
+
lst = []
|
|
783
|
+
start = 0
|
|
784
|
+
stop = 0
|
|
785
|
+
for count in sizes:
|
|
786
|
+
stop += count
|
|
787
|
+
lst.append(flatten_lst[start:stop])
|
|
788
|
+
start = stop
|
|
789
|
+
return lst
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def union_dicts(dicts: Iterable[Dict[K, V]]) -> Dict[K, V]:
|
|
793
|
+
"""Performs union of dictionaries."""
|
|
794
|
+
if Version.python() >= Version("3.9.0"):
|
|
795
|
+
return reduce_or(*dicts)
|
|
796
|
+
|
|
797
|
+
it = iter(dicts)
|
|
798
|
+
try:
|
|
799
|
+
dic0 = next(it)
|
|
800
|
+
except StopIteration:
|
|
801
|
+
return {}
|
|
802
|
+
for dic in it:
|
|
803
|
+
dic0.update(dic)
|
|
804
|
+
return dic0
|
|
805
|
+
|
|
806
|
+
|
|
807
|
+
def union_lists(lst_of_lst: Iterable[Iterable[K]]) -> List[K]:
|
|
808
|
+
"""Performs union of elements in lists (like set union), but keep their original order."""
|
|
809
|
+
out = {}
|
|
810
|
+
for lst_i in lst_of_lst:
|
|
811
|
+
out.update(dict.fromkeys(lst_i))
|
|
812
|
+
out = list(out)
|
|
813
|
+
return out
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
@overload
|
|
817
|
+
def unzip(lst: Iterable[Tuple[()]]) -> Tuple[()]:
|
|
818
|
+
"""Perform the unzip operation."""
|
|
819
|
+
...
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
@overload
|
|
823
|
+
def unzip(lst: Iterable[Tuple[T]]) -> Tuple[List[T]]:
|
|
824
|
+
"""Perform the unzip operation."""
|
|
825
|
+
...
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
@overload
|
|
829
|
+
def unzip(lst: Iterable[Tuple[T, U]]) -> Tuple[List[T], List[U]]:
|
|
830
|
+
"""Perform the unzip operation."""
|
|
831
|
+
...
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
@overload
|
|
835
|
+
def unzip(lst: Iterable[Tuple[T, U, V]]) -> Tuple[List[T], List[U], List[V]]:
|
|
836
|
+
"""Perform the unzip operation."""
|
|
837
|
+
...
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
@overload
|
|
841
|
+
def unzip(
|
|
842
|
+
lst: Iterable[Tuple[T, U, V, W]],
|
|
843
|
+
) -> Tuple[List[T], List[U], List[V], List[W]]:
|
|
844
|
+
"""Perform the unzip operation."""
|
|
845
|
+
...
|
|
846
|
+
|
|
847
|
+
|
|
848
|
+
@overload
|
|
849
|
+
def unzip(
|
|
850
|
+
lst: Iterable[Tuple[T, U, V, W, X]],
|
|
851
|
+
) -> Tuple[List[T], List[U], List[V], List[W], List[X]]:
|
|
852
|
+
"""Perform the unzip operation."""
|
|
853
|
+
...
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
@overload
|
|
857
|
+
def unzip(
|
|
858
|
+
lst: Iterable[Tuple[T, ...]],
|
|
859
|
+
) -> Tuple[List[T], ...]:
|
|
860
|
+
"""Perform the unzip operation."""
|
|
861
|
+
...
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
def unzip(lst):
|
|
865
|
+
"""Invert function of builtin zip().
|
|
866
|
+
|
|
867
|
+
Example
|
|
868
|
+
-------
|
|
869
|
+
>>> lst1 = [1, 2, 3, 4]
|
|
870
|
+
>>> lst2 = [5, 6, 7, 8]
|
|
871
|
+
>>> zipped_list = list(zip(lst1, lst2))
|
|
872
|
+
>>> zipped_list
|
|
873
|
+
... [(1, 5), (2, 6), (3, 7), (4, 8)]
|
|
874
|
+
>>> unzip(zipped_list)
|
|
875
|
+
... [1, 2, 3, 4], [5, 6, 7, 8]
|
|
876
|
+
"""
|
|
877
|
+
return tuple(map(list, zip(*lst)))
|
|
878
|
+
|
|
879
|
+
|
|
880
|
+
def duplicate_list(lst: List[T], sizes: List[int]) -> List[T]:
|
|
881
|
+
"""Duplicate elements elements of a list with the corresponding sizes.
|
|
882
|
+
|
|
883
|
+
Example
|
|
884
|
+
-------
|
|
885
|
+
>>> lst = ["a", "b", "c", "d", "e"]
|
|
886
|
+
>>> sizes = [1, 0, 2, 1, 3]
|
|
887
|
+
>>> duplicate_list(lst, sizes)
|
|
888
|
+
... ["a", "c", "c", "d", "e", "e", "e"]
|
|
889
|
+
"""
|
|
890
|
+
if len(lst) != len(sizes):
|
|
891
|
+
msg = f"Invalid arguments lengths. (found {len(lst)=} != {len(sizes)=})"
|
|
892
|
+
raise ValueError(msg)
|
|
893
|
+
|
|
894
|
+
out_size = sum(sizes)
|
|
895
|
+
out: List[T] = [None for _ in range(out_size)] # type: ignore
|
|
896
|
+
curidx = 0
|
|
897
|
+
for size, elt in zip(sizes, lst):
|
|
898
|
+
out[curidx : curidx + size] = [elt] * size
|
|
899
|
+
curidx += size
|
|
900
|
+
return out
|