iker-python-common 1.0.57__py3-none-any.whl → 1.0.59__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iker/common/utils/funcutils.py +14 -21
- iker/common/utils/jsonutils.py +11 -19
- iker/common/utils/numutils.py +10 -18
- iker/common/utils/randutils.py +16 -14
- iker/common/utils/sequtils.py +95 -105
- iker/common/utils/shutils.py +9 -5
- iker/common/utils/span.py +3 -6
- {iker_python_common-1.0.57.dist-info → iker_python_common-1.0.59.dist-info}/METADATA +4 -9
- {iker_python_common-1.0.57.dist-info → iker_python_common-1.0.59.dist-info}/RECORD +11 -12
- iker/common/utils/s3utils.py +0 -270
- {iker_python_common-1.0.57.dist-info → iker_python_common-1.0.59.dist-info}/WHEEL +0 -0
- {iker_python_common-1.0.57.dist-info → iker_python_common-1.0.59.dist-info}/top_level.txt +0 -0
iker/common/utils/funcutils.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import functools
|
|
2
2
|
from collections.abc import Callable
|
|
3
|
-
from typing import Protocol
|
|
3
|
+
from typing import Protocol
|
|
4
4
|
|
|
5
5
|
__all__ = [
|
|
6
6
|
"identity",
|
|
@@ -11,12 +11,8 @@ __all__ = [
|
|
|
11
11
|
"unique_returns",
|
|
12
12
|
]
|
|
13
13
|
|
|
14
|
-
T = TypeVar("T")
|
|
15
|
-
U = TypeVar("U")
|
|
16
|
-
RT = TypeVar("RT")
|
|
17
14
|
|
|
18
|
-
|
|
19
|
-
def identity(instance: T) -> T:
|
|
15
|
+
def identity[T](instance: T) -> T:
|
|
20
16
|
"""
|
|
21
17
|
Returns the input ``instance`` unchanged. This is a utility function often used as a default or placeholder.
|
|
22
18
|
|
|
@@ -26,22 +22,19 @@ def identity(instance: T) -> T:
|
|
|
26
22
|
return instance
|
|
27
23
|
|
|
28
24
|
|
|
29
|
-
class Composable
|
|
25
|
+
class Composable[T, R](Protocol):
|
|
30
26
|
"""
|
|
31
27
|
Protocol for composable callables, supporting composition and chaining with other callables.
|
|
32
|
-
|
|
33
|
-
:param x: The input value for the callable.
|
|
34
|
-
:return: The result of the callable.
|
|
35
28
|
"""
|
|
36
29
|
|
|
37
|
-
def __call__(self, x: T) ->
|
|
30
|
+
def __call__(self, x: T) -> R: ...
|
|
38
31
|
|
|
39
|
-
def compose(self, func: "Callable[[U], T] | Composable[U, T]") -> "Composable[U,
|
|
32
|
+
def compose[U](self, func: "Callable[[U], T] | Composable[U, T]") -> "Composable[U, R]": ...
|
|
40
33
|
|
|
41
|
-
def and_then(self, func: "Callable[[
|
|
34
|
+
def and_then[U](self, func: "Callable[[R], U] | Composable[R, U]") -> "Composable[T, U]": ...
|
|
42
35
|
|
|
43
36
|
|
|
44
|
-
def composable(func: Callable[[T],
|
|
37
|
+
def composable[T, R](func: Callable[[T], R]) -> Composable[T, R]:
|
|
45
38
|
"""
|
|
46
39
|
Wraps a function to make it composable, allowing chaining with compose and and_then methods.
|
|
47
40
|
|
|
@@ -49,13 +42,13 @@ def composable(func: Callable[[T], RT]) -> Composable[T, RT]:
|
|
|
49
42
|
:return: A composable version of the function.
|
|
50
43
|
"""
|
|
51
44
|
|
|
52
|
-
def compose(another_func: Callable[[U], T] | Composable[U, T]) -> Composable[U,
|
|
53
|
-
def chained(x: U) ->
|
|
45
|
+
def compose[U](another_func: Callable[[U], T] | Composable[U, T]) -> Composable[U, R]:
|
|
46
|
+
def chained(x: U) -> R:
|
|
54
47
|
return func(another_func(x))
|
|
55
48
|
|
|
56
49
|
return composable(chained)
|
|
57
50
|
|
|
58
|
-
def and_then(another_func: Callable[[
|
|
51
|
+
def and_then[U](another_func: Callable[[R], U] | Composable[R, U]) -> Composable[T, U]:
|
|
59
52
|
def chained(x: T) -> U:
|
|
60
53
|
return another_func(func(x))
|
|
61
54
|
|
|
@@ -66,7 +59,7 @@ def composable(func: Callable[[T], RT]) -> Composable[T, RT]:
|
|
|
66
59
|
return func
|
|
67
60
|
|
|
68
61
|
|
|
69
|
-
def singleton(tar: Callable[...,
|
|
62
|
+
def singleton[R](tar: Callable[..., R] = None):
|
|
70
63
|
"""
|
|
71
64
|
Decorator to ensure a function or class is only instantiated once. Subsequent calls return the same instance.
|
|
72
65
|
|
|
@@ -91,7 +84,7 @@ def singleton(tar: Callable[..., RT] = None):
|
|
|
91
84
|
return decorator if tar is None else decorator(tar)
|
|
92
85
|
|
|
93
86
|
|
|
94
|
-
def memorized(tar: Callable[...,
|
|
87
|
+
def memorized[R](tar: Callable[..., R] = None, *, ordered: bool = False, typed: bool = False):
|
|
95
88
|
"""
|
|
96
89
|
Decorator to cache the results of a function based on its arguments. Supports options for argument order and type.
|
|
97
90
|
|
|
@@ -134,7 +127,7 @@ def memorized(tar: Callable[..., RT] = None, *, ordered: bool = False, typed: bo
|
|
|
134
127
|
return decorator if tar is None else decorator(tar)
|
|
135
128
|
|
|
136
129
|
|
|
137
|
-
def lazy(tar: Callable[...,
|
|
130
|
+
def lazy[R](tar: Callable[..., R] = None):
|
|
138
131
|
"""
|
|
139
132
|
Decorator to defer the execution of a function until its result is explicitly requested.
|
|
140
133
|
|
|
@@ -155,7 +148,7 @@ def lazy(tar: Callable[..., RT] = None):
|
|
|
155
148
|
return decorator if tar is None else decorator(tar)
|
|
156
149
|
|
|
157
150
|
|
|
158
|
-
def unique_returns(tar: Callable[...,
|
|
151
|
+
def unique_returns[R](tar: Callable[..., R] = None, *, max_trials: int | None = None):
|
|
159
152
|
"""
|
|
160
153
|
Decorator to ensure a function produces unique return values. If no unique value is found within max_trials,
|
|
161
154
|
raises an error.
|
iker/common/utils/jsonutils.py
CHANGED
|
@@ -1,12 +1,6 @@
|
|
|
1
1
|
import math
|
|
2
|
-
import sys
|
|
3
2
|
from collections.abc import Callable, Mapping, MutableMapping, MutableSequence, Sequence, Set
|
|
4
|
-
from typing import Any, SupportsFloat, SupportsInt
|
|
5
|
-
|
|
6
|
-
if sys.version_info < (3, 12):
|
|
7
|
-
from typing_extensions import TypeAliasType
|
|
8
|
-
else:
|
|
9
|
-
from typing import TypeAliasType
|
|
3
|
+
from typing import Any, SupportsFloat, SupportsInt
|
|
10
4
|
|
|
11
5
|
from iker.common.utils.numutils import is_normal_real
|
|
12
6
|
|
|
@@ -35,28 +29,26 @@ __all__ = [
|
|
|
35
29
|
|
|
36
30
|
JsonKey = str
|
|
37
31
|
JsonValue = str | bool | float | int | None
|
|
38
|
-
JsonObject
|
|
39
|
-
JsonArray
|
|
40
|
-
JsonType
|
|
32
|
+
type JsonObject = dict[JsonKey, JsonType]
|
|
33
|
+
type JsonArray = list[JsonType]
|
|
34
|
+
type JsonType = JsonValue | JsonObject | JsonArray
|
|
35
|
+
|
|
41
36
|
JsonKeyCompatible = str
|
|
42
37
|
JsonValueCompatible = str | bool | SupportsFloat | SupportsInt | None
|
|
43
|
-
JsonObjectCompatible
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
"Sequence[JsonTypeCompatible]")
|
|
47
|
-
JsonTypeCompatible: TypeAlias = TypeAliasType("JsonTypeCompatible",
|
|
48
|
-
"JsonValueCompatible | JsonObjectCompatible | JsonArrayCompatible")
|
|
38
|
+
type JsonObjectCompatible = Mapping[JsonKeyCompatible, JsonTypeCompatible]
|
|
39
|
+
type JsonArrayCompatible = Sequence[JsonTypeCompatible]
|
|
40
|
+
type JsonTypeCompatible = JsonValueCompatible | JsonObjectCompatible | JsonArrayCompatible
|
|
49
41
|
|
|
50
42
|
|
|
51
43
|
class NodeAsterisk(object):
|
|
52
44
|
pass
|
|
53
45
|
|
|
54
46
|
|
|
55
|
-
Node = str | int | NodeAsterisk
|
|
56
|
-
NodePath: TypeAlias = TypeAliasType("NodePath", "list[Node]")
|
|
57
|
-
|
|
58
47
|
asterisk = NodeAsterisk()
|
|
59
48
|
|
|
49
|
+
Node = str | int | NodeAsterisk
|
|
50
|
+
type NodePath = list[Node]
|
|
51
|
+
|
|
60
52
|
|
|
61
53
|
def json_get(
|
|
62
54
|
obj: JsonTypeCompatible,
|
iker/common/utils/numutils.py
CHANGED
|
@@ -2,7 +2,7 @@ import math
|
|
|
2
2
|
from collections.abc import Callable, Sequence
|
|
3
3
|
from decimal import Decimal
|
|
4
4
|
from numbers import Real
|
|
5
|
-
from typing import Any
|
|
5
|
+
from typing import Any
|
|
6
6
|
|
|
7
7
|
import numpy as np
|
|
8
8
|
|
|
@@ -26,8 +26,6 @@ __all__ = [
|
|
|
26
26
|
"real_nan_std",
|
|
27
27
|
]
|
|
28
28
|
|
|
29
|
-
RealT = TypeVar("RealT", bound=Real)
|
|
30
|
-
|
|
31
29
|
|
|
32
30
|
def to_decimal(x: float | str | None) -> Decimal | None:
|
|
33
31
|
"""
|
|
@@ -72,10 +70,7 @@ def is_normal_real(v: Any) -> bool:
|
|
|
72
70
|
return is_real(v) and not math.isnan(v) and not math.isinf(v)
|
|
73
71
|
|
|
74
72
|
|
|
75
|
-
def make_real_unary(
|
|
76
|
-
op: Callable[[RealT], RealT],
|
|
77
|
-
fb: RealT | None = None,
|
|
78
|
-
) -> Callable[[RealT], RealT | None]:
|
|
73
|
+
def make_real_unary[T: Real](op: Callable[[T], T], fb: T | None = None) -> Callable[[T], T | None]:
|
|
79
74
|
"""
|
|
80
75
|
Creates a unary function that applies the given operation to a real number, returning a fallback if not real.
|
|
81
76
|
|
|
@@ -84,7 +79,7 @@ def make_real_unary(
|
|
|
84
79
|
:return: A function that applies ``op`` to a real number or returns ``fb``.
|
|
85
80
|
"""
|
|
86
81
|
|
|
87
|
-
def func(x:
|
|
82
|
+
def func(x: T) -> T | None:
|
|
88
83
|
if not is_real(x):
|
|
89
84
|
return fb
|
|
90
85
|
return op(x)
|
|
@@ -92,10 +87,7 @@ def make_real_unary(
|
|
|
92
87
|
return func
|
|
93
88
|
|
|
94
89
|
|
|
95
|
-
def make_real_binary(
|
|
96
|
-
op: Callable[[RealT, RealT], RealT],
|
|
97
|
-
fb: RealT | None = None,
|
|
98
|
-
) -> Callable[[RealT, RealT], RealT | None]:
|
|
90
|
+
def make_real_binary[T: Real](op: Callable[[T, T], T], fb: T | None = None) -> Callable[[T, T], T | None]:
|
|
99
91
|
"""
|
|
100
92
|
Creates a binary function that applies the given operation to two real numbers, with fallback logic.
|
|
101
93
|
|
|
@@ -104,7 +96,7 @@ def make_real_binary(
|
|
|
104
96
|
:return: A function that applies ``op`` to two real numbers or returns ``fb``.
|
|
105
97
|
"""
|
|
106
98
|
|
|
107
|
-
def func(a:
|
|
99
|
+
def func(a: T, b: T) -> T | None:
|
|
108
100
|
if not is_real(a) and not is_real(b):
|
|
109
101
|
return fb
|
|
110
102
|
if not is_real(a):
|
|
@@ -116,10 +108,10 @@ def make_real_binary(
|
|
|
116
108
|
return func
|
|
117
109
|
|
|
118
110
|
|
|
119
|
-
def make_real_reducer(
|
|
120
|
-
op: Callable[[Sequence[
|
|
121
|
-
fb:
|
|
122
|
-
) -> Callable[[Sequence[
|
|
111
|
+
def make_real_reducer[T: Real](
|
|
112
|
+
op: Callable[[Sequence[T], ...], T],
|
|
113
|
+
fb: T | None = None,
|
|
114
|
+
) -> Callable[[Sequence[T], ...], T | None]:
|
|
123
115
|
"""
|
|
124
116
|
Creates a reducer function that applies the given operation to a sequence of real numbers, filtering out non-reals.
|
|
125
117
|
|
|
@@ -128,7 +120,7 @@ def make_real_reducer(
|
|
|
128
120
|
:return: A function that reduces a sequence of real numbers or returns ``fb``.
|
|
129
121
|
"""
|
|
130
122
|
|
|
131
|
-
def func(xs: Sequence[
|
|
123
|
+
def func(xs: Sequence[T], *args, **kwargs) -> T | None:
|
|
132
124
|
xs_new = list(filter(is_real, xs))
|
|
133
125
|
return op(xs_new, *args, **kwargs) if len(xs_new) > 0 else fb
|
|
134
126
|
|
iker/common/utils/randutils.py
CHANGED
|
@@ -4,7 +4,6 @@ import random
|
|
|
4
4
|
import string
|
|
5
5
|
import sys
|
|
6
6
|
from collections.abc import Callable, Sequence
|
|
7
|
-
from typing import TypeVar
|
|
8
7
|
from typing import overload
|
|
9
8
|
|
|
10
9
|
from iker.common.utils.dtutils import dt_utc_max, dt_utc_min
|
|
@@ -19,8 +18,6 @@ __all__ = [
|
|
|
19
18
|
"randomizer",
|
|
20
19
|
]
|
|
21
20
|
|
|
22
|
-
T = TypeVar("T")
|
|
23
|
-
|
|
24
21
|
|
|
25
22
|
@singleton
|
|
26
23
|
def max_int() -> int:
|
|
@@ -268,22 +265,22 @@ class Randomizer(object):
|
|
|
268
265
|
return generate_json_object(0)
|
|
269
266
|
|
|
270
267
|
@overload
|
|
271
|
-
def sample(self, population: Sequence[T], count_func: Callable[[T], int], k: int) -> list[T]:
|
|
268
|
+
def sample[T](self, population: Sequence[T], count_func: Callable[[T], int], k: int) -> list[T]:
|
|
272
269
|
...
|
|
273
270
|
|
|
274
271
|
@overload
|
|
275
|
-
def sample(self, population: Sequence[T], k: int) -> list[T]:
|
|
272
|
+
def sample[T](self, population: Sequence[T], k: int) -> list[T]:
|
|
276
273
|
...
|
|
277
274
|
|
|
278
275
|
@overload
|
|
279
|
-
def sample(self, population: Sequence[T], count_func: Callable[[T], int], k: None = None) -> T:
|
|
276
|
+
def sample[T](self, population: Sequence[T], count_func: Callable[[T], int], k: None = None) -> T:
|
|
280
277
|
...
|
|
281
278
|
|
|
282
279
|
@overload
|
|
283
|
-
def sample(self, population: Sequence[T], k: None = None) -> T:
|
|
280
|
+
def sample[T](self, population: Sequence[T], k: None = None) -> T:
|
|
284
281
|
...
|
|
285
282
|
|
|
286
|
-
def sample(self, population: Sequence[T], count_func: Callable[[T], int] = None, k: int = None) -> list[T] | T:
|
|
283
|
+
def sample[T](self, population: Sequence[T], count_func: Callable[[T], int] = None, k: int = None) -> list[T] | T:
|
|
287
284
|
"""
|
|
288
285
|
Returns a random sample from the population, optionally weighted by a count function.
|
|
289
286
|
|
|
@@ -297,22 +294,27 @@ class Randomizer(object):
|
|
|
297
294
|
return result if k is not None else head_or_none(result)
|
|
298
295
|
|
|
299
296
|
@overload
|
|
300
|
-
def choose(self, population: Sequence[T], weight_func: Callable[[T], float], k: int) -> list[T]:
|
|
297
|
+
def choose[T](self, population: Sequence[T], weight_func: Callable[[T], float], k: int) -> list[T]:
|
|
301
298
|
...
|
|
302
299
|
|
|
303
300
|
@overload
|
|
304
|
-
def choose(self, population: Sequence[T], k: int) -> list[T]:
|
|
301
|
+
def choose[T](self, population: Sequence[T], k: int) -> list[T]:
|
|
305
302
|
...
|
|
306
303
|
|
|
307
304
|
@overload
|
|
308
|
-
def choose(self, population: Sequence[T], weight_func: Callable[[T], float], k: None = None) -> T:
|
|
305
|
+
def choose[T](self, population: Sequence[T], weight_func: Callable[[T], float], k: None = None) -> T:
|
|
309
306
|
...
|
|
310
307
|
|
|
311
308
|
@overload
|
|
312
|
-
def choose(self, population: Sequence[T], k: None = None) -> T:
|
|
309
|
+
def choose[T](self, population: Sequence[T], k: None = None) -> T:
|
|
313
310
|
...
|
|
314
311
|
|
|
315
|
-
def choose
|
|
312
|
+
def choose[T](
|
|
313
|
+
self,
|
|
314
|
+
population: Sequence[T],
|
|
315
|
+
weight_func: Callable[[T], float] = None,
|
|
316
|
+
k: int = None,
|
|
317
|
+
) -> list[T] | T:
|
|
316
318
|
"""
|
|
317
319
|
Returns a random selection from the population, optionally weighted by a weight function.
|
|
318
320
|
|
|
@@ -325,7 +327,7 @@ class Randomizer(object):
|
|
|
325
327
|
result = self.random.choices(population, weights=weights, k=k or 1)
|
|
326
328
|
return result if k is not None else head_or_none(result)
|
|
327
329
|
|
|
328
|
-
def shuffle(self, data: Sequence[T]) -> list[T]:
|
|
330
|
+
def shuffle[T](self, data: Sequence[T]) -> list[T]:
|
|
329
331
|
"""
|
|
330
332
|
Returns a shuffled copy of the input ``data`` sequence.
|
|
331
333
|
|
iker/common/utils/sequtils.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import itertools
|
|
2
2
|
from collections import deque
|
|
3
3
|
from collections.abc import Callable, Generator, Iterable, Iterator, Sequence, Sized
|
|
4
|
-
from typing import Generic, TypeAlias, TypeVar
|
|
5
4
|
from typing import overload
|
|
6
5
|
|
|
7
6
|
__all__ = [
|
|
@@ -32,19 +31,13 @@ __all__ = [
|
|
|
32
31
|
"seq",
|
|
33
32
|
]
|
|
34
33
|
|
|
35
|
-
T =
|
|
36
|
-
KT = TypeVar("KT")
|
|
37
|
-
VT = TypeVar("VT")
|
|
38
|
-
TT = TypeVar("TT")
|
|
39
|
-
Tco = TypeVar("Tco", covariant=True)
|
|
40
|
-
|
|
41
|
-
NestedIterable: TypeAlias = "T | Iterable[NestedSequence[T]]"
|
|
34
|
+
type NestedIterable[T] = T | Iterable[NestedIterable[T]]
|
|
42
35
|
|
|
43
36
|
|
|
44
37
|
# See Haskell's list operations head, tail, init, and last
|
|
45
38
|
# which is also provided in Scala list operations
|
|
46
39
|
|
|
47
|
-
def head(ms: Sequence[T]) -> T:
|
|
40
|
+
def head[T](ms: Sequence[T]) -> T:
|
|
48
41
|
"""
|
|
49
42
|
Returns the first element of the sequence ``ms``.
|
|
50
43
|
|
|
@@ -54,7 +47,7 @@ def head(ms: Sequence[T]) -> T:
|
|
|
54
47
|
return ms[0]
|
|
55
48
|
|
|
56
49
|
|
|
57
|
-
def head_or_none(ms: Sequence[T]) -> T | None:
|
|
50
|
+
def head_or_none[T](ms: Sequence[T]) -> T | None:
|
|
58
51
|
"""
|
|
59
52
|
Returns the first element of the sequence ``ms``, or ``None`` if the sequence is empty.
|
|
60
53
|
|
|
@@ -66,7 +59,7 @@ def head_or_none(ms: Sequence[T]) -> T | None:
|
|
|
66
59
|
return None
|
|
67
60
|
|
|
68
61
|
|
|
69
|
-
def last(ms: Sequence[T]) -> T:
|
|
62
|
+
def last[T](ms: Sequence[T]) -> T:
|
|
70
63
|
"""
|
|
71
64
|
Returns the last element of the sequence ``ms``.
|
|
72
65
|
|
|
@@ -76,7 +69,7 @@ def last(ms: Sequence[T]) -> T:
|
|
|
76
69
|
return ms[-1]
|
|
77
70
|
|
|
78
71
|
|
|
79
|
-
def last_or_none(ms: Sequence[T]) -> T | None:
|
|
72
|
+
def last_or_none[T](ms: Sequence[T]) -> T | None:
|
|
80
73
|
"""
|
|
81
74
|
Returns the last element of the sequence ``ms``, or ``None`` if the sequence is empty.
|
|
82
75
|
|
|
@@ -88,7 +81,7 @@ def last_or_none(ms: Sequence[T]) -> T | None:
|
|
|
88
81
|
return None
|
|
89
82
|
|
|
90
83
|
|
|
91
|
-
def tail(ms: Sequence[T]) -> Sequence[T]:
|
|
84
|
+
def tail[T](ms: Sequence[T]) -> Sequence[T]:
|
|
92
85
|
"""
|
|
93
86
|
Returns all elements of the sequence ``ms`` except the first.
|
|
94
87
|
|
|
@@ -98,7 +91,7 @@ def tail(ms: Sequence[T]) -> Sequence[T]:
|
|
|
98
91
|
return ms[1:]
|
|
99
92
|
|
|
100
93
|
|
|
101
|
-
def tail_iter(ms: Iterable[T]) -> Generator[T, None, None]:
|
|
94
|
+
def tail_iter[T](ms: Iterable[T]) -> Generator[T, None, None]:
|
|
102
95
|
"""
|
|
103
96
|
Returns an iterator over all elements of the iterable ``ms`` except the first.
|
|
104
97
|
|
|
@@ -114,7 +107,7 @@ def tail_iter(ms: Iterable[T]) -> Generator[T, None, None]:
|
|
|
114
107
|
return
|
|
115
108
|
|
|
116
109
|
|
|
117
|
-
def init(ms: Sequence[T]) -> Sequence[T]:
|
|
110
|
+
def init[T](ms: Sequence[T]) -> Sequence[T]:
|
|
118
111
|
"""
|
|
119
112
|
Returns all elements of the sequence ``ms`` except the last.
|
|
120
113
|
|
|
@@ -124,7 +117,7 @@ def init(ms: Sequence[T]) -> Sequence[T]:
|
|
|
124
117
|
return ms[:-1]
|
|
125
118
|
|
|
126
119
|
|
|
127
|
-
def init_iter(ms: Iterable[T]) -> Generator[T, None, None]:
|
|
120
|
+
def init_iter[T](ms: Iterable[T]) -> Generator[T, None, None]:
|
|
128
121
|
"""
|
|
129
122
|
Returns an iterator over all elements of the iterable ``ms`` except the last.
|
|
130
123
|
|
|
@@ -141,7 +134,7 @@ def init_iter(ms: Iterable[T]) -> Generator[T, None, None]:
|
|
|
141
134
|
prev = this
|
|
142
135
|
|
|
143
136
|
|
|
144
|
-
def some(x: T, test: Callable[[T], bool] = lambda x: x is not None) -> Generator[T, None, None]:
|
|
137
|
+
def some[T](x: T, test: Callable[[T], bool] = lambda x: x is not None) -> Generator[T, None, None]:
|
|
145
138
|
"""
|
|
146
139
|
Yields the value ``x`` if it passes the ``test`` function.
|
|
147
140
|
|
|
@@ -153,7 +146,7 @@ def some(x: T, test: Callable[[T], bool] = lambda x: x is not None) -> Generator
|
|
|
153
146
|
yield x
|
|
154
147
|
|
|
155
148
|
|
|
156
|
-
def flatten(ms: "NestedIterable[T] | None") -> Generator[T, None, None]:
|
|
149
|
+
def flatten[T](ms: "NestedIterable[T] | None") -> Generator[T, None, None]:
|
|
157
150
|
"""
|
|
158
151
|
Flattens a nested iterable ``ms`` into a generator of elements.
|
|
159
152
|
|
|
@@ -169,7 +162,7 @@ def flatten(ms: "NestedIterable[T] | None") -> Generator[T, None, None]:
|
|
|
169
162
|
yield from flatten(m)
|
|
170
163
|
|
|
171
164
|
|
|
172
|
-
def scan_left(ms: Sequence[T], zero:
|
|
165
|
+
def scan_left[T, R](ms: Sequence[T], zero: R | None, func: Callable[[R, T], R]) -> Generator[R, None, None]:
|
|
173
166
|
"""
|
|
174
167
|
Applies a function cumulatively to the items of a sequence from left to right, yielding the intermediate results.
|
|
175
168
|
|
|
@@ -184,7 +177,7 @@ def scan_left(ms: Sequence[T], zero: TT | None, func: Callable[[TT, T], TT]) ->
|
|
|
184
177
|
yield accum
|
|
185
178
|
|
|
186
179
|
|
|
187
|
-
def scan_right(ms: Sequence[T], zero:
|
|
180
|
+
def scan_right[T, R](ms: Sequence[T], zero: R | None, func: Callable[[R, T], R]) -> Generator[R, None, None]:
|
|
188
181
|
"""
|
|
189
182
|
Applies a function cumulatively to the items of a sequence from right to left, yielding the intermediate results.
|
|
190
183
|
|
|
@@ -199,7 +192,7 @@ def scan_right(ms: Sequence[T], zero: TT | None, func: Callable[[TT, T], TT]) ->
|
|
|
199
192
|
yield accum
|
|
200
193
|
|
|
201
194
|
|
|
202
|
-
def fold_left(ms: Sequence[T], zero:
|
|
195
|
+
def fold_left[T, R](ms: Sequence[T], zero: R | None, func: Callable[[R, T], R]) -> R | None:
|
|
203
196
|
"""
|
|
204
197
|
Applies a function cumulatively to the items of a sequence ``ms`` from left to right, reducing the sequence to a
|
|
205
198
|
single value. If the initial value (``zero``) is ``None``, the first element of ``ms`` is used as the starting
|
|
@@ -219,7 +212,7 @@ def fold_left(ms: Sequence[T], zero: TT | None, func: Callable[[TT, T], TT]) ->
|
|
|
219
212
|
return accum
|
|
220
213
|
|
|
221
214
|
|
|
222
|
-
def fold_right(ms: Sequence[T], zero:
|
|
215
|
+
def fold_right[T, R](ms: Sequence[T], zero: R | None, func: Callable[[R, T], R]) -> R | None:
|
|
223
216
|
"""
|
|
224
217
|
Applies a function cumulatively to the items of a sequence ``ms`` from right to left, reducing the sequence to a
|
|
225
218
|
single value. If the initial value (``zero``) is ``None``, the last element of ``ms`` is used as the starting
|
|
@@ -239,14 +232,14 @@ def fold_right(ms: Sequence[T], zero: TT | None, func: Callable[[TT, T], TT]) ->
|
|
|
239
232
|
return accum
|
|
240
233
|
|
|
241
234
|
|
|
242
|
-
def slide_left(
|
|
235
|
+
def slide_left[T, R](
|
|
243
236
|
ms: Sequence[T],
|
|
244
237
|
window: int,
|
|
245
|
-
func: Callable[[Iterable[T]],
|
|
238
|
+
func: Callable[[Iterable[T]], R] = lambda x: x,
|
|
246
239
|
*,
|
|
247
240
|
allow_partial: tuple[bool, bool] = (False, False),
|
|
248
241
|
padding: tuple[T, T] = (None, None),
|
|
249
|
-
) -> Generator[
|
|
242
|
+
) -> Generator[R, None, None]:
|
|
250
243
|
"""
|
|
251
244
|
Generates a sliding window over the input sequence from left to right.
|
|
252
245
|
|
|
@@ -295,14 +288,14 @@ def slide_left(
|
|
|
295
288
|
yield from (func(ms[-i:]) for i in reversed(range(1, min(window, len(ms)))))
|
|
296
289
|
|
|
297
290
|
|
|
298
|
-
def slide_right(
|
|
291
|
+
def slide_right[T, R](
|
|
299
292
|
ms: Sequence[T],
|
|
300
293
|
window: int,
|
|
301
|
-
func: Callable[[Iterable[T]],
|
|
294
|
+
func: Callable[[Iterable[T]], R] = lambda x: x,
|
|
302
295
|
*,
|
|
303
296
|
allow_partial: tuple[bool, bool] = (False, False),
|
|
304
297
|
padding: tuple[T, T] = (None, None),
|
|
305
|
-
) -> Generator[
|
|
298
|
+
) -> Generator[R, None, None]:
|
|
306
299
|
"""
|
|
307
300
|
Generates a sliding window over the input sequence from right to left.
|
|
308
301
|
|
|
@@ -352,28 +345,28 @@ def slide_right(
|
|
|
352
345
|
|
|
353
346
|
|
|
354
347
|
@overload
|
|
355
|
-
def grouped(
|
|
348
|
+
def grouped[T, K](
|
|
356
349
|
ms: Sequence[T],
|
|
357
|
-
key_func: Callable[[T],
|
|
350
|
+
key_func: Callable[[T], K],
|
|
358
351
|
keys_ordered: bool,
|
|
359
352
|
values_only: False = False,
|
|
360
|
-
) -> Generator[tuple[
|
|
353
|
+
) -> Generator[tuple[K, list[T]], None, None]:
|
|
361
354
|
...
|
|
362
355
|
|
|
363
356
|
|
|
364
357
|
@overload
|
|
365
|
-
def grouped(
|
|
358
|
+
def grouped[T, K](
|
|
366
359
|
ms: Sequence[T],
|
|
367
|
-
key_func: Callable[[T],
|
|
360
|
+
key_func: Callable[[T], K],
|
|
368
361
|
values_only: False = False,
|
|
369
|
-
) -> Generator[tuple[
|
|
362
|
+
) -> Generator[tuple[K, list[T]], None, None]:
|
|
370
363
|
...
|
|
371
364
|
|
|
372
365
|
|
|
373
366
|
@overload
|
|
374
|
-
def grouped(
|
|
367
|
+
def grouped[T, K](
|
|
375
368
|
ms: Sequence[T],
|
|
376
|
-
key_func: Callable[[T],
|
|
369
|
+
key_func: Callable[[T], K],
|
|
377
370
|
keys_ordered: bool,
|
|
378
371
|
values_only: True = True,
|
|
379
372
|
) -> Generator[list[T], None, None]:
|
|
@@ -381,20 +374,20 @@ def grouped(
|
|
|
381
374
|
|
|
382
375
|
|
|
383
376
|
@overload
|
|
384
|
-
def grouped(
|
|
377
|
+
def grouped[T, K](
|
|
385
378
|
ms: Sequence[T],
|
|
386
|
-
key_func: Callable[[T],
|
|
379
|
+
key_func: Callable[[T], K],
|
|
387
380
|
values_only: True = True,
|
|
388
381
|
) -> Generator[list[T], None, None]:
|
|
389
382
|
...
|
|
390
383
|
|
|
391
384
|
|
|
392
|
-
def grouped(
|
|
385
|
+
def grouped[T, K](
|
|
393
386
|
ms: Sequence[T],
|
|
394
|
-
key_func: Callable[[T],
|
|
387
|
+
key_func: Callable[[T], K],
|
|
395
388
|
keys_ordered: bool = False,
|
|
396
389
|
values_only: bool = False,
|
|
397
|
-
) -> Generator[tuple[
|
|
390
|
+
) -> Generator[tuple[K, list[T]] | list[T], None, None]:
|
|
398
391
|
"""
|
|
399
392
|
Groups elements of a sequence by a key function.
|
|
400
393
|
|
|
@@ -406,7 +399,7 @@ def grouped(
|
|
|
406
399
|
"""
|
|
407
400
|
if ms is None or len(ms) == 0:
|
|
408
401
|
return
|
|
409
|
-
grouped_ms: dict[
|
|
402
|
+
grouped_ms: dict[K, list[T]] = {}
|
|
410
403
|
for m in ms:
|
|
411
404
|
k = key_func(m)
|
|
412
405
|
grouped_ms.setdefault(k, []).append(m)
|
|
@@ -414,7 +407,7 @@ def grouped(
|
|
|
414
407
|
yield from (d for _, d in groups) if values_only else groups
|
|
415
408
|
|
|
416
409
|
|
|
417
|
-
def deduped(ms: Sequence[T], comp_func: Callable[[T, T], bool]) -> Generator[T, None, None]:
|
|
410
|
+
def deduped[T](ms: Sequence[T], comp_func: Callable[[T, T], bool]) -> Generator[T, None, None]:
|
|
418
411
|
"""
|
|
419
412
|
Yields elements from the input sequence, removing consecutive duplicates as determined by the comparison function.
|
|
420
413
|
|
|
@@ -432,7 +425,7 @@ def deduped(ms: Sequence[T], comp_func: Callable[[T, T], bool]) -> Generator[T,
|
|
|
432
425
|
prev = m
|
|
433
426
|
|
|
434
427
|
|
|
435
|
-
def batched(ms: Iterable[T], batch_size: int) -> Generator[list[T], None, None]:
|
|
428
|
+
def batched[T](ms: Iterable[T], batch_size: int) -> Generator[list[T], None, None]:
|
|
436
429
|
"""
|
|
437
430
|
Splits an iterable into consecutive batches of a specified size.
|
|
438
431
|
|
|
@@ -456,7 +449,7 @@ def batched(ms: Iterable[T], batch_size: int) -> Generator[list[T], None, None]:
|
|
|
456
449
|
batch_yield = batched
|
|
457
450
|
|
|
458
451
|
|
|
459
|
-
def chunk(
|
|
452
|
+
def chunk[T](
|
|
460
453
|
ms: Sequence[T],
|
|
461
454
|
chunk_func: Callable[[Sequence[T], T], bool],
|
|
462
455
|
exclusive_end: bool = False,
|
|
@@ -485,7 +478,7 @@ def chunk(
|
|
|
485
478
|
yield prev
|
|
486
479
|
|
|
487
480
|
|
|
488
|
-
def chunk_between(
|
|
481
|
+
def chunk_between[T](
|
|
489
482
|
ms: Sequence[T],
|
|
490
483
|
chunk_func: Callable[[T, T], bool],
|
|
491
484
|
exclusive_end: bool = False,
|
|
@@ -504,9 +497,9 @@ def chunk_between(
|
|
|
504
497
|
yield from chunk(ms, lambda x, y: chunk_func(last(x), y), exclusive_end)
|
|
505
498
|
|
|
506
499
|
|
|
507
|
-
def chunk_with_key(
|
|
500
|
+
def chunk_with_key[T, K](
|
|
508
501
|
ms: Sequence[T],
|
|
509
|
-
key_func: Callable[[T],
|
|
502
|
+
key_func: Callable[[T], K],
|
|
510
503
|
exclusive_end: bool = False,
|
|
511
504
|
) -> Generator[list[T], None, None]:
|
|
512
505
|
"""
|
|
@@ -521,7 +514,7 @@ def chunk_with_key(
|
|
|
521
514
|
yield from chunk_between(ms, lambda x, y: key_func(x) != key_func(y), exclusive_end)
|
|
522
515
|
|
|
523
516
|
|
|
524
|
-
def merge_chunks(
|
|
517
|
+
def merge_chunks[T](
|
|
525
518
|
chunks: Sequence[Sequence[T]],
|
|
526
519
|
merge_func: Callable[[Sequence[T], Sequence[T]], bool],
|
|
527
520
|
drop_exclusive_end: bool = False,
|
|
@@ -546,11 +539,8 @@ def merge_chunks(
|
|
|
546
539
|
chunk_between(chunks, lambda a, b: not merge_func(a, b)))
|
|
547
540
|
|
|
548
541
|
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
class Seq(Generic[Tco], Sequence[Tco], Sized):
|
|
553
|
-
def __init__(self, data: Iterable[Tco] | SeqT):
|
|
542
|
+
class Seq[T](Sequence[T], Sized):
|
|
543
|
+
def __init__(self, data: Iterable[T]):
|
|
554
544
|
if isinstance(data, Seq):
|
|
555
545
|
self.data = data.data
|
|
556
546
|
elif isinstance(data, Iterable):
|
|
@@ -566,15 +556,15 @@ class Seq(Generic[Tco], Sequence[Tco], Sized):
|
|
|
566
556
|
def empty(self) -> bool:
|
|
567
557
|
return self.size == 0
|
|
568
558
|
|
|
569
|
-
def __add__(self: SeqT, other: SeqT) -> SeqT:
|
|
559
|
+
def __add__[SeqT: Seq](self: SeqT, other: SeqT) -> SeqT:
|
|
570
560
|
return self.concat(other)
|
|
571
561
|
|
|
572
|
-
def __getitem__(self: SeqT, item: int | slice) -> SeqT:
|
|
562
|
+
def __getitem__[SeqT: Seq](self: SeqT, item: int | slice) -> SeqT:
|
|
573
563
|
if isinstance(item, slice):
|
|
574
564
|
return type(self)(self.data[item])
|
|
575
565
|
elif isinstance(item, int):
|
|
576
566
|
return type(self)(some(self.data[item]))
|
|
577
|
-
raise
|
|
567
|
+
raise IndexError("unsupported index type")
|
|
578
568
|
|
|
579
569
|
def __len__(self) -> int:
|
|
580
570
|
return len(self.data)
|
|
@@ -582,104 +572,104 @@ class Seq(Generic[Tco], Sequence[Tco], Sized):
|
|
|
582
572
|
def __contains__(self, item) -> bool:
|
|
583
573
|
return item in self.data
|
|
584
574
|
|
|
585
|
-
def __iter__(self) -> Iterator[
|
|
575
|
+
def __iter__(self) -> Iterator[T]:
|
|
586
576
|
return iter(self.data)
|
|
587
577
|
|
|
588
|
-
def __reversed__(self) -> Iterator[
|
|
578
|
+
def __reversed__(self) -> Iterator[T]:
|
|
589
579
|
return reversed(self.data)
|
|
590
580
|
|
|
591
|
-
def count(self, elem:
|
|
581
|
+
def count(self, elem: T):
|
|
592
582
|
return self.count_if(lambda x: x == elem)
|
|
593
583
|
|
|
594
|
-
def count_if(self, func: Callable[[
|
|
584
|
+
def count_if(self, func: Callable[[T], bool]):
|
|
595
585
|
return sum(1 for item in self.data if func(item))
|
|
596
586
|
|
|
597
|
-
def concat(self: SeqT, other: SeqT) -> SeqT:
|
|
587
|
+
def concat[SeqT: Seq](self: SeqT, other: SeqT) -> SeqT:
|
|
598
588
|
return type(self)(itertools.chain(self.data, other.data))
|
|
599
589
|
|
|
600
|
-
def pad_left(self: SeqT, n: int, value:
|
|
590
|
+
def pad_left[SeqT: Seq](self: SeqT, n: int, value: T) -> SeqT:
|
|
601
591
|
return self if n <= 0 else type(self)(itertools.repeat(value, n)).concat(self)
|
|
602
592
|
|
|
603
|
-
def pad_right(self: SeqT, n: int, value:
|
|
593
|
+
def pad_right[SeqT: Seq](self: SeqT, n: int, value: T) -> SeqT:
|
|
604
594
|
return self if n <= 0 else self.concat(type(self)(itertools.repeat(value, n)))
|
|
605
595
|
|
|
606
|
-
def pad_left_head(self: SeqT, n: int) -> SeqT:
|
|
596
|
+
def pad_left_head[SeqT: Seq](self: SeqT, n: int) -> SeqT:
|
|
607
597
|
return self if self.empty else self.pad_left(n, head(self.data))
|
|
608
598
|
|
|
609
|
-
def pad_right_last(self: SeqT, n: int) -> SeqT:
|
|
599
|
+
def pad_right_last[SeqT: Seq](self: SeqT, n: int) -> SeqT:
|
|
610
600
|
return self if self.empty else self.pad_right(n, last(self.data))
|
|
611
601
|
|
|
612
|
-
def take_left(self: SeqT, n: int) -> SeqT:
|
|
602
|
+
def take_left[SeqT: Seq](self: SeqT, n: int) -> SeqT:
|
|
613
603
|
if n <= 0:
|
|
614
604
|
return type(self)([])
|
|
615
605
|
return self[:n]
|
|
616
606
|
|
|
617
|
-
def take_right(self: SeqT, n: int) -> SeqT:
|
|
607
|
+
def take_right[SeqT: Seq](self: SeqT, n: int) -> SeqT:
|
|
618
608
|
if n <= 0:
|
|
619
609
|
return type(self)([])
|
|
620
610
|
return self[-n:]
|
|
621
611
|
|
|
622
612
|
take = take_left
|
|
623
613
|
|
|
624
|
-
def reverse(self: SeqT) -> SeqT:
|
|
614
|
+
def reverse[SeqT: Seq](self: SeqT) -> SeqT:
|
|
625
615
|
return type(self)(reversed(self.data))
|
|
626
616
|
|
|
627
|
-
def distinct(self: SeqT) -> SeqT:
|
|
617
|
+
def distinct[SeqT: Seq](self: SeqT) -> SeqT:
|
|
628
618
|
return type(self)(sorted(set(self.data)))
|
|
629
619
|
|
|
630
|
-
def scan_left(self, zero:
|
|
620
|
+
def scan_left[R](self, zero: R | None, func: Callable[[R, T], R]) -> "Seq[R]":
|
|
631
621
|
return Seq(scan_left(self.data, zero, func))
|
|
632
622
|
|
|
633
|
-
def scan_right(self, zero:
|
|
623
|
+
def scan_right[R](self, zero: R | None, func: Callable[[R, T], R]) -> "Seq[R]":
|
|
634
624
|
return Seq(scan_right(self.data, zero, func))
|
|
635
625
|
|
|
636
626
|
scan = scan_left
|
|
637
627
|
|
|
638
|
-
def map(self, func: Callable[[
|
|
628
|
+
def map[R](self, func: Callable[[T], R]) -> "Seq[R]":
|
|
639
629
|
return self.scan(None, lambda x, y: func(y))
|
|
640
630
|
|
|
641
|
-
def fold_left(self, zero:
|
|
631
|
+
def fold_left[R](self, zero: R | None, func: Callable[[R, T], R]) -> "Seq[R]":
|
|
642
632
|
return Seq(some(fold_left(self.data, zero, func)))
|
|
643
633
|
|
|
644
|
-
def fold_right(self, zero:
|
|
634
|
+
def fold_right[R](self, zero: R | None, func: Callable[[R, T], R]) -> "Seq[R]":
|
|
645
635
|
return Seq(some(fold_right(self.data, zero, func)))
|
|
646
636
|
|
|
647
637
|
fold = fold_left
|
|
648
638
|
|
|
649
|
-
def slide_left(
|
|
639
|
+
def slide_left[R](
|
|
650
640
|
self,
|
|
651
641
|
window: int,
|
|
652
|
-
func: Callable[[Iterable[
|
|
642
|
+
func: Callable[[Iterable[T]], R] = lambda x: x,
|
|
653
643
|
*,
|
|
654
644
|
allow_partial: tuple[bool, bool] = (False, False),
|
|
655
|
-
padding: tuple[
|
|
656
|
-
) -> "Seq[
|
|
645
|
+
padding: tuple[T, T] = (None, None),
|
|
646
|
+
) -> "Seq[R]":
|
|
657
647
|
return Seq(slide_left(self.data, window, func, allow_partial=allow_partial, padding=padding))
|
|
658
648
|
|
|
659
|
-
def slide_right(
|
|
649
|
+
def slide_right[R](
|
|
660
650
|
self,
|
|
661
651
|
window: int,
|
|
662
|
-
func: Callable[[Iterable[
|
|
652
|
+
func: Callable[[Iterable[T]], R] = lambda x: x,
|
|
663
653
|
*,
|
|
664
654
|
allow_partial: tuple[bool, bool] = (False, False),
|
|
665
|
-
padding: tuple[
|
|
666
|
-
) -> "Seq[
|
|
655
|
+
padding: tuple[T, T] = (None, None),
|
|
656
|
+
) -> "Seq[R]":
|
|
667
657
|
return Seq(slide_right(self.data, window, func, allow_partial=allow_partial, padding=padding))
|
|
668
658
|
|
|
669
659
|
slide = slide_left
|
|
670
660
|
|
|
671
|
-
def reduce(self: SeqT, func: Callable[[
|
|
661
|
+
def reduce[SeqT: Seq](self: SeqT, func: Callable[[T, T], T]) -> SeqT:
|
|
672
662
|
return type(self)(self.fold(None, lambda x, y: func(x, y)))
|
|
673
663
|
|
|
674
|
-
def max(self: SeqT, func: Callable[[
|
|
664
|
+
def max[SeqT: Seq](self: SeqT, func: Callable[[T, T], bool] = None) -> SeqT:
|
|
675
665
|
func = func or (lambda x, y: x > y)
|
|
676
666
|
return self.reduce(lambda x, y: x if func(x, y) else y)
|
|
677
667
|
|
|
678
|
-
def min(self: SeqT, func: Callable[[
|
|
668
|
+
def min[SeqT: Seq](self: SeqT, func: Callable[[T, T], bool] = None) -> SeqT:
|
|
679
669
|
func = func or (lambda x, y: x < y)
|
|
680
670
|
return self.reduce(lambda x, y: x if func(x, y) else y)
|
|
681
671
|
|
|
682
|
-
def group(self, func: Callable[[
|
|
672
|
+
def group[K](self, func: Callable[[T], K]) -> "Seq[tuple[K, list[T]]]":
|
|
683
673
|
return Seq(grouped(self.data, key_func=func, keys_ordered=True))
|
|
684
674
|
|
|
685
675
|
def keys(self):
|
|
@@ -691,63 +681,63 @@ class Seq(Generic[Tco], Sequence[Tco], Sized):
|
|
|
691
681
|
def swap(self):
|
|
692
682
|
return Seq((value, key) for key, value in self.data)
|
|
693
683
|
|
|
694
|
-
def map_keys(self, func: Callable[[
|
|
684
|
+
def map_keys[R](self, func: Callable[[T], R]):
|
|
695
685
|
return Seq((func(key), value) for key, value in self.data)
|
|
696
686
|
|
|
697
|
-
def map_values(self, func: Callable[[
|
|
687
|
+
def map_values[R](self, func: Callable[[T], R]):
|
|
698
688
|
return Seq((key, func(value)) for key, value in self.data)
|
|
699
689
|
|
|
700
|
-
def flat_map(self, func: Callable[[
|
|
690
|
+
def flat_map[R](self, func: Callable[[T], Iterable[R]]) -> "Seq[R]":
|
|
701
691
|
return Seq(flatten(map(lambda x: func(x), self.data)))
|
|
702
692
|
|
|
703
693
|
def flatten(self):
|
|
704
694
|
return self.flat_map(lambda x: list(x))
|
|
705
695
|
|
|
706
|
-
def group_map(self, group_func: Callable[[
|
|
696
|
+
def group_map[K, R](self, group_func: Callable[[T], K], map_func: Callable[[T], R]) -> "Seq[tuple[K, list[R]]]":
|
|
707
697
|
return self.group(group_func).map_values(lambda x: list(map(map_func, x)))
|
|
708
698
|
|
|
709
|
-
def filter(self: SeqT, func: Callable[[
|
|
699
|
+
def filter[SeqT: Seq](self: SeqT, func: Callable[[T], bool]) -> SeqT:
|
|
710
700
|
return type(self)(filter(func, self.data))
|
|
711
701
|
|
|
712
|
-
def filter_not(self: SeqT, func: Callable[[
|
|
702
|
+
def filter_not[SeqT: Seq](self: SeqT, func: Callable[[T], bool]) -> SeqT:
|
|
713
703
|
return self.filter(lambda x: not func(x))
|
|
714
704
|
|
|
715
|
-
def sort(self: SeqT, func: Callable[[
|
|
705
|
+
def sort[SeqT: Seq, K](self: SeqT, func: Callable[[T], K]) -> SeqT:
|
|
716
706
|
return type(self)(sorted(self.data, key=func))
|
|
717
707
|
|
|
718
|
-
def head(self: SeqT) -> SeqT:
|
|
708
|
+
def head[SeqT: Seq](self: SeqT) -> SeqT:
|
|
719
709
|
return type(self)(some(head_or_none(self.data)))
|
|
720
710
|
|
|
721
|
-
def last(self: SeqT) -> SeqT:
|
|
711
|
+
def last[SeqT: Seq](self: SeqT) -> SeqT:
|
|
722
712
|
return type(self)(some(last_or_none(self.data)))
|
|
723
713
|
|
|
724
|
-
def init(self: SeqT) -> SeqT:
|
|
714
|
+
def init[SeqT: Seq](self: SeqT) -> SeqT:
|
|
725
715
|
return type(self)(init_iter(self.data))
|
|
726
716
|
|
|
727
|
-
def tail(self: SeqT) -> SeqT:
|
|
717
|
+
def tail[SeqT: Seq](self: SeqT) -> SeqT:
|
|
728
718
|
return type(self)(tail_iter(self.data))
|
|
729
719
|
|
|
730
|
-
def foreach(self: SeqT, func: Callable[[
|
|
720
|
+
def foreach[SeqT: Seq](self: SeqT, func: Callable[[T], None]) -> SeqT:
|
|
731
721
|
for elem in self.data:
|
|
732
722
|
func(elem)
|
|
733
723
|
return self
|
|
734
724
|
|
|
735
|
-
def exists(self, func: Callable[[
|
|
725
|
+
def exists(self, func: Callable[[T], bool]) -> "Seq[bool]":
|
|
736
726
|
return Seq(some(any(map(func, self.data))))
|
|
737
727
|
|
|
738
|
-
def forall(self, func: Callable[[
|
|
728
|
+
def forall(self, func: Callable[[T], bool]) -> "Seq[bool]":
|
|
739
729
|
return Seq(some(all(map(func, self.data))))
|
|
740
730
|
|
|
741
|
-
def union(self: SeqT, other: SeqT) -> SeqT:
|
|
731
|
+
def union[SeqT: Seq](self: SeqT, other: SeqT) -> SeqT:
|
|
742
732
|
return type(self)(sorted(set(self.data).union(set(other.data))))
|
|
743
733
|
|
|
744
|
-
def intersect(self: SeqT, other: SeqT) -> SeqT:
|
|
734
|
+
def intersect[SeqT: Seq](self: SeqT, other: SeqT) -> SeqT:
|
|
745
735
|
return type(self)(sorted(set(self.data).intersection(set(other.data))))
|
|
746
736
|
|
|
747
|
-
def zip(self, other: "Seq[
|
|
737
|
+
def zip[U](self, other: "Seq[U]") -> "Seq[tuple[T, U]]":
|
|
748
738
|
return Seq(zip(self.data, other.data))
|
|
749
739
|
|
|
750
|
-
def zip_fill(self, other: "Seq[
|
|
740
|
+
def zip_fill[U](self, other: "Seq[U]", fill: T | U | None = None) -> "Seq[tuple[T, U]]":
|
|
751
741
|
return Seq(itertools.zip_longest(self.data, other.data, fillvalue=fill))
|
|
752
742
|
|
|
753
743
|
|
iker/common/utils/shutils.py
CHANGED
|
@@ -95,7 +95,11 @@ def path_depth(root: str, child: str) -> int:
|
|
|
95
95
|
return child_expanded[len(root_expanded):].count(os.sep)
|
|
96
96
|
|
|
97
97
|
|
|
98
|
-
def glob_match(
|
|
98
|
+
def glob_match(
|
|
99
|
+
names: list[str],
|
|
100
|
+
include_patterns: list[str] | None = None,
|
|
101
|
+
exclude_patterns: list[str] | None = None,
|
|
102
|
+
) -> list[str]:
|
|
99
103
|
"""
|
|
100
104
|
Applies the given inclusive and exclusive glob patterns to the given ``names`` and returns the filtered result.
|
|
101
105
|
|
|
@@ -121,8 +125,8 @@ class CopyFuncProtocol(Protocol):
|
|
|
121
125
|
def listfile(
|
|
122
126
|
path: str,
|
|
123
127
|
*,
|
|
124
|
-
include_patterns: list[str] = None,
|
|
125
|
-
exclude_patterns: list[str] = None,
|
|
128
|
+
include_patterns: list[str] | None = None,
|
|
129
|
+
exclude_patterns: list[str] | None = None,
|
|
126
130
|
depth: int = 0,
|
|
127
131
|
) -> list[str]:
|
|
128
132
|
"""
|
|
@@ -153,8 +157,8 @@ def copy(
|
|
|
153
157
|
src: str,
|
|
154
158
|
dst: str,
|
|
155
159
|
*,
|
|
156
|
-
include_patterns: list[str] = None,
|
|
157
|
-
exclude_patterns: list[str] = None,
|
|
160
|
+
include_patterns: list[str] | None = None,
|
|
161
|
+
exclude_patterns: list[str] | None = None,
|
|
158
162
|
depth: int = 0,
|
|
159
163
|
follow_symlinks: bool = False,
|
|
160
164
|
ignore_dangling_symlinks: bool = False,
|
iker/common/utils/span.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import enum
|
|
2
2
|
import itertools
|
|
3
3
|
from collections.abc import Sequence
|
|
4
|
-
from typing import TypeVar
|
|
5
4
|
|
|
6
5
|
__all__ = [
|
|
7
6
|
"SpanRelation",
|
|
@@ -11,8 +10,6 @@ __all__ = [
|
|
|
11
10
|
"spans_subtract",
|
|
12
11
|
]
|
|
13
12
|
|
|
14
|
-
T = TypeVar("T")
|
|
15
|
-
|
|
16
13
|
|
|
17
14
|
class SpanRelation(enum.IntEnum):
|
|
18
15
|
LeftIn = 0x1
|
|
@@ -69,7 +66,7 @@ def span_relation(a: tuple[float, float], b: tuple[float, float]) -> int:
|
|
|
69
66
|
return rel
|
|
70
67
|
|
|
71
68
|
|
|
72
|
-
def spans_union(a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tuple[T, T]]:
|
|
69
|
+
def spans_union[T](a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tuple[T, T]]:
|
|
73
70
|
"""
|
|
74
71
|
Computes the union of the given span lists. The spans in each of the lists must be sorted and must not
|
|
75
72
|
mutually overlap.
|
|
@@ -123,7 +120,7 @@ def spans_union(a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tu
|
|
|
123
120
|
return a
|
|
124
121
|
|
|
125
122
|
|
|
126
|
-
def spans_intersect(a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tuple[T, T]]:
|
|
123
|
+
def spans_intersect[T](a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tuple[T, T]]:
|
|
127
124
|
"""
|
|
128
125
|
Computes the intersection of the given span lists. The spans in each of the lists must be sorted and must not
|
|
129
126
|
mutually overlap.
|
|
@@ -161,7 +158,7 @@ def spans_intersect(a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> lis
|
|
|
161
158
|
return a
|
|
162
159
|
|
|
163
160
|
|
|
164
|
-
def spans_subtract(a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tuple[T, T]]:
|
|
161
|
+
def spans_subtract[T](a: Sequence[tuple[T, T]], *bs: Sequence[tuple[T, T]]) -> list[tuple[T, T]]:
|
|
165
162
|
"""
|
|
166
163
|
Computes the subtraction on the first span list by the remaining span lists. The spans in each of the lists must be
|
|
167
164
|
sorted and must not mutually overlap.
|
|
@@ -1,27 +1,22 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: iker-python-common
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.59
|
|
4
4
|
Classifier: Programming Language :: Python :: 3
|
|
5
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
6
5
|
Classifier: Programming Language :: Python :: 3.12
|
|
7
6
|
Classifier: Programming Language :: Python :: 3.13
|
|
8
7
|
Classifier: Programming Language :: Python :: 3.14
|
|
9
|
-
Requires-Python: <3.15,>=3.
|
|
10
|
-
Requires-Dist: boto3>=1.35
|
|
8
|
+
Requires-Python: <3.15,>=3.12
|
|
11
9
|
Requires-Dist: docker>=7.1
|
|
12
|
-
Requires-Dist: numpy>=
|
|
10
|
+
Requires-Dist: numpy>=2.3
|
|
13
11
|
Requires-Dist: psycopg>=3.2
|
|
14
12
|
Requires-Dist: pymysql>=1.1
|
|
15
|
-
Requires-Dist: sqlalchemy>=
|
|
16
|
-
Requires-Dist: typing-extensions; python_version < "3.12"
|
|
13
|
+
Requires-Dist: sqlalchemy>=2.0
|
|
17
14
|
Provides-Extra: all
|
|
18
15
|
Requires-Dist: iker-python-common; extra == "all"
|
|
19
16
|
Provides-Extra: test
|
|
20
17
|
Requires-Dist: ddt>=1.7; extra == "test"
|
|
21
|
-
Requires-Dist: moto[all,ec2,s3]>=5.0; extra == "test"
|
|
22
18
|
Requires-Dist: pytest-cov>=5.0; extra == "test"
|
|
23
19
|
Requires-Dist: pytest-mysql>=3.0; extra == "test"
|
|
24
20
|
Requires-Dist: pytest-order>=1.3; extra == "test"
|
|
25
21
|
Requires-Dist: pytest-postgresql>=6.1; extra == "test"
|
|
26
22
|
Requires-Dist: pytest>=8.3; extra == "test"
|
|
27
|
-
Requires-Dist: sqlalchemy>=2.0; extra == "test"
|
|
@@ -6,20 +6,19 @@ iker/common/utils/csv.py,sha256=_V9OUrKcojec2L-hWagEIVnL2uvGjyJAFTrD7tHNr48,7573
|
|
|
6
6
|
iker/common/utils/dbutils.py,sha256=zXZVJCz7HZPityFRF7sHRRMpMraegV_hyYnzApUUPhY,11852
|
|
7
7
|
iker/common/utils/dockerutils.py,sha256=n2WuzXaZB6_WocSljvPOnfExSIjIHRUbuWp2oBbaPKQ,8004
|
|
8
8
|
iker/common/utils/dtutils.py,sha256=86vbaa4pgcBWERZvTfJ92PKB3IimxP6tf0O11ho2Ffk,12554
|
|
9
|
-
iker/common/utils/funcutils.py,sha256=
|
|
10
|
-
iker/common/utils/jsonutils.py,sha256=
|
|
9
|
+
iker/common/utils/funcutils.py,sha256=hHk7UHI-h92F8CFff-SZGvfhSlZ-CkO2dI8GWa6V81Q,5734
|
|
10
|
+
iker/common/utils/jsonutils.py,sha256=Df4uxEgIejEhZdrfmiT4DX2aewEX0Pqbl1n4OCN4YdI,15970
|
|
11
11
|
iker/common/utils/logger.py,sha256=FJaai6Sbchy4wKHcUMUCrrkBcXvIxq4qByERZ_TJBps,3881
|
|
12
|
-
iker/common/utils/numutils.py,sha256=
|
|
13
|
-
iker/common/utils/randutils.py,sha256=
|
|
12
|
+
iker/common/utils/numutils.py,sha256=p6Rz1qyCcUru3v1zDy2PM-nds2NWJdL5A_vLmG-kswk,4294
|
|
13
|
+
iker/common/utils/randutils.py,sha256=Sxf852B18CJ-MfrEDsv1ROO_brmz79dRZ4jpJiH65v4,12843
|
|
14
14
|
iker/common/utils/retry.py,sha256=H9lR6pp_jzgOwKTM-dOWIddjTlQbK-ijcwuDmVvurZM,8938
|
|
15
|
-
iker/common/utils/
|
|
16
|
-
iker/common/utils/
|
|
17
|
-
iker/common/utils/
|
|
18
|
-
iker/common/utils/span.py,sha256=yiXqk86cLKxkMdkO3pAHEfU5bUvHsGo3p--pAWo_yfM,5999
|
|
15
|
+
iker/common/utils/sequtils.py,sha256=Wc8RcbNjVYSJYZv_07SOKWfYjhmGWz9_RXWbG2-tE1o,25060
|
|
16
|
+
iker/common/utils/shutils.py,sha256=dUm1Y7m8u1Ri_R5598oQJsxwgQaBnVzhtpcsL7_Vzp0,7916
|
|
17
|
+
iker/common/utils/span.py,sha256=u_KuWi2U7QDMUotl4AeW2_57ItL3YhVDSeCwaOiFDvs,5963
|
|
19
18
|
iker/common/utils/strutils.py,sha256=Tu_qFeH3K-SfwvMxdrZAc9iLPV8ZmtX4ntyyFGNslf8,5094
|
|
20
19
|
iker/common/utils/testutils.py,sha256=2VieV5yeCDntSKQSpIeyqRT8BZmZYE_ArMeQz3g7fXY,5568
|
|
21
20
|
iker/common/utils/typeutils.py,sha256=RVkYkFRgDrx77OHFH7PavMV0AIB0S8ly40rs4g7JWE4,8220
|
|
22
|
-
iker_python_common-1.0.
|
|
23
|
-
iker_python_common-1.0.
|
|
24
|
-
iker_python_common-1.0.
|
|
25
|
-
iker_python_common-1.0.
|
|
21
|
+
iker_python_common-1.0.59.dist-info/METADATA,sha256=C3LhE0abnoE8e5KeGHrKj2DmN8RCL8fP7M9aWY6KVE0,813
|
|
22
|
+
iker_python_common-1.0.59.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
23
|
+
iker_python_common-1.0.59.dist-info/top_level.txt,sha256=4_B8Prfc_lxFafFYTQThIU1ZqOYQ4pHHHnJ_fQ_oHs8,5
|
|
24
|
+
iker_python_common-1.0.59.dist-info/RECORD,,
|
iker/common/utils/s3utils.py
DELETED
|
@@ -1,270 +0,0 @@
|
|
|
1
|
-
import concurrent.futures
|
|
2
|
-
import contextlib
|
|
3
|
-
import dataclasses
|
|
4
|
-
import datetime
|
|
5
|
-
import mimetypes
|
|
6
|
-
import os
|
|
7
|
-
import tempfile
|
|
8
|
-
|
|
9
|
-
import boto3
|
|
10
|
-
from botocore.client import BaseClient
|
|
11
|
-
|
|
12
|
-
from iker.common.utils.shutils import glob_match, listfile, path_depth
|
|
13
|
-
from iker.common.utils.strutils import is_empty, trim_to_none
|
|
14
|
-
|
|
15
|
-
__all__ = [
|
|
16
|
-
"S3ObjectMeta",
|
|
17
|
-
"s3_make_client",
|
|
18
|
-
"s3_list_objects",
|
|
19
|
-
"s3_listfile",
|
|
20
|
-
"s3_cp_download",
|
|
21
|
-
"s3_cp_upload",
|
|
22
|
-
"s3_sync_download",
|
|
23
|
-
"s3_sync_upload",
|
|
24
|
-
"s3_pull_text",
|
|
25
|
-
"s3_push_text",
|
|
26
|
-
]
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
@dataclasses.dataclass
|
|
30
|
-
class S3ObjectMeta(object):
|
|
31
|
-
key: str
|
|
32
|
-
last_modified: datetime.datetime
|
|
33
|
-
size: int
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
def s3_make_client(
|
|
37
|
-
access_key_id: str = None,
|
|
38
|
-
secret_access_key: str = None,
|
|
39
|
-
region_name: str = None,
|
|
40
|
-
endpoint_url: str = None,
|
|
41
|
-
) -> contextlib.AbstractContextManager[BaseClient]:
|
|
42
|
-
"""
|
|
43
|
-
Creates an AWS S3 client as a context manager for safe resource handling.
|
|
44
|
-
|
|
45
|
-
:param access_key_id: AWS access key ID.
|
|
46
|
-
:param secret_access_key: AWS secret access key.
|
|
47
|
-
:param region_name: AWS service region name.
|
|
48
|
-
:param endpoint_url: AWS service endpoint URL.
|
|
49
|
-
:return: A context manager yielding an S3 client instance.
|
|
50
|
-
"""
|
|
51
|
-
client = boto3.client("s3",
|
|
52
|
-
region_name=trim_to_none(region_name),
|
|
53
|
-
endpoint_url=trim_to_none(endpoint_url),
|
|
54
|
-
aws_access_key_id=trim_to_none(access_key_id),
|
|
55
|
-
aws_secret_access_key=trim_to_none(secret_access_key))
|
|
56
|
-
return contextlib.closing(client)
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
def s3_list_objects(client: BaseClient, bucket: str, prefix: str, limit: int = None) -> list[S3ObjectMeta]:
|
|
60
|
-
"""
|
|
61
|
-
Lists all objects from the given S3 ``bucket`` and ``prefix``.
|
|
62
|
-
|
|
63
|
-
:param client: AWS S3 client instance.
|
|
64
|
-
:param bucket: Bucket name.
|
|
65
|
-
:param prefix: Object keys prefix.
|
|
66
|
-
:param limit: Maximum number of objects to return (``None`` for all).
|
|
67
|
-
:return: List of ``S3ObjectMeta`` items.
|
|
68
|
-
"""
|
|
69
|
-
entries = []
|
|
70
|
-
|
|
71
|
-
next_marker = None
|
|
72
|
-
while True:
|
|
73
|
-
if is_empty(next_marker):
|
|
74
|
-
response = client.list_objects(MaxKeys=1000, Bucket=bucket, Prefix=prefix)
|
|
75
|
-
else:
|
|
76
|
-
response = client.list_objects(MaxKeys=1000, Bucket=bucket, Prefix=prefix, Marker=next_marker)
|
|
77
|
-
|
|
78
|
-
entries.extend(response.get("Contents", []))
|
|
79
|
-
|
|
80
|
-
if limit is not None and len(entries) >= limit:
|
|
81
|
-
entries = entries[:limit]
|
|
82
|
-
|
|
83
|
-
if not response.get("IsTruncated"):
|
|
84
|
-
break
|
|
85
|
-
|
|
86
|
-
next_marker = response.get("NextMarker")
|
|
87
|
-
if is_empty(next_marker):
|
|
88
|
-
next_marker = entries[-1]["Key"]
|
|
89
|
-
|
|
90
|
-
return [S3ObjectMeta(key=e["Key"], last_modified=e["LastModified"], size=e["Size"]) for e in entries]
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
def s3_listfile(
|
|
94
|
-
client: BaseClient,
|
|
95
|
-
bucket: str,
|
|
96
|
-
prefix: str,
|
|
97
|
-
*,
|
|
98
|
-
include_patterns: list[str] = None,
|
|
99
|
-
exclude_patterns: list[str] = None,
|
|
100
|
-
depth: int = 0,
|
|
101
|
-
) -> list[S3ObjectMeta]:
|
|
102
|
-
"""
|
|
103
|
-
Lists all objects from the given S3 ``bucket`` and ``prefix``, filtered by patterns and directory depth.
|
|
104
|
-
|
|
105
|
-
:param client: AWS S3 client instance.
|
|
106
|
-
:param bucket: Bucket name.
|
|
107
|
-
:param prefix: Object keys prefix.
|
|
108
|
-
:param include_patterns: Inclusive glob patterns applied to filenames.
|
|
109
|
-
:param exclude_patterns: Exclusive glob patterns applied to filenames.
|
|
110
|
-
:param depth: Maximum depth of subdirectories to include in the scan.
|
|
111
|
-
:return: List of ``S3ObjectMeta`` items.
|
|
112
|
-
"""
|
|
113
|
-
|
|
114
|
-
# We add trailing slash "/" to the prefix if it is absent
|
|
115
|
-
if not prefix.endswith("/"):
|
|
116
|
-
prefix = prefix + "/"
|
|
117
|
-
|
|
118
|
-
objects = s3_list_objects(client, bucket, prefix)
|
|
119
|
-
|
|
120
|
-
def filter_object_meta(object_meta: S3ObjectMeta) -> bool:
|
|
121
|
-
if 0 < depth <= path_depth(prefix, os.path.dirname(object_meta.key)):
|
|
122
|
-
return False
|
|
123
|
-
if len(glob_match([os.path.basename(object_meta.key)], include_patterns, exclude_patterns)) == 0:
|
|
124
|
-
return False
|
|
125
|
-
return True
|
|
126
|
-
|
|
127
|
-
return list(filter(filter_object_meta, objects))
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def s3_cp_download(client: BaseClient, bucket: str, key: str, file_path: str):
|
|
131
|
-
"""
|
|
132
|
-
Downloads an object from the given S3 ``bucket`` and ``key`` to a local file path.
|
|
133
|
-
|
|
134
|
-
:param client: AWS S3 client instance.
|
|
135
|
-
:param bucket: Bucket name.
|
|
136
|
-
:param key: Object key.
|
|
137
|
-
:param file_path: Local file path to save the object.
|
|
138
|
-
"""
|
|
139
|
-
client.download_file(bucket, key, file_path)
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
def s3_cp_upload(client: BaseClient, file_path: str, bucket: str, key: str):
|
|
143
|
-
"""
|
|
144
|
-
Uploads a local file to the given S3 ``bucket`` and ``key``.
|
|
145
|
-
|
|
146
|
-
:param client: AWS S3 client instance.
|
|
147
|
-
:param file_path: Local file path to upload.
|
|
148
|
-
:param bucket: Bucket name.
|
|
149
|
-
:param key: Object key for the uploaded file.
|
|
150
|
-
"""
|
|
151
|
-
t, _ = mimetypes.MimeTypes().guess_type(file_path)
|
|
152
|
-
client.upload_file(file_path, bucket, key, ExtraArgs={"ContentType": "binary/octet-stream" if t is None else t})
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
def s3_sync_download(
|
|
156
|
-
client: BaseClient,
|
|
157
|
-
bucket: str,
|
|
158
|
-
prefix: str,
|
|
159
|
-
dir_path: str,
|
|
160
|
-
*,
|
|
161
|
-
max_workers: int = None,
|
|
162
|
-
include_patterns: list[str] = None,
|
|
163
|
-
exclude_patterns: list[str] = None,
|
|
164
|
-
depth: int = 0,
|
|
165
|
-
):
|
|
166
|
-
"""
|
|
167
|
-
Recursively downloads all objects from the given S3 ``bucket`` and ``prefix`` to a local directory path, using a thread pool.
|
|
168
|
-
|
|
169
|
-
:param client: AWS S3 client instance.
|
|
170
|
-
:param bucket: Bucket name.
|
|
171
|
-
:param prefix: Object keys prefix.
|
|
172
|
-
:param dir_path: Local directory path to save objects.
|
|
173
|
-
:param max_workers: Maximum number of worker threads.
|
|
174
|
-
:param include_patterns: Inclusive glob patterns applied to filenames.
|
|
175
|
-
:param exclude_patterns: Exclusive glob patterns applied to filenames.
|
|
176
|
-
:param depth: Maximum depth of subdirectories to include in the scan.
|
|
177
|
-
"""
|
|
178
|
-
|
|
179
|
-
# We add trailing slash "/" to the prefix if it is absent
|
|
180
|
-
if not prefix.endswith("/"):
|
|
181
|
-
prefix = prefix + "/"
|
|
182
|
-
|
|
183
|
-
objects = s3_listfile(client,
|
|
184
|
-
bucket,
|
|
185
|
-
prefix,
|
|
186
|
-
include_patterns=include_patterns,
|
|
187
|
-
exclude_patterns=exclude_patterns,
|
|
188
|
-
depth=depth)
|
|
189
|
-
|
|
190
|
-
def download_file(key: str):
|
|
191
|
-
file_path = os.path.join(dir_path, key[len(prefix):])
|
|
192
|
-
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
|
193
|
-
s3_cp_download(client, bucket, key, file_path)
|
|
194
|
-
|
|
195
|
-
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
196
|
-
concurrent.futures.wait([executor.submit(download_file, obj.key) for obj in objects],
|
|
197
|
-
return_when=concurrent.futures.FIRST_EXCEPTION)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
def s3_sync_upload(
|
|
201
|
-
client: BaseClient,
|
|
202
|
-
dir_path: str,
|
|
203
|
-
bucket: str,
|
|
204
|
-
prefix: str,
|
|
205
|
-
*,
|
|
206
|
-
max_workers: int = None,
|
|
207
|
-
include_patterns: list[str] = None,
|
|
208
|
-
exclude_patterns: list[str] = None,
|
|
209
|
-
depth: int = 0,
|
|
210
|
-
):
|
|
211
|
-
"""
|
|
212
|
-
Recursively uploads all files from a local directory to the given S3 ``bucket`` and ``prefix``, using a thread pool.
|
|
213
|
-
|
|
214
|
-
:param client: AWS S3 client instance.
|
|
215
|
-
:param dir_path: Local directory path to upload from.
|
|
216
|
-
:param bucket: Bucket name.
|
|
217
|
-
:param prefix: Object keys prefix for uploaded files.
|
|
218
|
-
:param max_workers: Maximum number of worker threads.
|
|
219
|
-
:param include_patterns: Inclusive glob patterns applied to filenames.
|
|
220
|
-
:param exclude_patterns: Exclusive glob patterns applied to filenames.
|
|
221
|
-
:param depth: Maximum depth of subdirectories to include in the scan.
|
|
222
|
-
"""
|
|
223
|
-
|
|
224
|
-
# We add trailing slash "/" to the prefix if it is absent
|
|
225
|
-
if not prefix.endswith("/"):
|
|
226
|
-
prefix = prefix + "/"
|
|
227
|
-
|
|
228
|
-
file_paths = listfile(dir_path,
|
|
229
|
-
include_patterns=include_patterns,
|
|
230
|
-
exclude_patterns=exclude_patterns,
|
|
231
|
-
depth=depth)
|
|
232
|
-
|
|
233
|
-
def upload_file(file_path: str):
|
|
234
|
-
s3_cp_upload(client, file_path, bucket, prefix + os.path.relpath(file_path, dir_path))
|
|
235
|
-
|
|
236
|
-
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
237
|
-
concurrent.futures.wait([executor.submit(upload_file, file_path) for file_path in file_paths],
|
|
238
|
-
return_when=concurrent.futures.FIRST_EXCEPTION)
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
def s3_pull_text(client: BaseClient, bucket: str, key: str, encoding: str = None) -> str:
|
|
242
|
-
"""
|
|
243
|
-
Downloads and decodes text content stored as an object in the given S3 ``bucket`` and ``key``.
|
|
244
|
-
|
|
245
|
-
:param client: AWS S3 client instance.
|
|
246
|
-
:param bucket: Bucket name.
|
|
247
|
-
:param key: Object key storing the text.
|
|
248
|
-
:param encoding: String encoding to use (defaults to UTF-8).
|
|
249
|
-
:return: The decoded text content.
|
|
250
|
-
"""
|
|
251
|
-
with tempfile.TemporaryFile() as fp:
|
|
252
|
-
client.download_fileobj(bucket, key, fp)
|
|
253
|
-
fp.seek(0)
|
|
254
|
-
return fp.read().decode(encoding or "utf-8")
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
def s3_push_text(client: BaseClient, text: str, bucket: str, key: str, encoding: str = None):
|
|
258
|
-
"""
|
|
259
|
-
Uploads the given text as an object to the specified S3 ``bucket`` and ``key``.
|
|
260
|
-
|
|
261
|
-
:param client: AWS S3 client instance.
|
|
262
|
-
:param text: Text content to upload.
|
|
263
|
-
:param bucket: Bucket name.
|
|
264
|
-
:param key: Object key to store the text.
|
|
265
|
-
:param encoding: String encoding to use (defaults to UTF-8).
|
|
266
|
-
"""
|
|
267
|
-
with tempfile.TemporaryFile() as fp:
|
|
268
|
-
fp.write(text.encode(encoding or "utf-8"))
|
|
269
|
-
fp.seek(0)
|
|
270
|
-
client.upload_fileobj(fp, bucket, key)
|
|
File without changes
|
|
File without changes
|