ialdev-core 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iad/core/__init__.py +9 -0
- iad/core/array.py +1961 -0
- iad/core/binary.py +377 -0
- iad/core/cache.py +903 -0
- iad/core/codetools.py +203 -0
- iad/core/datatools.py +671 -0
- iad/core/docs/locators.ipynb +754 -0
- iad/core/dotstyle.py +99 -0
- iad/core/env.py +271 -0
- iad/core/events.py +650 -0
- iad/core/filesproc.py +1046 -0
- iad/core/fnctools.py +390 -0
- iad/core/label.py +240 -0
- iad/core/logs.py +182 -0
- iad/core/nptools.py +449 -0
- iad/core/one_dark.puml +881 -0
- iad/core/param/__init__.py +17 -0
- iad/core/param/confargparse.py +55 -0
- iad/core/param/paramaze.py +339 -0
- iad/core/param/tbox.py +277 -0
- iad/core/paths.py +563 -0
- iad/core/pdtools.py +2570 -0
- iad/core/pydantools/__init__.py +5 -0
- iad/core/pydantools/fixed_pydantic_yaml/__init__.py +32 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/__init__.py +0 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/hacks.py +76 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/old_enums.py +37 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/representers.py +92 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/types.py +122 -0
- iad/core/pydantools/fixed_pydantic_yaml/compat/yaml_lib.py +104 -0
- iad/core/pydantools/fixed_pydantic_yaml/ext/__init__.py +1 -0
- iad/core/pydantools/fixed_pydantic_yaml/ext/semver.py +152 -0
- iad/core/pydantools/fixed_pydantic_yaml/ext/versioned_model.py +113 -0
- iad/core/pydantools/fixed_pydantic_yaml/main.py +30 -0
- iad/core/pydantools/fixed_pydantic_yaml/mixin.py +281 -0
- iad/core/pydantools/fixed_pydantic_yaml/model.py +20 -0
- iad/core/pydantools/fixed_pydantic_yaml/py.typed +1 -0
- iad/core/pydantools/fixed_pydantic_yaml/version.py +1 -0
- iad/core/pydantools/models.py +560 -0
- iad/core/regexp.py +348 -0
- iad/core/short.py +308 -0
- iad/core/strings.py +635 -0
- iad/core/unc_panda.py +270 -0
- iad/core/units.py +58 -0
- iad/core/wrap.py +420 -0
- ialdev_core-0.1.0.dist-info/METADATA +73 -0
- ialdev_core-0.1.0.dist-info/RECORD +48 -0
- ialdev_core-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
from frozendict import frozendict as fzdict
|
|
2
|
+
|
|
3
|
+
from . import paramaze as pa
|
|
4
|
+
from .tbox import TBox
|
|
5
|
+
|
|
6
|
+
__all__ = ['TBox', 'model_arguments', 'YamlModel', 'fzdict']
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def __getattr__(name: str):
|
|
10
|
+
"""Lazy imports for YAML/Pydantic helpers shipped in ``iad.core.pydantools``."""
|
|
11
|
+
if name == 'YamlModel':
|
|
12
|
+
from iad.core.pydantools.models import YamlModel as _YamlModel
|
|
13
|
+
return _YamlModel
|
|
14
|
+
if name == 'model_arguments':
|
|
15
|
+
from iad.core.pydantools.models import model_arguments as _model_arguments
|
|
16
|
+
return _model_arguments
|
|
17
|
+
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import configargparse as cap
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class ArgumentParserYAML(cap.ArgumentParser):
|
|
5
|
+
def __init__(self, *, hash_groups=None, add_env_var_help=False, **kws):
|
|
6
|
+
super().__init__(config_file_parser_class=cap.YAMLConfigFileParser,
|
|
7
|
+
add_env_var_help=add_env_var_help,
|
|
8
|
+
**kws)
|
|
9
|
+
self.hash_groups = hash_groups or {}
|
|
10
|
+
|
|
11
|
+
def save_config(self, namespace, config_file):
|
|
12
|
+
if BUG_FIXED := False: # TODO: write_config_file prints all values as str!
|
|
13
|
+
self.write_config_file(namespace, [str(config_file)])
|
|
14
|
+
else:
|
|
15
|
+
op_map = {op.dest: op.option_strings[-1] for op in self._get_optional_actions()}
|
|
16
|
+
|
|
17
|
+
def items_to_save():
|
|
18
|
+
for k, v in namespace.__dict__.items():
|
|
19
|
+
if not (v is None or k in ('cmd', 'config')):
|
|
20
|
+
yield op_map[k].strip('-'), v
|
|
21
|
+
|
|
22
|
+
file_contents = self._config_file_parser.serialize(items_to_save())
|
|
23
|
+
with self._config_file_open_func(config_file, "w") as output_file:
|
|
24
|
+
output_file.write(file_contents)
|
|
25
|
+
print(f"Config file saved: {config_file}")
|
|
26
|
+
|
|
27
|
+
def hashes(self, ns: cap.Namespace, length=4) -> dict:
|
|
28
|
+
"""
|
|
29
|
+
Calculate hash values for the defined groups of arguments.
|
|
30
|
+
Return dict with hashes py group.
|
|
31
|
+
If groups are not defined the namespace is used and named 'cfg'.
|
|
32
|
+
|
|
33
|
+
:param ns: Namespace with parsed arguments
|
|
34
|
+
:param length: length of hash string tail to used (0 - all of it)
|
|
35
|
+
|
|
36
|
+
:return: dict {grp_name: hash_str}
|
|
37
|
+
"""
|
|
38
|
+
from .tbox import TBox
|
|
39
|
+
groups = self.hash_groups or {'cfg': [*vars(ns)]}
|
|
40
|
+
ns = vars(ns)
|
|
41
|
+
|
|
42
|
+
def hash_group(names):
|
|
43
|
+
group_params = {name: ns[name] for name in names if name in ns}
|
|
44
|
+
return TBox(group_params).hash_str(length)
|
|
45
|
+
|
|
46
|
+
return {grp: hash_group(names) for grp, names in groups.items()}
|
|
47
|
+
|
|
48
|
+
def hash_stamp(self, ns: cap.Namespace, length=4) -> str:
|
|
49
|
+
"""
|
|
50
|
+
Hash stamp string unique for the ns content by arguments groups.
|
|
51
|
+
:param ns: namespace with arguments
|
|
52
|
+
:param length: length of hash strings
|
|
53
|
+
:return: "grp1_hash1_grp2_hash2_..."
|
|
54
|
+
"""
|
|
55
|
+
return '_'.join(map(lambda it: f"{it[0]}_{it[1]}", self.hashes(ns, length).items()))
|
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
import inspect
|
|
2
|
+
from contextlib import contextmanager
|
|
3
|
+
from enum import IntEnum, unique
|
|
4
|
+
from typing import Union, Callable, Optional, Dict, Sequence
|
|
5
|
+
|
|
6
|
+
from .. import as_list, fnctools as fnt, wrap
|
|
7
|
+
from .tbox import TBox
|
|
8
|
+
|
|
9
|
+
Rule = Union[Callable, str, None]
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def sub(*args, **kws):
|
|
13
|
+
func_defs = {f.paramaze.alias: f.paramaze.defaults for f in args}
|
|
14
|
+
return TBox(frozen_box=True, **func_defs, **kws)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@unique
|
|
18
|
+
class Validate(IntEnum):
|
|
19
|
+
OFF = 0 # no validation
|
|
20
|
+
REPORT = 1 # report about errors but not raise
|
|
21
|
+
RAISE = 2 # validate, but not report - only raise on errors
|
|
22
|
+
VERBOSE = 3 # raise on errors
|
|
23
|
+
|
|
24
|
+
@classmethod
|
|
25
|
+
@contextmanager
|
|
26
|
+
def level(cls, vld_level: Union['Validate', str]):
|
|
27
|
+
"""
|
|
28
|
+
Create validation context to execute code with specific validation level
|
|
29
|
+
and restore the current one on exit
|
|
30
|
+
:param vld_level: as name str or `Validate` object
|
|
31
|
+
"""
|
|
32
|
+
if isinstance(vld_level, str):
|
|
33
|
+
vld = Validate[vld_level]
|
|
34
|
+
old_vld = FuncPar.validation(vld)
|
|
35
|
+
try:
|
|
36
|
+
yield vld
|
|
37
|
+
finally:
|
|
38
|
+
FuncPar.validation(old_vld)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class FuncPar:
|
|
42
|
+
"""
|
|
43
|
+
Class cls::`FuncPar` treats arguments of a function as dictionary of
|
|
44
|
+
parameters with additional properties tools, like
|
|
45
|
+
- defaults
|
|
46
|
+
- validation conditions
|
|
47
|
+
- automatic or explicit validation
|
|
48
|
+
- automatic extraction of parameters found arbitrary processing flows
|
|
49
|
+
- and more
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
_validate = Validate.RAISE
|
|
53
|
+
|
|
54
|
+
@classmethod
|
|
55
|
+
def validation(cls, state: Optional[Validate] = None) -> Validate:
|
|
56
|
+
"""Set new parameters validation level and return the previous one"""
|
|
57
|
+
if state is None:
|
|
58
|
+
return cls._validate
|
|
59
|
+
else:
|
|
60
|
+
old, cls._validate = cls._validate, Validate(state)
|
|
61
|
+
return old
|
|
62
|
+
|
|
63
|
+
class Tracker:
|
|
64
|
+
"""Helper sub-class of cls::`FuncPar` to track nested calls of
|
|
65
|
+
parametrized functions."""
|
|
66
|
+
|
|
67
|
+
def __init__(self):
|
|
68
|
+
self._track = False
|
|
69
|
+
self._frame = None
|
|
70
|
+
self._depth = None
|
|
71
|
+
self._record = {}
|
|
72
|
+
self._stack = []
|
|
73
|
+
|
|
74
|
+
reset = __init__
|
|
75
|
+
|
|
76
|
+
def start(self):
|
|
77
|
+
"""Start tracking of the execution flow for all nested calls"""
|
|
78
|
+
if self._frame is None:
|
|
79
|
+
st = inspect.stack()
|
|
80
|
+
self._frame = st[1].frame
|
|
81
|
+
self._depth = len(st)
|
|
82
|
+
self._track = True
|
|
83
|
+
|
|
84
|
+
def stop(self):
|
|
85
|
+
"""Stop tracking - but keep collected record (unlike `reset`)"""
|
|
86
|
+
self._track = False
|
|
87
|
+
|
|
88
|
+
def restart(self):
|
|
89
|
+
"""Clear context and restart the tracking from current frame"""
|
|
90
|
+
self.__init__()
|
|
91
|
+
self.start()
|
|
92
|
+
|
|
93
|
+
def report(self) -> Dict[str, 'FuncPar']:
|
|
94
|
+
"""Produce dictionary of all calls to functions with `FuncPar`
|
|
95
|
+
with keys as dot separated names of nested called functions.
|
|
96
|
+
"""
|
|
97
|
+
return self._record.copy()
|
|
98
|
+
|
|
99
|
+
def __bool__(self):
|
|
100
|
+
return self._track
|
|
101
|
+
|
|
102
|
+
def track_call(self, arg_par: 'FuncPar'):
|
|
103
|
+
"""Track specific call provided its `FuncPar` context"""
|
|
104
|
+
func_name = arg_par.function.__qualname__
|
|
105
|
+
self._stack.append(func_name)
|
|
106
|
+
if self._track:
|
|
107
|
+
st = inspect.stack()
|
|
108
|
+
depth = len(st) - self._depth + 1
|
|
109
|
+
if depth > 0 and st[depth].frame is self._frame:
|
|
110
|
+
self._record['.'.join(self._stack)] = arg_par
|
|
111
|
+
|
|
112
|
+
track = Tracker()
|
|
113
|
+
|
|
114
|
+
def __init__(self, func: Callable, *, alias: str = None,
|
|
115
|
+
validate: Union[Rule, Sequence[Rule]] = None,
|
|
116
|
+
directives: Dict[str, Union[Rule]]):
|
|
117
|
+
"""Construct `FuncPar` instance for specific function using
|
|
118
|
+
provided validation rules and condition directives.
|
|
119
|
+
|
|
120
|
+
:param func: function object to associate with
|
|
121
|
+
:param alias: optional (short) version of name to associate with
|
|
122
|
+
the parameters structure instead of the function name
|
|
123
|
+
:param validate: single or sequence of validation rules in form of
|
|
124
|
+
a python expression using variables named by the parameters
|
|
125
|
+
or callable accepting keywords arguments or those parameters.
|
|
126
|
+
:param directives: dictionary per parameter of validation conditions
|
|
127
|
+
in form of expression using single variable denoted as "{}" or
|
|
128
|
+
callable with a single argument to receive the parameter value.
|
|
129
|
+
|
|
130
|
+
Examples:
|
|
131
|
+
Associates conditions with function arguments.
|
|
132
|
+
>>> def func(x=10, y=20): pass
|
|
133
|
+
>>> p_func = FuncPar(func, validate='x < y',\
|
|
134
|
+
directives={'y': "{} > 0"}).function
|
|
135
|
+
>>> assert p_func(20, 10) # will fail on x < y
|
|
136
|
+
Traceback (most recent call last):
|
|
137
|
+
...
|
|
138
|
+
AssertionError
|
|
139
|
+
>>> assert p_func(-10, -2) # will fail on y > 0
|
|
140
|
+
"""
|
|
141
|
+
from functools import wraps
|
|
142
|
+
|
|
143
|
+
self._name = func.__qualname__
|
|
144
|
+
self._params = TBox() # arguments which are parameters with defaults
|
|
145
|
+
self._cond = {} # parameters with assigned conditions
|
|
146
|
+
self._alias = alias or self._name
|
|
147
|
+
|
|
148
|
+
# First parse all the function's arguments to determine:
|
|
149
|
+
# - which have defaults - candidates to be parameters
|
|
150
|
+
# - which are mentioned in the special directives
|
|
151
|
+
all_defaults = {} # default values for ALL the arguments
|
|
152
|
+
invalid_references = set(directives) # ensure directives refer to existing arguments
|
|
153
|
+
|
|
154
|
+
pos_kws_start = 0 # location of first not position-only argument
|
|
155
|
+
pos_kws = [] # names (ordered) of all the pos_or_kw arguments
|
|
156
|
+
try:
|
|
157
|
+
for i, (name, arg) in enumerate(inspect.signature(func).parameters.items()):
|
|
158
|
+
if arg.kind is arg.VAR_POSITIONAL:
|
|
159
|
+
raise NotImplementedError("Variadic positional arguments")
|
|
160
|
+
if arg.kind in (arg.POSITIONAL_ONLY, arg.VAR_KEYWORD):
|
|
161
|
+
continue # positional and variadic arguments are not parametrized!
|
|
162
|
+
|
|
163
|
+
# other kinds with provided default values are be turned in parameters
|
|
164
|
+
# unless explicitly excluded in directives
|
|
165
|
+
if arg.kind is arg.POSITIONAL_OR_KEYWORD: # keep track on ambiguous arguments
|
|
166
|
+
if not pos_kws: # to separate position-only when called
|
|
167
|
+
pos_kws_start = i
|
|
168
|
+
pos_kws.append(name)
|
|
169
|
+
|
|
170
|
+
if arg.default is not arg.empty:
|
|
171
|
+
all_defaults[name] = arg.default # to implement defaults logistics
|
|
172
|
+
self._params[name] = arg.default # may be exclude below
|
|
173
|
+
if name in directives: # either provides condition or excludes from params
|
|
174
|
+
invalid_references.remove(name) # remove valid names from the black list
|
|
175
|
+
cond = directives[name]
|
|
176
|
+
if cond:
|
|
177
|
+
if isinstance(cond, str):
|
|
178
|
+
cond = fnt.express_to_lambda(cond.format(name), name)
|
|
179
|
+
self._cond[name] = cond
|
|
180
|
+
else: # exclude from params if explicitly set to False or None
|
|
181
|
+
self._params.pop(name)
|
|
182
|
+
|
|
183
|
+
if invalid_references:
|
|
184
|
+
raise ReferenceError(f"Conditions defined for invalid arguments: {invalid_references}")
|
|
185
|
+
|
|
186
|
+
self._validators = [v if callable(v) else fnt.express_to_kw_func(v) for v in
|
|
187
|
+
(as_list(validate) if validate else [])]
|
|
188
|
+
except SyntaxError as e:
|
|
189
|
+
e.msg = f"Invalid validation condition for function <{self._name}>"
|
|
190
|
+
raise e
|
|
191
|
+
|
|
192
|
+
def separate_kw_args(args):
|
|
193
|
+
"""Separate variadic positional argument into position only and pos_or_kw."""
|
|
194
|
+
found_pos_kws = len(args) - pos_kws_start
|
|
195
|
+
if found_pos_kws > len(pos_kws):
|
|
196
|
+
raise SyntaxError(f"Positional args to <{self._name}> exceed {len(pos_kws)}!")
|
|
197
|
+
if found_pos_kws < 0:
|
|
198
|
+
raise SyntaxError(f"Missing {-found_pos_kws} positional args in <{self._name}>!")
|
|
199
|
+
# create kw form of the args passed by position
|
|
200
|
+
silent_kw_arg = dict(zip(pos_kws[:found_pos_kws], args[-found_pos_kws:]))
|
|
201
|
+
return silent_kw_arg
|
|
202
|
+
|
|
203
|
+
def filter_keys_in(d, seq, *, keep_in=True):
|
|
204
|
+
"""Filter dictionary by keys belonging or not to given sequence"""
|
|
205
|
+
seq = set(seq)
|
|
206
|
+
condition = (lambda x: x in seq) if keep_in else (lambda x: x not in seq)
|
|
207
|
+
return {k: v for k, v in d.items() if condition(k)}
|
|
208
|
+
|
|
209
|
+
@wraps(func)
|
|
210
|
+
def pz_func(*args, _par: dict = None, **kws):
|
|
211
|
+
# Responsibilities of the wrapper:
|
|
212
|
+
# 1. Optional validation of the parameters values
|
|
213
|
+
# 2. Inform the ArgPar.tracker about the call to collect flow information.
|
|
214
|
+
# 3. Automatically fetch parameters from repository through special argument _par
|
|
215
|
+
# 4. (No Harm) Maintain same syntax as the original function in spite of different signature
|
|
216
|
+
# - make variadic argument behave like named in the original function
|
|
217
|
+
# - support the additional (optional) _par argument
|
|
218
|
+
silent_kws = separate_kw_args(args)
|
|
219
|
+
|
|
220
|
+
# optionally keyword parameters may be passed inside special _par
|
|
221
|
+
if _par: # automatic fetch of relevant parameters from the repo
|
|
222
|
+
# dict-like argument as item under the key `func.name` or `func.alias`
|
|
223
|
+
# which `dict`-like value would contain any subset of them.
|
|
224
|
+
_par = getattr(_par, self._name, getattr(_par, self._alias, {}))
|
|
225
|
+
# make sure no arguments are not passed twice from different sources
|
|
226
|
+
if self.validation() > Validate.RAISE:
|
|
227
|
+
if twice := set(_par).intersection(set(silent_kws).union(kws)):
|
|
228
|
+
from warnings import warn
|
|
229
|
+
warn(f"Double {twice} in {self._name}: as args and (ignored) in _par")
|
|
230
|
+
|
|
231
|
+
kws = {**filter_keys_in(_par, silent_kws, keep_in=False), **kws} # _par-silent-kws+kws
|
|
232
|
+
|
|
233
|
+
self.validate({**silent_kws, **kws}) # add silent to validation
|
|
234
|
+
FuncPar.track.track_call(self)
|
|
235
|
+
return func(*args, **kws)
|
|
236
|
+
|
|
237
|
+
self._func = pz_func
|
|
238
|
+
self.__call__ = pz_func
|
|
239
|
+
|
|
240
|
+
setattr(pz_func, 'paramaze', self)
|
|
241
|
+
setattr(pz_func, 'defaults', self.defaults)
|
|
242
|
+
setattr(pz_func, 'validate', self.validate)
|
|
243
|
+
|
|
244
|
+
def __repr__(self):
|
|
245
|
+
return f'Params in function `{self.name}`:\n' \
|
|
246
|
+
f'{self._params.to_yaml()}'
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def function(self):
|
|
250
|
+
return self._func
|
|
251
|
+
|
|
252
|
+
@property
|
|
253
|
+
def name(self):
|
|
254
|
+
return self._name
|
|
255
|
+
|
|
256
|
+
@property
|
|
257
|
+
def alias(self):
|
|
258
|
+
return self._alias
|
|
259
|
+
|
|
260
|
+
@property
|
|
261
|
+
def defaults(self):
|
|
262
|
+
"""Return editable copy of the default parameters"""
|
|
263
|
+
return TBox(self._params, frozen_box=False)
|
|
264
|
+
|
|
265
|
+
def validate(self, par, throw=True):
|
|
266
|
+
vld = self.validation()
|
|
267
|
+
if vld is Validate.OFF:
|
|
268
|
+
return True
|
|
269
|
+
if vld is Validate.VERBOSE:
|
|
270
|
+
print(f'Validating {self.name} parameters: {tuple(par)}...', end='')
|
|
271
|
+
|
|
272
|
+
def try_cond(context, cond, *args, **kws):
|
|
273
|
+
try:
|
|
274
|
+
return cond(*args, **kws)
|
|
275
|
+
except Exception as e:
|
|
276
|
+
raise SyntaxError("Invalid validation condition for function "
|
|
277
|
+
f"<{self._func.__qualname__}>\n"
|
|
278
|
+
f"{context} {cond.__qualname__}\n{e}")
|
|
279
|
+
|
|
280
|
+
failed_pars = [f"{k}={v} fails condition: {self._cond[k].__qualname__}"
|
|
281
|
+
for k, v in par.items() if k in self._cond and
|
|
282
|
+
not try_cond(f"for parameter {k}", self._cond[k], v)]
|
|
283
|
+
|
|
284
|
+
merged_par = {**self._params, **par}
|
|
285
|
+
failed_rules = [rule.__qualname__ for rule in self._validators if
|
|
286
|
+
not try_cond("in validation rule", rule, **merged_par)]
|
|
287
|
+
|
|
288
|
+
msg = '\n\t'.join(['Invalid Parameters:', *failed_pars]) + '\n' if failed_pars else ''
|
|
289
|
+
msg += '\n\t'.join(['Failed rules:', *failed_rules]) if failed_rules else ''
|
|
290
|
+
if msg:
|
|
291
|
+
msg = f'Failed parameters validation for <{self.name}>:\n{msg}'
|
|
292
|
+
vld is Validate.VERBOSE and print() # additional line in verbose case
|
|
293
|
+
print(msg, repr(TBox(par)))
|
|
294
|
+
if throw and vld is not Validate.REPORT:
|
|
295
|
+
raise ValueError(msg)
|
|
296
|
+
return False
|
|
297
|
+
if vld is not Validate.RAISE:
|
|
298
|
+
print('OK!')
|
|
299
|
+
return True
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
@wrap.double_wrap
|
|
303
|
+
def paramaze(func, _alias: str = None,
|
|
304
|
+
_validate: Union[Rule, Sequence[Rule]] = None,
|
|
305
|
+
**kws: Rule):
|
|
306
|
+
"""
|
|
307
|
+
Decorator wrapping function to associate it with `FuncPar` and
|
|
308
|
+
provide all the related functionality: arguments as parameters with
|
|
309
|
+
defaults, validation conditions, automatic collection of flow parameters,...
|
|
310
|
+
|
|
311
|
+
By default all keyword arguments with defaults are considered parameters.
|
|
312
|
+
Decorator mau be used with or without arguments, then no conditions or
|
|
313
|
+
alias is associated with the function, but parameters collections is
|
|
314
|
+
still available.
|
|
315
|
+
|
|
316
|
+
:param func: function to wrap
|
|
317
|
+
:param _alias: optional (short) version of name to associate with
|
|
318
|
+
the parameters structure instead of the function name
|
|
319
|
+
:param _validate: single or sequence of validation rules in form of
|
|
320
|
+
a python expression using variables named by the parameters
|
|
321
|
+
or callable accepting keywords arguments or those parameters.
|
|
322
|
+
:param kws: dictionary per parameter of validation conditions
|
|
323
|
+
in form of expression using single variable denoted as "{}" or
|
|
324
|
+
callable with a single argument to receive the parameter value.
|
|
325
|
+
|
|
326
|
+
:return: wrapped function
|
|
327
|
+
|
|
328
|
+
Example:
|
|
329
|
+
|
|
330
|
+
>>> @paramaze
|
|
331
|
+
>>> def func(x=10, y=20): pass
|
|
332
|
+
>>> assert func.defaults == {'x': 10, 'y': 20}
|
|
333
|
+
|
|
334
|
+
>>> @paramaze(y="{} > 0",
|
|
335
|
+
>>> _validate_type=['x < y', 'x**2 + y**2 < 100'])
|
|
336
|
+
>>> def func(x=10, y=20): pass
|
|
337
|
+
>>> assert func.parz.validate(x=2, y=3)
|
|
338
|
+
"""
|
|
339
|
+
return FuncPar(func, alias=_alias, validate=_validate, directives=kws).function
|
iad/core/param/tbox.py
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Iterable, Any
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
import pydantic.v1 as pydantic
|
|
7
|
+
from box import Box
|
|
8
|
+
|
|
9
|
+
from ..datatools import complete_missing, UndefCond, UndefTypes
|
|
10
|
+
from ..strings import compact_repr, hash_str
|
|
11
|
+
from ..wrap import name_tuple
|
|
12
|
+
|
|
13
|
+
__all__ = ['TBox', 'UndefCond']
|
|
14
|
+
|
|
15
|
+
_intact_types = (tuple, list, pd.DataFrame, pd.Series)
|
|
16
|
+
UND_EMPTY = UndefCond(empty_dict=True)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class TBox(Box):
|
|
20
|
+
_protected_keys = Box._protected_keys + [
|
|
21
|
+
"_flatten",
|
|
22
|
+
"_repr_json_",
|
|
23
|
+
"diff",
|
|
24
|
+
"issubset",
|
|
25
|
+
"issuperset",
|
|
26
|
+
"find_key",
|
|
27
|
+
"remove",
|
|
28
|
+
"discard",
|
|
29
|
+
"hash_str"
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
def __init__(self, *args: pydantic.BaseModel | dict | Iterable[tuple[str, Any]],
|
|
33
|
+
undef: UndefTypes = UND_EMPTY, **kw_box):
|
|
34
|
+
"""
|
|
35
|
+
Initialize hierarchical dict-like structure.
|
|
36
|
+
|
|
37
|
+
Note, that argument ``undef`` controlling behavior of ``setdefault``method
|
|
38
|
+
may be provided in various forms, but is eventually converted into
|
|
39
|
+
the most complete one: instance of ``UndefCond``:
|
|
40
|
+
- Collection of undef objects
|
|
41
|
+
- Callable -> bool
|
|
42
|
+
- UndefCond (allows to control empty_dict)
|
|
43
|
+
|
|
44
|
+
:param args: dicts to initialize with
|
|
45
|
+
:param kw_box: either key=value pairs as a data items, or Box kw args
|
|
46
|
+
:param undef: describes condition to consider a node undefined
|
|
47
|
+
"""
|
|
48
|
+
kw_box.setdefault('box_intact_types', _intact_types)
|
|
49
|
+
kw_box.setdefault('box_dots', True) # dotted - default behaviour for TBox !
|
|
50
|
+
|
|
51
|
+
if args: # special conversions
|
|
52
|
+
args = tuple(arg.dict() if isinstance(arg, pydantic.BaseModel) else arg for arg in args)
|
|
53
|
+
|
|
54
|
+
if len(args) > 1: # Box supports only 1 positional argument, so for more than that
|
|
55
|
+
dict_arg = {} # convert all the args into dicts and then merge them into one
|
|
56
|
+
for arg in args: # here we deal with dict and Iterable[tuple[k, v]]
|
|
57
|
+
dict_arg.update(arg)
|
|
58
|
+
args = (dict_arg,)
|
|
59
|
+
|
|
60
|
+
super().__init__(*args, **kw_box)
|
|
61
|
+
self._box_config['undef'] = undef if isinstance(undef, UndefCond) else UndefCond(undef=undef)
|
|
62
|
+
|
|
63
|
+
def __setitem__(self, key, value):
|
|
64
|
+
# ToDo risky - what about '1.32' or 'file.ext' or relative annotation ..
|
|
65
|
+
if isinstance(key, str) and '.' in key:
|
|
66
|
+
node, tail = key.split('.', 1)
|
|
67
|
+
if node not in self:
|
|
68
|
+
super().__setitem__(node, {})
|
|
69
|
+
self[node].__setitem__(tail, value)
|
|
70
|
+
else:
|
|
71
|
+
super().__setitem__(key, value)
|
|
72
|
+
|
|
73
|
+
def flatten(self):
|
|
74
|
+
res = []
|
|
75
|
+
for key, val in self.items():
|
|
76
|
+
if isinstance(val, Box) and val:
|
|
77
|
+
for subs, v in val.flatten():
|
|
78
|
+
res.append((f'{key}.{subs}', v))
|
|
79
|
+
else:
|
|
80
|
+
res.append((key, val))
|
|
81
|
+
return res
|
|
82
|
+
|
|
83
|
+
def _repr_json_(self):
|
|
84
|
+
jbox = TBox(default_box=True)
|
|
85
|
+
is_basic = lambda x: x is None or isinstance(x, (str, int, float))
|
|
86
|
+
for k, v in self.items(True):
|
|
87
|
+
jbox[k] = v if is_basic(v) or (
|
|
88
|
+
isinstance(v, (list, set, tuple))
|
|
89
|
+
and len(v) < 5 and all(map(is_basic, v))
|
|
90
|
+
) else compact_repr(v)
|
|
91
|
+
return TBox(jbox)
|
|
92
|
+
|
|
93
|
+
def __repr__(self):
|
|
94
|
+
if not self: return f"{type(self).__name__}()"
|
|
95
|
+
return '\n'.join(f'{k}: {compact_repr(v)}' for k, v in self.flatten())
|
|
96
|
+
|
|
97
|
+
def hash_str(self, length: int = None):
|
|
98
|
+
"""Return hex hash string as unique id of the content.
|
|
99
|
+
|
|
100
|
+
:param len: if provided limit string to this length (keep tail)
|
|
101
|
+
"""
|
|
102
|
+
s = ''
|
|
103
|
+
for k, v in sorted(self.items(True), key=lambda _: _[0]):
|
|
104
|
+
if isinstance(v, list):
|
|
105
|
+
v = [TBox(x).hash_str(length=1000)
|
|
106
|
+
if isinstance(x, dict) else x
|
|
107
|
+
for x in v]
|
|
108
|
+
s += f"{k}:{str(v)}\n"
|
|
109
|
+
|
|
110
|
+
return hash_str(s, length)
|
|
111
|
+
|
|
112
|
+
def __copy__(self, **kws):
|
|
113
|
+
old_kws = {k: v for k, v in self._box_config.items() if not k.startswith('_')}
|
|
114
|
+
assert set(kws).issubset(old_kws), "copy only accepts Box standard arguments"
|
|
115
|
+
old_kws.update(kws)
|
|
116
|
+
return self.__class__(self.to_dict(), **old_kws)
|
|
117
|
+
|
|
118
|
+
# def __deepcopy__(self, memodict={}):
|
|
119
|
+
# return self.__copy__()
|
|
120
|
+
|
|
121
|
+
def copy(self, **kws):
|
|
122
|
+
from copy import deepcopy
|
|
123
|
+
return deepcopy(self)
|
|
124
|
+
|
|
125
|
+
def diff(self, other: 'TBox'):
|
|
126
|
+
""" Difference between two tree forms as tuple of
|
|
127
|
+
- keys missing in the other,
|
|
128
|
+
- keys missing in self,
|
|
129
|
+
- keys of incomparable or not equal values
|
|
130
|
+
:returns tuple(dif_keys, dif_values)
|
|
131
|
+
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
def not_same_value(k):
|
|
135
|
+
try:
|
|
136
|
+
v1, v2 = other[k], self[k]
|
|
137
|
+
same = (v1 == v2)
|
|
138
|
+
if type(same) is not bool:
|
|
139
|
+
import numpy as np
|
|
140
|
+
same = np.allclose(v1, v2)
|
|
141
|
+
except Exception:
|
|
142
|
+
same = False
|
|
143
|
+
return not same
|
|
144
|
+
|
|
145
|
+
other = other if isinstance(other, TBox) else TBox(other)
|
|
146
|
+
my_keys, other_keys = set(self.keys(True)), set(other.keys(True))
|
|
147
|
+
missing = my_keys.difference(other_keys)
|
|
148
|
+
extra = other_keys.difference(my_keys)
|
|
149
|
+
unequal = {*filter(not_same_value, my_keys.intersection(other_keys))}
|
|
150
|
+
return name_tuple('TBoxDiffKeys', missing=missing, extra=extra, unequal=unequal)
|
|
151
|
+
|
|
152
|
+
def to_yaml(self, filename=None, *, default_flow_style=False,
|
|
153
|
+
encoding="utf-8", errors="strict", **yaml_kwargs):
|
|
154
|
+
from ..filesproc import prepare_parent_folder, Path
|
|
155
|
+
|
|
156
|
+
par = self.copy() # type: TBox
|
|
157
|
+
for k, v in par.items(True):
|
|
158
|
+
if isinstance(v, Path):
|
|
159
|
+
par[k] = str(v)
|
|
160
|
+
if hasattr(v, '__qualname__'):
|
|
161
|
+
par[k] = v.__qualname__
|
|
162
|
+
|
|
163
|
+
if filename:
|
|
164
|
+
prepare_parent_folder(filename)
|
|
165
|
+
filename = str(filename)
|
|
166
|
+
|
|
167
|
+
return Box.to_yaml(par, filename=filename, default_flow_style=default_flow_style,
|
|
168
|
+
encoding=encoding, errors=errors, **yaml_kwargs)
|
|
169
|
+
|
|
170
|
+
def setdefault(self, key: str, default: dict | Any, *, undef: UndefTypes = None):
|
|
171
|
+
"""
|
|
172
|
+
Include support for dotted (nested) keys for `key` arg,
|
|
173
|
+
and dict-like (also nested) structure for default argument.
|
|
174
|
+
|
|
175
|
+
If key is None - setting default for the entire tree, in which case
|
|
176
|
+
default must be nested dict as well.
|
|
177
|
+
|
|
178
|
+
May include check that mandatory fields (values set as TBox.REQUIRED
|
|
179
|
+
in the default dict) are already defined in self, or raise ValueError.
|
|
180
|
+
|
|
181
|
+
:param key: name (also nested) of the field or None for entire tree.
|
|
182
|
+
:param default: a value to set to specified key
|
|
183
|
+
or dict (nested) if key points to a subtree
|
|
184
|
+
:param undef: override TBox.undef
|
|
185
|
+
:return: the value been set (or the (sub)tree if default dict is dict
|
|
186
|
+
"""
|
|
187
|
+
if (sep := '.') in key:
|
|
188
|
+
assert sep not in (key[0], key[-1]), f"Invalid dotted {key=}!"
|
|
189
|
+
node = self # Ensure dotted key node exists and not a leaf
|
|
190
|
+
for k in key.split(sep):
|
|
191
|
+
if not isinstance(node, dict): # every next level must be a dict
|
|
192
|
+
raise KeyError(f'Position {k} in {key=} must be a node!')
|
|
193
|
+
node = Box.setdefault(node, k, {}) # existed or just created
|
|
194
|
+
|
|
195
|
+
if isinstance(node, dict) and not node: # don't consider empty node as set!
|
|
196
|
+
self[key] = default
|
|
197
|
+
return self[key]
|
|
198
|
+
|
|
199
|
+
default = TBox({key: default} if key else default)
|
|
200
|
+
undef = getattr(self, '_box_config')['undef'] if undef is None else undef
|
|
201
|
+
return complete_missing(self, default, undef=undef)
|
|
202
|
+
|
|
203
|
+
def issubset(self, other: dict):
|
|
204
|
+
if not isinstance(other, self.__class__):
|
|
205
|
+
other = self.__class__(other)
|
|
206
|
+
return set(self.keys(True)).issubset(other.keys(True))
|
|
207
|
+
|
|
208
|
+
def issuperset(self, other: dict):
|
|
209
|
+
if not isinstance(other, self.__class__):
|
|
210
|
+
other = self.__class__(other)
|
|
211
|
+
return set(other.keys(True)).issuperset(self.keys(True))
|
|
212
|
+
|
|
213
|
+
def find_key(self, name, *, part='any', multi=False) -> str | list:
|
|
214
|
+
"""
|
|
215
|
+
Find key(s) with deep (dotted) naming matching given name.
|
|
216
|
+
|
|
217
|
+
Return full (dotted) key found or '' if not.
|
|
218
|
+
|
|
219
|
+
If multiple matching keys are found return depending on *multi*:
|
|
220
|
+
- list of them if ``True``
|
|
221
|
+
- only the first if ``None``
|
|
222
|
+
- raise ``KeyError`` if ``False``
|
|
223
|
+
|
|
224
|
+
:param name: may include dots inside: "x.y"
|
|
225
|
+
:param part: key's part to match: ['any'] | 'start' | 'end'
|
|
226
|
+
:param multi: allow multiple keys to return
|
|
227
|
+
|
|
228
|
+
:return: a string or list of strings (keys found)
|
|
229
|
+
"""
|
|
230
|
+
import re
|
|
231
|
+
if not re.fullmatch(r'(\w+(\.\w+)?)+', name):
|
|
232
|
+
return [] if multi else ''
|
|
233
|
+
|
|
234
|
+
reg = re.compile({'any': rf'\b{name}\b',
|
|
235
|
+
'start': rf'^{name}\b',
|
|
236
|
+
'end': rf'\b{name}$'
|
|
237
|
+
}[part])
|
|
238
|
+
|
|
239
|
+
found = [*filter(reg.search, self.keys(True))]
|
|
240
|
+
if multi:
|
|
241
|
+
return found
|
|
242
|
+
elif len(found) > 1:
|
|
243
|
+
raise KeyError(f"Multiple matches to {name=} been {found=}!")
|
|
244
|
+
else:
|
|
245
|
+
return found[0]
|
|
246
|
+
|
|
247
|
+
def remove(self, keys: Iterable[str], *, strict=True):
|
|
248
|
+
"""Remove nodes addressed by the keys (dotted).
|
|
249
|
+
:param keys: iterable over dotted keys to remove
|
|
250
|
+
:param strict: if True - raise KeyError if any of keys is missing
|
|
251
|
+
otherwise - ignore and remove only found
|
|
252
|
+
:return: a new TBox with keys removed
|
|
253
|
+
"""
|
|
254
|
+
if isinstance(keys, str): keys = [keys]
|
|
255
|
+
res = self.copy()
|
|
256
|
+
for key in keys:
|
|
257
|
+
try:
|
|
258
|
+
del res[key]
|
|
259
|
+
except KeyError as ex:
|
|
260
|
+
if strict: raise ex
|
|
261
|
+
return res
|
|
262
|
+
|
|
263
|
+
def discard(self, keys: Iterable[str], *, strict=True):
|
|
264
|
+
"""Discard from this box nodes addressed by given keys (dotted).
|
|
265
|
+
:param keys: iterable over dotted keys to remove
|
|
266
|
+
:param strict: if True - raise KeyError if any of keys is missing
|
|
267
|
+
otherwise - ignore and remove only found
|
|
268
|
+
:return: None
|
|
269
|
+
"""
|
|
270
|
+
if isinstance(keys, str): keys = [keys]
|
|
271
|
+
if not hasattr(keys, '__len__'):
|
|
272
|
+
keys = [*keys]
|
|
273
|
+
valid = [*filter(self.__contains__, keys)]
|
|
274
|
+
if strict and len(valid) < len(keys):
|
|
275
|
+
raise KeyError(f"Missing keys: {set(self.keys(True)).difference(valid)}")
|
|
276
|
+
for key in valid:
|
|
277
|
+
del self[key]
|