ialdev-core 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. iad/core/__init__.py +9 -0
  2. iad/core/array.py +1961 -0
  3. iad/core/binary.py +377 -0
  4. iad/core/cache.py +903 -0
  5. iad/core/codetools.py +203 -0
  6. iad/core/datatools.py +671 -0
  7. iad/core/docs/locators.ipynb +754 -0
  8. iad/core/dotstyle.py +99 -0
  9. iad/core/env.py +271 -0
  10. iad/core/events.py +650 -0
  11. iad/core/filesproc.py +1046 -0
  12. iad/core/fnctools.py +390 -0
  13. iad/core/label.py +240 -0
  14. iad/core/logs.py +182 -0
  15. iad/core/nptools.py +449 -0
  16. iad/core/one_dark.puml +881 -0
  17. iad/core/param/__init__.py +17 -0
  18. iad/core/param/confargparse.py +55 -0
  19. iad/core/param/paramaze.py +339 -0
  20. iad/core/param/tbox.py +277 -0
  21. iad/core/paths.py +563 -0
  22. iad/core/pdtools.py +2570 -0
  23. iad/core/pydantools/__init__.py +5 -0
  24. iad/core/pydantools/fixed_pydantic_yaml/__init__.py +32 -0
  25. iad/core/pydantools/fixed_pydantic_yaml/compat/__init__.py +0 -0
  26. iad/core/pydantools/fixed_pydantic_yaml/compat/hacks.py +76 -0
  27. iad/core/pydantools/fixed_pydantic_yaml/compat/old_enums.py +37 -0
  28. iad/core/pydantools/fixed_pydantic_yaml/compat/representers.py +92 -0
  29. iad/core/pydantools/fixed_pydantic_yaml/compat/types.py +122 -0
  30. iad/core/pydantools/fixed_pydantic_yaml/compat/yaml_lib.py +104 -0
  31. iad/core/pydantools/fixed_pydantic_yaml/ext/__init__.py +1 -0
  32. iad/core/pydantools/fixed_pydantic_yaml/ext/semver.py +152 -0
  33. iad/core/pydantools/fixed_pydantic_yaml/ext/versioned_model.py +113 -0
  34. iad/core/pydantools/fixed_pydantic_yaml/main.py +30 -0
  35. iad/core/pydantools/fixed_pydantic_yaml/mixin.py +281 -0
  36. iad/core/pydantools/fixed_pydantic_yaml/model.py +20 -0
  37. iad/core/pydantools/fixed_pydantic_yaml/py.typed +1 -0
  38. iad/core/pydantools/fixed_pydantic_yaml/version.py +1 -0
  39. iad/core/pydantools/models.py +560 -0
  40. iad/core/regexp.py +348 -0
  41. iad/core/short.py +308 -0
  42. iad/core/strings.py +635 -0
  43. iad/core/unc_panda.py +270 -0
  44. iad/core/units.py +58 -0
  45. iad/core/wrap.py +420 -0
  46. ialdev_core-0.1.0.dist-info/METADATA +73 -0
  47. ialdev_core-0.1.0.dist-info/RECORD +48 -0
  48. ialdev_core-0.1.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,17 @@
1
+ from frozendict import frozendict as fzdict
2
+
3
+ from . import paramaze as pa
4
+ from .tbox import TBox
5
+
6
+ __all__ = ['TBox', 'model_arguments', 'YamlModel', 'fzdict']
7
+
8
+
9
+ def __getattr__(name: str):
10
+ """Lazy imports for YAML/Pydantic helpers shipped in ``iad.core.pydantools``."""
11
+ if name == 'YamlModel':
12
+ from iad.core.pydantools.models import YamlModel as _YamlModel
13
+ return _YamlModel
14
+ if name == 'model_arguments':
15
+ from iad.core.pydantools.models import model_arguments as _model_arguments
16
+ return _model_arguments
17
+ raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
@@ -0,0 +1,55 @@
1
+ import configargparse as cap
2
+
3
+
4
+ class ArgumentParserYAML(cap.ArgumentParser):
5
+ def __init__(self, *, hash_groups=None, add_env_var_help=False, **kws):
6
+ super().__init__(config_file_parser_class=cap.YAMLConfigFileParser,
7
+ add_env_var_help=add_env_var_help,
8
+ **kws)
9
+ self.hash_groups = hash_groups or {}
10
+
11
+ def save_config(self, namespace, config_file):
12
+ if BUG_FIXED := False: # TODO: write_config_file prints all values as str!
13
+ self.write_config_file(namespace, [str(config_file)])
14
+ else:
15
+ op_map = {op.dest: op.option_strings[-1] for op in self._get_optional_actions()}
16
+
17
+ def items_to_save():
18
+ for k, v in namespace.__dict__.items():
19
+ if not (v is None or k in ('cmd', 'config')):
20
+ yield op_map[k].strip('-'), v
21
+
22
+ file_contents = self._config_file_parser.serialize(items_to_save())
23
+ with self._config_file_open_func(config_file, "w") as output_file:
24
+ output_file.write(file_contents)
25
+ print(f"Config file saved: {config_file}")
26
+
27
+ def hashes(self, ns: cap.Namespace, length=4) -> dict:
28
+ """
29
+ Calculate hash values for the defined groups of arguments.
30
+ Return dict with hashes py group.
31
+ If groups are not defined the namespace is used and named 'cfg'.
32
+
33
+ :param ns: Namespace with parsed arguments
34
+ :param length: length of hash string tail to used (0 - all of it)
35
+
36
+ :return: dict {grp_name: hash_str}
37
+ """
38
+ from .tbox import TBox
39
+ groups = self.hash_groups or {'cfg': [*vars(ns)]}
40
+ ns = vars(ns)
41
+
42
+ def hash_group(names):
43
+ group_params = {name: ns[name] for name in names if name in ns}
44
+ return TBox(group_params).hash_str(length)
45
+
46
+ return {grp: hash_group(names) for grp, names in groups.items()}
47
+
48
+ def hash_stamp(self, ns: cap.Namespace, length=4) -> str:
49
+ """
50
+ Hash stamp string unique for the ns content by arguments groups.
51
+ :param ns: namespace with arguments
52
+ :param length: length of hash strings
53
+ :return: "grp1_hash1_grp2_hash2_..."
54
+ """
55
+ return '_'.join(map(lambda it: f"{it[0]}_{it[1]}", self.hashes(ns, length).items()))
@@ -0,0 +1,339 @@
1
+ import inspect
2
+ from contextlib import contextmanager
3
+ from enum import IntEnum, unique
4
+ from typing import Union, Callable, Optional, Dict, Sequence
5
+
6
+ from .. import as_list, fnctools as fnt, wrap
7
+ from .tbox import TBox
8
+
9
+ Rule = Union[Callable, str, None]
10
+
11
+
12
+ def sub(*args, **kws):
13
+ func_defs = {f.paramaze.alias: f.paramaze.defaults for f in args}
14
+ return TBox(frozen_box=True, **func_defs, **kws)
15
+
16
+
17
+ @unique
18
+ class Validate(IntEnum):
19
+ OFF = 0 # no validation
20
+ REPORT = 1 # report about errors but not raise
21
+ RAISE = 2 # validate, but not report - only raise on errors
22
+ VERBOSE = 3 # raise on errors
23
+
24
+ @classmethod
25
+ @contextmanager
26
+ def level(cls, vld_level: Union['Validate', str]):
27
+ """
28
+ Create validation context to execute code with specific validation level
29
+ and restore the current one on exit
30
+ :param vld_level: as name str or `Validate` object
31
+ """
32
+ if isinstance(vld_level, str):
33
+ vld = Validate[vld_level]
34
+ old_vld = FuncPar.validation(vld)
35
+ try:
36
+ yield vld
37
+ finally:
38
+ FuncPar.validation(old_vld)
39
+
40
+
41
+ class FuncPar:
42
+ """
43
+ Class cls::`FuncPar` treats arguments of a function as dictionary of
44
+ parameters with additional properties tools, like
45
+ - defaults
46
+ - validation conditions
47
+ - automatic or explicit validation
48
+ - automatic extraction of parameters found arbitrary processing flows
49
+ - and more
50
+ """
51
+
52
+ _validate = Validate.RAISE
53
+
54
+ @classmethod
55
+ def validation(cls, state: Optional[Validate] = None) -> Validate:
56
+ """Set new parameters validation level and return the previous one"""
57
+ if state is None:
58
+ return cls._validate
59
+ else:
60
+ old, cls._validate = cls._validate, Validate(state)
61
+ return old
62
+
63
+ class Tracker:
64
+ """Helper sub-class of cls::`FuncPar` to track nested calls of
65
+ parametrized functions."""
66
+
67
+ def __init__(self):
68
+ self._track = False
69
+ self._frame = None
70
+ self._depth = None
71
+ self._record = {}
72
+ self._stack = []
73
+
74
+ reset = __init__
75
+
76
+ def start(self):
77
+ """Start tracking of the execution flow for all nested calls"""
78
+ if self._frame is None:
79
+ st = inspect.stack()
80
+ self._frame = st[1].frame
81
+ self._depth = len(st)
82
+ self._track = True
83
+
84
+ def stop(self):
85
+ """Stop tracking - but keep collected record (unlike `reset`)"""
86
+ self._track = False
87
+
88
+ def restart(self):
89
+ """Clear context and restart the tracking from current frame"""
90
+ self.__init__()
91
+ self.start()
92
+
93
+ def report(self) -> Dict[str, 'FuncPar']:
94
+ """Produce dictionary of all calls to functions with `FuncPar`
95
+ with keys as dot separated names of nested called functions.
96
+ """
97
+ return self._record.copy()
98
+
99
+ def __bool__(self):
100
+ return self._track
101
+
102
+ def track_call(self, arg_par: 'FuncPar'):
103
+ """Track specific call provided its `FuncPar` context"""
104
+ func_name = arg_par.function.__qualname__
105
+ self._stack.append(func_name)
106
+ if self._track:
107
+ st = inspect.stack()
108
+ depth = len(st) - self._depth + 1
109
+ if depth > 0 and st[depth].frame is self._frame:
110
+ self._record['.'.join(self._stack)] = arg_par
111
+
112
+ track = Tracker()
113
+
114
+ def __init__(self, func: Callable, *, alias: str = None,
115
+ validate: Union[Rule, Sequence[Rule]] = None,
116
+ directives: Dict[str, Union[Rule]]):
117
+ """Construct `FuncPar` instance for specific function using
118
+ provided validation rules and condition directives.
119
+
120
+ :param func: function object to associate with
121
+ :param alias: optional (short) version of name to associate with
122
+ the parameters structure instead of the function name
123
+ :param validate: single or sequence of validation rules in form of
124
+ a python expression using variables named by the parameters
125
+ or callable accepting keywords arguments or those parameters.
126
+ :param directives: dictionary per parameter of validation conditions
127
+ in form of expression using single variable denoted as "{}" or
128
+ callable with a single argument to receive the parameter value.
129
+
130
+ Examples:
131
+ Associates conditions with function arguments.
132
+ >>> def func(x=10, y=20): pass
133
+ >>> p_func = FuncPar(func, validate='x < y',\
134
+ directives={'y': "{} > 0"}).function
135
+ >>> assert p_func(20, 10) # will fail on x < y
136
+ Traceback (most recent call last):
137
+ ...
138
+ AssertionError
139
+ >>> assert p_func(-10, -2) # will fail on y > 0
140
+ """
141
+ from functools import wraps
142
+
143
+ self._name = func.__qualname__
144
+ self._params = TBox() # arguments which are parameters with defaults
145
+ self._cond = {} # parameters with assigned conditions
146
+ self._alias = alias or self._name
147
+
148
+ # First parse all the function's arguments to determine:
149
+ # - which have defaults - candidates to be parameters
150
+ # - which are mentioned in the special directives
151
+ all_defaults = {} # default values for ALL the arguments
152
+ invalid_references = set(directives) # ensure directives refer to existing arguments
153
+
154
+ pos_kws_start = 0 # location of first not position-only argument
155
+ pos_kws = [] # names (ordered) of all the pos_or_kw arguments
156
+ try:
157
+ for i, (name, arg) in enumerate(inspect.signature(func).parameters.items()):
158
+ if arg.kind is arg.VAR_POSITIONAL:
159
+ raise NotImplementedError("Variadic positional arguments")
160
+ if arg.kind in (arg.POSITIONAL_ONLY, arg.VAR_KEYWORD):
161
+ continue # positional and variadic arguments are not parametrized!
162
+
163
+ # other kinds with provided default values are be turned in parameters
164
+ # unless explicitly excluded in directives
165
+ if arg.kind is arg.POSITIONAL_OR_KEYWORD: # keep track on ambiguous arguments
166
+ if not pos_kws: # to separate position-only when called
167
+ pos_kws_start = i
168
+ pos_kws.append(name)
169
+
170
+ if arg.default is not arg.empty:
171
+ all_defaults[name] = arg.default # to implement defaults logistics
172
+ self._params[name] = arg.default # may be exclude below
173
+ if name in directives: # either provides condition or excludes from params
174
+ invalid_references.remove(name) # remove valid names from the black list
175
+ cond = directives[name]
176
+ if cond:
177
+ if isinstance(cond, str):
178
+ cond = fnt.express_to_lambda(cond.format(name), name)
179
+ self._cond[name] = cond
180
+ else: # exclude from params if explicitly set to False or None
181
+ self._params.pop(name)
182
+
183
+ if invalid_references:
184
+ raise ReferenceError(f"Conditions defined for invalid arguments: {invalid_references}")
185
+
186
+ self._validators = [v if callable(v) else fnt.express_to_kw_func(v) for v in
187
+ (as_list(validate) if validate else [])]
188
+ except SyntaxError as e:
189
+ e.msg = f"Invalid validation condition for function <{self._name}>"
190
+ raise e
191
+
192
+ def separate_kw_args(args):
193
+ """Separate variadic positional argument into position only and pos_or_kw."""
194
+ found_pos_kws = len(args) - pos_kws_start
195
+ if found_pos_kws > len(pos_kws):
196
+ raise SyntaxError(f"Positional args to <{self._name}> exceed {len(pos_kws)}!")
197
+ if found_pos_kws < 0:
198
+ raise SyntaxError(f"Missing {-found_pos_kws} positional args in <{self._name}>!")
199
+ # create kw form of the args passed by position
200
+ silent_kw_arg = dict(zip(pos_kws[:found_pos_kws], args[-found_pos_kws:]))
201
+ return silent_kw_arg
202
+
203
+ def filter_keys_in(d, seq, *, keep_in=True):
204
+ """Filter dictionary by keys belonging or not to given sequence"""
205
+ seq = set(seq)
206
+ condition = (lambda x: x in seq) if keep_in else (lambda x: x not in seq)
207
+ return {k: v for k, v in d.items() if condition(k)}
208
+
209
+ @wraps(func)
210
+ def pz_func(*args, _par: dict = None, **kws):
211
+ # Responsibilities of the wrapper:
212
+ # 1. Optional validation of the parameters values
213
+ # 2. Inform the ArgPar.tracker about the call to collect flow information.
214
+ # 3. Automatically fetch parameters from repository through special argument _par
215
+ # 4. (No Harm) Maintain same syntax as the original function in spite of different signature
216
+ # - make variadic argument behave like named in the original function
217
+ # - support the additional (optional) _par argument
218
+ silent_kws = separate_kw_args(args)
219
+
220
+ # optionally keyword parameters may be passed inside special _par
221
+ if _par: # automatic fetch of relevant parameters from the repo
222
+ # dict-like argument as item under the key `func.name` or `func.alias`
223
+ # which `dict`-like value would contain any subset of them.
224
+ _par = getattr(_par, self._name, getattr(_par, self._alias, {}))
225
+ # make sure no arguments are not passed twice from different sources
226
+ if self.validation() > Validate.RAISE:
227
+ if twice := set(_par).intersection(set(silent_kws).union(kws)):
228
+ from warnings import warn
229
+ warn(f"Double {twice} in {self._name}: as args and (ignored) in _par")
230
+
231
+ kws = {**filter_keys_in(_par, silent_kws, keep_in=False), **kws} # _par-silent-kws+kws
232
+
233
+ self.validate({**silent_kws, **kws}) # add silent to validation
234
+ FuncPar.track.track_call(self)
235
+ return func(*args, **kws)
236
+
237
+ self._func = pz_func
238
+ self.__call__ = pz_func
239
+
240
+ setattr(pz_func, 'paramaze', self)
241
+ setattr(pz_func, 'defaults', self.defaults)
242
+ setattr(pz_func, 'validate', self.validate)
243
+
244
+ def __repr__(self):
245
+ return f'Params in function `{self.name}`:\n' \
246
+ f'{self._params.to_yaml()}'
247
+
248
+ @property
249
+ def function(self):
250
+ return self._func
251
+
252
+ @property
253
+ def name(self):
254
+ return self._name
255
+
256
+ @property
257
+ def alias(self):
258
+ return self._alias
259
+
260
+ @property
261
+ def defaults(self):
262
+ """Return editable copy of the default parameters"""
263
+ return TBox(self._params, frozen_box=False)
264
+
265
+ def validate(self, par, throw=True):
266
+ vld = self.validation()
267
+ if vld is Validate.OFF:
268
+ return True
269
+ if vld is Validate.VERBOSE:
270
+ print(f'Validating {self.name} parameters: {tuple(par)}...', end='')
271
+
272
+ def try_cond(context, cond, *args, **kws):
273
+ try:
274
+ return cond(*args, **kws)
275
+ except Exception as e:
276
+ raise SyntaxError("Invalid validation condition for function "
277
+ f"<{self._func.__qualname__}>\n"
278
+ f"{context} {cond.__qualname__}\n{e}")
279
+
280
+ failed_pars = [f"{k}={v} fails condition: {self._cond[k].__qualname__}"
281
+ for k, v in par.items() if k in self._cond and
282
+ not try_cond(f"for parameter {k}", self._cond[k], v)]
283
+
284
+ merged_par = {**self._params, **par}
285
+ failed_rules = [rule.__qualname__ for rule in self._validators if
286
+ not try_cond("in validation rule", rule, **merged_par)]
287
+
288
+ msg = '\n\t'.join(['Invalid Parameters:', *failed_pars]) + '\n' if failed_pars else ''
289
+ msg += '\n\t'.join(['Failed rules:', *failed_rules]) if failed_rules else ''
290
+ if msg:
291
+ msg = f'Failed parameters validation for <{self.name}>:\n{msg}'
292
+ vld is Validate.VERBOSE and print() # additional line in verbose case
293
+ print(msg, repr(TBox(par)))
294
+ if throw and vld is not Validate.REPORT:
295
+ raise ValueError(msg)
296
+ return False
297
+ if vld is not Validate.RAISE:
298
+ print('OK!')
299
+ return True
300
+
301
+
302
+ @wrap.double_wrap
303
+ def paramaze(func, _alias: str = None,
304
+ _validate: Union[Rule, Sequence[Rule]] = None,
305
+ **kws: Rule):
306
+ """
307
+ Decorator wrapping function to associate it with `FuncPar` and
308
+ provide all the related functionality: arguments as parameters with
309
+ defaults, validation conditions, automatic collection of flow parameters,...
310
+
311
+ By default all keyword arguments with defaults are considered parameters.
312
+ Decorator mau be used with or without arguments, then no conditions or
313
+ alias is associated with the function, but parameters collections is
314
+ still available.
315
+
316
+ :param func: function to wrap
317
+ :param _alias: optional (short) version of name to associate with
318
+ the parameters structure instead of the function name
319
+ :param _validate: single or sequence of validation rules in form of
320
+ a python expression using variables named by the parameters
321
+ or callable accepting keywords arguments or those parameters.
322
+ :param kws: dictionary per parameter of validation conditions
323
+ in form of expression using single variable denoted as "{}" or
324
+ callable with a single argument to receive the parameter value.
325
+
326
+ :return: wrapped function
327
+
328
+ Example:
329
+
330
+ >>> @paramaze
331
+ >>> def func(x=10, y=20): pass
332
+ >>> assert func.defaults == {'x': 10, 'y': 20}
333
+
334
+ >>> @paramaze(y="{} > 0",
335
+ >>> _validate_type=['x < y', 'x**2 + y**2 < 100'])
336
+ >>> def func(x=10, y=20): pass
337
+ >>> assert func.parz.validate(x=2, y=3)
338
+ """
339
+ return FuncPar(func, alias=_alias, validate=_validate, directives=kws).function
iad/core/param/tbox.py ADDED
@@ -0,0 +1,277 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Iterable, Any
4
+
5
+ import pandas as pd
6
+ import pydantic.v1 as pydantic
7
+ from box import Box
8
+
9
+ from ..datatools import complete_missing, UndefCond, UndefTypes
10
+ from ..strings import compact_repr, hash_str
11
+ from ..wrap import name_tuple
12
+
13
+ __all__ = ['TBox', 'UndefCond']
14
+
15
+ _intact_types = (tuple, list, pd.DataFrame, pd.Series)
16
+ UND_EMPTY = UndefCond(empty_dict=True)
17
+
18
+
19
+ class TBox(Box):
20
+ _protected_keys = Box._protected_keys + [
21
+ "_flatten",
22
+ "_repr_json_",
23
+ "diff",
24
+ "issubset",
25
+ "issuperset",
26
+ "find_key",
27
+ "remove",
28
+ "discard",
29
+ "hash_str"
30
+ ]
31
+
32
+ def __init__(self, *args: pydantic.BaseModel | dict | Iterable[tuple[str, Any]],
33
+ undef: UndefTypes = UND_EMPTY, **kw_box):
34
+ """
35
+ Initialize hierarchical dict-like structure.
36
+
37
+ Note, that argument ``undef`` controlling behavior of ``setdefault``method
38
+ may be provided in various forms, but is eventually converted into
39
+ the most complete one: instance of ``UndefCond``:
40
+ - Collection of undef objects
41
+ - Callable -> bool
42
+ - UndefCond (allows to control empty_dict)
43
+
44
+ :param args: dicts to initialize with
45
+ :param kw_box: either key=value pairs as a data items, or Box kw args
46
+ :param undef: describes condition to consider a node undefined
47
+ """
48
+ kw_box.setdefault('box_intact_types', _intact_types)
49
+ kw_box.setdefault('box_dots', True) # dotted - default behaviour for TBox !
50
+
51
+ if args: # special conversions
52
+ args = tuple(arg.dict() if isinstance(arg, pydantic.BaseModel) else arg for arg in args)
53
+
54
+ if len(args) > 1: # Box supports only 1 positional argument, so for more than that
55
+ dict_arg = {} # convert all the args into dicts and then merge them into one
56
+ for arg in args: # here we deal with dict and Iterable[tuple[k, v]]
57
+ dict_arg.update(arg)
58
+ args = (dict_arg,)
59
+
60
+ super().__init__(*args, **kw_box)
61
+ self._box_config['undef'] = undef if isinstance(undef, UndefCond) else UndefCond(undef=undef)
62
+
63
+ def __setitem__(self, key, value):
64
+ # ToDo risky - what about '1.32' or 'file.ext' or relative annotation ..
65
+ if isinstance(key, str) and '.' in key:
66
+ node, tail = key.split('.', 1)
67
+ if node not in self:
68
+ super().__setitem__(node, {})
69
+ self[node].__setitem__(tail, value)
70
+ else:
71
+ super().__setitem__(key, value)
72
+
73
+ def flatten(self):
74
+ res = []
75
+ for key, val in self.items():
76
+ if isinstance(val, Box) and val:
77
+ for subs, v in val.flatten():
78
+ res.append((f'{key}.{subs}', v))
79
+ else:
80
+ res.append((key, val))
81
+ return res
82
+
83
+ def _repr_json_(self):
84
+ jbox = TBox(default_box=True)
85
+ is_basic = lambda x: x is None or isinstance(x, (str, int, float))
86
+ for k, v in self.items(True):
87
+ jbox[k] = v if is_basic(v) or (
88
+ isinstance(v, (list, set, tuple))
89
+ and len(v) < 5 and all(map(is_basic, v))
90
+ ) else compact_repr(v)
91
+ return TBox(jbox)
92
+
93
+ def __repr__(self):
94
+ if not self: return f"{type(self).__name__}()"
95
+ return '\n'.join(f'{k}: {compact_repr(v)}' for k, v in self.flatten())
96
+
97
+ def hash_str(self, length: int = None):
98
+ """Return hex hash string as unique id of the content.
99
+
100
+ :param len: if provided limit string to this length (keep tail)
101
+ """
102
+ s = ''
103
+ for k, v in sorted(self.items(True), key=lambda _: _[0]):
104
+ if isinstance(v, list):
105
+ v = [TBox(x).hash_str(length=1000)
106
+ if isinstance(x, dict) else x
107
+ for x in v]
108
+ s += f"{k}:{str(v)}\n"
109
+
110
+ return hash_str(s, length)
111
+
112
+ def __copy__(self, **kws):
113
+ old_kws = {k: v for k, v in self._box_config.items() if not k.startswith('_')}
114
+ assert set(kws).issubset(old_kws), "copy only accepts Box standard arguments"
115
+ old_kws.update(kws)
116
+ return self.__class__(self.to_dict(), **old_kws)
117
+
118
+ # def __deepcopy__(self, memodict={}):
119
+ # return self.__copy__()
120
+
121
+ def copy(self, **kws):
122
+ from copy import deepcopy
123
+ return deepcopy(self)
124
+
125
+ def diff(self, other: 'TBox'):
126
+ """ Difference between two tree forms as tuple of
127
+ - keys missing in the other,
128
+ - keys missing in self,
129
+ - keys of incomparable or not equal values
130
+ :returns tuple(dif_keys, dif_values)
131
+
132
+ """
133
+
134
+ def not_same_value(k):
135
+ try:
136
+ v1, v2 = other[k], self[k]
137
+ same = (v1 == v2)
138
+ if type(same) is not bool:
139
+ import numpy as np
140
+ same = np.allclose(v1, v2)
141
+ except Exception:
142
+ same = False
143
+ return not same
144
+
145
+ other = other if isinstance(other, TBox) else TBox(other)
146
+ my_keys, other_keys = set(self.keys(True)), set(other.keys(True))
147
+ missing = my_keys.difference(other_keys)
148
+ extra = other_keys.difference(my_keys)
149
+ unequal = {*filter(not_same_value, my_keys.intersection(other_keys))}
150
+ return name_tuple('TBoxDiffKeys', missing=missing, extra=extra, unequal=unequal)
151
+
152
+ def to_yaml(self, filename=None, *, default_flow_style=False,
153
+ encoding="utf-8", errors="strict", **yaml_kwargs):
154
+ from ..filesproc import prepare_parent_folder, Path
155
+
156
+ par = self.copy() # type: TBox
157
+ for k, v in par.items(True):
158
+ if isinstance(v, Path):
159
+ par[k] = str(v)
160
+ if hasattr(v, '__qualname__'):
161
+ par[k] = v.__qualname__
162
+
163
+ if filename:
164
+ prepare_parent_folder(filename)
165
+ filename = str(filename)
166
+
167
+ return Box.to_yaml(par, filename=filename, default_flow_style=default_flow_style,
168
+ encoding=encoding, errors=errors, **yaml_kwargs)
169
+
170
+ def setdefault(self, key: str, default: dict | Any, *, undef: UndefTypes = None):
171
+ """
172
+ Include support for dotted (nested) keys for `key` arg,
173
+ and dict-like (also nested) structure for default argument.
174
+
175
+ If key is None - setting default for the entire tree, in which case
176
+ default must be nested dict as well.
177
+
178
+ May include check that mandatory fields (values set as TBox.REQUIRED
179
+ in the default dict) are already defined in self, or raise ValueError.
180
+
181
+ :param key: name (also nested) of the field or None for entire tree.
182
+ :param default: a value to set to specified key
183
+ or dict (nested) if key points to a subtree
184
+ :param undef: override TBox.undef
185
+ :return: the value been set (or the (sub)tree if default dict is dict
186
+ """
187
+ if (sep := '.') in key:
188
+ assert sep not in (key[0], key[-1]), f"Invalid dotted {key=}!"
189
+ node = self # Ensure dotted key node exists and not a leaf
190
+ for k in key.split(sep):
191
+ if not isinstance(node, dict): # every next level must be a dict
192
+ raise KeyError(f'Position {k} in {key=} must be a node!')
193
+ node = Box.setdefault(node, k, {}) # existed or just created
194
+
195
+ if isinstance(node, dict) and not node: # don't consider empty node as set!
196
+ self[key] = default
197
+ return self[key]
198
+
199
+ default = TBox({key: default} if key else default)
200
+ undef = getattr(self, '_box_config')['undef'] if undef is None else undef
201
+ return complete_missing(self, default, undef=undef)
202
+
203
+ def issubset(self, other: dict):
204
+ if not isinstance(other, self.__class__):
205
+ other = self.__class__(other)
206
+ return set(self.keys(True)).issubset(other.keys(True))
207
+
208
+ def issuperset(self, other: dict):
209
+ if not isinstance(other, self.__class__):
210
+ other = self.__class__(other)
211
+ return set(other.keys(True)).issuperset(self.keys(True))
212
+
213
+ def find_key(self, name, *, part='any', multi=False) -> str | list:
214
+ """
215
+ Find key(s) with deep (dotted) naming matching given name.
216
+
217
+ Return full (dotted) key found or '' if not.
218
+
219
+ If multiple matching keys are found return depending on *multi*:
220
+ - list of them if ``True``
221
+ - only the first if ``None``
222
+ - raise ``KeyError`` if ``False``
223
+
224
+ :param name: may include dots inside: "x.y"
225
+ :param part: key's part to match: ['any'] | 'start' | 'end'
226
+ :param multi: allow multiple keys to return
227
+
228
+ :return: a string or list of strings (keys found)
229
+ """
230
+ import re
231
+ if not re.fullmatch(r'(\w+(\.\w+)?)+', name):
232
+ return [] if multi else ''
233
+
234
+ reg = re.compile({'any': rf'\b{name}\b',
235
+ 'start': rf'^{name}\b',
236
+ 'end': rf'\b{name}$'
237
+ }[part])
238
+
239
+ found = [*filter(reg.search, self.keys(True))]
240
+ if multi:
241
+ return found
242
+ elif len(found) > 1:
243
+ raise KeyError(f"Multiple matches to {name=} been {found=}!")
244
+ else:
245
+ return found[0]
246
+
247
+ def remove(self, keys: Iterable[str], *, strict=True):
248
+ """Remove nodes addressed by the keys (dotted).
249
+ :param keys: iterable over dotted keys to remove
250
+ :param strict: if True - raise KeyError if any of keys is missing
251
+ otherwise - ignore and remove only found
252
+ :return: a new TBox with keys removed
253
+ """
254
+ if isinstance(keys, str): keys = [keys]
255
+ res = self.copy()
256
+ for key in keys:
257
+ try:
258
+ del res[key]
259
+ except KeyError as ex:
260
+ if strict: raise ex
261
+ return res
262
+
263
+ def discard(self, keys: Iterable[str], *, strict=True):
264
+ """Discard from this box nodes addressed by given keys (dotted).
265
+ :param keys: iterable over dotted keys to remove
266
+ :param strict: if True - raise KeyError if any of keys is missing
267
+ otherwise - ignore and remove only found
268
+ :return: None
269
+ """
270
+ if isinstance(keys, str): keys = [keys]
271
+ if not hasattr(keys, '__len__'):
272
+ keys = [*keys]
273
+ valid = [*filter(self.__contains__, keys)]
274
+ if strict and len(valid) < len(keys):
275
+ raise KeyError(f"Missing keys: {set(self.keys(True)).difference(valid)}")
276
+ for key in valid:
277
+ del self[key]