cdxcore 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cdxcore might be problematic. Click here for more details.
- cdxcore/__init__.py +15 -0
- cdxcore/config.py +1633 -0
- cdxcore/crman.py +105 -0
- cdxcore/deferred.py +220 -0
- cdxcore/dynaplot.py +1155 -0
- cdxcore/filelock.py +430 -0
- cdxcore/jcpool.py +411 -0
- cdxcore/logger.py +319 -0
- cdxcore/np.py +1098 -0
- cdxcore/npio.py +270 -0
- cdxcore/prettydict.py +388 -0
- cdxcore/prettyobject.py +64 -0
- cdxcore/sharedarray.py +285 -0
- cdxcore/subdir.py +2963 -0
- cdxcore/uniquehash.py +970 -0
- cdxcore/util.py +1041 -0
- cdxcore/verbose.py +403 -0
- cdxcore/version.py +402 -0
- cdxcore-0.1.5.dist-info/METADATA +1418 -0
- cdxcore-0.1.5.dist-info/RECORD +30 -0
- cdxcore-0.1.5.dist-info/WHEEL +5 -0
- cdxcore-0.1.5.dist-info/licenses/LICENSE +21 -0
- cdxcore-0.1.5.dist-info/top_level.txt +4 -0
- conda/conda_exists.py +10 -0
- conda/conda_modify_yaml.py +42 -0
- tests/_cdxbasics.py +1086 -0
- tests/test_uniquehash.py +469 -0
- tests/test_util.py +329 -0
- up/git_message.py +7 -0
- up/pip_modify_setup.py +55 -0
cdxcore/config.py
ADDED
|
@@ -0,0 +1,1633 @@
|
|
|
1
|
+
"""
|
|
2
|
+
config
|
|
3
|
+
Utility object for ML project configuration
|
|
4
|
+
Hans Buehler 2022
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from collections import OrderedDict
|
|
8
|
+
from collections.abc import Mapping
|
|
9
|
+
from sortedcontainers import SortedDict
|
|
10
|
+
import warnings as warnings
|
|
11
|
+
from dataclasses import Field
|
|
12
|
+
from .util import mt as txtfmt
|
|
13
|
+
from .uniquehash import UniqueHash
|
|
14
|
+
from .prettydict import PrettyDict as pdct
|
|
15
|
+
|
|
16
|
+
__all__ = ["Config", "Int", "Float", "ConfigField", "NotDoneError"]
|
|
17
|
+
|
|
18
|
+
class _ID(object):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
def error( text, *args, exception = RuntimeError, **kwargs ):
|
|
22
|
+
raise exception( txtfmt(text, *args, **kwargs) )
|
|
23
|
+
def verify( cond, text, *args, exception = RuntimeError, **kwargs ):
|
|
24
|
+
if not cond:
|
|
25
|
+
error( text, *args, **kwargs, exception=exception )
|
|
26
|
+
def warn( text, *args, warning=warnings.RuntimeWarning, stack_level=1, **kwargs ):
|
|
27
|
+
warnings.warn( txtfmt(text, *args, **kwargs), warning, stack_level=stack_level )
|
|
28
|
+
def warn_if( cond, text, *args, warning=warnings.RuntimeWarning, stack_level=1, **kwargs ):
|
|
29
|
+
if cond:
|
|
30
|
+
warn( text, *args, warning=warning, stack_level=stack_level, **kwargs )
|
|
31
|
+
|
|
32
|
+
class NotDoneError( RuntimeError ):
|
|
33
|
+
"""
|
|
34
|
+
Raised when done() finds that some arguments have not been read.
|
|
35
|
+
Those arguments are accessible via the 'not_done' attribute
|
|
36
|
+
"""
|
|
37
|
+
def __init__(self, not_done, message):
|
|
38
|
+
self.not_done = not_done
|
|
39
|
+
RuntimeError.__init__(self, message)
|
|
40
|
+
|
|
41
|
+
class InconsistencyError( RuntimeError ):
|
|
42
|
+
"""
|
|
43
|
+
Raised when __call__ used inconsistently for a given parameter, ie when the defaults are different.
|
|
44
|
+
Such inconsistencies must be fixed for safe usage of any configuration strategy.
|
|
45
|
+
"""
|
|
46
|
+
def __init__(self, field, message ):
|
|
47
|
+
self.field = field
|
|
48
|
+
RuntimeError.__init__(self, message)
|
|
49
|
+
|
|
50
|
+
class CastError(RuntimeError):
|
|
51
|
+
def __init__(self, *, config_name : str, key_name : str, message : str):
|
|
52
|
+
header = f"Error in cast definition for key '{key_name}' in config '{config_name}': " if len(config_name) > 0 else f"Error in cast definition for key '{key_name}': "
|
|
53
|
+
RuntimeError.__init__(self, header+message)
|
|
54
|
+
self.config_name = config_name
|
|
55
|
+
self.key_name = key_name
|
|
56
|
+
|
|
57
|
+
#@private
|
|
58
|
+
no_default = _ID() #@private creates a unique object which can be used to detect if a default value was provided
|
|
59
|
+
|
|
60
|
+
# ==============================================================================
|
|
61
|
+
#
|
|
62
|
+
# Actual Config class
|
|
63
|
+
#
|
|
64
|
+
# ==============================================================================
|
|
65
|
+
|
|
66
|
+
class Config(OrderedDict):
|
|
67
|
+
"""
|
|
68
|
+
A simple Config class.
|
|
69
|
+
Main features
|
|
70
|
+
|
|
71
|
+
Write:
|
|
72
|
+
|
|
73
|
+
Set data as usual:
|
|
74
|
+
config = Config()
|
|
75
|
+
config['features'] = [ 'time', 'spot' ]
|
|
76
|
+
config['weights'] = [ 1, 2, 3 ]
|
|
77
|
+
|
|
78
|
+
Use member notation
|
|
79
|
+
config.network.samples = 10000
|
|
80
|
+
config.network.activation = 'relu'
|
|
81
|
+
|
|
82
|
+
Read:
|
|
83
|
+
|
|
84
|
+
def read_config( confg ):
|
|
85
|
+
features = config("features", [], list ) # reads features and returns a list
|
|
86
|
+
weights = config("weights", [], np.ndarray ) # reads features and returns a nump array
|
|
87
|
+
|
|
88
|
+
network = config.network
|
|
89
|
+
samples = network('samples', 10000) # networks samples
|
|
90
|
+
config.done() # returns an error as we haven't read 'network.activitation'
|
|
91
|
+
|
|
92
|
+
Detaching child configs
|
|
93
|
+
You can also detach a child config, which allows you to store it for later
|
|
94
|
+
use without triggering done() errors for its parent.
|
|
95
|
+
|
|
96
|
+
def read_config( confg ):
|
|
97
|
+
features = config("features", [], list ) # reads features and returns a list
|
|
98
|
+
weights = config("weights", [], np.ndarray ) # reads features and returns a nump array
|
|
99
|
+
|
|
100
|
+
network = config.network.detach()
|
|
101
|
+
samples = network('samples', 10000) # networks samples
|
|
102
|
+
config.done() # no error as 'network' was detached
|
|
103
|
+
network.done() # error as network.activation was not read
|
|
104
|
+
|
|
105
|
+
Self-recording "help"
|
|
106
|
+
When reading a value, specify an optional help text:
|
|
107
|
+
|
|
108
|
+
def read_config( confg ):
|
|
109
|
+
|
|
110
|
+
features = config("features", [], list, help="Defines the features" )
|
|
111
|
+
weights = config("weights", [], np.ndarray, help="Network weights" )
|
|
112
|
+
|
|
113
|
+
network = config.network
|
|
114
|
+
samples = network('samples', 10000, int, help="Number of samples")
|
|
115
|
+
activt = network('activation', "relu", str, help="Activation function")
|
|
116
|
+
config.done()
|
|
117
|
+
|
|
118
|
+
config.usage_report() # prints help
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
def __init__(self, *args, config_name : str = None, **kwargs):
|
|
122
|
+
"""
|
|
123
|
+
See help(Config) for a description of this class.
|
|
124
|
+
|
|
125
|
+
Two patterns:
|
|
126
|
+
__init__(config):
|
|
127
|
+
is the copy constructor; see copy()
|
|
128
|
+
or
|
|
129
|
+
__init__( dict, x=1, y=2 ):
|
|
130
|
+
creates a new config by first iteratively loading all positional dictionary arguments,
|
|
131
|
+
and then copying the keyword arguments as provided.
|
|
132
|
+
|
|
133
|
+
See also Config.config_kwargs().
|
|
134
|
+
|
|
135
|
+
Parameters
|
|
136
|
+
----------
|
|
137
|
+
*args : list
|
|
138
|
+
List of dictionaries to create a new config with, iteratively.
|
|
139
|
+
If the first element is a config, and no other parameters are passed,
|
|
140
|
+
then this object will be full copy of that config.
|
|
141
|
+
It then shares all usage recording. See copy().
|
|
142
|
+
config_name : str, optional
|
|
143
|
+
Name of the configuration for report_usage. Default is 'config'
|
|
144
|
+
**kwargs : dict
|
|
145
|
+
Used to initialize the config, e.g.
|
|
146
|
+
Config(a=1, b=2)
|
|
147
|
+
"""
|
|
148
|
+
if len(args) == 1 and isinstance(args[0], Config) and config_name is None and len(kwargs) == 0:
|
|
149
|
+
source = args[0]
|
|
150
|
+
self._done = source._done
|
|
151
|
+
self._name = source._name
|
|
152
|
+
self._recorder = source._recorder
|
|
153
|
+
self._children = source._children
|
|
154
|
+
self.update(source)
|
|
155
|
+
return
|
|
156
|
+
|
|
157
|
+
OrderedDict.__init__(self)
|
|
158
|
+
self._done = set()
|
|
159
|
+
self._name = config_name if not config_name is None else "config"
|
|
160
|
+
self._children = OrderedDict()
|
|
161
|
+
self._recorder = SortedDict()
|
|
162
|
+
self._recorder._name = self._name
|
|
163
|
+
for k in args:
|
|
164
|
+
if not k is None:
|
|
165
|
+
self.update(k)
|
|
166
|
+
self.update(kwargs)
|
|
167
|
+
|
|
168
|
+
# Information
|
|
169
|
+
# -----------
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def config_name(self) -> str:
|
|
173
|
+
""" Returns the fully qualified name of this config """
|
|
174
|
+
return self._name
|
|
175
|
+
|
|
176
|
+
@property
|
|
177
|
+
def children(self) -> OrderedDict:
|
|
178
|
+
""" Returns dictionary of children """
|
|
179
|
+
return self._children
|
|
180
|
+
|
|
181
|
+
def __str__(self) -> str:
|
|
182
|
+
""" Print myself as dictionary """
|
|
183
|
+
s = self.config_name + str(self.as_dict(mark_done=False))
|
|
184
|
+
return s
|
|
185
|
+
|
|
186
|
+
def __repr__(self) -> str:
|
|
187
|
+
""" Print myself as reconstructable object """
|
|
188
|
+
s = repr(self.as_dict(mark_done=False))
|
|
189
|
+
s = "Config( **" + s + ", config_name='" + self.config_name + "' )"
|
|
190
|
+
return s
|
|
191
|
+
|
|
192
|
+
@property
|
|
193
|
+
def is_empty(self) -> bool:
|
|
194
|
+
""" Checks whether any variables have been set """
|
|
195
|
+
return len(self) + len(self._children) == 0
|
|
196
|
+
|
|
197
|
+
# conversion
|
|
198
|
+
# ----------
|
|
199
|
+
|
|
200
|
+
def as_dict(self, mark_done : bool = True ) -> dict:
|
|
201
|
+
"""
|
|
202
|
+
Convert into dictionary of dictionaries
|
|
203
|
+
|
|
204
|
+
Parameters
|
|
205
|
+
----------
|
|
206
|
+
mark_done : bool
|
|
207
|
+
If True, then all members of this config will be considered read ('done').
|
|
208
|
+
|
|
209
|
+
Returns
|
|
210
|
+
-------
|
|
211
|
+
Dict of dict's
|
|
212
|
+
"""
|
|
213
|
+
d = { key : self.get(key) if mark_done else self.get_raw(key) for key in self }
|
|
214
|
+
for n, c in self._children.items():
|
|
215
|
+
if n == '_ipython_canary_method_should_not_exist_':
|
|
216
|
+
continue
|
|
217
|
+
c = c.as_dict(mark_done)
|
|
218
|
+
verify( not n in d, "Cannot convert config to dictionary: found both a regular value, and a child with name '%s'", n)
|
|
219
|
+
d[n] = c
|
|
220
|
+
return d
|
|
221
|
+
|
|
222
|
+
def as_field(self) -> Field:
|
|
223
|
+
"""
|
|
224
|
+
Returns a ConfigField wrapped around self for dataclasses and flax nn.Mpodule support.
|
|
225
|
+
See ConfigField documentation for an example.
|
|
226
|
+
"""
|
|
227
|
+
return ConfigField(self)
|
|
228
|
+
|
|
229
|
+
# handle finishing config use
|
|
230
|
+
# ---------------------------
|
|
231
|
+
|
|
232
|
+
def done(self, include_children : bool = True, mark_done : bool = True ):
|
|
233
|
+
"""
|
|
234
|
+
Closes the config and checks that no unread parameters remain.
|
|
235
|
+
By default this function also validates that all child configs were "done" and raises a NotDoneError() if not the case
|
|
236
|
+
(NotDoneError is derived from RuntimeError).
|
|
237
|
+
|
|
238
|
+
If you want to make a copy of a child config for later processing use detach() first
|
|
239
|
+
config = Config()
|
|
240
|
+
config.a = 1
|
|
241
|
+
config.child.b = 2
|
|
242
|
+
|
|
243
|
+
_ = config.a # read a
|
|
244
|
+
config.done() # error because confg.child.b has not been read yet
|
|
245
|
+
|
|
246
|
+
Instead use:
|
|
247
|
+
config = Config()
|
|
248
|
+
config.a = 1
|
|
249
|
+
config.child.b = 2
|
|
250
|
+
|
|
251
|
+
_ = config.a # read a
|
|
252
|
+
child = config.child.detach()
|
|
253
|
+
config.done() # no error, even though confg.child.b has not been read yet
|
|
254
|
+
|
|
255
|
+
You can force 'done' status by calling mark_done()
|
|
256
|
+
|
|
257
|
+
Parameters
|
|
258
|
+
----------
|
|
259
|
+
include_children:
|
|
260
|
+
Validate child configs, too.
|
|
261
|
+
mark_done:
|
|
262
|
+
Upon completion mark this config as 'done'.
|
|
263
|
+
This stops it being modified; subsequent calls to done() will be successful.
|
|
264
|
+
|
|
265
|
+
Raises
|
|
266
|
+
------
|
|
267
|
+
NotDoneError if not all elements were read.
|
|
268
|
+
"""
|
|
269
|
+
inputs = set(self)
|
|
270
|
+
rest = inputs - self._done
|
|
271
|
+
if len(rest) > 0:
|
|
272
|
+
raise NotDoneError( rest, txtfmt("Error closing config '%s': the following config arguments were not read: %s\n\n"\
|
|
273
|
+
"Summary of all variables read from this object:\n%s", \
|
|
274
|
+
self._name, list(rest), \
|
|
275
|
+
self.usage_report(filter_path=self._name ) ))
|
|
276
|
+
if include_children:
|
|
277
|
+
for _, c in self._children.items():
|
|
278
|
+
c.done(include_children=include_children,mark_done=False)
|
|
279
|
+
if mark_done:
|
|
280
|
+
self.mark_done(include_children=include_children)
|
|
281
|
+
return
|
|
282
|
+
|
|
283
|
+
def reset(self):
|
|
284
|
+
"""
|
|
285
|
+
Reset all usage information
|
|
286
|
+
|
|
287
|
+
Use reset_done() to only reset the information whether a key was used, but to keep information on previously used default values.
|
|
288
|
+
This avoids inconsistency in default values between function calls.
|
|
289
|
+
"""
|
|
290
|
+
self._done.clear()
|
|
291
|
+
self._recorder.clear()
|
|
292
|
+
|
|
293
|
+
def reset_done(self):
|
|
294
|
+
"""
|
|
295
|
+
Reset the internal list of which are 'done', e.g. read.
|
|
296
|
+
This function does not reset the recording of previous uses of each key. This ensures consistency of default values between uses of keys.
|
|
297
|
+
Use reset() to reset both 'done' and create a new recorder.
|
|
298
|
+
"""
|
|
299
|
+
self._done.clear()
|
|
300
|
+
|
|
301
|
+
def mark_done(self, include_children : bool = True ):
|
|
302
|
+
""" Mark all members as being read. Once called calling done() will no longer trigger an error """
|
|
303
|
+
self._done.update( self )
|
|
304
|
+
if include_children:
|
|
305
|
+
for _, c in self._children.items():
|
|
306
|
+
c.mark_done(include_children=include_children)
|
|
307
|
+
|
|
308
|
+
# making copies
|
|
309
|
+
# -------------
|
|
310
|
+
|
|
311
|
+
def _detach( self, *, mark_self_done : bool, copy_done : bool, new_recorder ):
|
|
312
|
+
"""
|
|
313
|
+
Creates a copy of the current config, with a number of options how to share usage information.
|
|
314
|
+
Use the functions
|
|
315
|
+
detach()
|
|
316
|
+
copy()
|
|
317
|
+
clean_copy()
|
|
318
|
+
instead.
|
|
319
|
+
|
|
320
|
+
Parameters
|
|
321
|
+
----------
|
|
322
|
+
mark_self_done : bool
|
|
323
|
+
If True mark 'self' as 'read', otherwise not.
|
|
324
|
+
copy_done : bool
|
|
325
|
+
If True, create a copy of self._done, else remove all 'done' information
|
|
326
|
+
new_recorder :
|
|
327
|
+
<recorder> if a recorder is specified, use it.
|
|
328
|
+
"clean": use a new, empty recorder
|
|
329
|
+
"copy": use a new recorder which is a copy of self.recorder
|
|
330
|
+
"share": share the same recorder
|
|
331
|
+
|
|
332
|
+
Returns
|
|
333
|
+
-------
|
|
334
|
+
A new config
|
|
335
|
+
"""
|
|
336
|
+
config = Config()
|
|
337
|
+
config.update(self)
|
|
338
|
+
config._done = set( self._done ) if copy_done else config._done
|
|
339
|
+
config._name = self._name
|
|
340
|
+
if isinstance( new_recorder, SortedDict ):
|
|
341
|
+
config._recorder = new_recorder
|
|
342
|
+
elif new_recorder == "clean":
|
|
343
|
+
new_recorder = config._recorder
|
|
344
|
+
elif new_recorder == "share":
|
|
345
|
+
config._recorder = self._recorder
|
|
346
|
+
else:
|
|
347
|
+
assert new_recorder == "copy", "Invalid value for 'new_recorder': %s" % new_recorder
|
|
348
|
+
config._recorder.update( self._recorder )
|
|
349
|
+
|
|
350
|
+
config._children = { k: c._detach(mark_self_done=mark_self_done, copy_done=copy_done, new_recorder=new_recorder) for k, c in self._children.items() }
|
|
351
|
+
config._children = OrderedDict( config._children )
|
|
352
|
+
|
|
353
|
+
if mark_self_done:
|
|
354
|
+
self.mark_done()
|
|
355
|
+
return config
|
|
356
|
+
|
|
357
|
+
def detach( self ):
|
|
358
|
+
"""
|
|
359
|
+
Returns a copy of 'self': the purpose of this function is to defer using a config to a later point, while maintaining consistency of usage.
|
|
360
|
+
|
|
361
|
+
- The copy has the same 'done' status at the time of calling detach. It does not share 'done' afterwards since 'self' will be marked as done.
|
|
362
|
+
- The copy shares the recorded to keep track of consistency of usage
|
|
363
|
+
- The function flags 'self' as done
|
|
364
|
+
|
|
365
|
+
For example:
|
|
366
|
+
|
|
367
|
+
class Example(object):
|
|
368
|
+
|
|
369
|
+
def __init__( config ):
|
|
370
|
+
|
|
371
|
+
self.a = config('a', 1, Int>=0, "'a' value")
|
|
372
|
+
self.later = config.later.detach() # detach sub-config
|
|
373
|
+
self._cache = None
|
|
374
|
+
config.done()
|
|
375
|
+
|
|
376
|
+
def function(self):
|
|
377
|
+
if self._cache is None:
|
|
378
|
+
self._cache = Cache(self.later) # deferred use of the self.later config. Cache() calls done() on self.later
|
|
379
|
+
return self._cache
|
|
380
|
+
|
|
381
|
+
See also the examples in Deep Hedging which make extensive use of this feature.
|
|
382
|
+
"""
|
|
383
|
+
return self._detach(mark_self_done=True, copy_done=True, new_recorder="share")
|
|
384
|
+
|
|
385
|
+
def copy( self ):
|
|
386
|
+
"""
|
|
387
|
+
Return a copy of 'self': the purpose of this function is to create a copy of the current state of 'self', which is then independent of 'self'
|
|
388
|
+
-- The copy shares a copy of the 'done' status of 'self'
|
|
389
|
+
-- The copy has a copy of the usage of 'self', but will not share furhter usage
|
|
390
|
+
-- 'self' will not be flagged as 'done'
|
|
391
|
+
|
|
392
|
+
As an example, this allows using different default values for
|
|
393
|
+
config members of the same name:
|
|
394
|
+
|
|
395
|
+
base = Config()
|
|
396
|
+
base.a = 1
|
|
397
|
+
_ = base('a', 1) # use a
|
|
398
|
+
|
|
399
|
+
copy = base.copy() # copy will know 'a' as used with default 1
|
|
400
|
+
|
|
401
|
+
_ = base("x", 1)
|
|
402
|
+
_ = copy("x", 2) # will not fail, as usage tracking is not shared after copy()
|
|
403
|
+
|
|
404
|
+
_ = copy('a', 2) # will fail, as default value differs from previous use of 'a' prior to copy()
|
|
405
|
+
"""
|
|
406
|
+
return self._detach( mark_self_done=False, copy_done=True, new_recorder="copy" )
|
|
407
|
+
|
|
408
|
+
def clean_copy( self ):
|
|
409
|
+
"""
|
|
410
|
+
Return a copy of 'self': the purpose of this function is to create a clean, unused copy of 'self'.
|
|
411
|
+
|
|
412
|
+
As an example, this allows using different default values for
|
|
413
|
+
config members of the same name:
|
|
414
|
+
|
|
415
|
+
base = Config()
|
|
416
|
+
base.a = 1
|
|
417
|
+
_ = base('a', 1) # use a
|
|
418
|
+
|
|
419
|
+
copy = base.copy() # copy will know 'a' as used with default 1
|
|
420
|
+
|
|
421
|
+
_ = base("x", 1)
|
|
422
|
+
_ = copy("x", 2) # will not fail, as no usage is shared
|
|
423
|
+
|
|
424
|
+
_ = copy('a', 2) # will not fail, as no usage is shared
|
|
425
|
+
"""
|
|
426
|
+
return self._detach( mark_self_done=False, copy_done=False, new_recorder="clean" )
|
|
427
|
+
|
|
428
|
+
def clone(self):
|
|
429
|
+
"""
|
|
430
|
+
Return a copy of 'self' which shares all usage tracking with 'self'.
|
|
431
|
+
-- The copy shares the 'done' status of 'self'
|
|
432
|
+
-- The copy shares the 'usage' status of 'self'
|
|
433
|
+
-- 'self' will not be flagged as 'done'
|
|
434
|
+
"""
|
|
435
|
+
return Config(self)
|
|
436
|
+
|
|
437
|
+
# Read
|
|
438
|
+
# -----
|
|
439
|
+
|
|
440
|
+
def __call__(self, key : str,
|
|
441
|
+
default = no_default,
|
|
442
|
+
cast : type = None,
|
|
443
|
+
help : str = None,
|
|
444
|
+
help_default : str = None,
|
|
445
|
+
help_cast : str = None,
|
|
446
|
+
mark_done : bool = True,
|
|
447
|
+
record : bool = True ):
|
|
448
|
+
"""
|
|
449
|
+
Reads 'key' from the config. If not found, return 'default' if specified.
|
|
450
|
+
|
|
451
|
+
config("key") - returns the value for 'key' or if not found raises an exception
|
|
452
|
+
config("key", 1) - returns the value for 'key' or if not found returns 1
|
|
453
|
+
config("key", 1, int) - if 'key' is not found, return 1. If it is found cast the result with int().
|
|
454
|
+
config("key", 1, int, "A number" - also stores an optional help text.
|
|
455
|
+
Call usage_report() after the config has been read to a get a full
|
|
456
|
+
summary of all data requested from this config.
|
|
457
|
+
|
|
458
|
+
Advanced casting
|
|
459
|
+
----------------
|
|
460
|
+
|
|
461
|
+
Int, Float allow to bind the range of numbers:
|
|
462
|
+
config("positive_int", 1, Int>=1, "A positive integer")
|
|
463
|
+
config("ranged_int", 1, (Int>=0)&(Int<=10), "An integer between 0 and 10, inclusive")
|
|
464
|
+
config("positive_float", 1, Float>0., "A positive integerg"
|
|
465
|
+
|
|
466
|
+
Choices are implemented with lists:
|
|
467
|
+
config("difficulty", 'easy', ['easy','medium','hard'], "Choose one")
|
|
468
|
+
|
|
469
|
+
Alternative types are implemented with tuples:
|
|
470
|
+
config("difficulty", None, (None, ['easy','medium','hard']), "None or a level of difficulty")
|
|
471
|
+
config("level", None, (None, Int>=0), "None or a non-negative level")
|
|
472
|
+
|
|
473
|
+
Parameters
|
|
474
|
+
----------
|
|
475
|
+
key : string
|
|
476
|
+
Keyword to read
|
|
477
|
+
default : optional
|
|
478
|
+
Default value.
|
|
479
|
+
Set to 'Config.no_default' to avoid defaulting. If then 'key' cannot be found a KeyError is raised.
|
|
480
|
+
cast : object, optional
|
|
481
|
+
If None, any value will be acceptable.
|
|
482
|
+
If not None, the function will attempt to cast the value provided with the provided value.
|
|
483
|
+
E.g. if cast = int, then it will run int(x)
|
|
484
|
+
This function now also allows passing the following complex arguemts:
|
|
485
|
+
* A list, in which case it is assumed that the 'key' must be from this list. The type of the first element of the list will be used to cast values
|
|
486
|
+
* Int or Float which allow defining constrained integers and floating numbers.
|
|
487
|
+
* A tuple of types, in which case any of the types is acceptable. A None here means that the value 'None' is acceptable (it does not mean that any value is acceptable)
|
|
488
|
+
help : str, optional
|
|
489
|
+
If provied adds a help text when self documentation is used.
|
|
490
|
+
help_default : str, optional
|
|
491
|
+
If provided, specifies the default value in plain text.
|
|
492
|
+
If not provided, help_default is equal to the string representation of the default value, if any.
|
|
493
|
+
Use this for complex default values which are hard to read.
|
|
494
|
+
help_cast : str, optional
|
|
495
|
+
If provided, specifies a description of the cast type.
|
|
496
|
+
If not provided, help_cast is set to the string representation of 'cast', or "None" if 'cast' is None. Complex casts are supported.
|
|
497
|
+
Use this for complex cast types which are hard to read.
|
|
498
|
+
mark_done : bool, optional
|
|
499
|
+
If true, marks the respective element as read.
|
|
500
|
+
record : bool, optional
|
|
501
|
+
If True, records usage of the key and validates that previous usage of the key is consistent with
|
|
502
|
+
the current usage, e.g. that the default values are consistent and that if help was provided it is the same.
|
|
503
|
+
|
|
504
|
+
Returns
|
|
505
|
+
-------
|
|
506
|
+
Value.
|
|
507
|
+
|
|
508
|
+
Raises
|
|
509
|
+
------
|
|
510
|
+
KeyError
|
|
511
|
+
if 'key' could not be found.
|
|
512
|
+
InconsistencyError
|
|
513
|
+
if 'key' was previously accessed with different default, help, help_default or help_cast values.
|
|
514
|
+
For all the help texts empty strings are not compared, ie __call__("x", default=1) will succeed even if a previous call was __call__("x", default=1, help="value for x").
|
|
515
|
+
Note that 'cast' is not validated.
|
|
516
|
+
CastError:
|
|
517
|
+
If an error occcurs casting a provided value.
|
|
518
|
+
ValueError:
|
|
519
|
+
For input errors.
|
|
520
|
+
"""
|
|
521
|
+
verify( isinstance(key, str), "'key' must be a string. Found type %s. Details: %s", type(key), key, exception=ValueError )
|
|
522
|
+
verify( key.find('.') == -1 , "Error using config '%s': key name cannot contain '.'. Found %s", self._name, key, exception=ValueError )
|
|
523
|
+
|
|
524
|
+
# determine raw value
|
|
525
|
+
if not key in self:
|
|
526
|
+
if default == no_default:
|
|
527
|
+
raise KeyError(key, "Error using config '%s': key '%s' not found " % (self._name, key))
|
|
528
|
+
value = default
|
|
529
|
+
else:
|
|
530
|
+
value = OrderedDict.get(self,key)
|
|
531
|
+
|
|
532
|
+
# has user only specified 'help' but not 'cast' ?
|
|
533
|
+
if isinstance(cast, str) and help is None:
|
|
534
|
+
help = cast
|
|
535
|
+
cast = None
|
|
536
|
+
|
|
537
|
+
# cast
|
|
538
|
+
caster = _create_caster( cast, self._name, key, none_is_any = True )
|
|
539
|
+
try:
|
|
540
|
+
value = caster( value, self._name, key )
|
|
541
|
+
except Exception as e:
|
|
542
|
+
raise CastError( config_name=self._name, key_name=key, message=e )
|
|
543
|
+
|
|
544
|
+
# mark key as read
|
|
545
|
+
if mark_done:
|
|
546
|
+
self._done.add(key)
|
|
547
|
+
|
|
548
|
+
# avoid recording
|
|
549
|
+
if not record:
|
|
550
|
+
return value
|
|
551
|
+
# record?
|
|
552
|
+
record_key = self.record_key( key ) # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
|
|
553
|
+
help = str(help) if not help is None and len(help) > 0 else ""
|
|
554
|
+
help = help[:-1] if help[-1:] == "." else help # remove trailing '.'
|
|
555
|
+
help_default = str(help_default) if not help_default is None else ""
|
|
556
|
+
help_default = str(default) if default != no_default and len(help_default) == 0 else help_default
|
|
557
|
+
help_cast = str(help_cast) if not help_cast is None else str(caster)
|
|
558
|
+
verify( default != no_default or help_default == "", "Config %s setup error for key %s: cannot specify 'help_default' if no default is given", self._name, key, exception=ValueError )
|
|
559
|
+
|
|
560
|
+
raw_use = help == "" and help_cast == "" and help_default == "" # raw_use, e.g. simply get() or []. Including internal use
|
|
561
|
+
|
|
562
|
+
exst_value = self._recorder.get(record_key, None)
|
|
563
|
+
|
|
564
|
+
if exst_value is None:
|
|
565
|
+
# no previous recorded use --> record this one, even if 'raw'
|
|
566
|
+
record = SortedDict(value=value,
|
|
567
|
+
raw_use=raw_use,
|
|
568
|
+
help=help,
|
|
569
|
+
help_default=help_default,
|
|
570
|
+
help_cast=help_cast )
|
|
571
|
+
if default != no_default:
|
|
572
|
+
record['default'] = default
|
|
573
|
+
self._recorder[record_key] = record
|
|
574
|
+
return value
|
|
575
|
+
|
|
576
|
+
if raw_use:
|
|
577
|
+
# do not compare raw_use with any other use
|
|
578
|
+
return value
|
|
579
|
+
|
|
580
|
+
if exst_value['raw_use']:
|
|
581
|
+
# previous usesage was 'raw'. Record this new use.
|
|
582
|
+
record = SortedDict(value=value,
|
|
583
|
+
raw_use=raw_use,
|
|
584
|
+
help=help,
|
|
585
|
+
help_default=help_default,
|
|
586
|
+
help_cast=help_cast )
|
|
587
|
+
if default != no_default:
|
|
588
|
+
record['default'] = default
|
|
589
|
+
self._recorder[record_key] = record
|
|
590
|
+
return value
|
|
591
|
+
|
|
592
|
+
# Both current and past were bona fide recorded uses.
|
|
593
|
+
# Ensure that their usage is consistent.
|
|
594
|
+
if default != no_default:
|
|
595
|
+
if 'default' in exst_value:
|
|
596
|
+
if exst_value['default'] != default:
|
|
597
|
+
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different default values '%s' and '%s'" % ( key, self._name, record_key, exst_value['default'], default ))
|
|
598
|
+
else:
|
|
599
|
+
exst_value['default'] = default
|
|
600
|
+
|
|
601
|
+
if help != "":
|
|
602
|
+
if exst_value['help'] != "":
|
|
603
|
+
if exst_value['help'] != help:
|
|
604
|
+
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help' texts '%s' and '%s'" % ( key, self._name, record_key, exst_value['help'], help ) )
|
|
605
|
+
else:
|
|
606
|
+
exst_value['help'] = help
|
|
607
|
+
|
|
608
|
+
if help_default != "":
|
|
609
|
+
if exst_value['help_default'] != "":
|
|
610
|
+
# we do not insist on the same 'help_default'
|
|
611
|
+
if exst_value['help_default'] != help_default:
|
|
612
|
+
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help_default' texts '%s' and '%s'" % ( key, self._name, record_key, exst_value['help_default'], help_default ) )
|
|
613
|
+
else:
|
|
614
|
+
exst_value['help_default'] = help_default
|
|
615
|
+
|
|
616
|
+
if help_cast != "" and help_cast != _Simple.STR_NONE_CAST:
|
|
617
|
+
if exst_value['help_cast'] != "" and exst_value['help_cast'] != _Simple.STR_NONE_CAST:
|
|
618
|
+
if exst_value['help_cast'] != help_cast:
|
|
619
|
+
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help_cast' texts '%s' and '%s'" % ( key, self._name, record_key, exst_value['help_cast'], help_cast ))
|
|
620
|
+
else:
|
|
621
|
+
exst_value['help_cast'] = help_cast
|
|
622
|
+
# done
|
|
623
|
+
return value
|
|
624
|
+
|
|
625
|
+
def __getitem__(self, key : str):
|
|
626
|
+
"""
|
|
627
|
+
Returns the item for 'key' *without* recording its usage which for example means that done() will assume it hasn't been read.
|
|
628
|
+
Equivalent to get_raw(key).
|
|
629
|
+
|
|
630
|
+
__getitem__ does not record usage as many Python functions will use to iterate through objects (recall that Config is derived from OrderedDict)
|
|
631
|
+
"""
|
|
632
|
+
return self.get_raw(key)
|
|
633
|
+
|
|
634
|
+
def __getattr__(self, key : str):
|
|
635
|
+
"""
|
|
636
|
+
Returns either the value for 'key', if it exists, or creates on-the-fly a child config with the name 'key' and returns it.
|
|
637
|
+
If an exsting child value is returned, its usage is *not* recorded which for example means that done() will assume it hasn't been read.
|
|
638
|
+
|
|
639
|
+
Because __getattr__ will generte child configs on the fly it means the following is a legitmate use:
|
|
640
|
+
config = Config()
|
|
641
|
+
config.sub.x = 1 # <-- create 'sub' on the fly
|
|
642
|
+
"""
|
|
643
|
+
verify( key.find('.') == -1 , "Error using config '%s': key name cannot contain '.'. Found %s", self._name, key, exception=ValueError )
|
|
644
|
+
if key in self._children:
|
|
645
|
+
return self._children[key]
|
|
646
|
+
verify( key.find(" ") == -1, "Error using config '%s': sub-config names cannot contain spaces. Found %s", self._name, key, exception=ValueError )
|
|
647
|
+
config = Config()
|
|
648
|
+
config._name = self._name + "." + key
|
|
649
|
+
config._recorder = self._recorder
|
|
650
|
+
self._children[key] = config
|
|
651
|
+
return config
|
|
652
|
+
|
|
653
|
+
def get(self, *kargs, **kwargs ):
|
|
654
|
+
"""
|
|
655
|
+
Returns __call__(*kargs, **kwargs)
|
|
656
|
+
"""
|
|
657
|
+
return self(*kargs, **kwargs)
|
|
658
|
+
|
|
659
|
+
def get_default(self, *kargs, **kwargs ):
|
|
660
|
+
"""
|
|
661
|
+
Returns __call__(*kargs, **kwargs)
|
|
662
|
+
"""
|
|
663
|
+
return self(*kargs, **kwargs)
|
|
664
|
+
|
|
665
|
+
def get_raw(self, key : str, default = no_default ):
|
|
666
|
+
"""
|
|
667
|
+
Reads the respectitve element without marking the element as read, and without recording access to the element.
|
|
668
|
+
Equivalent to __call__(key, default, mark_done=False, record=False )
|
|
669
|
+
"""
|
|
670
|
+
return self(key, default, mark_done=False, record=False)
|
|
671
|
+
|
|
672
|
+
def get_recorded(self, key : str ):
|
|
673
|
+
"""
|
|
674
|
+
Returns the recorded used value of key, e.g. the value returned when the config was used:
|
|
675
|
+
If key is part of the input data, return that value
|
|
676
|
+
If key is not part of the input data, and a default was provided when the config was read, return the default.
|
|
677
|
+
This function:
|
|
678
|
+
Throws a KeyError if the key was never read successfully from the config (e.g. it is not used in the calling stack)
|
|
679
|
+
"""
|
|
680
|
+
verify( key.find('.') == -1 , "Error using config '%s': key name cannot contain '.'. Found %s", self._name, key )
|
|
681
|
+
record_key = self._name + "['" + key + "']" # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
|
|
682
|
+
record = self._recorder.get(record_key, None)
|
|
683
|
+
if record is None:
|
|
684
|
+
raise KeyError(key)
|
|
685
|
+
return record['value']
|
|
686
|
+
|
|
687
|
+
def keys(self):
|
|
688
|
+
"""
|
|
689
|
+
Returns the keys for the immediate keys of this config.
|
|
690
|
+
This call will *not* return the names of config children
|
|
691
|
+
"""
|
|
692
|
+
return OrderedDict.keys(self)
|
|
693
|
+
|
|
694
|
+
# Write
|
|
695
|
+
# -----
|
|
696
|
+
|
|
697
|
+
def __setattr__(self, key, value):
|
|
698
|
+
"""
|
|
699
|
+
Assign value using member notation, i.e. self.key = value
|
|
700
|
+
Identical to self[key] = value
|
|
701
|
+
Do not use leading underscores for config variables, see below
|
|
702
|
+
|
|
703
|
+
Parameters
|
|
704
|
+
----------
|
|
705
|
+
key : str
|
|
706
|
+
Key to store. Note that keys with underscores are *not* stored as standard values,
|
|
707
|
+
but become classic members of the object (self.__dict__)
|
|
708
|
+
value :
|
|
709
|
+
If value is a Config object, them its usage information will be reset, and
|
|
710
|
+
the recorder will be set to the current recorder.
|
|
711
|
+
This way the following works as expected
|
|
712
|
+
|
|
713
|
+
config = Config()
|
|
714
|
+
sub = Config(a=1)
|
|
715
|
+
config.sub = sub
|
|
716
|
+
a = config.sub("a", 0, int, "Test")
|
|
717
|
+
config.done() # <- no error is reported, usage_report() is correct
|
|
718
|
+
"""
|
|
719
|
+
self.__setitem__(key,value)
|
|
720
|
+
|
|
721
|
+
def __setitem__(self, key, value):
|
|
722
|
+
"""
|
|
723
|
+
Assign value using array notation, i.e. self[key] = value
|
|
724
|
+
Identical to self.key = value
|
|
725
|
+
|
|
726
|
+
Parameters
|
|
727
|
+
----------
|
|
728
|
+
key : str
|
|
729
|
+
Key to store. Note that keys with underscores are *not* stored as standard values,
|
|
730
|
+
but become classic members of the object (self.__dict__)
|
|
731
|
+
'key' may contain '.' for hierarchical access.
|
|
732
|
+
value :
|
|
733
|
+
If value is a Config object, them its usage information will be reset, and
|
|
734
|
+
the recorder will be set to the current recorder.
|
|
735
|
+
This way the following works as expected
|
|
736
|
+
|
|
737
|
+
config = Config()
|
|
738
|
+
sub = Config(a=1)
|
|
739
|
+
config.sub = sub
|
|
740
|
+
a = config.sub("a", 0, int, "Test")
|
|
741
|
+
config.done() # <- no error is reported, usage_report() is correct
|
|
742
|
+
"""
|
|
743
|
+
if key[0] == "_" or key in self.__dict__:
|
|
744
|
+
OrderedDict.__setattr__(self, key, value )
|
|
745
|
+
elif isinstance( value, Config ):
|
|
746
|
+
warn_if( len(value._recorder) > 0, "Warning: when assigning a used Config to another Config, all existing usage will be reset. "
|
|
747
|
+
"The 'recorder' of the assignee will be set ot the recorder of the receiving Config. "
|
|
748
|
+
"Make a 'clean_copy()' to avoid this warning.")
|
|
749
|
+
value._name = self._name + "." + key
|
|
750
|
+
def update_recorder( config ):
|
|
751
|
+
config._recorder = self._recorder
|
|
752
|
+
config._done.clear()
|
|
753
|
+
for k, c in config._children.items():
|
|
754
|
+
c._name = config._name + "." + k
|
|
755
|
+
update_recorder(c)
|
|
756
|
+
update_recorder(value)
|
|
757
|
+
self._children[key] = value
|
|
758
|
+
else:
|
|
759
|
+
keys = key.split(".")
|
|
760
|
+
if len(keys) == 1:
|
|
761
|
+
OrderedDict.__setitem__(self, key, value)
|
|
762
|
+
else:
|
|
763
|
+
c = self
|
|
764
|
+
for key in keys[:1]:
|
|
765
|
+
c = c.__getattr__(key)
|
|
766
|
+
OrderedDict.__setitem__(c, key, value)
|
|
767
|
+
|
|
768
|
+
def update( self, other=None, **kwargs ):
|
|
769
|
+
"""
|
|
770
|
+
Overwrite values of 'self' new values.
|
|
771
|
+
Accepts the two main formats
|
|
772
|
+
|
|
773
|
+
update( dictionary )
|
|
774
|
+
update( config )
|
|
775
|
+
update( a=1, b=2 )
|
|
776
|
+
update( {'x.a':1 } ) # hierarchical assignment self.x.a = 1
|
|
777
|
+
|
|
778
|
+
Parameters
|
|
779
|
+
----------
|
|
780
|
+
other : dict, Config, optional
|
|
781
|
+
Copy all content of 'other' into 'self'.
|
|
782
|
+
If 'other' is a config: elements will be clean_copy()ed.
|
|
783
|
+
'other' will not be marked as 'used'
|
|
784
|
+
If 'other' is a dictionary, then '.' notation can be used for hierarchical assignments
|
|
785
|
+
**kwargs
|
|
786
|
+
Allows assigning specific values.
|
|
787
|
+
"""
|
|
788
|
+
if not other is None:
|
|
789
|
+
if isinstance( other, Config ):
|
|
790
|
+
# copy() children
|
|
791
|
+
# and reset recorder to ours.
|
|
792
|
+
def set_recorder(config, recorder):
|
|
793
|
+
config._recorder = recorder
|
|
794
|
+
for _,c in config._children.items():
|
|
795
|
+
set_recorder( c, recorder )
|
|
796
|
+
for sub, child in other._children.items():
|
|
797
|
+
assert isinstance(child,Config)
|
|
798
|
+
if sub in self._children:
|
|
799
|
+
self._children[sub].update( child )
|
|
800
|
+
else:
|
|
801
|
+
self[sub] = child.clean_copy() # see above for assigning config
|
|
802
|
+
assert sub in self._children
|
|
803
|
+
assert not sub in self
|
|
804
|
+
# copy elements from other.
|
|
805
|
+
# we do not mark elements from another config as 'used'
|
|
806
|
+
for key in other:
|
|
807
|
+
if key in self._children:
|
|
808
|
+
del self._children[key]
|
|
809
|
+
self[key] = other.get_raw(key)
|
|
810
|
+
assert key in self
|
|
811
|
+
assert not key in self._children
|
|
812
|
+
else:
|
|
813
|
+
verify( isinstance(other, Mapping), "Cannot update config with an object of type '%s'. Expected 'Mapping' type.", type(other).__name__, exception=ValueError )
|
|
814
|
+
for key in other:
|
|
815
|
+
if key[:1] == "_" or key in self.__dict__:
|
|
816
|
+
continue
|
|
817
|
+
if isinstance(other[key], Mapping):
|
|
818
|
+
if key in self:
|
|
819
|
+
del self[key]
|
|
820
|
+
elif not key in self._children:
|
|
821
|
+
self.__getattr__(key) # creates child
|
|
822
|
+
self._children[key].update( other[key] )
|
|
823
|
+
else:
|
|
824
|
+
if key in self._children:
|
|
825
|
+
del self._children[key]
|
|
826
|
+
self[key] = other[key]
|
|
827
|
+
|
|
828
|
+
if len(kwargs) > 0:
|
|
829
|
+
self.update( other=kwargs )
|
|
830
|
+
|
|
831
|
+
# delete
|
|
832
|
+
# ------
|
|
833
|
+
|
|
834
|
+
def delete_children( self, names : list ):
|
|
835
|
+
"""
|
|
836
|
+
Delete one or several children.
|
|
837
|
+
This function does not delete 'record' information.
|
|
838
|
+
"""
|
|
839
|
+
if isinstance(names, str):
|
|
840
|
+
names = [ names ]
|
|
841
|
+
|
|
842
|
+
for name in names:
|
|
843
|
+
del self._children[name]
|
|
844
|
+
|
|
845
|
+
# Usage information & reports
|
|
846
|
+
# ---------------------------
|
|
847
|
+
|
|
848
|
+
@property
|
|
849
|
+
def recorder(self) -> SortedDict:
|
|
850
|
+
""" Returns the top level recorder """
|
|
851
|
+
return self._recorder
|
|
852
|
+
|
|
853
|
+
def usage_report(self, with_values : bool = True,
|
|
854
|
+
with_help : bool = True,
|
|
855
|
+
with_defaults: bool = True,
|
|
856
|
+
with_cast : bool = False,
|
|
857
|
+
filter_path : str = None ) -> str:
|
|
858
|
+
"""
|
|
859
|
+
Generate a human readable report of all variables read from this config.
|
|
860
|
+
|
|
861
|
+
Parameters
|
|
862
|
+
----------
|
|
863
|
+
with_values : bool, optional
|
|
864
|
+
Whether to also print values. This can be hard to read
|
|
865
|
+
if values are complex objects
|
|
866
|
+
|
|
867
|
+
with_help: bool, optional
|
|
868
|
+
Whether to print help
|
|
869
|
+
|
|
870
|
+
with_defaults: bool, optional
|
|
871
|
+
Whether to print default values
|
|
872
|
+
|
|
873
|
+
with_cast: bool, optional
|
|
874
|
+
Whether to print types
|
|
875
|
+
|
|
876
|
+
filter_path : str, optional
|
|
877
|
+
If provided, will match all children names vs this string.
|
|
878
|
+
Most useful with filter_path = self._name
|
|
879
|
+
|
|
880
|
+
Returns
|
|
881
|
+
-------
|
|
882
|
+
str
|
|
883
|
+
Report.
|
|
884
|
+
"""
|
|
885
|
+
with_values = bool(with_values)
|
|
886
|
+
with_help = bool(with_help)
|
|
887
|
+
with_defaults = bool(with_defaults)
|
|
888
|
+
with_cast = bool(with_cast)
|
|
889
|
+
l = len(filter_path) if not filter_path is None else 0
|
|
890
|
+
rep_here = ""
|
|
891
|
+
reported = ""
|
|
892
|
+
|
|
893
|
+
for key, record in self._recorder.items():
|
|
894
|
+
value = record['value']
|
|
895
|
+
help = record['help']
|
|
896
|
+
help_default = record['help_default']
|
|
897
|
+
help_cast = record['help_cast']
|
|
898
|
+
report = key + " = " + str(value) if with_values else key
|
|
899
|
+
|
|
900
|
+
do_help = with_help and help != ""
|
|
901
|
+
do_cast = with_cast and help_cast != ""
|
|
902
|
+
do_defaults = with_defaults and help_default != ""
|
|
903
|
+
|
|
904
|
+
if do_help or do_cast or do_defaults:
|
|
905
|
+
report += " # "
|
|
906
|
+
if do_cast:
|
|
907
|
+
report += "(" + help_cast + ") "
|
|
908
|
+
if do_help:
|
|
909
|
+
report += help
|
|
910
|
+
if do_defaults:
|
|
911
|
+
report += "; default: " + help_default
|
|
912
|
+
elif do_defaults:
|
|
913
|
+
report += "Default: " + help_default
|
|
914
|
+
|
|
915
|
+
if l > 0 and key[:l] == filter_path:
|
|
916
|
+
rep_here += report + "\n"
|
|
917
|
+
else:
|
|
918
|
+
reported += report + "\n"
|
|
919
|
+
|
|
920
|
+
if len(reported) == 0:
|
|
921
|
+
return rep_here
|
|
922
|
+
if len(rep_here) == 0:
|
|
923
|
+
return reported
|
|
924
|
+
return rep_here + "# \n" + reported
|
|
925
|
+
|
|
926
|
+
def usage_reproducer(self) -> str:
|
|
927
|
+
"""
|
|
928
|
+
Returns a string expression which will reproduce the current
|
|
929
|
+
configuration tree as long as each 'value' handles
|
|
930
|
+
repr() correctly.
|
|
931
|
+
"""
|
|
932
|
+
report = ""
|
|
933
|
+
for key, record in self._recorder.items():
|
|
934
|
+
value = record['value']
|
|
935
|
+
report += key + " = " + repr(value) + "\n"
|
|
936
|
+
return report
|
|
937
|
+
|
|
938
|
+
def input_report(self) -> str:
|
|
939
|
+
"""
|
|
940
|
+
Returns a report of all inputs in a readable format, as long as all values
|
|
941
|
+
are as such.
|
|
942
|
+
"""
|
|
943
|
+
inputs = []
|
|
944
|
+
def ireport(self, inputs):
|
|
945
|
+
for key in self:
|
|
946
|
+
value = self.get_raw(key)
|
|
947
|
+
report_key = self._name + "['" + key + "'] = %s" % str(value)
|
|
948
|
+
inputs.append( report_key )
|
|
949
|
+
for c in self._children.values():
|
|
950
|
+
ireport(c, inputs)
|
|
951
|
+
ireport(self, inputs)
|
|
952
|
+
|
|
953
|
+
inputs = sorted(inputs)
|
|
954
|
+
report = ""
|
|
955
|
+
for i in inputs:
|
|
956
|
+
report += i + "\n"
|
|
957
|
+
return report
|
|
958
|
+
|
|
959
|
+
@property
|
|
960
|
+
def not_done(self) -> dict:
|
|
961
|
+
""" Returns a dictionary of keys which were not read yet """
|
|
962
|
+
h = { key : False for key in self if not key in self._done }
|
|
963
|
+
for k,c in self._children.items():
|
|
964
|
+
ch = c.not_done
|
|
965
|
+
if len(ch) > 0:
|
|
966
|
+
h[k] = ch
|
|
967
|
+
return h
|
|
968
|
+
|
|
969
|
+
def input_dict(self, ignore_underscore = True ) -> dict:
|
|
970
|
+
""" Returns a (pretty) dictionary of all inputs into this config. """
|
|
971
|
+
inputs = pdct()
|
|
972
|
+
for key in self:
|
|
973
|
+
if ignore_underscore and key[:1] == "_":
|
|
974
|
+
continue
|
|
975
|
+
inputs[key] = self.get_raw(key)
|
|
976
|
+
for k,c in self._children.items():
|
|
977
|
+
if ignore_underscore and k[:1] == "_":
|
|
978
|
+
continue
|
|
979
|
+
inputs[k] = c.input_dict()
|
|
980
|
+
return inputs
|
|
981
|
+
|
|
982
|
+
def unique_id(self, *, uniqueHash = None, debug_trace = None, **unique_hash_parameters ) -> str:
|
|
983
|
+
"""
|
|
984
|
+
Returns a unique hash key for this object, based on its provided inputs /not/ based on its usage.
|
|
985
|
+
Please consult the documentation for cdxbasics.uniquehash.UniqueHashExt
|
|
986
|
+
** WARNING **
|
|
987
|
+
By default function ignores
|
|
988
|
+
1) Config keys or children with leading '_'s are ignored unless 'parse_underscore' is set to 'protected' or 'private'.
|
|
989
|
+
2) Functions and properties are ignored unless parse_functions is True
|
|
990
|
+
In the latter case function code will be used to distinguish
|
|
991
|
+
functions assigned to the config.
|
|
992
|
+
See uniquehash.unqiueHashExt() for further information.
|
|
993
|
+
|
|
994
|
+
Parameters
|
|
995
|
+
----------
|
|
996
|
+
|
|
997
|
+
Returns
|
|
998
|
+
-------
|
|
999
|
+
String ID
|
|
1000
|
+
"""
|
|
1001
|
+
if uniqueHash is None:
|
|
1002
|
+
uniqueHash = UniqueHash( **unique_hash_parameters )
|
|
1003
|
+
else:
|
|
1004
|
+
if len(unique_hash_parameters) != 0: raise ValueError("Cannot provide 'unique_hash_parameters' if 'uniqueHashExt' is provided")
|
|
1005
|
+
|
|
1006
|
+
def rec(config):
|
|
1007
|
+
""" Recursive version which returns an empty string for empty sub configs """
|
|
1008
|
+
inputs = {}
|
|
1009
|
+
for key in config:
|
|
1010
|
+
if key[:1] == "_":
|
|
1011
|
+
continue
|
|
1012
|
+
inputs[key] = config.get_raw(key)
|
|
1013
|
+
for c, child in config._children.items():
|
|
1014
|
+
if c[:1] == "_":
|
|
1015
|
+
continue
|
|
1016
|
+
# collect ID for the child
|
|
1017
|
+
child_data = rec(child)
|
|
1018
|
+
# we only register children if they have keys.
|
|
1019
|
+
# this way we do not trigger a change in ID simply due to a failed read access.
|
|
1020
|
+
if child_data != "":
|
|
1021
|
+
inputs[c] = child_data
|
|
1022
|
+
if len(inputs) == 0:
|
|
1023
|
+
return ""
|
|
1024
|
+
return uniqueHashExt(inputs)
|
|
1025
|
+
uid = rec(self)
|
|
1026
|
+
return uid if uid!="" else uniqueHashExt("")
|
|
1027
|
+
|
|
1028
|
+
def used_info(self, key : str) -> tuple:
|
|
1029
|
+
"""Returns the usage stats for a given key in the form of a tuple (done, record) where 'done' is a boolean and 'record' is a dictionary of information on the key """
|
|
1030
|
+
done = key in self._done
|
|
1031
|
+
record = self._recorder.get( self.record_key(key), None )
|
|
1032
|
+
return (done, record)
|
|
1033
|
+
|
|
1034
|
+
def record_key(self, key):
|
|
1035
|
+
"""
|
|
1036
|
+
Returns the fully qualified 'record' key for a relative 'key'.
|
|
1037
|
+
It has the form config1.config['entry']
|
|
1038
|
+
"""
|
|
1039
|
+
return self._name + "['" + key + "']" # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
|
|
1040
|
+
|
|
1041
|
+
# magic
|
|
1042
|
+
# -----
|
|
1043
|
+
|
|
1044
|
+
def __iter__(self):
|
|
1045
|
+
"""
|
|
1046
|
+
Iterate. For some odd reason, adding this override will make
|
|
1047
|
+
using f(**self) call our __getitem__() function.
|
|
1048
|
+
"""
|
|
1049
|
+
return OrderedDict.__iter__(self)
|
|
1050
|
+
|
|
1051
|
+
# pickling
|
|
1052
|
+
# --------
|
|
1053
|
+
|
|
1054
|
+
def __reduce__(self):
|
|
1055
|
+
"""
|
|
1056
|
+
Pickling this object explicitly
|
|
1057
|
+
See https://docs.python.org/3/library/pickle.html#object.__reduce__
|
|
1058
|
+
"""
|
|
1059
|
+
keys = [ k for k in self ]
|
|
1060
|
+
data = [ self.get_raw(k) for k in keys ]
|
|
1061
|
+
state = dict(done = self._done,
|
|
1062
|
+
name = self._name,
|
|
1063
|
+
children = self._children,
|
|
1064
|
+
recorder = self._recorder,
|
|
1065
|
+
keys = keys,
|
|
1066
|
+
data = data )
|
|
1067
|
+
return (Config, (), state)
|
|
1068
|
+
|
|
1069
|
+
def __setstate__(self, state):
|
|
1070
|
+
""" Supports unpickling """
|
|
1071
|
+
self._name = state['name']
|
|
1072
|
+
self._done = state['done']
|
|
1073
|
+
self._children = state['children']
|
|
1074
|
+
self._recorder = state['recorder']
|
|
1075
|
+
data = state['data']
|
|
1076
|
+
keys = state['keys']
|
|
1077
|
+
for (k,d) in zip(keys,data):
|
|
1078
|
+
self[k] = d
|
|
1079
|
+
|
|
1080
|
+
# casting
|
|
1081
|
+
# -------
|
|
1082
|
+
|
|
1083
|
+
@staticmethod
|
|
1084
|
+
def to_config( kwargs : dict, config_name : str = "kwargs"):
|
|
1085
|
+
"""
|
|
1086
|
+
Makes sure an object is a config, and otherwise tries to convert it into one
|
|
1087
|
+
Classic use case is to transform 'kwargs' to a Config
|
|
1088
|
+
"""
|
|
1089
|
+
return kwargs if isinstance(kwargs,Config) else Config( kwargs,config_name=config_name )
|
|
1090
|
+
|
|
1091
|
+
@staticmethod
|
|
1092
|
+
def config_kwargs( config, kwargs : dict, config_name : str = "kwargs"):
|
|
1093
|
+
"""
|
|
1094
|
+
Default implementation for a usage pattern where the user can use both a 'config' and kwargs.
|
|
1095
|
+
This function 'detaches' the current config from 'self' which means done() must be called again.
|
|
1096
|
+
|
|
1097
|
+
Example
|
|
1098
|
+
-------
|
|
1099
|
+
|
|
1100
|
+
def f(config, **kwargs):
|
|
1101
|
+
config = Config.config_kwargs( config, kwargs )
|
|
1102
|
+
...
|
|
1103
|
+
x = config("x", 1, ...)
|
|
1104
|
+
config.done() # <-- important to do this here. Remembert that config_kwargs() calls 'detach'
|
|
1105
|
+
|
|
1106
|
+
and then one can use either
|
|
1107
|
+
|
|
1108
|
+
config = Config()
|
|
1109
|
+
config.x = 1
|
|
1110
|
+
f(config)
|
|
1111
|
+
|
|
1112
|
+
or
|
|
1113
|
+
f(x=1)
|
|
1114
|
+
|
|
1115
|
+
Parameters
|
|
1116
|
+
----------
|
|
1117
|
+
config : a Config object or None
|
|
1118
|
+
kwargs : a dictionary. If 'config' is provided, the function will call config.update(kwargs).
|
|
1119
|
+
config_name : a declarative name for the config if 'config' is not proivded
|
|
1120
|
+
|
|
1121
|
+
Returns
|
|
1122
|
+
-------
|
|
1123
|
+
A Config
|
|
1124
|
+
"""
|
|
1125
|
+
assert isinstance( config_name, str ), "'config_name' must be a string"
|
|
1126
|
+
if type(config).__name__ == Config.__name__: # we allow for import inconsistencies
|
|
1127
|
+
config = config.detach()
|
|
1128
|
+
config.update(kwargs)
|
|
1129
|
+
else:
|
|
1130
|
+
if not config is None: raise TypeError("'config' must be of type 'Config'")
|
|
1131
|
+
config = Config.to_config( kwargs=kwargs, config_name=config_name )
|
|
1132
|
+
return config
|
|
1133
|
+
|
|
1134
|
+
# for uniqueHash
|
|
1135
|
+
# --------------
|
|
1136
|
+
|
|
1137
|
+
def __unique_hash__(self, uniqueHashExt, debug_trace ) -> str:
|
|
1138
|
+
"""
|
|
1139
|
+
Returns a unique hash for this object
|
|
1140
|
+
This function is required because by default uniqueHash() ignores members starting with '_', which
|
|
1141
|
+
in the case of Config means that no children are hashed.
|
|
1142
|
+
"""
|
|
1143
|
+
return self.unique_id( uniqueHashExt=uniqueHashExt, debug_trace=debug_trace, )
|
|
1144
|
+
|
|
1145
|
+
|
|
1146
|
+
# Comparison
|
|
1147
|
+
# -----------
|
|
1148
|
+
|
|
1149
|
+
def __eq__(self, other):
|
|
1150
|
+
""" Equality operator comparing 'name' and standard dictionary content """
|
|
1151
|
+
if type(self).__name__ != type(other).__name__: # allow comparison betweenn different imports
|
|
1152
|
+
return False
|
|
1153
|
+
if self._name != other._name:
|
|
1154
|
+
return False
|
|
1155
|
+
return OrderedDict.__eq__(self, other)
|
|
1156
|
+
|
|
1157
|
+
def __hash__(self):
|
|
1158
|
+
return hash(self._name) ^ OrderedDict.__hash__(self)
|
|
1159
|
+
|
|
1160
|
+
to_config = Config.to_config
|
|
1161
|
+
config_kwargs = Config.config_kwargs
|
|
1162
|
+
Config.no_default = no_default
|
|
1163
|
+
|
|
1164
|
+
# ==============================================================================
|
|
1165
|
+
# New in version 0.1.45
|
|
1166
|
+
# Support for conditional types, e.g. we can write
|
|
1167
|
+
#
|
|
1168
|
+
# x = config(x, 0.1, Float >= 0., "An 'x' which cannot be negative")
|
|
1169
|
+
# ==============================================================================
|
|
1170
|
+
|
|
1171
|
+
class ConfigField(object):
|
|
1172
|
+
"""
|
|
1173
|
+
Simplististc 'read only' wrapper for Config objects.
|
|
1174
|
+
Useful for Flax
|
|
1175
|
+
|
|
1176
|
+
import dataclasses as dataclasses
|
|
1177
|
+
import jax.numpy as jnp
|
|
1178
|
+
import jax as jax
|
|
1179
|
+
from cdxbasics.config import Config, ConfigField
|
|
1180
|
+
import types as types
|
|
1181
|
+
|
|
1182
|
+
class A( nn.Module ):
|
|
1183
|
+
config : ConfigField = ConfigField.Field()
|
|
1184
|
+
|
|
1185
|
+
def setup(self):
|
|
1186
|
+
self.dense = nn.Dense(1)
|
|
1187
|
+
|
|
1188
|
+
def __call__(self, x):
|
|
1189
|
+
a = self.config("y", 0.1 ,float)
|
|
1190
|
+
return self.dense(x)*a
|
|
1191
|
+
|
|
1192
|
+
print("Default")
|
|
1193
|
+
a = A()
|
|
1194
|
+
key1, key2 = jax.random.split(jax.random.key(0))
|
|
1195
|
+
x = jnp.zeros((10,10))
|
|
1196
|
+
param = a.init( key1, x )
|
|
1197
|
+
y = a.apply( param, x )
|
|
1198
|
+
|
|
1199
|
+
print("Value")
|
|
1200
|
+
w = ConfigField(y=1.)
|
|
1201
|
+
a = A(config=w)
|
|
1202
|
+
|
|
1203
|
+
key1, key2 = jax.random.split(jax.random.key(0))
|
|
1204
|
+
x = jnp.zeros((10,10))
|
|
1205
|
+
param = a.init( key1, x )
|
|
1206
|
+
y = a.apply( param, x )
|
|
1207
|
+
|
|
1208
|
+
class A( nn.Module ):
|
|
1209
|
+
config : ConfigField = ConfigField.field()
|
|
1210
|
+
|
|
1211
|
+
@nn.compact
|
|
1212
|
+
def __call__(self, x):
|
|
1213
|
+
a = self.config.x("y", 0.1 ,float)
|
|
1214
|
+
self.config.done()
|
|
1215
|
+
return nn.Dense(1)(x)*a
|
|
1216
|
+
|
|
1217
|
+
print("Config")
|
|
1218
|
+
c = Config()
|
|
1219
|
+
c.x.y = 1.
|
|
1220
|
+
w = ConfigField(c)
|
|
1221
|
+
a = A(config=w)
|
|
1222
|
+
|
|
1223
|
+
key1, key2 = jax.random.split(jax.random.key(0))
|
|
1224
|
+
x = jnp.zeros((10,10))
|
|
1225
|
+
param = a.init( key1, x )
|
|
1226
|
+
y = a.apply( param, x )
|
|
1227
|
+
y = a.apply( param, x )
|
|
1228
|
+
"""
|
|
1229
|
+
def __init__(self, config : Config = None, **kwargs):
|
|
1230
|
+
if not config is None:
|
|
1231
|
+
config = config if type(config).__name__ != type(self).__name__ else config.__config
|
|
1232
|
+
self.__config = Config.config_kwargs( config, kwargs )
|
|
1233
|
+
def __call__(self, *kargs, **kwargs):
|
|
1234
|
+
return self.__config(*kargs,**kwargs)
|
|
1235
|
+
def __getattr__(self, key):
|
|
1236
|
+
if key[:2] == "__":
|
|
1237
|
+
return object.__getattr__(self, key)
|
|
1238
|
+
return getattr(self.__config, key)
|
|
1239
|
+
def __getitem__(self, key):
|
|
1240
|
+
return self.__config[key]
|
|
1241
|
+
def __eq__(self, other):
|
|
1242
|
+
if type(other).__name__ == "Config":
|
|
1243
|
+
return self.__config == other
|
|
1244
|
+
else:
|
|
1245
|
+
return self.__config == other.config
|
|
1246
|
+
def __hash__(self):
|
|
1247
|
+
h = 0
|
|
1248
|
+
for k, v in self.items():
|
|
1249
|
+
h ^= hash(k) ^ hash(v)
|
|
1250
|
+
return h
|
|
1251
|
+
def __unique_hash__(self, *kargs, **kwargs):
|
|
1252
|
+
return self.__config.__unique_hash__(*kargs, **kwargs)
|
|
1253
|
+
def __str__(self):
|
|
1254
|
+
return self.__pdct.__str__()
|
|
1255
|
+
def __repr__(self):
|
|
1256
|
+
return self.__pdct.__repr__()
|
|
1257
|
+
def as_dict(self):
|
|
1258
|
+
return self.__config.as_dict(mark_done=False)
|
|
1259
|
+
def done(self):
|
|
1260
|
+
return self.__config.done()
|
|
1261
|
+
|
|
1262
|
+
@property
|
|
1263
|
+
def config(self) -> Config:
|
|
1264
|
+
return self.__config
|
|
1265
|
+
|
|
1266
|
+
@staticmethod
|
|
1267
|
+
def default():
|
|
1268
|
+
return ConfigField()
|
|
1269
|
+
|
|
1270
|
+
@staticmethod
|
|
1271
|
+
def Field():
|
|
1272
|
+
import dataclasses as dataclasses
|
|
1273
|
+
return dataclasses.field( default_factory=ConfigField )
|
|
1274
|
+
|
|
1275
|
+
# ==============================================================================
|
|
1276
|
+
# New in version 0.1.45
|
|
1277
|
+
# Support for conditional types, e.g. we can write
|
|
1278
|
+
#
|
|
1279
|
+
# x = config(x, 0.1, Float >= 0., "An 'x' which cannot be negative")
|
|
1280
|
+
# ==============================================================================
|
|
1281
|
+
|
|
1282
|
+
class _Cast(object):
|
|
1283
|
+
|
|
1284
|
+
def __call__( self, value, *, config_name : str, key_name : str ):
|
|
1285
|
+
""" cast 'value' to the proper type """
|
|
1286
|
+
raise NotImplementedError("Internal error")
|
|
1287
|
+
|
|
1288
|
+
def __str__(self) -> str:
|
|
1289
|
+
""" Returns readable string description of 'self' """
|
|
1290
|
+
raise NotImplementedError("Internal error")
|
|
1291
|
+
|
|
1292
|
+
def _cast_name( cast : type ) -> str:
|
|
1293
|
+
""" Returns the class name of 'cast' """
|
|
1294
|
+
if cast is None:
|
|
1295
|
+
return ""
|
|
1296
|
+
return getattr(cast,"__name__", str(cast))
|
|
1297
|
+
|
|
1298
|
+
def _cast_err_header(*, config_name : str, key_name : str):
|
|
1299
|
+
if len(config_name) > 0:
|
|
1300
|
+
return f"Error using config '{config_name}' for key '{key_name}': "
|
|
1301
|
+
else:
|
|
1302
|
+
return f"Config error for key '{key_name}': "
|
|
1303
|
+
|
|
1304
|
+
# ================================
|
|
1305
|
+
# Simple wrapper
|
|
1306
|
+
# ================================
|
|
1307
|
+
|
|
1308
|
+
class _Simple(_Cast):# NOQA
|
|
1309
|
+
"""
|
|
1310
|
+
Default case where the 'cast' argument for a config call() is simply a type, a Callable, or None.
|
|
1311
|
+
Cast to an actual underlying type.
|
|
1312
|
+
"""
|
|
1313
|
+
|
|
1314
|
+
STR_NONE_CAST = "any"
|
|
1315
|
+
|
|
1316
|
+
def __init__(self, *, cast : type, config_name : str, key_name : str, none_is_any : bool ):
|
|
1317
|
+
""" Simple atomic caster """
|
|
1318
|
+
if not cast is None:
|
|
1319
|
+
if isinstance(cast, str):
|
|
1320
|
+
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name)+\
|
|
1321
|
+
"'cast' must be a type. Found a string. Most likely this happened because a help string was defined as positional argument, but no 'cast' type was specified. "+\
|
|
1322
|
+
"In this case, use 'help=' to specify the help text.")
|
|
1323
|
+
assert not isinstance(cast, _Cast), lambda : "Internal error in definition for key '%s' in config '%s': 'cast' should not be derived from _Cast. Object is %s" % ( key_name, config_name, str(cast) )
|
|
1324
|
+
self.cast = cast
|
|
1325
|
+
self.none_is_any = none_is_any
|
|
1326
|
+
assert not none_is_any is None or not cast is None, "Must set 'none_is_any' to bool value if cast is 'None'."
|
|
1327
|
+
|
|
1328
|
+
def __call__(self, value, *, config_name : str, key_name : str ):
|
|
1329
|
+
""" Cast 'value' to the proper type """
|
|
1330
|
+
if self.cast is None:
|
|
1331
|
+
if value is None or self.none_is_any:
|
|
1332
|
+
return value
|
|
1333
|
+
if not value is None:
|
|
1334
|
+
raise ValueError(f"'None' value expected, found value of type {type(value).__name__}")
|
|
1335
|
+
return self.cast(value)
|
|
1336
|
+
|
|
1337
|
+
def __str__(self) -> str:
|
|
1338
|
+
""" Returns readable string """
|
|
1339
|
+
if self.cast is None:
|
|
1340
|
+
return self.STR_NONE_CAST if self.none_is_any else "None"
|
|
1341
|
+
return _cast_name(self.cast)
|
|
1342
|
+
|
|
1343
|
+
# ================================
|
|
1344
|
+
# Conditional types such as Int>0
|
|
1345
|
+
# ================================
|
|
1346
|
+
|
|
1347
|
+
class _Condition(_Cast):
|
|
1348
|
+
""" Represents a simple operator condition such as 'Float >= 0.' """
|
|
1349
|
+
|
|
1350
|
+
def __init__(self, cast, op, other):
|
|
1351
|
+
""" Initialize the condition for a base type 'cast' and an 'op' with an 'other' """
|
|
1352
|
+
self.cast = cast
|
|
1353
|
+
self.op = op
|
|
1354
|
+
self.other = other
|
|
1355
|
+
self.l_and = None
|
|
1356
|
+
|
|
1357
|
+
def __and__(self, cond):
|
|
1358
|
+
"""
|
|
1359
|
+
Combines to conditions with logical AND .
|
|
1360
|
+
Requires the left hand 'self' to be a > or >=; the right hand must be < or <=
|
|
1361
|
+
This means you can write
|
|
1362
|
+
|
|
1363
|
+
x = config("x", 0.5, (Float >= 0.) & (Float < 1.), "Variable x")
|
|
1364
|
+
|
|
1365
|
+
"""
|
|
1366
|
+
if not self.l_and is None:
|
|
1367
|
+
raise NotImplementedError("Cannot combine more than two conditions")
|
|
1368
|
+
if not self.op[0] == 'g':
|
|
1369
|
+
raise NotImplementedError("The left hand condition when using '&' must be > or >=. Found %s" % self._op_str)
|
|
1370
|
+
if not cond.op[0] == 'l':
|
|
1371
|
+
raise NotImplementedError("The right hand condition when using '&' must be < or <=. Found %s" % cond._op_str)
|
|
1372
|
+
if self.cast != cond.cast:
|
|
1373
|
+
raise NotImplementedError("Cannot '&' conditions for types %s and %s" % (self.cast.__name__, cond.cast.__name__))
|
|
1374
|
+
op_new = _Condition(self.cast, self.op, self.other)
|
|
1375
|
+
op_new.l_and = cond
|
|
1376
|
+
return op_new
|
|
1377
|
+
|
|
1378
|
+
def __call__(self, value, *, config_name : str, key_name : str ):
|
|
1379
|
+
""" Test whether 'value' satisfies the condition """
|
|
1380
|
+
value = self.cast(value)
|
|
1381
|
+
|
|
1382
|
+
if self.op == "ge":
|
|
1383
|
+
ok = value >= self.other
|
|
1384
|
+
elif self.op == "gt":
|
|
1385
|
+
ok = value > self.other
|
|
1386
|
+
elif self.op == "le":
|
|
1387
|
+
ok = value <= self.other
|
|
1388
|
+
elif self.op == "lt":
|
|
1389
|
+
ok = value < self.other
|
|
1390
|
+
else:
|
|
1391
|
+
raise RuntimeError("Internal error: unknown operator %s" % str(self.op))
|
|
1392
|
+
if not ok:
|
|
1393
|
+
raise ValueError( "value for key '%s' %s. Found %s" % ( key_name, self.err_str, value ))
|
|
1394
|
+
return self.l_and( value, config_name=config_name, key_name=key_name) if not self.l_and is None else value
|
|
1395
|
+
|
|
1396
|
+
def __str__(self) -> str:
|
|
1397
|
+
""" Returns readable string """
|
|
1398
|
+
s = _cast_name(self.cast) + self._op_str + str(self.other)
|
|
1399
|
+
if not self.l_and is None:
|
|
1400
|
+
s += " and " + str(self.l_and)
|
|
1401
|
+
return s
|
|
1402
|
+
|
|
1403
|
+
@property
|
|
1404
|
+
def _op_str(self) -> str:
|
|
1405
|
+
""" Returns a string for the operator of this conditon """
|
|
1406
|
+
if self.op == "ge":
|
|
1407
|
+
return ">="
|
|
1408
|
+
elif self.op == "gt":
|
|
1409
|
+
return ">"
|
|
1410
|
+
elif self.op == "le":
|
|
1411
|
+
return "<="
|
|
1412
|
+
elif self.op == "lt":
|
|
1413
|
+
return "<"
|
|
1414
|
+
raise RuntimeError("Internal error: unknown operator %s" % str(self.op))
|
|
1415
|
+
|
|
1416
|
+
@property
|
|
1417
|
+
def err_str(self) -> str:
|
|
1418
|
+
""" Nice error string """
|
|
1419
|
+
zero = self.cast(0)
|
|
1420
|
+
def mk_txt(cond):
|
|
1421
|
+
if cond.op == "ge":
|
|
1422
|
+
s = ("not be lower than %s" % cond.other) if cond.other != zero else ("not be negative")
|
|
1423
|
+
elif cond.op == "gt":
|
|
1424
|
+
s = ("be bigger than %s" % cond.other) if cond.other != zero else ("be positive")
|
|
1425
|
+
elif cond.op == "le":
|
|
1426
|
+
s = ("not exceed %s" % cond.other) if cond.other != zero else ("not be positive")
|
|
1427
|
+
elif cond.op == "lt":
|
|
1428
|
+
s = ("be lower than %s" % cond.other) if cond.other != zero else ("be negative")
|
|
1429
|
+
else:
|
|
1430
|
+
raise RuntimeError("Internal error: unknown operator %s" % str(cond.op))
|
|
1431
|
+
return s
|
|
1432
|
+
|
|
1433
|
+
s = "must " + mk_txt(self)
|
|
1434
|
+
if not self.l_and is None:
|
|
1435
|
+
s += ", and " + mk_txt(self.l_and)
|
|
1436
|
+
return s
|
|
1437
|
+
|
|
1438
|
+
class _CastCond(_Cast): # NOQA
|
|
1439
|
+
"""
|
|
1440
|
+
Generates compound _Condition's
|
|
1441
|
+
See the two members Float and Int
|
|
1442
|
+
"""
|
|
1443
|
+
|
|
1444
|
+
def __init__(self, cast):# NOQA
|
|
1445
|
+
self.cast = cast
|
|
1446
|
+
def __ge__(self, other) -> bool:# NOQA
|
|
1447
|
+
return _Condition( self.cast, 'ge', self.cast(other) )
|
|
1448
|
+
def __gt__(self, other) -> bool:# NOQA
|
|
1449
|
+
return _Condition( self.cast, 'gt', self.cast(other) )
|
|
1450
|
+
def __le__(self, other) -> bool:# NOQA
|
|
1451
|
+
return _Condition( self.cast, 'le', self.cast(other) )
|
|
1452
|
+
def __lt__(self, other) -> bool:# NOQA
|
|
1453
|
+
return _Condition( self.cast, 'lt', self.cast(other) )
|
|
1454
|
+
def __call__(self, value, *, config_name : str, key_name : str ):
|
|
1455
|
+
""" This gets called if the type was used without operators """
|
|
1456
|
+
cast = _Simple(self.cast, config_name=config_name, key_name=key_name, none_is_any=None )
|
|
1457
|
+
return cast( value, config_name=config_name, key_name=key_name )
|
|
1458
|
+
def __str__(self) -> str:
|
|
1459
|
+
""" This gets called if the type was used without operators """
|
|
1460
|
+
return _cast_name(self.cast)
|
|
1461
|
+
|
|
1462
|
+
Float = _CastCond(float)
|
|
1463
|
+
"""
|
|
1464
|
+
Allows to apply conditions to float's as part of config's.
|
|
1465
|
+
For example
|
|
1466
|
+
```
|
|
1467
|
+
timeout = config("timeout", 0.5, Float>=0., "Timeout")
|
|
1468
|
+
```
|
|
1469
|
+
|
|
1470
|
+
In combination with `&` we can limit a float to a range:
|
|
1471
|
+
```
|
|
1472
|
+
probability = config("probability", 0.5, (Float>=0.) & (Float <= 1.), "Probability")
|
|
1473
|
+
```
|
|
1474
|
+
"""
|
|
1475
|
+
|
|
1476
|
+
|
|
1477
|
+
Int = _CastCond(int)
|
|
1478
|
+
"""
|
|
1479
|
+
Allows to apply conditions to int' as part of config's.
|
|
1480
|
+
For example
|
|
1481
|
+
```
|
|
1482
|
+
num_steps = config("num_steps", 1, Int>0., "Number of steps")
|
|
1483
|
+
```
|
|
1484
|
+
|
|
1485
|
+
In combination with `&` we can limit an int to a range:
|
|
1486
|
+
```
|
|
1487
|
+
bus_days_per_year = config("bus_days_per_year", 255, (Int > 0) & (Int < 365), "Business days per year")
|
|
1488
|
+
```
|
|
1489
|
+
"""
|
|
1490
|
+
|
|
1491
|
+
# ================================
|
|
1492
|
+
# Enum type for list 'cast's
|
|
1493
|
+
# ================================
|
|
1494
|
+
|
|
1495
|
+
class _Enum(_Cast):
|
|
1496
|
+
"""
|
|
1497
|
+
Utility class to support enumerator types.
|
|
1498
|
+
No need to use this classs directly. It will be automatically instantiated if a list is passed as type, e.g.
|
|
1499
|
+
|
|
1500
|
+
code = config("code", "Python", ['C++', 'Python'], "Which language do we love")
|
|
1501
|
+
|
|
1502
|
+
Note that all list members must be of the same type.
|
|
1503
|
+
"""
|
|
1504
|
+
|
|
1505
|
+
def __init__(self, enum : list, *, config_name : str, key_name : str ):
|
|
1506
|
+
""" Initializes an enumerator casting type. """
|
|
1507
|
+
self.enum = list(enum)
|
|
1508
|
+
if len(self.enum) == 0:
|
|
1509
|
+
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1510
|
+
f"'cast' for key '{key_name}' is an empty list. Lists are used for enumerator types, hence passing empty list is not defined")
|
|
1511
|
+
if self.enum[0] is None:
|
|
1512
|
+
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1513
|
+
f"'cast' for key '{key_name}' is an list, with first element 'None'. Lists are used for enumerator types, and the first element defines their underlying type. Hence you cannot use 'None'. (Did you want to use alternative notation with tuples?)")
|
|
1514
|
+
self.cast = _Simple( type(self.enum[0]), config_name=config_name, key_name=key_name, none_is_any=None )
|
|
1515
|
+
for i in range(1,len(self.enum)):
|
|
1516
|
+
try:
|
|
1517
|
+
self.enum[i] = self.cast( self.enum[i], config_name=config_name, key_name=key_name )
|
|
1518
|
+
except:
|
|
1519
|
+
other_name = _cast_name(type(self.enum[i]))
|
|
1520
|
+
raise ValueError( _cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1521
|
+
f"'cast' for key {key_name}: members of the 'cast' list are not of consistent type. Found {self.cast} for the first element which does match the type {other_name} of the {i}th element" )
|
|
1522
|
+
|
|
1523
|
+
def __call__( self, value, *, config_name, key_name ):
|
|
1524
|
+
"""
|
|
1525
|
+
Cast 'value' to the proper type and check is one of the list members
|
|
1526
|
+
Raises a KeyError if the value was not found in our enum
|
|
1527
|
+
"""
|
|
1528
|
+
value = self.cast(value, config_name=config_name, key_name=key_name)
|
|
1529
|
+
if not value in self.num:
|
|
1530
|
+
raise ValueError( f"Value for key '{key_name}' {self.err_str}; found 'str(value)[:20]'" )
|
|
1531
|
+
return value
|
|
1532
|
+
|
|
1533
|
+
@property
|
|
1534
|
+
def err_str(self) -> str:
|
|
1535
|
+
""" Nice error string """
|
|
1536
|
+
if len(self.enum) == 1:
|
|
1537
|
+
return f"must be '{str(self.enum[0])}'"
|
|
1538
|
+
|
|
1539
|
+
s = "must be one of: '" + str(self.enum[0]) + "'"
|
|
1540
|
+
for i in range(1,len(self.enum)-1):
|
|
1541
|
+
s += ", '" + str(self.enum[i]) + "'"
|
|
1542
|
+
s += " or '" + str(self.enum[-1]) + "'"
|
|
1543
|
+
return s
|
|
1544
|
+
|
|
1545
|
+
def __str__(self) -> str:
|
|
1546
|
+
""" Returns readable string """
|
|
1547
|
+
s = "[ "
|
|
1548
|
+
for i in range(len(self.enum)):
|
|
1549
|
+
s += ( ", " + self.enum[i] ) if i > 0 else self.enum[i]
|
|
1550
|
+
s += " ]"
|
|
1551
|
+
return s
|
|
1552
|
+
|
|
1553
|
+
# ================================
|
|
1554
|
+
# Multiple types
|
|
1555
|
+
# ================================
|
|
1556
|
+
|
|
1557
|
+
class _Alt(_Cast):
|
|
1558
|
+
"""
|
|
1559
|
+
Initialize a casting compund "alternative" type, e.g. it the variable may contain several types, each of which is acceptable.
|
|
1560
|
+
None here means that 'None' is an accepted value.
|
|
1561
|
+
This is invokved when a tuple is passed, e.g
|
|
1562
|
+
|
|
1563
|
+
config("spread", None, ( None, float ), "Float or None")
|
|
1564
|
+
config("spread", 1, ( Int<=-1, Int>=1. ), "A variable which has to be outside (-1,+1)")
|
|
1565
|
+
"""
|
|
1566
|
+
|
|
1567
|
+
def __init__(self, casts : list, *, config_name : str, key_name : str ):
|
|
1568
|
+
""" Initialize a compound cast """
|
|
1569
|
+
if len(casts) == 0:
|
|
1570
|
+
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1571
|
+
f"'cast' for key '{key_name}' is an empty tuple. Tuples are used for alternative types, hence passing empty tuple is not defined")
|
|
1572
|
+
self.casts = [ _create_caster(cast=cast, config_name=config_name, key_name=key_name, none_is_any=False) for cast in enumerate(casts) ]
|
|
1573
|
+
|
|
1574
|
+
def __call__( self, value, *, config_name : str, key_name : str ):
|
|
1575
|
+
""" Cast 'value' to the proper type """
|
|
1576
|
+
e0 = None
|
|
1577
|
+
for cast in self.casts:
|
|
1578
|
+
# None means that value == None is acceptable
|
|
1579
|
+
try:
|
|
1580
|
+
return cast(value, config_name=config_name, key_namkey_name=key_name )
|
|
1581
|
+
except Exception as e:
|
|
1582
|
+
e0 = e if e0 is None else e0
|
|
1583
|
+
raise ValueError(f"Error using config '{config_name}': value for key '{key_name}' {self.err_str}. Found '{str(value)}' of type '{type(value).__name__}'")
|
|
1584
|
+
|
|
1585
|
+
def test(self, value):
|
|
1586
|
+
""" Test whether 'value' satisfies the condition """
|
|
1587
|
+
raise self.test
|
|
1588
|
+
|
|
1589
|
+
@property
|
|
1590
|
+
def err_str(self):
|
|
1591
|
+
""" Returns readable string """
|
|
1592
|
+
return "must be one of the following types: " + self.__str__()
|
|
1593
|
+
|
|
1594
|
+
def __str__(self):
|
|
1595
|
+
""" Returns readable string """
|
|
1596
|
+
s = ""
|
|
1597
|
+
for cast in self.casts[:-1]:
|
|
1598
|
+
s += str(cast) + " or "
|
|
1599
|
+
s += str(self.casts[-1])
|
|
1600
|
+
return s
|
|
1601
|
+
|
|
1602
|
+
# ================================
|
|
1603
|
+
# Manage casting
|
|
1604
|
+
# ================================
|
|
1605
|
+
|
|
1606
|
+
def _create_caster( *, cast : type, config_name : str, key_name : str, none_is_any : bool ):
|
|
1607
|
+
"""
|
|
1608
|
+
Implements casting.
|
|
1609
|
+
|
|
1610
|
+
Parameters
|
|
1611
|
+
----------
|
|
1612
|
+
value: value, either from the user or the default value if provided
|
|
1613
|
+
cast : cast input to call() from the user, or None.
|
|
1614
|
+
key_name : name of the config
|
|
1615
|
+
key : name of the key being access
|
|
1616
|
+
none_is_any :If True, then None means that any type is accepted. If False, the None means that only None is accepted.
|
|
1617
|
+
|
|
1618
|
+
Returns
|
|
1619
|
+
-------
|
|
1620
|
+
value : casted value
|
|
1621
|
+
__str__ : casting help text. Empty if 'cast' is None
|
|
1622
|
+
"""
|
|
1623
|
+
if isinstance( cast, str ):
|
|
1624
|
+
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1625
|
+
f"string '{cast}' provided as 'cast'. Strings are not supported. To pass sets of characters, use a list of strings of those single characters.")
|
|
1626
|
+
if isinstance(cast, list):
|
|
1627
|
+
return _Enum( cast, config_name=config_name, key_name=key_name )
|
|
1628
|
+
elif isinstance(cast, tuple):
|
|
1629
|
+
return _Alt( cast, config_name=config_name, key_name=key_name )
|
|
1630
|
+
elif isinstance(cast,_Cast):
|
|
1631
|
+
return cast
|
|
1632
|
+
return _Simple( cast, config_name=config_name, key_name=key_name, none_is_any=none_is_any )
|
|
1633
|
+
|