cdxcore 0.1.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of cdxcore might be problematic. Click here for more details.

cdxcore/config.py ADDED
@@ -0,0 +1,1633 @@
1
+ """
2
+ config
3
+ Utility object for ML project configuration
4
+ Hans Buehler 2022
5
+ """
6
+
7
+ from collections import OrderedDict
8
+ from collections.abc import Mapping
9
+ from sortedcontainers import SortedDict
10
+ import warnings as warnings
11
+ from dataclasses import Field
12
+ from .util import mt as txtfmt
13
+ from .uniquehash import UniqueHash
14
+ from .prettydict import PrettyDict as pdct
15
+
16
+ __all__ = ["Config", "Int", "Float", "ConfigField", "NotDoneError"]
17
+
18
+ class _ID(object):
19
+ pass
20
+
21
+ def error( text, *args, exception = RuntimeError, **kwargs ):
22
+ raise exception( txtfmt(text, *args, **kwargs) )
23
+ def verify( cond, text, *args, exception = RuntimeError, **kwargs ):
24
+ if not cond:
25
+ error( text, *args, **kwargs, exception=exception )
26
+ def warn( text, *args, warning=warnings.RuntimeWarning, stack_level=1, **kwargs ):
27
+ warnings.warn( txtfmt(text, *args, **kwargs), warning, stack_level=stack_level )
28
+ def warn_if( cond, text, *args, warning=warnings.RuntimeWarning, stack_level=1, **kwargs ):
29
+ if cond:
30
+ warn( text, *args, warning=warning, stack_level=stack_level, **kwargs )
31
+
32
+ class NotDoneError( RuntimeError ):
33
+ """
34
+ Raised when done() finds that some arguments have not been read.
35
+ Those arguments are accessible via the 'not_done' attribute
36
+ """
37
+ def __init__(self, not_done, message):
38
+ self.not_done = not_done
39
+ RuntimeError.__init__(self, message)
40
+
41
+ class InconsistencyError( RuntimeError ):
42
+ """
43
+ Raised when __call__ used inconsistently for a given parameter, ie when the defaults are different.
44
+ Such inconsistencies must be fixed for safe usage of any configuration strategy.
45
+ """
46
+ def __init__(self, field, message ):
47
+ self.field = field
48
+ RuntimeError.__init__(self, message)
49
+
50
+ class CastError(RuntimeError):
51
+ def __init__(self, *, config_name : str, key_name : str, message : str):
52
+ header = f"Error in cast definition for key '{key_name}' in config '{config_name}': " if len(config_name) > 0 else f"Error in cast definition for key '{key_name}': "
53
+ RuntimeError.__init__(self, header+message)
54
+ self.config_name = config_name
55
+ self.key_name = key_name
56
+
57
+ #@private
58
+ no_default = _ID() #@private creates a unique object which can be used to detect if a default value was provided
59
+
60
+ # ==============================================================================
61
+ #
62
+ # Actual Config class
63
+ #
64
+ # ==============================================================================
65
+
66
+ class Config(OrderedDict):
67
+ """
68
+ A simple Config class.
69
+ Main features
70
+
71
+ Write:
72
+
73
+ Set data as usual:
74
+ config = Config()
75
+ config['features'] = [ 'time', 'spot' ]
76
+ config['weights'] = [ 1, 2, 3 ]
77
+
78
+ Use member notation
79
+ config.network.samples = 10000
80
+ config.network.activation = 'relu'
81
+
82
+ Read:
83
+
84
+ def read_config( confg ):
85
+ features = config("features", [], list ) # reads features and returns a list
86
+ weights = config("weights", [], np.ndarray ) # reads features and returns a nump array
87
+
88
+ network = config.network
89
+ samples = network('samples', 10000) # networks samples
90
+ config.done() # returns an error as we haven't read 'network.activitation'
91
+
92
+ Detaching child configs
93
+ You can also detach a child config, which allows you to store it for later
94
+ use without triggering done() errors for its parent.
95
+
96
+ def read_config( confg ):
97
+ features = config("features", [], list ) # reads features and returns a list
98
+ weights = config("weights", [], np.ndarray ) # reads features and returns a nump array
99
+
100
+ network = config.network.detach()
101
+ samples = network('samples', 10000) # networks samples
102
+ config.done() # no error as 'network' was detached
103
+ network.done() # error as network.activation was not read
104
+
105
+ Self-recording "help"
106
+ When reading a value, specify an optional help text:
107
+
108
+ def read_config( confg ):
109
+
110
+ features = config("features", [], list, help="Defines the features" )
111
+ weights = config("weights", [], np.ndarray, help="Network weights" )
112
+
113
+ network = config.network
114
+ samples = network('samples', 10000, int, help="Number of samples")
115
+ activt = network('activation', "relu", str, help="Activation function")
116
+ config.done()
117
+
118
+ config.usage_report() # prints help
119
+ """
120
+
121
+ def __init__(self, *args, config_name : str = None, **kwargs):
122
+ """
123
+ See help(Config) for a description of this class.
124
+
125
+ Two patterns:
126
+ __init__(config):
127
+ is the copy constructor; see copy()
128
+ or
129
+ __init__( dict, x=1, y=2 ):
130
+ creates a new config by first iteratively loading all positional dictionary arguments,
131
+ and then copying the keyword arguments as provided.
132
+
133
+ See also Config.config_kwargs().
134
+
135
+ Parameters
136
+ ----------
137
+ *args : list
138
+ List of dictionaries to create a new config with, iteratively.
139
+ If the first element is a config, and no other parameters are passed,
140
+ then this object will be full copy of that config.
141
+ It then shares all usage recording. See copy().
142
+ config_name : str, optional
143
+ Name of the configuration for report_usage. Default is 'config'
144
+ **kwargs : dict
145
+ Used to initialize the config, e.g.
146
+ Config(a=1, b=2)
147
+ """
148
+ if len(args) == 1 and isinstance(args[0], Config) and config_name is None and len(kwargs) == 0:
149
+ source = args[0]
150
+ self._done = source._done
151
+ self._name = source._name
152
+ self._recorder = source._recorder
153
+ self._children = source._children
154
+ self.update(source)
155
+ return
156
+
157
+ OrderedDict.__init__(self)
158
+ self._done = set()
159
+ self._name = config_name if not config_name is None else "config"
160
+ self._children = OrderedDict()
161
+ self._recorder = SortedDict()
162
+ self._recorder._name = self._name
163
+ for k in args:
164
+ if not k is None:
165
+ self.update(k)
166
+ self.update(kwargs)
167
+
168
+ # Information
169
+ # -----------
170
+
171
+ @property
172
+ def config_name(self) -> str:
173
+ """ Returns the fully qualified name of this config """
174
+ return self._name
175
+
176
+ @property
177
+ def children(self) -> OrderedDict:
178
+ """ Returns dictionary of children """
179
+ return self._children
180
+
181
+ def __str__(self) -> str:
182
+ """ Print myself as dictionary """
183
+ s = self.config_name + str(self.as_dict(mark_done=False))
184
+ return s
185
+
186
+ def __repr__(self) -> str:
187
+ """ Print myself as reconstructable object """
188
+ s = repr(self.as_dict(mark_done=False))
189
+ s = "Config( **" + s + ", config_name='" + self.config_name + "' )"
190
+ return s
191
+
192
+ @property
193
+ def is_empty(self) -> bool:
194
+ """ Checks whether any variables have been set """
195
+ return len(self) + len(self._children) == 0
196
+
197
+ # conversion
198
+ # ----------
199
+
200
+ def as_dict(self, mark_done : bool = True ) -> dict:
201
+ """
202
+ Convert into dictionary of dictionaries
203
+
204
+ Parameters
205
+ ----------
206
+ mark_done : bool
207
+ If True, then all members of this config will be considered read ('done').
208
+
209
+ Returns
210
+ -------
211
+ Dict of dict's
212
+ """
213
+ d = { key : self.get(key) if mark_done else self.get_raw(key) for key in self }
214
+ for n, c in self._children.items():
215
+ if n == '_ipython_canary_method_should_not_exist_':
216
+ continue
217
+ c = c.as_dict(mark_done)
218
+ verify( not n in d, "Cannot convert config to dictionary: found both a regular value, and a child with name '%s'", n)
219
+ d[n] = c
220
+ return d
221
+
222
+ def as_field(self) -> Field:
223
+ """
224
+ Returns a ConfigField wrapped around self for dataclasses and flax nn.Mpodule support.
225
+ See ConfigField documentation for an example.
226
+ """
227
+ return ConfigField(self)
228
+
229
+ # handle finishing config use
230
+ # ---------------------------
231
+
232
+ def done(self, include_children : bool = True, mark_done : bool = True ):
233
+ """
234
+ Closes the config and checks that no unread parameters remain.
235
+ By default this function also validates that all child configs were "done" and raises a NotDoneError() if not the case
236
+ (NotDoneError is derived from RuntimeError).
237
+
238
+ If you want to make a copy of a child config for later processing use detach() first
239
+ config = Config()
240
+ config.a = 1
241
+ config.child.b = 2
242
+
243
+ _ = config.a # read a
244
+ config.done() # error because confg.child.b has not been read yet
245
+
246
+ Instead use:
247
+ config = Config()
248
+ config.a = 1
249
+ config.child.b = 2
250
+
251
+ _ = config.a # read a
252
+ child = config.child.detach()
253
+ config.done() # no error, even though confg.child.b has not been read yet
254
+
255
+ You can force 'done' status by calling mark_done()
256
+
257
+ Parameters
258
+ ----------
259
+ include_children:
260
+ Validate child configs, too.
261
+ mark_done:
262
+ Upon completion mark this config as 'done'.
263
+ This stops it being modified; subsequent calls to done() will be successful.
264
+
265
+ Raises
266
+ ------
267
+ NotDoneError if not all elements were read.
268
+ """
269
+ inputs = set(self)
270
+ rest = inputs - self._done
271
+ if len(rest) > 0:
272
+ raise NotDoneError( rest, txtfmt("Error closing config '%s': the following config arguments were not read: %s\n\n"\
273
+ "Summary of all variables read from this object:\n%s", \
274
+ self._name, list(rest), \
275
+ self.usage_report(filter_path=self._name ) ))
276
+ if include_children:
277
+ for _, c in self._children.items():
278
+ c.done(include_children=include_children,mark_done=False)
279
+ if mark_done:
280
+ self.mark_done(include_children=include_children)
281
+ return
282
+
283
+ def reset(self):
284
+ """
285
+ Reset all usage information
286
+
287
+ Use reset_done() to only reset the information whether a key was used, but to keep information on previously used default values.
288
+ This avoids inconsistency in default values between function calls.
289
+ """
290
+ self._done.clear()
291
+ self._recorder.clear()
292
+
293
+ def reset_done(self):
294
+ """
295
+ Reset the internal list of which are 'done', e.g. read.
296
+ This function does not reset the recording of previous uses of each key. This ensures consistency of default values between uses of keys.
297
+ Use reset() to reset both 'done' and create a new recorder.
298
+ """
299
+ self._done.clear()
300
+
301
+ def mark_done(self, include_children : bool = True ):
302
+ """ Mark all members as being read. Once called calling done() will no longer trigger an error """
303
+ self._done.update( self )
304
+ if include_children:
305
+ for _, c in self._children.items():
306
+ c.mark_done(include_children=include_children)
307
+
308
+ # making copies
309
+ # -------------
310
+
311
+ def _detach( self, *, mark_self_done : bool, copy_done : bool, new_recorder ):
312
+ """
313
+ Creates a copy of the current config, with a number of options how to share usage information.
314
+ Use the functions
315
+ detach()
316
+ copy()
317
+ clean_copy()
318
+ instead.
319
+
320
+ Parameters
321
+ ----------
322
+ mark_self_done : bool
323
+ If True mark 'self' as 'read', otherwise not.
324
+ copy_done : bool
325
+ If True, create a copy of self._done, else remove all 'done' information
326
+ new_recorder :
327
+ <recorder> if a recorder is specified, use it.
328
+ "clean": use a new, empty recorder
329
+ "copy": use a new recorder which is a copy of self.recorder
330
+ "share": share the same recorder
331
+
332
+ Returns
333
+ -------
334
+ A new config
335
+ """
336
+ config = Config()
337
+ config.update(self)
338
+ config._done = set( self._done ) if copy_done else config._done
339
+ config._name = self._name
340
+ if isinstance( new_recorder, SortedDict ):
341
+ config._recorder = new_recorder
342
+ elif new_recorder == "clean":
343
+ new_recorder = config._recorder
344
+ elif new_recorder == "share":
345
+ config._recorder = self._recorder
346
+ else:
347
+ assert new_recorder == "copy", "Invalid value for 'new_recorder': %s" % new_recorder
348
+ config._recorder.update( self._recorder )
349
+
350
+ config._children = { k: c._detach(mark_self_done=mark_self_done, copy_done=copy_done, new_recorder=new_recorder) for k, c in self._children.items() }
351
+ config._children = OrderedDict( config._children )
352
+
353
+ if mark_self_done:
354
+ self.mark_done()
355
+ return config
356
+
357
+ def detach( self ):
358
+ """
359
+ Returns a copy of 'self': the purpose of this function is to defer using a config to a later point, while maintaining consistency of usage.
360
+
361
+ - The copy has the same 'done' status at the time of calling detach. It does not share 'done' afterwards since 'self' will be marked as done.
362
+ - The copy shares the recorded to keep track of consistency of usage
363
+ - The function flags 'self' as done
364
+
365
+ For example:
366
+
367
+ class Example(object):
368
+
369
+ def __init__( config ):
370
+
371
+ self.a = config('a', 1, Int>=0, "'a' value")
372
+ self.later = config.later.detach() # detach sub-config
373
+ self._cache = None
374
+ config.done()
375
+
376
+ def function(self):
377
+ if self._cache is None:
378
+ self._cache = Cache(self.later) # deferred use of the self.later config. Cache() calls done() on self.later
379
+ return self._cache
380
+
381
+ See also the examples in Deep Hedging which make extensive use of this feature.
382
+ """
383
+ return self._detach(mark_self_done=True, copy_done=True, new_recorder="share")
384
+
385
+ def copy( self ):
386
+ """
387
+ Return a copy of 'self': the purpose of this function is to create a copy of the current state of 'self', which is then independent of 'self'
388
+ -- The copy shares a copy of the 'done' status of 'self'
389
+ -- The copy has a copy of the usage of 'self', but will not share furhter usage
390
+ -- 'self' will not be flagged as 'done'
391
+
392
+ As an example, this allows using different default values for
393
+ config members of the same name:
394
+
395
+ base = Config()
396
+ base.a = 1
397
+ _ = base('a', 1) # use a
398
+
399
+ copy = base.copy() # copy will know 'a' as used with default 1
400
+
401
+ _ = base("x", 1)
402
+ _ = copy("x", 2) # will not fail, as usage tracking is not shared after copy()
403
+
404
+ _ = copy('a', 2) # will fail, as default value differs from previous use of 'a' prior to copy()
405
+ """
406
+ return self._detach( mark_self_done=False, copy_done=True, new_recorder="copy" )
407
+
408
+ def clean_copy( self ):
409
+ """
410
+ Return a copy of 'self': the purpose of this function is to create a clean, unused copy of 'self'.
411
+
412
+ As an example, this allows using different default values for
413
+ config members of the same name:
414
+
415
+ base = Config()
416
+ base.a = 1
417
+ _ = base('a', 1) # use a
418
+
419
+ copy = base.copy() # copy will know 'a' as used with default 1
420
+
421
+ _ = base("x", 1)
422
+ _ = copy("x", 2) # will not fail, as no usage is shared
423
+
424
+ _ = copy('a', 2) # will not fail, as no usage is shared
425
+ """
426
+ return self._detach( mark_self_done=False, copy_done=False, new_recorder="clean" )
427
+
428
+ def clone(self):
429
+ """
430
+ Return a copy of 'self' which shares all usage tracking with 'self'.
431
+ -- The copy shares the 'done' status of 'self'
432
+ -- The copy shares the 'usage' status of 'self'
433
+ -- 'self' will not be flagged as 'done'
434
+ """
435
+ return Config(self)
436
+
437
+ # Read
438
+ # -----
439
+
440
+ def __call__(self, key : str,
441
+ default = no_default,
442
+ cast : type = None,
443
+ help : str = None,
444
+ help_default : str = None,
445
+ help_cast : str = None,
446
+ mark_done : bool = True,
447
+ record : bool = True ):
448
+ """
449
+ Reads 'key' from the config. If not found, return 'default' if specified.
450
+
451
+ config("key") - returns the value for 'key' or if not found raises an exception
452
+ config("key", 1) - returns the value for 'key' or if not found returns 1
453
+ config("key", 1, int) - if 'key' is not found, return 1. If it is found cast the result with int().
454
+ config("key", 1, int, "A number" - also stores an optional help text.
455
+ Call usage_report() after the config has been read to a get a full
456
+ summary of all data requested from this config.
457
+
458
+ Advanced casting
459
+ ----------------
460
+
461
+ Int, Float allow to bind the range of numbers:
462
+ config("positive_int", 1, Int>=1, "A positive integer")
463
+ config("ranged_int", 1, (Int>=0)&(Int<=10), "An integer between 0 and 10, inclusive")
464
+ config("positive_float", 1, Float>0., "A positive integerg"
465
+
466
+ Choices are implemented with lists:
467
+ config("difficulty", 'easy', ['easy','medium','hard'], "Choose one")
468
+
469
+ Alternative types are implemented with tuples:
470
+ config("difficulty", None, (None, ['easy','medium','hard']), "None or a level of difficulty")
471
+ config("level", None, (None, Int>=0), "None or a non-negative level")
472
+
473
+ Parameters
474
+ ----------
475
+ key : string
476
+ Keyword to read
477
+ default : optional
478
+ Default value.
479
+ Set to 'Config.no_default' to avoid defaulting. If then 'key' cannot be found a KeyError is raised.
480
+ cast : object, optional
481
+ If None, any value will be acceptable.
482
+ If not None, the function will attempt to cast the value provided with the provided value.
483
+ E.g. if cast = int, then it will run int(x)
484
+ This function now also allows passing the following complex arguemts:
485
+ * A list, in which case it is assumed that the 'key' must be from this list. The type of the first element of the list will be used to cast values
486
+ * Int or Float which allow defining constrained integers and floating numbers.
487
+ * A tuple of types, in which case any of the types is acceptable. A None here means that the value 'None' is acceptable (it does not mean that any value is acceptable)
488
+ help : str, optional
489
+ If provied adds a help text when self documentation is used.
490
+ help_default : str, optional
491
+ If provided, specifies the default value in plain text.
492
+ If not provided, help_default is equal to the string representation of the default value, if any.
493
+ Use this for complex default values which are hard to read.
494
+ help_cast : str, optional
495
+ If provided, specifies a description of the cast type.
496
+ If not provided, help_cast is set to the string representation of 'cast', or "None" if 'cast' is None. Complex casts are supported.
497
+ Use this for complex cast types which are hard to read.
498
+ mark_done : bool, optional
499
+ If true, marks the respective element as read.
500
+ record : bool, optional
501
+ If True, records usage of the key and validates that previous usage of the key is consistent with
502
+ the current usage, e.g. that the default values are consistent and that if help was provided it is the same.
503
+
504
+ Returns
505
+ -------
506
+ Value.
507
+
508
+ Raises
509
+ ------
510
+ KeyError
511
+ if 'key' could not be found.
512
+ InconsistencyError
513
+ if 'key' was previously accessed with different default, help, help_default or help_cast values.
514
+ For all the help texts empty strings are not compared, ie __call__("x", default=1) will succeed even if a previous call was __call__("x", default=1, help="value for x").
515
+ Note that 'cast' is not validated.
516
+ CastError:
517
+ If an error occcurs casting a provided value.
518
+ ValueError:
519
+ For input errors.
520
+ """
521
+ verify( isinstance(key, str), "'key' must be a string. Found type %s. Details: %s", type(key), key, exception=ValueError )
522
+ verify( key.find('.') == -1 , "Error using config '%s': key name cannot contain '.'. Found %s", self._name, key, exception=ValueError )
523
+
524
+ # determine raw value
525
+ if not key in self:
526
+ if default == no_default:
527
+ raise KeyError(key, "Error using config '%s': key '%s' not found " % (self._name, key))
528
+ value = default
529
+ else:
530
+ value = OrderedDict.get(self,key)
531
+
532
+ # has user only specified 'help' but not 'cast' ?
533
+ if isinstance(cast, str) and help is None:
534
+ help = cast
535
+ cast = None
536
+
537
+ # cast
538
+ caster = _create_caster( cast, self._name, key, none_is_any = True )
539
+ try:
540
+ value = caster( value, self._name, key )
541
+ except Exception as e:
542
+ raise CastError( config_name=self._name, key_name=key, message=e )
543
+
544
+ # mark key as read
545
+ if mark_done:
546
+ self._done.add(key)
547
+
548
+ # avoid recording
549
+ if not record:
550
+ return value
551
+ # record?
552
+ record_key = self.record_key( key ) # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
553
+ help = str(help) if not help is None and len(help) > 0 else ""
554
+ help = help[:-1] if help[-1:] == "." else help # remove trailing '.'
555
+ help_default = str(help_default) if not help_default is None else ""
556
+ help_default = str(default) if default != no_default and len(help_default) == 0 else help_default
557
+ help_cast = str(help_cast) if not help_cast is None else str(caster)
558
+ verify( default != no_default or help_default == "", "Config %s setup error for key %s: cannot specify 'help_default' if no default is given", self._name, key, exception=ValueError )
559
+
560
+ raw_use = help == "" and help_cast == "" and help_default == "" # raw_use, e.g. simply get() or []. Including internal use
561
+
562
+ exst_value = self._recorder.get(record_key, None)
563
+
564
+ if exst_value is None:
565
+ # no previous recorded use --> record this one, even if 'raw'
566
+ record = SortedDict(value=value,
567
+ raw_use=raw_use,
568
+ help=help,
569
+ help_default=help_default,
570
+ help_cast=help_cast )
571
+ if default != no_default:
572
+ record['default'] = default
573
+ self._recorder[record_key] = record
574
+ return value
575
+
576
+ if raw_use:
577
+ # do not compare raw_use with any other use
578
+ return value
579
+
580
+ if exst_value['raw_use']:
581
+ # previous usesage was 'raw'. Record this new use.
582
+ record = SortedDict(value=value,
583
+ raw_use=raw_use,
584
+ help=help,
585
+ help_default=help_default,
586
+ help_cast=help_cast )
587
+ if default != no_default:
588
+ record['default'] = default
589
+ self._recorder[record_key] = record
590
+ return value
591
+
592
+ # Both current and past were bona fide recorded uses.
593
+ # Ensure that their usage is consistent.
594
+ if default != no_default:
595
+ if 'default' in exst_value:
596
+ if exst_value['default'] != default:
597
+ raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different default values '%s' and '%s'" % ( key, self._name, record_key, exst_value['default'], default ))
598
+ else:
599
+ exst_value['default'] = default
600
+
601
+ if help != "":
602
+ if exst_value['help'] != "":
603
+ if exst_value['help'] != help:
604
+ raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help' texts '%s' and '%s'" % ( key, self._name, record_key, exst_value['help'], help ) )
605
+ else:
606
+ exst_value['help'] = help
607
+
608
+ if help_default != "":
609
+ if exst_value['help_default'] != "":
610
+ # we do not insist on the same 'help_default'
611
+ if exst_value['help_default'] != help_default:
612
+ raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help_default' texts '%s' and '%s'" % ( key, self._name, record_key, exst_value['help_default'], help_default ) )
613
+ else:
614
+ exst_value['help_default'] = help_default
615
+
616
+ if help_cast != "" and help_cast != _Simple.STR_NONE_CAST:
617
+ if exst_value['help_cast'] != "" and exst_value['help_cast'] != _Simple.STR_NONE_CAST:
618
+ if exst_value['help_cast'] != help_cast:
619
+ raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help_cast' texts '%s' and '%s'" % ( key, self._name, record_key, exst_value['help_cast'], help_cast ))
620
+ else:
621
+ exst_value['help_cast'] = help_cast
622
+ # done
623
+ return value
624
+
625
+ def __getitem__(self, key : str):
626
+ """
627
+ Returns the item for 'key' *without* recording its usage which for example means that done() will assume it hasn't been read.
628
+ Equivalent to get_raw(key).
629
+
630
+ __getitem__ does not record usage as many Python functions will use to iterate through objects (recall that Config is derived from OrderedDict)
631
+ """
632
+ return self.get_raw(key)
633
+
634
+ def __getattr__(self, key : str):
635
+ """
636
+ Returns either the value for 'key', if it exists, or creates on-the-fly a child config with the name 'key' and returns it.
637
+ If an exsting child value is returned, its usage is *not* recorded which for example means that done() will assume it hasn't been read.
638
+
639
+ Because __getattr__ will generte child configs on the fly it means the following is a legitmate use:
640
+ config = Config()
641
+ config.sub.x = 1 # <-- create 'sub' on the fly
642
+ """
643
+ verify( key.find('.') == -1 , "Error using config '%s': key name cannot contain '.'. Found %s", self._name, key, exception=ValueError )
644
+ if key in self._children:
645
+ return self._children[key]
646
+ verify( key.find(" ") == -1, "Error using config '%s': sub-config names cannot contain spaces. Found %s", self._name, key, exception=ValueError )
647
+ config = Config()
648
+ config._name = self._name + "." + key
649
+ config._recorder = self._recorder
650
+ self._children[key] = config
651
+ return config
652
+
653
+ def get(self, *kargs, **kwargs ):
654
+ """
655
+ Returns __call__(*kargs, **kwargs)
656
+ """
657
+ return self(*kargs, **kwargs)
658
+
659
+ def get_default(self, *kargs, **kwargs ):
660
+ """
661
+ Returns __call__(*kargs, **kwargs)
662
+ """
663
+ return self(*kargs, **kwargs)
664
+
665
+ def get_raw(self, key : str, default = no_default ):
666
+ """
667
+ Reads the respectitve element without marking the element as read, and without recording access to the element.
668
+ Equivalent to __call__(key, default, mark_done=False, record=False )
669
+ """
670
+ return self(key, default, mark_done=False, record=False)
671
+
672
+ def get_recorded(self, key : str ):
673
+ """
674
+ Returns the recorded used value of key, e.g. the value returned when the config was used:
675
+ If key is part of the input data, return that value
676
+ If key is not part of the input data, and a default was provided when the config was read, return the default.
677
+ This function:
678
+ Throws a KeyError if the key was never read successfully from the config (e.g. it is not used in the calling stack)
679
+ """
680
+ verify( key.find('.') == -1 , "Error using config '%s': key name cannot contain '.'. Found %s", self._name, key )
681
+ record_key = self._name + "['" + key + "']" # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
682
+ record = self._recorder.get(record_key, None)
683
+ if record is None:
684
+ raise KeyError(key)
685
+ return record['value']
686
+
687
+ def keys(self):
688
+ """
689
+ Returns the keys for the immediate keys of this config.
690
+ This call will *not* return the names of config children
691
+ """
692
+ return OrderedDict.keys(self)
693
+
694
+ # Write
695
+ # -----
696
+
697
+ def __setattr__(self, key, value):
698
+ """
699
+ Assign value using member notation, i.e. self.key = value
700
+ Identical to self[key] = value
701
+ Do not use leading underscores for config variables, see below
702
+
703
+ Parameters
704
+ ----------
705
+ key : str
706
+ Key to store. Note that keys with underscores are *not* stored as standard values,
707
+ but become classic members of the object (self.__dict__)
708
+ value :
709
+ If value is a Config object, them its usage information will be reset, and
710
+ the recorder will be set to the current recorder.
711
+ This way the following works as expected
712
+
713
+ config = Config()
714
+ sub = Config(a=1)
715
+ config.sub = sub
716
+ a = config.sub("a", 0, int, "Test")
717
+ config.done() # <- no error is reported, usage_report() is correct
718
+ """
719
+ self.__setitem__(key,value)
720
+
721
+ def __setitem__(self, key, value):
722
+ """
723
+ Assign value using array notation, i.e. self[key] = value
724
+ Identical to self.key = value
725
+
726
+ Parameters
727
+ ----------
728
+ key : str
729
+ Key to store. Note that keys with underscores are *not* stored as standard values,
730
+ but become classic members of the object (self.__dict__)
731
+ 'key' may contain '.' for hierarchical access.
732
+ value :
733
+ If value is a Config object, them its usage information will be reset, and
734
+ the recorder will be set to the current recorder.
735
+ This way the following works as expected
736
+
737
+ config = Config()
738
+ sub = Config(a=1)
739
+ config.sub = sub
740
+ a = config.sub("a", 0, int, "Test")
741
+ config.done() # <- no error is reported, usage_report() is correct
742
+ """
743
+ if key[0] == "_" or key in self.__dict__:
744
+ OrderedDict.__setattr__(self, key, value )
745
+ elif isinstance( value, Config ):
746
+ warn_if( len(value._recorder) > 0, "Warning: when assigning a used Config to another Config, all existing usage will be reset. "
747
+ "The 'recorder' of the assignee will be set ot the recorder of the receiving Config. "
748
+ "Make a 'clean_copy()' to avoid this warning.")
749
+ value._name = self._name + "." + key
750
+ def update_recorder( config ):
751
+ config._recorder = self._recorder
752
+ config._done.clear()
753
+ for k, c in config._children.items():
754
+ c._name = config._name + "." + k
755
+ update_recorder(c)
756
+ update_recorder(value)
757
+ self._children[key] = value
758
+ else:
759
+ keys = key.split(".")
760
+ if len(keys) == 1:
761
+ OrderedDict.__setitem__(self, key, value)
762
+ else:
763
+ c = self
764
+ for key in keys[:1]:
765
+ c = c.__getattr__(key)
766
+ OrderedDict.__setitem__(c, key, value)
767
+
768
+ def update( self, other=None, **kwargs ):
769
+ """
770
+ Overwrite values of 'self' new values.
771
+ Accepts the two main formats
772
+
773
+ update( dictionary )
774
+ update( config )
775
+ update( a=1, b=2 )
776
+ update( {'x.a':1 } ) # hierarchical assignment self.x.a = 1
777
+
778
+ Parameters
779
+ ----------
780
+ other : dict, Config, optional
781
+ Copy all content of 'other' into 'self'.
782
+ If 'other' is a config: elements will be clean_copy()ed.
783
+ 'other' will not be marked as 'used'
784
+ If 'other' is a dictionary, then '.' notation can be used for hierarchical assignments
785
+ **kwargs
786
+ Allows assigning specific values.
787
+ """
788
+ if not other is None:
789
+ if isinstance( other, Config ):
790
+ # copy() children
791
+ # and reset recorder to ours.
792
+ def set_recorder(config, recorder):
793
+ config._recorder = recorder
794
+ for _,c in config._children.items():
795
+ set_recorder( c, recorder )
796
+ for sub, child in other._children.items():
797
+ assert isinstance(child,Config)
798
+ if sub in self._children:
799
+ self._children[sub].update( child )
800
+ else:
801
+ self[sub] = child.clean_copy() # see above for assigning config
802
+ assert sub in self._children
803
+ assert not sub in self
804
+ # copy elements from other.
805
+ # we do not mark elements from another config as 'used'
806
+ for key in other:
807
+ if key in self._children:
808
+ del self._children[key]
809
+ self[key] = other.get_raw(key)
810
+ assert key in self
811
+ assert not key in self._children
812
+ else:
813
+ verify( isinstance(other, Mapping), "Cannot update config with an object of type '%s'. Expected 'Mapping' type.", type(other).__name__, exception=ValueError )
814
+ for key in other:
815
+ if key[:1] == "_" or key in self.__dict__:
816
+ continue
817
+ if isinstance(other[key], Mapping):
818
+ if key in self:
819
+ del self[key]
820
+ elif not key in self._children:
821
+ self.__getattr__(key) # creates child
822
+ self._children[key].update( other[key] )
823
+ else:
824
+ if key in self._children:
825
+ del self._children[key]
826
+ self[key] = other[key]
827
+
828
+ if len(kwargs) > 0:
829
+ self.update( other=kwargs )
830
+
831
+ # delete
832
+ # ------
833
+
834
+ def delete_children( self, names : list ):
835
+ """
836
+ Delete one or several children.
837
+ This function does not delete 'record' information.
838
+ """
839
+ if isinstance(names, str):
840
+ names = [ names ]
841
+
842
+ for name in names:
843
+ del self._children[name]
844
+
845
+ # Usage information & reports
846
+ # ---------------------------
847
+
848
+ @property
849
+ def recorder(self) -> SortedDict:
850
+ """ Returns the top level recorder """
851
+ return self._recorder
852
+
853
+ def usage_report(self, with_values : bool = True,
854
+ with_help : bool = True,
855
+ with_defaults: bool = True,
856
+ with_cast : bool = False,
857
+ filter_path : str = None ) -> str:
858
+ """
859
+ Generate a human readable report of all variables read from this config.
860
+
861
+ Parameters
862
+ ----------
863
+ with_values : bool, optional
864
+ Whether to also print values. This can be hard to read
865
+ if values are complex objects
866
+
867
+ with_help: bool, optional
868
+ Whether to print help
869
+
870
+ with_defaults: bool, optional
871
+ Whether to print default values
872
+
873
+ with_cast: bool, optional
874
+ Whether to print types
875
+
876
+ filter_path : str, optional
877
+ If provided, will match all children names vs this string.
878
+ Most useful with filter_path = self._name
879
+
880
+ Returns
881
+ -------
882
+ str
883
+ Report.
884
+ """
885
+ with_values = bool(with_values)
886
+ with_help = bool(with_help)
887
+ with_defaults = bool(with_defaults)
888
+ with_cast = bool(with_cast)
889
+ l = len(filter_path) if not filter_path is None else 0
890
+ rep_here = ""
891
+ reported = ""
892
+
893
+ for key, record in self._recorder.items():
894
+ value = record['value']
895
+ help = record['help']
896
+ help_default = record['help_default']
897
+ help_cast = record['help_cast']
898
+ report = key + " = " + str(value) if with_values else key
899
+
900
+ do_help = with_help and help != ""
901
+ do_cast = with_cast and help_cast != ""
902
+ do_defaults = with_defaults and help_default != ""
903
+
904
+ if do_help or do_cast or do_defaults:
905
+ report += " # "
906
+ if do_cast:
907
+ report += "(" + help_cast + ") "
908
+ if do_help:
909
+ report += help
910
+ if do_defaults:
911
+ report += "; default: " + help_default
912
+ elif do_defaults:
913
+ report += "Default: " + help_default
914
+
915
+ if l > 0 and key[:l] == filter_path:
916
+ rep_here += report + "\n"
917
+ else:
918
+ reported += report + "\n"
919
+
920
+ if len(reported) == 0:
921
+ return rep_here
922
+ if len(rep_here) == 0:
923
+ return reported
924
+ return rep_here + "# \n" + reported
925
+
926
+ def usage_reproducer(self) -> str:
927
+ """
928
+ Returns a string expression which will reproduce the current
929
+ configuration tree as long as each 'value' handles
930
+ repr() correctly.
931
+ """
932
+ report = ""
933
+ for key, record in self._recorder.items():
934
+ value = record['value']
935
+ report += key + " = " + repr(value) + "\n"
936
+ return report
937
+
938
+ def input_report(self) -> str:
939
+ """
940
+ Returns a report of all inputs in a readable format, as long as all values
941
+ are as such.
942
+ """
943
+ inputs = []
944
+ def ireport(self, inputs):
945
+ for key in self:
946
+ value = self.get_raw(key)
947
+ report_key = self._name + "['" + key + "'] = %s" % str(value)
948
+ inputs.append( report_key )
949
+ for c in self._children.values():
950
+ ireport(c, inputs)
951
+ ireport(self, inputs)
952
+
953
+ inputs = sorted(inputs)
954
+ report = ""
955
+ for i in inputs:
956
+ report += i + "\n"
957
+ return report
958
+
959
+ @property
960
+ def not_done(self) -> dict:
961
+ """ Returns a dictionary of keys which were not read yet """
962
+ h = { key : False for key in self if not key in self._done }
963
+ for k,c in self._children.items():
964
+ ch = c.not_done
965
+ if len(ch) > 0:
966
+ h[k] = ch
967
+ return h
968
+
969
+ def input_dict(self, ignore_underscore = True ) -> dict:
970
+ """ Returns a (pretty) dictionary of all inputs into this config. """
971
+ inputs = pdct()
972
+ for key in self:
973
+ if ignore_underscore and key[:1] == "_":
974
+ continue
975
+ inputs[key] = self.get_raw(key)
976
+ for k,c in self._children.items():
977
+ if ignore_underscore and k[:1] == "_":
978
+ continue
979
+ inputs[k] = c.input_dict()
980
+ return inputs
981
+
982
+ def unique_id(self, *, uniqueHash = None, debug_trace = None, **unique_hash_parameters ) -> str:
983
+ """
984
+ Returns a unique hash key for this object, based on its provided inputs /not/ based on its usage.
985
+ Please consult the documentation for cdxbasics.uniquehash.UniqueHashExt
986
+ ** WARNING **
987
+ By default function ignores
988
+ 1) Config keys or children with leading '_'s are ignored unless 'parse_underscore' is set to 'protected' or 'private'.
989
+ 2) Functions and properties are ignored unless parse_functions is True
990
+ In the latter case function code will be used to distinguish
991
+ functions assigned to the config.
992
+ See uniquehash.unqiueHashExt() for further information.
993
+
994
+ Parameters
995
+ ----------
996
+
997
+ Returns
998
+ -------
999
+ String ID
1000
+ """
1001
+ if uniqueHash is None:
1002
+ uniqueHash = UniqueHash( **unique_hash_parameters )
1003
+ else:
1004
+ if len(unique_hash_parameters) != 0: raise ValueError("Cannot provide 'unique_hash_parameters' if 'uniqueHashExt' is provided")
1005
+
1006
+ def rec(config):
1007
+ """ Recursive version which returns an empty string for empty sub configs """
1008
+ inputs = {}
1009
+ for key in config:
1010
+ if key[:1] == "_":
1011
+ continue
1012
+ inputs[key] = config.get_raw(key)
1013
+ for c, child in config._children.items():
1014
+ if c[:1] == "_":
1015
+ continue
1016
+ # collect ID for the child
1017
+ child_data = rec(child)
1018
+ # we only register children if they have keys.
1019
+ # this way we do not trigger a change in ID simply due to a failed read access.
1020
+ if child_data != "":
1021
+ inputs[c] = child_data
1022
+ if len(inputs) == 0:
1023
+ return ""
1024
+ return uniqueHashExt(inputs)
1025
+ uid = rec(self)
1026
+ return uid if uid!="" else uniqueHashExt("")
1027
+
1028
+ def used_info(self, key : str) -> tuple:
1029
+ """Returns the usage stats for a given key in the form of a tuple (done, record) where 'done' is a boolean and 'record' is a dictionary of information on the key """
1030
+ done = key in self._done
1031
+ record = self._recorder.get( self.record_key(key), None )
1032
+ return (done, record)
1033
+
1034
+ def record_key(self, key):
1035
+ """
1036
+ Returns the fully qualified 'record' key for a relative 'key'.
1037
+ It has the form config1.config['entry']
1038
+ """
1039
+ return self._name + "['" + key + "']" # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
1040
+
1041
+ # magic
1042
+ # -----
1043
+
1044
+ def __iter__(self):
1045
+ """
1046
+ Iterate. For some odd reason, adding this override will make
1047
+ using f(**self) call our __getitem__() function.
1048
+ """
1049
+ return OrderedDict.__iter__(self)
1050
+
1051
+ # pickling
1052
+ # --------
1053
+
1054
+ def __reduce__(self):
1055
+ """
1056
+ Pickling this object explicitly
1057
+ See https://docs.python.org/3/library/pickle.html#object.__reduce__
1058
+ """
1059
+ keys = [ k for k in self ]
1060
+ data = [ self.get_raw(k) for k in keys ]
1061
+ state = dict(done = self._done,
1062
+ name = self._name,
1063
+ children = self._children,
1064
+ recorder = self._recorder,
1065
+ keys = keys,
1066
+ data = data )
1067
+ return (Config, (), state)
1068
+
1069
+ def __setstate__(self, state):
1070
+ """ Supports unpickling """
1071
+ self._name = state['name']
1072
+ self._done = state['done']
1073
+ self._children = state['children']
1074
+ self._recorder = state['recorder']
1075
+ data = state['data']
1076
+ keys = state['keys']
1077
+ for (k,d) in zip(keys,data):
1078
+ self[k] = d
1079
+
1080
+ # casting
1081
+ # -------
1082
+
1083
+ @staticmethod
1084
+ def to_config( kwargs : dict, config_name : str = "kwargs"):
1085
+ """
1086
+ Makes sure an object is a config, and otherwise tries to convert it into one
1087
+ Classic use case is to transform 'kwargs' to a Config
1088
+ """
1089
+ return kwargs if isinstance(kwargs,Config) else Config( kwargs,config_name=config_name )
1090
+
1091
+ @staticmethod
1092
+ def config_kwargs( config, kwargs : dict, config_name : str = "kwargs"):
1093
+ """
1094
+ Default implementation for a usage pattern where the user can use both a 'config' and kwargs.
1095
+ This function 'detaches' the current config from 'self' which means done() must be called again.
1096
+
1097
+ Example
1098
+ -------
1099
+
1100
+ def f(config, **kwargs):
1101
+ config = Config.config_kwargs( config, kwargs )
1102
+ ...
1103
+ x = config("x", 1, ...)
1104
+ config.done() # <-- important to do this here. Remembert that config_kwargs() calls 'detach'
1105
+
1106
+ and then one can use either
1107
+
1108
+ config = Config()
1109
+ config.x = 1
1110
+ f(config)
1111
+
1112
+ or
1113
+ f(x=1)
1114
+
1115
+ Parameters
1116
+ ----------
1117
+ config : a Config object or None
1118
+ kwargs : a dictionary. If 'config' is provided, the function will call config.update(kwargs).
1119
+ config_name : a declarative name for the config if 'config' is not proivded
1120
+
1121
+ Returns
1122
+ -------
1123
+ A Config
1124
+ """
1125
+ assert isinstance( config_name, str ), "'config_name' must be a string"
1126
+ if type(config).__name__ == Config.__name__: # we allow for import inconsistencies
1127
+ config = config.detach()
1128
+ config.update(kwargs)
1129
+ else:
1130
+ if not config is None: raise TypeError("'config' must be of type 'Config'")
1131
+ config = Config.to_config( kwargs=kwargs, config_name=config_name )
1132
+ return config
1133
+
1134
+ # for uniqueHash
1135
+ # --------------
1136
+
1137
+ def __unique_hash__(self, uniqueHashExt, debug_trace ) -> str:
1138
+ """
1139
+ Returns a unique hash for this object
1140
+ This function is required because by default uniqueHash() ignores members starting with '_', which
1141
+ in the case of Config means that no children are hashed.
1142
+ """
1143
+ return self.unique_id( uniqueHashExt=uniqueHashExt, debug_trace=debug_trace, )
1144
+
1145
+
1146
+ # Comparison
1147
+ # -----------
1148
+
1149
+ def __eq__(self, other):
1150
+ """ Equality operator comparing 'name' and standard dictionary content """
1151
+ if type(self).__name__ != type(other).__name__: # allow comparison betweenn different imports
1152
+ return False
1153
+ if self._name != other._name:
1154
+ return False
1155
+ return OrderedDict.__eq__(self, other)
1156
+
1157
+ def __hash__(self):
1158
+ return hash(self._name) ^ OrderedDict.__hash__(self)
1159
+
1160
+ to_config = Config.to_config
1161
+ config_kwargs = Config.config_kwargs
1162
+ Config.no_default = no_default
1163
+
1164
+ # ==============================================================================
1165
+ # New in version 0.1.45
1166
+ # Support for conditional types, e.g. we can write
1167
+ #
1168
+ # x = config(x, 0.1, Float >= 0., "An 'x' which cannot be negative")
1169
+ # ==============================================================================
1170
+
1171
+ class ConfigField(object):
1172
+ """
1173
+ Simplististc 'read only' wrapper for Config objects.
1174
+ Useful for Flax
1175
+
1176
+ import dataclasses as dataclasses
1177
+ import jax.numpy as jnp
1178
+ import jax as jax
1179
+ from cdxbasics.config import Config, ConfigField
1180
+ import types as types
1181
+
1182
+ class A( nn.Module ):
1183
+ config : ConfigField = ConfigField.Field()
1184
+
1185
+ def setup(self):
1186
+ self.dense = nn.Dense(1)
1187
+
1188
+ def __call__(self, x):
1189
+ a = self.config("y", 0.1 ,float)
1190
+ return self.dense(x)*a
1191
+
1192
+ print("Default")
1193
+ a = A()
1194
+ key1, key2 = jax.random.split(jax.random.key(0))
1195
+ x = jnp.zeros((10,10))
1196
+ param = a.init( key1, x )
1197
+ y = a.apply( param, x )
1198
+
1199
+ print("Value")
1200
+ w = ConfigField(y=1.)
1201
+ a = A(config=w)
1202
+
1203
+ key1, key2 = jax.random.split(jax.random.key(0))
1204
+ x = jnp.zeros((10,10))
1205
+ param = a.init( key1, x )
1206
+ y = a.apply( param, x )
1207
+
1208
+ class A( nn.Module ):
1209
+ config : ConfigField = ConfigField.field()
1210
+
1211
+ @nn.compact
1212
+ def __call__(self, x):
1213
+ a = self.config.x("y", 0.1 ,float)
1214
+ self.config.done()
1215
+ return nn.Dense(1)(x)*a
1216
+
1217
+ print("Config")
1218
+ c = Config()
1219
+ c.x.y = 1.
1220
+ w = ConfigField(c)
1221
+ a = A(config=w)
1222
+
1223
+ key1, key2 = jax.random.split(jax.random.key(0))
1224
+ x = jnp.zeros((10,10))
1225
+ param = a.init( key1, x )
1226
+ y = a.apply( param, x )
1227
+ y = a.apply( param, x )
1228
+ """
1229
+ def __init__(self, config : Config = None, **kwargs):
1230
+ if not config is None:
1231
+ config = config if type(config).__name__ != type(self).__name__ else config.__config
1232
+ self.__config = Config.config_kwargs( config, kwargs )
1233
+ def __call__(self, *kargs, **kwargs):
1234
+ return self.__config(*kargs,**kwargs)
1235
+ def __getattr__(self, key):
1236
+ if key[:2] == "__":
1237
+ return object.__getattr__(self, key)
1238
+ return getattr(self.__config, key)
1239
+ def __getitem__(self, key):
1240
+ return self.__config[key]
1241
+ def __eq__(self, other):
1242
+ if type(other).__name__ == "Config":
1243
+ return self.__config == other
1244
+ else:
1245
+ return self.__config == other.config
1246
+ def __hash__(self):
1247
+ h = 0
1248
+ for k, v in self.items():
1249
+ h ^= hash(k) ^ hash(v)
1250
+ return h
1251
+ def __unique_hash__(self, *kargs, **kwargs):
1252
+ return self.__config.__unique_hash__(*kargs, **kwargs)
1253
+ def __str__(self):
1254
+ return self.__pdct.__str__()
1255
+ def __repr__(self):
1256
+ return self.__pdct.__repr__()
1257
+ def as_dict(self):
1258
+ return self.__config.as_dict(mark_done=False)
1259
+ def done(self):
1260
+ return self.__config.done()
1261
+
1262
+ @property
1263
+ def config(self) -> Config:
1264
+ return self.__config
1265
+
1266
+ @staticmethod
1267
+ def default():
1268
+ return ConfigField()
1269
+
1270
+ @staticmethod
1271
+ def Field():
1272
+ import dataclasses as dataclasses
1273
+ return dataclasses.field( default_factory=ConfigField )
1274
+
1275
+ # ==============================================================================
1276
+ # New in version 0.1.45
1277
+ # Support for conditional types, e.g. we can write
1278
+ #
1279
+ # x = config(x, 0.1, Float >= 0., "An 'x' which cannot be negative")
1280
+ # ==============================================================================
1281
+
1282
+ class _Cast(object):
1283
+
1284
+ def __call__( self, value, *, config_name : str, key_name : str ):
1285
+ """ cast 'value' to the proper type """
1286
+ raise NotImplementedError("Internal error")
1287
+
1288
+ def __str__(self) -> str:
1289
+ """ Returns readable string description of 'self' """
1290
+ raise NotImplementedError("Internal error")
1291
+
1292
+ def _cast_name( cast : type ) -> str:
1293
+ """ Returns the class name of 'cast' """
1294
+ if cast is None:
1295
+ return ""
1296
+ return getattr(cast,"__name__", str(cast))
1297
+
1298
+ def _cast_err_header(*, config_name : str, key_name : str):
1299
+ if len(config_name) > 0:
1300
+ return f"Error using config '{config_name}' for key '{key_name}': "
1301
+ else:
1302
+ return f"Config error for key '{key_name}': "
1303
+
1304
+ # ================================
1305
+ # Simple wrapper
1306
+ # ================================
1307
+
1308
+ class _Simple(_Cast):# NOQA
1309
+ """
1310
+ Default case where the 'cast' argument for a config call() is simply a type, a Callable, or None.
1311
+ Cast to an actual underlying type.
1312
+ """
1313
+
1314
+ STR_NONE_CAST = "any"
1315
+
1316
+ def __init__(self, *, cast : type, config_name : str, key_name : str, none_is_any : bool ):
1317
+ """ Simple atomic caster """
1318
+ if not cast is None:
1319
+ if isinstance(cast, str):
1320
+ raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name)+\
1321
+ "'cast' must be a type. Found a string. Most likely this happened because a help string was defined as positional argument, but no 'cast' type was specified. "+\
1322
+ "In this case, use 'help=' to specify the help text.")
1323
+ assert not isinstance(cast, _Cast), lambda : "Internal error in definition for key '%s' in config '%s': 'cast' should not be derived from _Cast. Object is %s" % ( key_name, config_name, str(cast) )
1324
+ self.cast = cast
1325
+ self.none_is_any = none_is_any
1326
+ assert not none_is_any is None or not cast is None, "Must set 'none_is_any' to bool value if cast is 'None'."
1327
+
1328
+ def __call__(self, value, *, config_name : str, key_name : str ):
1329
+ """ Cast 'value' to the proper type """
1330
+ if self.cast is None:
1331
+ if value is None or self.none_is_any:
1332
+ return value
1333
+ if not value is None:
1334
+ raise ValueError(f"'None' value expected, found value of type {type(value).__name__}")
1335
+ return self.cast(value)
1336
+
1337
+ def __str__(self) -> str:
1338
+ """ Returns readable string """
1339
+ if self.cast is None:
1340
+ return self.STR_NONE_CAST if self.none_is_any else "None"
1341
+ return _cast_name(self.cast)
1342
+
1343
+ # ================================
1344
+ # Conditional types such as Int>0
1345
+ # ================================
1346
+
1347
+ class _Condition(_Cast):
1348
+ """ Represents a simple operator condition such as 'Float >= 0.' """
1349
+
1350
+ def __init__(self, cast, op, other):
1351
+ """ Initialize the condition for a base type 'cast' and an 'op' with an 'other' """
1352
+ self.cast = cast
1353
+ self.op = op
1354
+ self.other = other
1355
+ self.l_and = None
1356
+
1357
+ def __and__(self, cond):
1358
+ """
1359
+ Combines to conditions with logical AND .
1360
+ Requires the left hand 'self' to be a > or >=; the right hand must be < or <=
1361
+ This means you can write
1362
+
1363
+ x = config("x", 0.5, (Float >= 0.) & (Float < 1.), "Variable x")
1364
+
1365
+ """
1366
+ if not self.l_and is None:
1367
+ raise NotImplementedError("Cannot combine more than two conditions")
1368
+ if not self.op[0] == 'g':
1369
+ raise NotImplementedError("The left hand condition when using '&' must be > or >=. Found %s" % self._op_str)
1370
+ if not cond.op[0] == 'l':
1371
+ raise NotImplementedError("The right hand condition when using '&' must be < or <=. Found %s" % cond._op_str)
1372
+ if self.cast != cond.cast:
1373
+ raise NotImplementedError("Cannot '&' conditions for types %s and %s" % (self.cast.__name__, cond.cast.__name__))
1374
+ op_new = _Condition(self.cast, self.op, self.other)
1375
+ op_new.l_and = cond
1376
+ return op_new
1377
+
1378
+ def __call__(self, value, *, config_name : str, key_name : str ):
1379
+ """ Test whether 'value' satisfies the condition """
1380
+ value = self.cast(value)
1381
+
1382
+ if self.op == "ge":
1383
+ ok = value >= self.other
1384
+ elif self.op == "gt":
1385
+ ok = value > self.other
1386
+ elif self.op == "le":
1387
+ ok = value <= self.other
1388
+ elif self.op == "lt":
1389
+ ok = value < self.other
1390
+ else:
1391
+ raise RuntimeError("Internal error: unknown operator %s" % str(self.op))
1392
+ if not ok:
1393
+ raise ValueError( "value for key '%s' %s. Found %s" % ( key_name, self.err_str, value ))
1394
+ return self.l_and( value, config_name=config_name, key_name=key_name) if not self.l_and is None else value
1395
+
1396
+ def __str__(self) -> str:
1397
+ """ Returns readable string """
1398
+ s = _cast_name(self.cast) + self._op_str + str(self.other)
1399
+ if not self.l_and is None:
1400
+ s += " and " + str(self.l_and)
1401
+ return s
1402
+
1403
+ @property
1404
+ def _op_str(self) -> str:
1405
+ """ Returns a string for the operator of this conditon """
1406
+ if self.op == "ge":
1407
+ return ">="
1408
+ elif self.op == "gt":
1409
+ return ">"
1410
+ elif self.op == "le":
1411
+ return "<="
1412
+ elif self.op == "lt":
1413
+ return "<"
1414
+ raise RuntimeError("Internal error: unknown operator %s" % str(self.op))
1415
+
1416
+ @property
1417
+ def err_str(self) -> str:
1418
+ """ Nice error string """
1419
+ zero = self.cast(0)
1420
+ def mk_txt(cond):
1421
+ if cond.op == "ge":
1422
+ s = ("not be lower than %s" % cond.other) if cond.other != zero else ("not be negative")
1423
+ elif cond.op == "gt":
1424
+ s = ("be bigger than %s" % cond.other) if cond.other != zero else ("be positive")
1425
+ elif cond.op == "le":
1426
+ s = ("not exceed %s" % cond.other) if cond.other != zero else ("not be positive")
1427
+ elif cond.op == "lt":
1428
+ s = ("be lower than %s" % cond.other) if cond.other != zero else ("be negative")
1429
+ else:
1430
+ raise RuntimeError("Internal error: unknown operator %s" % str(cond.op))
1431
+ return s
1432
+
1433
+ s = "must " + mk_txt(self)
1434
+ if not self.l_and is None:
1435
+ s += ", and " + mk_txt(self.l_and)
1436
+ return s
1437
+
1438
+ class _CastCond(_Cast): # NOQA
1439
+ """
1440
+ Generates compound _Condition's
1441
+ See the two members Float and Int
1442
+ """
1443
+
1444
+ def __init__(self, cast):# NOQA
1445
+ self.cast = cast
1446
+ def __ge__(self, other) -> bool:# NOQA
1447
+ return _Condition( self.cast, 'ge', self.cast(other) )
1448
+ def __gt__(self, other) -> bool:# NOQA
1449
+ return _Condition( self.cast, 'gt', self.cast(other) )
1450
+ def __le__(self, other) -> bool:# NOQA
1451
+ return _Condition( self.cast, 'le', self.cast(other) )
1452
+ def __lt__(self, other) -> bool:# NOQA
1453
+ return _Condition( self.cast, 'lt', self.cast(other) )
1454
+ def __call__(self, value, *, config_name : str, key_name : str ):
1455
+ """ This gets called if the type was used without operators """
1456
+ cast = _Simple(self.cast, config_name=config_name, key_name=key_name, none_is_any=None )
1457
+ return cast( value, config_name=config_name, key_name=key_name )
1458
+ def __str__(self) -> str:
1459
+ """ This gets called if the type was used without operators """
1460
+ return _cast_name(self.cast)
1461
+
1462
+ Float = _CastCond(float)
1463
+ """
1464
+ Allows to apply conditions to float's as part of config's.
1465
+ For example
1466
+ ```
1467
+ timeout = config("timeout", 0.5, Float>=0., "Timeout")
1468
+ ```
1469
+
1470
+ In combination with `&` we can limit a float to a range:
1471
+ ```
1472
+ probability = config("probability", 0.5, (Float>=0.) & (Float <= 1.), "Probability")
1473
+ ```
1474
+ """
1475
+
1476
+
1477
+ Int = _CastCond(int)
1478
+ """
1479
+ Allows to apply conditions to int' as part of config's.
1480
+ For example
1481
+ ```
1482
+ num_steps = config("num_steps", 1, Int>0., "Number of steps")
1483
+ ```
1484
+
1485
+ In combination with `&` we can limit an int to a range:
1486
+ ```
1487
+ bus_days_per_year = config("bus_days_per_year", 255, (Int > 0) & (Int < 365), "Business days per year")
1488
+ ```
1489
+ """
1490
+
1491
+ # ================================
1492
+ # Enum type for list 'cast's
1493
+ # ================================
1494
+
1495
+ class _Enum(_Cast):
1496
+ """
1497
+ Utility class to support enumerator types.
1498
+ No need to use this classs directly. It will be automatically instantiated if a list is passed as type, e.g.
1499
+
1500
+ code = config("code", "Python", ['C++', 'Python'], "Which language do we love")
1501
+
1502
+ Note that all list members must be of the same type.
1503
+ """
1504
+
1505
+ def __init__(self, enum : list, *, config_name : str, key_name : str ):
1506
+ """ Initializes an enumerator casting type. """
1507
+ self.enum = list(enum)
1508
+ if len(self.enum) == 0:
1509
+ raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
1510
+ f"'cast' for key '{key_name}' is an empty list. Lists are used for enumerator types, hence passing empty list is not defined")
1511
+ if self.enum[0] is None:
1512
+ raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
1513
+ f"'cast' for key '{key_name}' is an list, with first element 'None'. Lists are used for enumerator types, and the first element defines their underlying type. Hence you cannot use 'None'. (Did you want to use alternative notation with tuples?)")
1514
+ self.cast = _Simple( type(self.enum[0]), config_name=config_name, key_name=key_name, none_is_any=None )
1515
+ for i in range(1,len(self.enum)):
1516
+ try:
1517
+ self.enum[i] = self.cast( self.enum[i], config_name=config_name, key_name=key_name )
1518
+ except:
1519
+ other_name = _cast_name(type(self.enum[i]))
1520
+ raise ValueError( _cast_err_header(config_name=config_name,key_name=key_name) +\
1521
+ f"'cast' for key {key_name}: members of the 'cast' list are not of consistent type. Found {self.cast} for the first element which does match the type {other_name} of the {i}th element" )
1522
+
1523
+ def __call__( self, value, *, config_name, key_name ):
1524
+ """
1525
+ Cast 'value' to the proper type and check is one of the list members
1526
+ Raises a KeyError if the value was not found in our enum
1527
+ """
1528
+ value = self.cast(value, config_name=config_name, key_name=key_name)
1529
+ if not value in self.num:
1530
+ raise ValueError( f"Value for key '{key_name}' {self.err_str}; found 'str(value)[:20]'" )
1531
+ return value
1532
+
1533
+ @property
1534
+ def err_str(self) -> str:
1535
+ """ Nice error string """
1536
+ if len(self.enum) == 1:
1537
+ return f"must be '{str(self.enum[0])}'"
1538
+
1539
+ s = "must be one of: '" + str(self.enum[0]) + "'"
1540
+ for i in range(1,len(self.enum)-1):
1541
+ s += ", '" + str(self.enum[i]) + "'"
1542
+ s += " or '" + str(self.enum[-1]) + "'"
1543
+ return s
1544
+
1545
+ def __str__(self) -> str:
1546
+ """ Returns readable string """
1547
+ s = "[ "
1548
+ for i in range(len(self.enum)):
1549
+ s += ( ", " + self.enum[i] ) if i > 0 else self.enum[i]
1550
+ s += " ]"
1551
+ return s
1552
+
1553
+ # ================================
1554
+ # Multiple types
1555
+ # ================================
1556
+
1557
+ class _Alt(_Cast):
1558
+ """
1559
+ Initialize a casting compund "alternative" type, e.g. it the variable may contain several types, each of which is acceptable.
1560
+ None here means that 'None' is an accepted value.
1561
+ This is invokved when a tuple is passed, e.g
1562
+
1563
+ config("spread", None, ( None, float ), "Float or None")
1564
+ config("spread", 1, ( Int<=-1, Int>=1. ), "A variable which has to be outside (-1,+1)")
1565
+ """
1566
+
1567
+ def __init__(self, casts : list, *, config_name : str, key_name : str ):
1568
+ """ Initialize a compound cast """
1569
+ if len(casts) == 0:
1570
+ raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
1571
+ f"'cast' for key '{key_name}' is an empty tuple. Tuples are used for alternative types, hence passing empty tuple is not defined")
1572
+ self.casts = [ _create_caster(cast=cast, config_name=config_name, key_name=key_name, none_is_any=False) for cast in enumerate(casts) ]
1573
+
1574
+ def __call__( self, value, *, config_name : str, key_name : str ):
1575
+ """ Cast 'value' to the proper type """
1576
+ e0 = None
1577
+ for cast in self.casts:
1578
+ # None means that value == None is acceptable
1579
+ try:
1580
+ return cast(value, config_name=config_name, key_namkey_name=key_name )
1581
+ except Exception as e:
1582
+ e0 = e if e0 is None else e0
1583
+ raise ValueError(f"Error using config '{config_name}': value for key '{key_name}' {self.err_str}. Found '{str(value)}' of type '{type(value).__name__}'")
1584
+
1585
+ def test(self, value):
1586
+ """ Test whether 'value' satisfies the condition """
1587
+ raise self.test
1588
+
1589
+ @property
1590
+ def err_str(self):
1591
+ """ Returns readable string """
1592
+ return "must be one of the following types: " + self.__str__()
1593
+
1594
+ def __str__(self):
1595
+ """ Returns readable string """
1596
+ s = ""
1597
+ for cast in self.casts[:-1]:
1598
+ s += str(cast) + " or "
1599
+ s += str(self.casts[-1])
1600
+ return s
1601
+
1602
+ # ================================
1603
+ # Manage casting
1604
+ # ================================
1605
+
1606
+ def _create_caster( *, cast : type, config_name : str, key_name : str, none_is_any : bool ):
1607
+ """
1608
+ Implements casting.
1609
+
1610
+ Parameters
1611
+ ----------
1612
+ value: value, either from the user or the default value if provided
1613
+ cast : cast input to call() from the user, or None.
1614
+ key_name : name of the config
1615
+ key : name of the key being access
1616
+ none_is_any :If True, then None means that any type is accepted. If False, the None means that only None is accepted.
1617
+
1618
+ Returns
1619
+ -------
1620
+ value : casted value
1621
+ __str__ : casting help text. Empty if 'cast' is None
1622
+ """
1623
+ if isinstance( cast, str ):
1624
+ raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
1625
+ f"string '{cast}' provided as 'cast'. Strings are not supported. To pass sets of characters, use a list of strings of those single characters.")
1626
+ if isinstance(cast, list):
1627
+ return _Enum( cast, config_name=config_name, key_name=key_name )
1628
+ elif isinstance(cast, tuple):
1629
+ return _Alt( cast, config_name=config_name, key_name=key_name )
1630
+ elif isinstance(cast,_Cast):
1631
+ return cast
1632
+ return _Simple( cast, config_name=config_name, key_name=key_name, none_is_any=none_is_any )
1633
+