cdxcore 0.1.6__py3-none-any.whl → 0.1.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cdxcore might be problematic. Click here for more details.
- cdxcore/__init__.py +1 -9
- cdxcore/config.py +1188 -521
- cdxcore/crman.py +95 -25
- cdxcore/err.py +371 -0
- cdxcore/pretty.py +468 -0
- cdxcore/pretty.py_bak.py +750 -0
- cdxcore/subdir.py +2225 -1334
- cdxcore/uniquehash.py +515 -363
- cdxcore/util.py +358 -417
- cdxcore/verbose.py +683 -248
- cdxcore/version.py +398 -139
- cdxcore-0.1.9.dist-info/METADATA +27 -0
- cdxcore-0.1.9.dist-info/RECORD +36 -0
- {cdxcore-0.1.6.dist-info → cdxcore-0.1.9.dist-info}/top_level.txt +3 -1
- docs/source/conf.py +123 -0
- docs2/source/conf.py +35 -0
- tests/test_config.py +502 -0
- tests/test_crman.py +54 -0
- tests/test_err.py +86 -0
- tests/test_pretty.py +404 -0
- tests/test_subdir.py +289 -0
- tests/test_uniquehash.py +159 -144
- tests/test_util.py +122 -83
- tests/test_verbose.py +119 -0
- tests/test_version.py +153 -0
- up/git_message.py +2 -2
- cdxcore/logger.py +0 -319
- cdxcore/prettydict.py +0 -388
- cdxcore/prettyobject.py +0 -64
- cdxcore-0.1.6.dist-info/METADATA +0 -1418
- cdxcore-0.1.6.dist-info/RECORD +0 -30
- conda/conda_exists.py +0 -10
- conda/conda_modify_yaml.py +0 -42
- tests/_cdxbasics.py +0 -1086
- {cdxcore-0.1.6.dist-info → cdxcore-0.1.9.dist-info}/WHEEL +0 -0
- {cdxcore-0.1.6.dist-info → cdxcore-0.1.9.dist-info}/licenses/LICENSE +0 -0
- {cdxcore → tmp}/deferred.py +0 -0
- {cdxcore → tmp}/dynaplot.py +0 -0
- {cdxcore → tmp}/filelock.py +0 -0
- {cdxcore → tmp}/jcpool.py +0 -0
- {cdxcore → tmp}/np.py +0 -0
- {cdxcore → tmp}/npio.py +0 -0
- {cdxcore → tmp}/sharedarray.py +0 -0
cdxcore/config.py
CHANGED
|
@@ -1,149 +1,474 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
Hans Buehler 2022
|
|
5
|
-
"""
|
|
2
|
+
Overview
|
|
3
|
+
--------
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
from sortedcontainers import SortedDict
|
|
10
|
-
import warnings as warnings
|
|
11
|
-
from dataclasses import Field
|
|
12
|
-
from .util import mt as txtfmt
|
|
13
|
-
from .uniquehash import UniqueHash
|
|
14
|
-
from .prettydict import PrettyDict as pdct
|
|
5
|
+
Tooling for setting up program-wide configuration hierachies.
|
|
6
|
+
Aimed at machine learning programs to ensure consistency of code accross experimentation.
|
|
15
7
|
|
|
16
|
-
|
|
8
|
+
**Basic config construction**::
|
|
17
9
|
|
|
18
|
-
|
|
19
|
-
|
|
10
|
+
from cdxbasics.config import Config, Int
|
|
11
|
+
config = Config()
|
|
12
|
+
config.num_batches = 1000 # object-like assigment of config values
|
|
13
|
+
config.network.depth = 3 # on-the-fly hierarchy generation: here `network` becomes a sub-config
|
|
14
|
+
config.network.width = 100
|
|
15
|
+
...
|
|
16
|
+
|
|
17
|
+
def train(config):
|
|
18
|
+
num_batches = config("num_batches", 10, Int>=2, "Number of batches. Must be at least 2")
|
|
19
|
+
...
|
|
20
|
+
|
|
21
|
+
Key features
|
|
22
|
+
^^^^^^^^^^^^
|
|
23
|
+
|
|
24
|
+
* Detect misspelled parameters by checking that all parameters provided via a `config` by a user have been read.
|
|
25
|
+
|
|
26
|
+
* Provide summary of all parameters used, including summary help for what they were for.
|
|
27
|
+
|
|
28
|
+
* Nicer object attribute synthax than dictionary notation, in particular for nested configurations.
|
|
29
|
+
|
|
30
|
+
* Automatic conversion including simple value validation to ensure user-provided values are within
|
|
31
|
+
a given range or from a list of options.
|
|
20
32
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
error( text, *args, **kwargs, exception=exception )
|
|
26
|
-
def warn( text, *args, warning=warnings.RuntimeWarning, stack_level=1, **kwargs ):
|
|
27
|
-
warnings.warn( txtfmt(text, *args, **kwargs), warning, stack_level=stack_level )
|
|
28
|
-
def warn_if( cond, text, *args, warning=warnings.RuntimeWarning, stack_level=1, **kwargs ):
|
|
29
|
-
if cond:
|
|
30
|
-
warn( text, *args, warning=warning, stack_level=stack_level, **kwargs )
|
|
33
|
+
Creating Configs
|
|
34
|
+
^^^^^^^^^^^^^^^^
|
|
35
|
+
|
|
36
|
+
Set data with both dictionary and member notation::
|
|
31
37
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
Those arguments are accessible via the 'not_done' attribute
|
|
36
|
-
"""
|
|
37
|
-
def __init__(self, not_done, message):
|
|
38
|
-
self.not_done = not_done
|
|
39
|
-
RuntimeError.__init__(self, message)
|
|
38
|
+
config = Config()
|
|
39
|
+
config['features'] = [ 'time', 'spot' ] # examplearray-type assignment
|
|
40
|
+
config.scaling = [ 1., 1000. ] # example object-type assignment
|
|
40
41
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
42
|
+
Reading a Config
|
|
43
|
+
^^^^^^^^^^^^^^^^^
|
|
44
|
+
|
|
45
|
+
When reading the value for a ``key`` from a config, :meth:`cdxcore.config.Config.__call__`
|
|
46
|
+
expects a ``key``, a ``default`` value, a ``cast`` type, and a brief ``help`` text.
|
|
47
|
+
The function first attempts to find ``key`` in the provided `Config`:
|
|
48
|
+
|
|
49
|
+
* If ``key`` is found, it casts the value provided for ``key`` using the ``cast`` type and returns.
|
|
50
|
+
|
|
51
|
+
* If ``key`` is not found, then the default value will be returned (after also being cast using ``cast``).
|
|
52
|
+
|
|
53
|
+
Example::
|
|
54
|
+
|
|
55
|
+
from cdxcore.config import Config
|
|
56
|
+
import numpy as np
|
|
57
|
+
|
|
58
|
+
class Model(object):
|
|
59
|
+
def __init__( self, config ):
|
|
60
|
+
# read top level parameters
|
|
61
|
+
self.features = config("features", [], list, "Features for the agent" )
|
|
62
|
+
self.scaling = config("scaling", [], np.asarray, "Scaling for the features", help_default="no scaling")
|
|
49
63
|
|
|
50
|
-
|
|
51
|
-
def __init__(self, *, config_name : str, key_name : str, message : str):
|
|
52
|
-
header = f"Error in cast definition for key '{key_name}' in config '{config_name}': " if len(config_name) > 0 else f"Error in cast definition for key '{key_name}': "
|
|
53
|
-
RuntimeError.__init__(self, header+message)
|
|
54
|
-
self.config_name = config_name
|
|
55
|
-
self.key_name = key_name
|
|
64
|
+
model = Model( config )
|
|
56
65
|
|
|
57
|
-
|
|
58
|
-
|
|
66
|
+
Most of the example is self-explanatory, but note that
|
|
67
|
+
the :class:'numpy.asarray` provided as ``cast`` parameter for
|
|
68
|
+
``weights`` means that any values passed by the user will be automatically
|
|
69
|
+
converted to :class:`numpy.ndarray` objects.
|
|
59
70
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
71
|
+
The ``help`` text parameter allows providing information on what variables
|
|
72
|
+
are read from the config. The latter can be displayed using the function
|
|
73
|
+
:meth:`cdxcore.config.Config.usage_report`. (There a number of further parameters to
|
|
74
|
+
:meth:`cdxcore.config.Config.__call__` to fine-tune this report such as the ``help_defaults``
|
|
75
|
+
parameter used above).
|
|
65
76
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
77
|
+
In the above example, ``print( config.usage_report() )`` will return::
|
|
78
|
+
|
|
79
|
+
config['features'] = ['time', 'spot'] # Features for the agent; default: []
|
|
80
|
+
config['scaling'] = [ 1. 1000.] # Weigths for the agent; default: no initial weights
|
|
81
|
+
|
|
82
|
+
Sub-Configs
|
|
83
|
+
^^^^^^^^^^^
|
|
84
|
+
|
|
85
|
+
You can write and read sub-configurations directly with member notation, without having
|
|
86
|
+
to explicitly create an entry for the sub-config:
|
|
87
|
+
|
|
88
|
+
Assume as before::
|
|
89
|
+
|
|
90
|
+
config = Config()
|
|
91
|
+
config['features'] = [ 'time', 'spot' ]
|
|
92
|
+
config.scaling = [ 1., 1000. ]
|
|
70
93
|
|
|
71
|
-
|
|
94
|
+
Then create a ``network`` sub configuration with member notation on the fly::
|
|
72
95
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
96
|
+
config.network.depth = 10
|
|
97
|
+
config.network.width = 100
|
|
98
|
+
config.network.activation = 'relu'
|
|
99
|
+
|
|
100
|
+
This is equivalent to::
|
|
101
|
+
|
|
102
|
+
config.network = Config()
|
|
103
|
+
config.network.depth = 10
|
|
104
|
+
config.network.width = 100
|
|
105
|
+
config.network.activation = 'relu'
|
|
106
|
+
|
|
107
|
+
Now use naturally as follows::
|
|
108
|
+
|
|
109
|
+
from cdxcore.config import Config
|
|
110
|
+
import numpy as np
|
|
111
|
+
|
|
112
|
+
class Network(object):
|
|
113
|
+
def __init__( self, config ):
|
|
114
|
+
self.depth = config("depth", 1, Int>0, "Depth of the network")
|
|
115
|
+
self.width = config("width", 1, Int>0, "Width of the network")
|
|
116
|
+
self.activation = config("activation", "selu", str, "Activation function")
|
|
117
|
+
config.done() # see below
|
|
118
|
+
|
|
119
|
+
class Model(object):
|
|
120
|
+
def __init__( self, config ):
|
|
121
|
+
# read top level parameters
|
|
122
|
+
self.features = config("features", [], list, "Features for the agent" )
|
|
123
|
+
self.weights = config("weights", [], np.asarray, "Weigths for the agent", help_default="no initial weights")
|
|
124
|
+
self.networks = Network( config.network )
|
|
125
|
+
config.done() # see below
|
|
126
|
+
|
|
127
|
+
model = Model( config )
|
|
128
|
+
|
|
129
|
+
Imposing Simple Restrictions on Values
|
|
130
|
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
131
|
+
|
|
132
|
+
The ``cast`` parameter to :meth:`cdxcore.config.Config.__call__` is a callable; this allows imposing
|
|
133
|
+
simple restrictions to any values read from a config.
|
|
134
|
+
To this end, import the respective type operators::
|
|
135
|
+
|
|
136
|
+
from cdxcore.config import Int, Float
|
|
137
|
+
|
|
138
|
+
Implement a one-sided restriction::
|
|
139
|
+
|
|
140
|
+
# example enforcing simple conditions
|
|
141
|
+
self.width = network('width', 100, Int>3, "Width for the network")
|
|
142
|
+
|
|
143
|
+
Restrictions on both sides of a scalar::
|
|
144
|
+
|
|
145
|
+
# example encorcing two-sided conditions
|
|
146
|
+
self.percentage = network('percentage', 0.5, ( Float >= 0. ) & ( Float <= 1.), "A percentage")
|
|
147
|
+
|
|
148
|
+
Enforce the value being a member of a list::
|
|
149
|
+
|
|
150
|
+
# example ensuring a returned type is from a list
|
|
151
|
+
self.ntype = network('ntype', 'fastforward', ['fastforward','recurrent','lstm'], "Type of network")
|
|
152
|
+
|
|
153
|
+
We can allow a returned value to be one of several casting types by using tuples.
|
|
154
|
+
The most common use case is that ``None`` is a valid value, too.
|
|
155
|
+
For example, assume that the ``name`` of the network model should be a string or ``None``.
|
|
156
|
+
This is implemented as::
|
|
77
157
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
158
|
+
# example allowing either None or a string
|
|
159
|
+
self.keras_name = network('name', None, (None, str), "Keras name of the network model")
|
|
160
|
+
|
|
161
|
+
We can combine conditional expressions with the tuple notation::
|
|
162
|
+
|
|
163
|
+
# example allowing either None or a positive int
|
|
164
|
+
self.batch_size = network('batch_size', None, (None, Int>0), "Batch size or None for TensorFlow's default 32", help_cast="Positive integer, or None")
|
|
165
|
+
|
|
166
|
+
Ensuring that we had no Typos & that all provided Data is meaningful
|
|
167
|
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
168
|
+
|
|
169
|
+
A common issue when using dictionary-based code configuration is that we might misspell one of the parameters.
|
|
170
|
+
Unless this is a mandatory parameter we might not notice that we have not actually
|
|
171
|
+
changed its value.
|
|
172
|
+
|
|
173
|
+
To check that all values of a `config` were read use :meth:`cdxcore.config.Config.done`.
|
|
174
|
+
It will alert you if there are keywords or children which have not been read.
|
|
175
|
+
Most likely, those will be typos. Consider the following example where ``width`` is misspelled in our config::
|
|
176
|
+
|
|
177
|
+
class Network(object):
|
|
178
|
+
def __init__( self, config ):
|
|
179
|
+
# read top level parameters
|
|
180
|
+
self.depth = config("depth", 1, Int>=1, "Depth of the network")
|
|
181
|
+
self.width = config("width", 3, Int>=1, "Width of the network")
|
|
182
|
+
self.activaton = config("activation", "relu", help="Activation function", help_cast="String with the function name, or function")
|
|
183
|
+
config.done() # <-- test that all members of config where read
|
|
184
|
+
|
|
185
|
+
config = Config()
|
|
186
|
+
config.features = ['time', 'spot']
|
|
187
|
+
config.network.depth = 10
|
|
188
|
+
config.network.activation = 'relu'
|
|
189
|
+
config.network.widht = 100 # (intentional typo)
|
|
190
|
+
|
|
191
|
+
n = Network(config.network)
|
|
192
|
+
|
|
193
|
+
Since ``width`` was misspelled in setting up the config,
|
|
194
|
+
a :class:`cdxcore.config.NotDoneError` exception is raised::
|
|
195
|
+
|
|
196
|
+
NotDoneError: Error closing Config 'config.network': the following config arguments were not read: widht
|
|
197
|
+
|
|
198
|
+
Summary of all variables read from this object:
|
|
199
|
+
config.network['activation'] = relu # Activation function; default: relu
|
|
200
|
+
config.network['depth'] = 10 # Depth of the network; default: 1
|
|
201
|
+
config.network['width'] = 3 # Width of the network; default: 3
|
|
81
202
|
|
|
82
|
-
|
|
203
|
+
Note that you can also call :meth:`cdxcore.config.Config.done` at top level::
|
|
83
204
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
205
|
+
class Network(object):
|
|
206
|
+
def __init__( self, config ):
|
|
207
|
+
# read top level parameters
|
|
208
|
+
self.depth = config("depth", 1, Int>=1, "Depth of the network")
|
|
209
|
+
self.width = config("width", 3, Int>=1, "Width of the network")
|
|
210
|
+
self.activaton = config("activation", "relu", help="Activation function", help_cast="String with the function name, or function")
|
|
87
211
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
212
|
+
config = Config()
|
|
213
|
+
config.features = ['time', 'spot']
|
|
214
|
+
config.network.depth = 10
|
|
215
|
+
config.network.activation = 'relu'
|
|
216
|
+
config.network.widht = 100 # (intentional typo)
|
|
91
217
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
218
|
+
n = Network(config.network)
|
|
219
|
+
test_features = config("features", [], list, "Features for my network")
|
|
220
|
+
config.done()
|
|
95
221
|
|
|
96
|
-
|
|
97
|
-
features = config("features", [], list ) # reads features and returns a list
|
|
98
|
-
weights = config("weights", [], np.ndarray ) # reads features and returns a nump array
|
|
222
|
+
produces::
|
|
99
223
|
|
|
100
|
-
|
|
101
|
-
samples = network('samples', 10000) # networks samples
|
|
102
|
-
config.done() # no error as 'network' was detached
|
|
103
|
-
network.done() # error as network.activation was not read
|
|
224
|
+
NotDoneError: Error closing Config 'config.network': the following config arguments were not read: widht
|
|
104
225
|
|
|
105
|
-
|
|
106
|
-
|
|
226
|
+
Summary of all variables read from this object:
|
|
227
|
+
config.network['activation'] = relu # Activation function; default: relu
|
|
228
|
+
config.network['depth'] = 10 # Depth of the network; default: 1
|
|
229
|
+
config.network['width'] = 3 # Width of the network; default: 3
|
|
230
|
+
#
|
|
231
|
+
config['features'] = ['time', 'spot'] # Features for my network; default: []
|
|
107
232
|
|
|
108
|
-
|
|
233
|
+
You can check the status of the use of the config by using the :attr:`cdxcore.config.Config.not_done` property.
|
|
109
234
|
|
|
110
|
-
|
|
111
|
-
|
|
235
|
+
Detaching Child Configs
|
|
236
|
+
^^^^^^^^^^^^^^^^^^^^^^^
|
|
112
237
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
238
|
+
You can also detach a child config,
|
|
239
|
+
which allows you to store it for later use without triggering :meth:`cdxcore.config.Config.done` errors::
|
|
240
|
+
|
|
241
|
+
def read_config( self, confg ):
|
|
242
|
+
...
|
|
243
|
+
self.config_training = config.training.detach()
|
|
116
244
|
config.done()
|
|
117
245
|
|
|
118
|
-
|
|
119
|
-
|
|
246
|
+
The function :meth:`cdxcore.config.Config.detach` will mark he original child but not the detached
|
|
247
|
+
child itself as 'done'.
|
|
248
|
+
Therefore, we will need to call :meth:`cdxcore.config.Config.done` for the detached child
|
|
249
|
+
when we finished processing it::
|
|
120
250
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
251
|
+
def training(self):
|
|
252
|
+
epochs = self.config_training("epochs", 100, int, "Epochs for training")
|
|
253
|
+
batch_size = self.config_training("batch_size", None, help="Batch size. Use None for default of 32" )
|
|
254
|
+
|
|
255
|
+
self.config_training.done()
|
|
256
|
+
|
|
257
|
+
Various Copy Operations
|
|
258
|
+
^^^^^^^^^^^^^^^^^^^^^^^
|
|
259
|
+
|
|
260
|
+
When making a copy of a `config` we will need to decide about the semantics of the operation.
|
|
261
|
+
A :class:`cdxcore.config.Config` object contains
|
|
262
|
+
|
|
263
|
+
* **Inputs**: the user's input hierarchy. This is accessible via :attr:`cdxcore.config.Config.children` and
|
|
264
|
+
:meth:`cdxcore.config.Config.keys`.
|
|
265
|
+
|
|
266
|
+
All copy operations share (and do not modify) the user's input.
|
|
267
|
+
See also :meth:`cdxcore.config.Config.input_report`.
|
|
268
|
+
|
|
269
|
+
* **Done Status**: to check whether all parameters provided by the users are read by some code `config` keeps
|
|
270
|
+
track of which parameters were read with :meth:`cdxcore.config.Config.__call__`. This list is
|
|
271
|
+
checked against when :meth:`cdxcore.config.Config.done` is called.
|
|
272
|
+
|
|
273
|
+
This list of elements
|
|
274
|
+
not yet read can be obtained using :meth:`cdxcore.config.Config.input_dict`.
|
|
275
|
+
|
|
276
|
+
* **Consistency**: a :class:`cdxcore.config.Config` object makes sure that if a parameter is requested
|
|
277
|
+
twice with :meth:`cdxcore.config.Config.__call__` then the respective ``default`` and ``help`` values
|
|
278
|
+
are consistency between function calls. This avoids typically divergence of code where one
|
|
279
|
+
part of code assumes a different default value than another.
|
|
280
|
+
|
|
281
|
+
Recorded consistency information are accessible via
|
|
282
|
+
:attr:`cdxcore.config.Config.recorder`.
|
|
283
|
+
|
|
284
|
+
Note that you can read a parameter "quietly" without recording any usage by using the ``[]`` operator.
|
|
285
|
+
|
|
286
|
+
Accordingly, when making a copy of ``self`` we need to determine the relationship of the copy with
|
|
287
|
+
above.
|
|
288
|
+
|
|
289
|
+
* :meth:`cdxcore.config.Config.detach`: use case is deferring usage of a config to a later point.
|
|
290
|
+
|
|
291
|
+
* *Done status*: ``self`` is marked as "done"; the copy is used keep track of usage of the remaining parameters.
|
|
292
|
+
|
|
293
|
+
* *Consistency*: both ``self`` and the copy share the same consistency recorder.
|
|
294
|
+
|
|
295
|
+
* :meth:`cdxcore.config.Config.copy`: make an indepedent copy of the current status of ``self``.
|
|
296
|
+
|
|
297
|
+
* *Done status*: the copy has an inpendent copy of the "done" status of ``self``.
|
|
298
|
+
|
|
299
|
+
* *Consistency*: the copy has an inpendent copy of the consistency recorder of ``self``.
|
|
300
|
+
|
|
301
|
+
* :meth:`cdxcore.config.Config.clean_copy`: make an indepedent copy of ``self``, and
|
|
302
|
+
reset all usage information.
|
|
303
|
+
|
|
304
|
+
* *Done status*: the copy has an empty "done" status.
|
|
305
|
+
|
|
306
|
+
* *Consistency*: the copy has an empty consistency recorder.
|
|
307
|
+
|
|
308
|
+
* :meth:`cdxcore.config.Config.shallow_copy`: make a shallow copy which shares all
|
|
309
|
+
future usage tracking with ``self``.
|
|
310
|
+
|
|
311
|
+
The copy acts as a view on ``self``. This is the semantic of the copy constructor.
|
|
312
|
+
|
|
313
|
+
* *Done status*: the copy and ``self`` share all "done" status; if a parameter is read with one, it is considered
|
|
314
|
+
"done" by both.
|
|
315
|
+
|
|
316
|
+
* *Consistency*: the copy and ``self`` share all consistency handling. If a parameter is read with one with a given
|
|
317
|
+
``default`` and ``help``, the other must use the same values when accessing the same parameter.
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
Self-Recording All Available Configuration Parameters
|
|
321
|
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
322
|
+
|
|
323
|
+
Once your program ran, you can read the summary of all values read, their defaults, and their help texts::
|
|
324
|
+
|
|
325
|
+
print( config.usage_report( with_cast=True ) )
|
|
124
326
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
327
|
+
Prints::
|
|
328
|
+
|
|
329
|
+
config.network['activation'] = relu # (str) Activation function for the network; default: relu
|
|
330
|
+
config.network['depth'] = 10 # (int) Depth for the network; default: 10000
|
|
331
|
+
config.network['width'] = 100 # (int>3) Width for the network; default: 100
|
|
332
|
+
config.network['percentage'] = 0.5 # (float>=0. and float<=1.) Width for the network; default: 0.5
|
|
333
|
+
config.network['ntype'] = 'fastforward' # (['fastforward','recurrent','lstm']) Type of network; default 'fastforward'
|
|
334
|
+
config.training['batch_size'] = None # () Batch size. Use None for default of 32; default: None
|
|
335
|
+
config.training['epochs'] = 100 # (int) Epochs for training; default: 100
|
|
336
|
+
config['features'] = ['time', 'spot'] # (list) Features for the agent; default: []
|
|
337
|
+
config['weights'] = [1 2 3] # (asarray) Weigths for the agent; default: no initial weights
|
|
338
|
+
|
|
339
|
+
Unique Hash
|
|
340
|
+
^^^^^^^^^^^
|
|
341
|
+
|
|
342
|
+
Another common use case is that we wish to cache the result of some complex operation.
|
|
343
|
+
Assuming that the `config` describes all relevant parameters, and is therefore a valid `ID` for
|
|
344
|
+
the data we wish to cache, we can use :meth:`cdxcore.config.Config.unique_hash`
|
|
345
|
+
to obtain a unique hash ID for the given config.
|
|
346
|
+
|
|
347
|
+
:class:`cdxcore.config.Config` also implements
|
|
348
|
+
the custom hashing protocol ``__unique_hash__`` defined by :class:`cdxcore.uniquehash.UniqueHash`,
|
|
349
|
+
which means that if a ``Config`` is used during a hashing function from :mod:`cdxcore.uniquehash`
|
|
350
|
+
the config will be hashed correctly.
|
|
351
|
+
|
|
352
|
+
A fully transparent caching framework which supports code versioning and transparent
|
|
353
|
+
hashing of function parameters is implemented with :meth:`cdxcore.subdir.SubDir.cache`.
|
|
354
|
+
|
|
355
|
+
Consistent ** kwargs Handling
|
|
356
|
+
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
357
|
+
|
|
358
|
+
The `Config` class can be used to improve ``** kwargs`` handling.
|
|
359
|
+
Assume we have::
|
|
360
|
+
|
|
361
|
+
def f(** kwargs):
|
|
362
|
+
a = kwargs.get("difficult_name", 10)
|
|
363
|
+
b = kwargs.get("b", 20)
|
|
364
|
+
|
|
365
|
+
We run the usual risk of a user mispronouncing a parameter name which we would never know.
|
|
366
|
+
Therefore we may improve upon the above with::
|
|
367
|
+
|
|
368
|
+
def f(**kwargs):
|
|
369
|
+
kwargs = Config(kwargs)
|
|
370
|
+
a = kwargs("difficult_name", 10)
|
|
371
|
+
b = kwargs("b", 20)
|
|
372
|
+
kwargs.done()
|
|
373
|
+
|
|
374
|
+
If now a user calls ``f`` with, say, ``config(difficlt_name=5)`` an error will be raised.
|
|
375
|
+
|
|
376
|
+
A more advanced pattern is to allow both ``config`` and ``kwargs`` function parameters. In this case, the user
|
|
377
|
+
can both provide a ``config`` or specify its parameters directory::
|
|
378
|
+
|
|
379
|
+
def f( config=None, **kwargs):
|
|
380
|
+
config = Config.config_kwargs(config,kwargs)
|
|
381
|
+
a = config("difficult_name", 10, int)
|
|
382
|
+
b = config("b", 20, int)
|
|
383
|
+
config.done()
|
|
384
|
+
|
|
385
|
+
Any of the following function calls are now valid::
|
|
386
|
+
|
|
387
|
+
f( Config(difficult_name=11, b=21) ) # use a Config
|
|
388
|
+
f( difficult_name=12, b=22 ) # use a kwargs
|
|
389
|
+
f( Config(difficult_name=11, b=21), b=22 ) # use both; kwargs overwrite config values
|
|
132
390
|
|
|
133
|
-
|
|
391
|
+
Dataclasses
|
|
392
|
+
^^^^^^^^^^^
|
|
134
393
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
394
|
+
:mod:`dataclasses` rely on default values of any member being "frozen" objects, which most user-defined objects and
|
|
395
|
+
:class:`cdxcore.config.Config` objects are not.
|
|
396
|
+
This limitation applies as well to `flax <https://flax-linen.readthedocs.io/en/latest/api_reference/flax.linen/module.html>`__ modules.
|
|
397
|
+
To use non-frozen default values, use the
|
|
398
|
+
:meth:`cdxcore.config.Config.as_field` function::
|
|
399
|
+
|
|
400
|
+
from cdxcore.config import Config
|
|
401
|
+
from dataclasses import dataclass
|
|
402
|
+
|
|
403
|
+
@dataclass
|
|
404
|
+
class Data:
|
|
405
|
+
data : Config = Config().as_field()
|
|
406
|
+
|
|
407
|
+
def f(self):
|
|
408
|
+
return self.data("x", 1, Int>0, "A positive integer")
|
|
409
|
+
|
|
410
|
+
d = Data() # default constructor used.
|
|
411
|
+
d.f()
|
|
412
|
+
|
|
413
|
+
Import
|
|
414
|
+
------
|
|
415
|
+
.. code-block:: python
|
|
416
|
+
|
|
417
|
+
from cdxcore.config import Config
|
|
418
|
+
|
|
419
|
+
Documentation
|
|
420
|
+
-------------
|
|
421
|
+
"""
|
|
422
|
+
|
|
423
|
+
from collections import OrderedDict
|
|
424
|
+
from collections.abc import Mapping, Callable
|
|
425
|
+
from sortedcontainers import SortedDict
|
|
426
|
+
import dataclasses as dataclasses
|
|
427
|
+
from dataclasses import Field
|
|
428
|
+
from .err import verify, warn_if
|
|
429
|
+
from .uniquehash import UniqueHash, DebugTrace
|
|
430
|
+
from .pretty import PrettyObject as pdct
|
|
431
|
+
from .util import fmt_list
|
|
432
|
+
|
|
433
|
+
class _ID(object):
|
|
434
|
+
pass
|
|
435
|
+
|
|
436
|
+
#: Value indicating no default is available for a given parameter.
|
|
437
|
+
no_default = _ID()
|
|
438
|
+
|
|
439
|
+
# ==============================================================================
|
|
440
|
+
#
|
|
441
|
+
# Actual Config class
|
|
442
|
+
#
|
|
443
|
+
# ==============================================================================
|
|
444
|
+
|
|
445
|
+
class Config(OrderedDict):
|
|
446
|
+
"""
|
|
447
|
+
A simple `Config` class for hierarchical dictionary-like configurations but with type checking, detecting
|
|
448
|
+
missspelled parameters, and simple built-in help.
|
|
449
|
+
|
|
450
|
+
See :mod:`cdxcore.config` for an extensive discussion of features.
|
|
451
|
+
|
|
452
|
+
Parameters
|
|
453
|
+
----------
|
|
454
|
+
*args : list
|
|
455
|
+
List of ``Mapping`` to iteratively create a new config with.
|
|
456
|
+
|
|
457
|
+
If the first element is a ``Config``, and no other parameters are passed,
|
|
458
|
+
then this object will be a shallow copy of that ``Config``.
|
|
459
|
+
It then shares all usage recording. See :meth:`cdxcore.config.Config.shallow_copy`.
|
|
460
|
+
|
|
461
|
+
config_name : str, optional
|
|
462
|
+
Name of the configuration for report_usage. Default is ``"config"``.
|
|
463
|
+
|
|
464
|
+
** kwargs : dict
|
|
465
|
+
Additional key/value pairs to initialize the config with, e.g.``Config(a=1, b=2)``.
|
|
466
|
+
|
|
467
|
+
"""
|
|
468
|
+
|
|
469
|
+
def __init__(self, *args, config_name : str = None, **kwargs):
|
|
470
|
+
"""
|
|
471
|
+
Create a :class:`cdxcore.config.Config`.
|
|
147
472
|
"""
|
|
148
473
|
if len(args) == 1 and isinstance(args[0], Config) and config_name is None and len(kwargs) == 0:
|
|
149
474
|
source = args[0]
|
|
@@ -170,80 +495,118 @@ class Config(OrderedDict):
|
|
|
170
495
|
|
|
171
496
|
@property
|
|
172
497
|
def config_name(self) -> str:
|
|
173
|
-
"""
|
|
498
|
+
""" Qualified name of this config. """
|
|
174
499
|
return self._name
|
|
175
500
|
|
|
176
501
|
@property
|
|
177
502
|
def children(self) -> OrderedDict:
|
|
178
|
-
"""
|
|
503
|
+
""" Dictionary of the child configs of ``self``. """
|
|
179
504
|
return self._children
|
|
180
505
|
|
|
181
506
|
def __str__(self) -> str:
|
|
182
|
-
""" Print myself as dictionary """
|
|
507
|
+
""" Print myself as dictionary. """
|
|
183
508
|
s = self.config_name + str(self.as_dict(mark_done=False))
|
|
184
509
|
return s
|
|
185
510
|
|
|
186
511
|
def __repr__(self) -> str:
|
|
187
|
-
""" Print myself as reconstructable object """
|
|
512
|
+
""" Print myself as reconstructable object. """
|
|
188
513
|
s = repr(self.as_dict(mark_done=False))
|
|
189
514
|
s = "Config( **" + s + ", config_name='" + self.config_name + "' )"
|
|
190
515
|
return s
|
|
191
516
|
|
|
192
517
|
@property
|
|
193
518
|
def is_empty(self) -> bool:
|
|
194
|
-
"""
|
|
195
|
-
|
|
196
|
-
|
|
519
|
+
""" Whether any parameters have been set, at parent level or at any child level. """
|
|
520
|
+
if len(self) > 0:
|
|
521
|
+
return False
|
|
522
|
+
for c in self._children.values():
|
|
523
|
+
if not c.is_empty:
|
|
524
|
+
return False
|
|
525
|
+
return True
|
|
526
|
+
|
|
197
527
|
# conversion
|
|
198
528
|
# ----------
|
|
199
529
|
|
|
200
530
|
def as_dict(self, mark_done : bool = True ) -> dict:
|
|
201
531
|
"""
|
|
202
|
-
Convert into dictionary of dictionaries
|
|
532
|
+
Convert ``self`` into a dictionary of dictionaries.
|
|
203
533
|
|
|
204
534
|
Parameters
|
|
205
535
|
----------
|
|
206
536
|
mark_done : bool
|
|
207
|
-
If True, then all members of this config will be considered
|
|
537
|
+
If True, then all members of this config will be considered "done"
|
|
538
|
+
upon return of this function.
|
|
208
539
|
|
|
209
540
|
Returns
|
|
210
541
|
-------
|
|
211
|
-
Dict
|
|
542
|
+
Dict : dict
|
|
543
|
+
Dictionary of dictionaries.
|
|
212
544
|
"""
|
|
213
545
|
d = { key : self.get(key) if mark_done else self.get_raw(key) for key in self }
|
|
214
546
|
for n, c in self._children.items():
|
|
215
547
|
if n == '_ipython_canary_method_should_not_exist_':
|
|
216
548
|
continue
|
|
217
549
|
c = c.as_dict(mark_done)
|
|
218
|
-
verify( not n in d, "Cannot convert
|
|
550
|
+
verify( not n in d, "Cannot convert Config to dictionary: found both a regular value, and a child with name '{n}'", n)
|
|
219
551
|
d[n] = c
|
|
220
552
|
return d
|
|
221
553
|
|
|
222
554
|
def as_field(self) -> Field:
|
|
223
555
|
"""
|
|
224
|
-
|
|
225
|
-
|
|
556
|
+
This function provides support for :class:`dataclasses.dataclass` fields
|
|
557
|
+
with ``Config`` default values.
|
|
558
|
+
|
|
559
|
+
When adding a field with a non-frozen default value to a ``@dataclass`` class,
|
|
560
|
+
a ``default_factory`` has to be provided.
|
|
561
|
+
The function ``as_field`` returns the corresponding :class:`dataclasses.Field`
|
|
562
|
+
element by returning simply::
|
|
563
|
+
|
|
564
|
+
def factory():
|
|
565
|
+
return self
|
|
566
|
+
return dataclasses.field( default_factory=factory )
|
|
567
|
+
|
|
568
|
+
Usage is as follows::
|
|
569
|
+
|
|
570
|
+
from dataclasses import dataclass
|
|
571
|
+
@dataclass
|
|
572
|
+
class A:
|
|
573
|
+
data : Config = Config(x=2).as_field()
|
|
574
|
+
|
|
575
|
+
a = A()
|
|
576
|
+
print(a.data['x']) # -> "2"
|
|
577
|
+
a = A(data=Config(x=3))
|
|
578
|
+
print(a.data['x']) # -> "3"
|
|
226
579
|
"""
|
|
227
|
-
|
|
580
|
+
def factory():
|
|
581
|
+
return self
|
|
582
|
+
return dataclasses.field( default_factory=factory )
|
|
228
583
|
|
|
229
584
|
# handle finishing config use
|
|
230
585
|
# ---------------------------
|
|
231
586
|
|
|
232
587
|
def done(self, include_children : bool = True, mark_done : bool = True ):
|
|
233
588
|
"""
|
|
234
|
-
Closes the config and checks that no unread parameters remain.
|
|
235
|
-
|
|
236
|
-
|
|
589
|
+
Closes the config and checks that no unread parameters remain. This is used
|
|
590
|
+
to detect typos in configuration files.
|
|
591
|
+
|
|
592
|
+
Raises a
|
|
593
|
+
:class:`cdxcore.config.NotDoneError` if there are unused parameters in ``self``.
|
|
237
594
|
|
|
238
|
-
|
|
595
|
+
Consider this example::
|
|
596
|
+
|
|
239
597
|
config = Config()
|
|
240
598
|
config.a = 1
|
|
241
599
|
config.child.b = 2
|
|
242
600
|
|
|
243
601
|
_ = config.a # read a
|
|
244
|
-
|
|
602
|
+
child = config.child
|
|
603
|
+
config.done() # error because config.child.b has not been read yet
|
|
604
|
+
|
|
605
|
+
print( child.b )
|
|
606
|
+
|
|
607
|
+
This example raises an error because ``config.child.b`` was not read. If you wish to process
|
|
608
|
+
the sub-config ``config.child`` later, use :meth:`cdxcore.config.Config.detach`::
|
|
245
609
|
|
|
246
|
-
Instead use:
|
|
247
610
|
config = Config()
|
|
248
611
|
config.a = 1
|
|
249
612
|
config.child.b = 2
|
|
@@ -252,27 +615,40 @@ class Config(OrderedDict):
|
|
|
252
615
|
child = config.child.detach()
|
|
253
616
|
config.done() # no error, even though confg.child.b has not been read yet
|
|
254
617
|
|
|
255
|
-
|
|
618
|
+
print( child.b )
|
|
619
|
+
child.done() # need to call done() for the child
|
|
620
|
+
|
|
621
|
+
By default this function also validates that all child configs were "done".
|
|
622
|
+
|
|
623
|
+
**See Also**
|
|
624
|
+
|
|
625
|
+
* :meth:`cdxcore.config.Config.mark_done` marks all parameters as "done" (used).
|
|
626
|
+
|
|
627
|
+
* :meth:`cdxcore.config.Config.reset_done` marks all parameters as "not done".
|
|
628
|
+
|
|
629
|
+
* :meth:`cdxcore.config.Config.clean_copy` makes a copy of ``self`` without any usage information.
|
|
630
|
+
|
|
631
|
+
* Introduction to the various copy operations in :mod:`cdxcore.config`.
|
|
256
632
|
|
|
257
633
|
Parameters
|
|
258
634
|
----------
|
|
259
|
-
include_children:
|
|
260
|
-
Validate child configs, too.
|
|
635
|
+
include_children: bool
|
|
636
|
+
Validate child configs, too. Stronly recommended default.
|
|
261
637
|
mark_done:
|
|
262
638
|
Upon completion mark this config as 'done'.
|
|
263
|
-
This stops it being modified; subsequent calls to done() will be successful.
|
|
639
|
+
This stops it being modified; that also means subsequent calls to done() will be successful.
|
|
264
640
|
|
|
265
641
|
Raises
|
|
266
642
|
------
|
|
267
|
-
NotDoneError
|
|
643
|
+
:class:`cdxcore.config.NotDoneError`
|
|
644
|
+
If not all elements were read.
|
|
268
645
|
"""
|
|
269
646
|
inputs = set(self)
|
|
270
647
|
rest = inputs - self._done
|
|
271
648
|
if len(rest) > 0:
|
|
272
|
-
raise NotDoneError( rest,
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
self.usage_report(filter_path=self._name ) ))
|
|
649
|
+
raise NotDoneError( rest, self.config_name,
|
|
650
|
+
f"Error closing Config '{self._name}': the following config arguments were not read: {fmt_list(rest)}\n\n"\
|
|
651
|
+
f"Summary of all variables read from this object:\n{self.usage_report(filter_path=self.config_name)}" )
|
|
276
652
|
if include_children:
|
|
277
653
|
for _, c in self._children.items():
|
|
278
654
|
c.done(include_children=include_children,mark_done=False)
|
|
@@ -282,24 +658,33 @@ class Config(OrderedDict):
|
|
|
282
658
|
|
|
283
659
|
def reset(self):
|
|
284
660
|
"""
|
|
285
|
-
Reset all usage information
|
|
661
|
+
Reset all usage information.
|
|
286
662
|
|
|
287
|
-
Use reset_done
|
|
288
|
-
|
|
663
|
+
Use :meth:`cdxcore.config.Config.reset_done` to only reset the information whether a key was used,
|
|
664
|
+
but to keep consistency information on previously used default and/or help values.
|
|
289
665
|
"""
|
|
290
666
|
self._done.clear()
|
|
291
667
|
self._recorder.clear()
|
|
292
668
|
|
|
293
669
|
def reset_done(self):
|
|
294
670
|
"""
|
|
295
|
-
Reset the internal list of which are
|
|
296
|
-
|
|
297
|
-
|
|
671
|
+
Reset the internal list of which are "done" (used).
|
|
672
|
+
|
|
673
|
+
Typically "done" means that a parameter
|
|
674
|
+
has been read using :meth:`cdxcore.config.Config.call`.
|
|
675
|
+
|
|
676
|
+
This function does not reset the consistency recording of previous uses of each key.
|
|
677
|
+
This ensures consistency of default values between uses of keys.
|
|
678
|
+
Use :meth:`cdxcore.config.Config.reset` to reset all "done" and reset all usage records.
|
|
679
|
+
|
|
680
|
+
See also the summary on various copy operations in :mod:`cdxcore.config`.
|
|
298
681
|
"""
|
|
299
682
|
self._done.clear()
|
|
300
683
|
|
|
301
684
|
def mark_done(self, include_children : bool = True ):
|
|
302
|
-
"""
|
|
685
|
+
"""
|
|
686
|
+
Mark all members as "done" (having been used).
|
|
687
|
+
"""
|
|
303
688
|
self._done.update( self )
|
|
304
689
|
if include_children:
|
|
305
690
|
for _, c in self._children.items():
|
|
@@ -311,6 +696,7 @@ class Config(OrderedDict):
|
|
|
311
696
|
def _detach( self, *, mark_self_done : bool, copy_done : bool, new_recorder ):
|
|
312
697
|
"""
|
|
313
698
|
Creates a copy of the current config, with a number of options how to share usage information.
|
|
699
|
+
|
|
314
700
|
Use the functions
|
|
315
701
|
detach()
|
|
316
702
|
copy()
|
|
@@ -356,81 +742,110 @@ class Config(OrderedDict):
|
|
|
356
742
|
|
|
357
743
|
def detach( self ):
|
|
358
744
|
"""
|
|
359
|
-
Returns a copy of
|
|
745
|
+
Returns a copy of ``self``, and sets ``self`` to "done".
|
|
746
|
+
|
|
747
|
+
The purpose of this function is to defer using a config (often a sub-config) to a later point,
|
|
748
|
+
while maintaining consistency of usage.
|
|
360
749
|
|
|
361
|
-
|
|
362
|
-
- The copy shares the recorded to keep track of consistency of usage
|
|
363
|
-
- The function flags 'self' as done
|
|
750
|
+
* The copy has the same "done" status as ``self`` at the time of calling ``detach()``.
|
|
364
751
|
|
|
365
|
-
|
|
752
|
+
* The copy shares usage consistency checks with ``self``, i.e. if the same parameter is
|
|
753
|
+
read with different ``default`` or ``help`` values an error is raised.
|
|
366
754
|
|
|
367
|
-
|
|
755
|
+
* The function flags ``self`` as "done" using :meth:`cdxcore.config.Config.mark_done`.
|
|
368
756
|
|
|
369
|
-
|
|
757
|
+
For example::
|
|
370
758
|
|
|
759
|
+
class Example(object):
|
|
760
|
+
|
|
761
|
+
def __init__( config ):
|
|
371
762
|
self.a = config('a', 1, Int>=0, "'a' value")
|
|
372
763
|
self.later = config.later.detach() # detach sub-config
|
|
373
764
|
self._cache = None
|
|
374
765
|
config.done()
|
|
375
|
-
|
|
766
|
+
|
|
376
767
|
def function(self):
|
|
377
768
|
if self._cache is None:
|
|
378
769
|
self._cache = Cache(self.later) # deferred use of the self.later config. Cache() calls done() on self.later
|
|
379
770
|
return self._cache
|
|
771
|
+
|
|
772
|
+
See also the summary on various copy operations in :mod:`cdxcore.config`.
|
|
380
773
|
|
|
381
|
-
|
|
774
|
+
Returns
|
|
775
|
+
-------
|
|
776
|
+
copy : Config
|
|
777
|
+
A copy of ``self``.
|
|
382
778
|
"""
|
|
383
779
|
return self._detach(mark_self_done=True, copy_done=True, new_recorder="share")
|
|
384
780
|
|
|
385
781
|
def copy( self ):
|
|
386
782
|
"""
|
|
387
|
-
Return a
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
783
|
+
Return a fully independent copy of ``self``.
|
|
784
|
+
|
|
785
|
+
* The copy has an independent "done" status of ``self``.
|
|
786
|
+
|
|
787
|
+
* The copy has an independent usage consistency status.
|
|
788
|
+
|
|
789
|
+
* ``self`` will remain untouched. In particular, in contrast to :meth:`cdxcore.config.Config.detach`
|
|
790
|
+
it will not be set to "done".
|
|
391
791
|
|
|
392
|
-
As an example,
|
|
393
|
-
config members of the same name
|
|
792
|
+
As an example, the following allows using different default values for
|
|
793
|
+
config members of the same name::
|
|
394
794
|
|
|
395
795
|
base = Config()
|
|
396
|
-
base
|
|
397
|
-
_ = base('a', 1) # use a
|
|
796
|
+
_ = base('a', 1) # read a with default 1
|
|
398
797
|
|
|
399
798
|
copy = base.copy() # copy will know 'a' as used with default 1
|
|
799
|
+
# 'b' was not used yet
|
|
400
800
|
|
|
401
|
-
_ = base(
|
|
402
|
-
_ = copy(
|
|
801
|
+
_ = base('b', 111) # read 'b' with default 111
|
|
802
|
+
_ = copy('b', 222) # read 'b' with default 222 -> ok
|
|
803
|
+
|
|
804
|
+
_ = copy('a', 2) # use 'a' with default 2 -> will fail
|
|
805
|
+
|
|
806
|
+
Use :meth:`cdxcore.config.Config.clean_copy` for making a copy which discards any prior
|
|
807
|
+
usage information.
|
|
403
808
|
|
|
404
|
-
|
|
809
|
+
See also the summary on various copy operations in :mod:`cdxcore.config`.
|
|
405
810
|
"""
|
|
406
811
|
return self._detach( mark_self_done=False, copy_done=True, new_recorder="copy" )
|
|
407
812
|
|
|
408
813
|
def clean_copy( self ):
|
|
409
814
|
"""
|
|
410
|
-
|
|
815
|
+
Make a copy of ``self``, and reset it to the original input state from the user.
|
|
411
816
|
|
|
412
|
-
As an example,
|
|
413
|
-
config members of the same name
|
|
817
|
+
As an example, the following allows using different default values for
|
|
818
|
+
config members of the same name::
|
|
414
819
|
|
|
415
820
|
base = Config()
|
|
416
|
-
base
|
|
417
|
-
_ = base('a', 1) # use a
|
|
821
|
+
_ = base('a', 1) # read a with default 1
|
|
418
822
|
|
|
419
823
|
copy = base.copy() # copy will know 'a' as used with default 1
|
|
824
|
+
# 'b' was not used yet
|
|
825
|
+
|
|
826
|
+
_ = base('b', 111) # read 'b' with default 111
|
|
827
|
+
_ = copy('b', 222) # read 'b' with default 222 -> ok
|
|
828
|
+
|
|
829
|
+
_ = copy('a', 2) # use 'a' with default 2 -> ok
|
|
420
830
|
|
|
421
|
-
|
|
422
|
-
|
|
831
|
+
Use :meth:`cdxcore.config.Config.copy` for a making a copy which
|
|
832
|
+
tracks prior usage information.
|
|
833
|
+
|
|
834
|
+
See also the summary on various copy operations in :mod:`cdxcore.config`.
|
|
423
835
|
|
|
424
|
-
_ = copy('a', 2) # will not fail, as no usage is shared
|
|
425
836
|
"""
|
|
426
837
|
return self._detach( mark_self_done=False, copy_done=False, new_recorder="clean" )
|
|
427
838
|
|
|
428
|
-
def
|
|
839
|
+
def shallow_copy( self ):
|
|
429
840
|
"""
|
|
430
|
-
Return a copy of
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
841
|
+
Return a shallow copy of ``self`` which shares all usage tracking with ``self``
|
|
842
|
+
going forward.
|
|
843
|
+
|
|
844
|
+
* The copy shares the "done" status of ``self``.
|
|
845
|
+
|
|
846
|
+
* The copy shares all consistency usage status of ``self``.
|
|
847
|
+
|
|
848
|
+
* ``self`` will not be flagged as 'done'
|
|
434
849
|
"""
|
|
435
850
|
return Config(self)
|
|
436
851
|
|
|
@@ -439,92 +854,121 @@ class Config(OrderedDict):
|
|
|
439
854
|
|
|
440
855
|
def __call__(self, key : str,
|
|
441
856
|
default = no_default,
|
|
442
|
-
cast :
|
|
857
|
+
cast : Callable = None,
|
|
443
858
|
help : str = None,
|
|
444
859
|
help_default : str = None,
|
|
445
860
|
help_cast : str = None,
|
|
446
861
|
mark_done : bool = True,
|
|
447
862
|
record : bool = True ):
|
|
448
863
|
"""
|
|
449
|
-
Reads
|
|
864
|
+
Reads a parameter ``key`` from the `config` subject to casting with ``cast``.
|
|
865
|
+
If not found, return ``default``
|
|
450
866
|
|
|
451
|
-
|
|
452
|
-
config("key", 1) - returns the value for 'key' or if not found returns 1
|
|
453
|
-
config("key", 1, int) - if 'key' is not found, return 1. If it is found cast the result with int().
|
|
454
|
-
config("key", 1, int, "A number" - also stores an optional help text.
|
|
455
|
-
Call usage_report() after the config has been read to a get a full
|
|
456
|
-
summary of all data requested from this config.
|
|
457
|
-
|
|
458
|
-
Advanced casting
|
|
459
|
-
----------------
|
|
460
|
-
|
|
461
|
-
Int, Float allow to bind the range of numbers:
|
|
462
|
-
config("positive_int", 1, Int>=1, "A positive integer")
|
|
463
|
-
config("ranged_int", 1, (Int>=0)&(Int<=10), "An integer between 0 and 10, inclusive")
|
|
464
|
-
config("positive_float", 1, Float>0., "A positive integerg"
|
|
867
|
+
Examples::
|
|
465
868
|
|
|
466
|
-
|
|
467
|
-
|
|
869
|
+
config("key") # returns the value for 'key' or if not found raises an exception
|
|
870
|
+
config("key", 1) # returns the value for 'key' or if not found returns 1
|
|
871
|
+
config("key", 1, int) # if 'key' is not found, return 1. If it is found cast the result with int().
|
|
872
|
+
config("key", 1, int, "A number" # also stores an optional help text.
|
|
873
|
+
# Call usage_report() after the config has been read to a get a full
|
|
874
|
+
# summary of all data requested from this config.
|
|
875
|
+
|
|
876
|
+
Use :attr:`cdxcore.config.Int` and :attr:`cdxcore.config.Float` to ensure a number
|
|
877
|
+
is within a given range::
|
|
878
|
+
|
|
879
|
+
config("positive_int", 1, Int>=1, "A positive integer")
|
|
880
|
+
config("ranged_int", 1, (Int>=0)&(Int<=10), "An integer between 0 and 10, inclusive")
|
|
881
|
+
config("positive_float", 1, Float>0., "A positive integerg"
|
|
882
|
+
|
|
883
|
+
Choices are implemented with lists::
|
|
884
|
+
|
|
885
|
+
config("difficulty", 'easy', ['easy','medium','hard'], "Choose one")
|
|
468
886
|
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
887
|
+
Alternative types are implemented with tuples::
|
|
888
|
+
|
|
889
|
+
config("difficulty", None, (None, ['easy','medium','hard']), "None or a level of difficulty")
|
|
890
|
+
config("level", None, (None, Int>=0), "None or a non-negative level")
|
|
472
891
|
|
|
473
892
|
Parameters
|
|
474
893
|
----------
|
|
475
894
|
key : string
|
|
476
|
-
Keyword to read
|
|
895
|
+
Keyword to read.
|
|
896
|
+
|
|
477
897
|
default : optional
|
|
478
898
|
Default value.
|
|
479
|
-
Set to
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
899
|
+
Set to :attr:`cdxcore.config.Config.no_default` for mandatory parameters without default.
|
|
900
|
+
If then 'key' cannot be found a :class:`KeyError` is raised.
|
|
901
|
+
|
|
902
|
+
cast : Callable, optional
|
|
903
|
+
|
|
904
|
+
If ``None``, any value provided by the user will be acceptable.
|
|
905
|
+
|
|
906
|
+
If not ``None``, the function will attempt to cast the value provided by the user
|
|
907
|
+
with ``cast()``.
|
|
908
|
+
For example, if ``cast = int``, then the function will apply ``int(x)`` to the user's input ``x``.
|
|
909
|
+
|
|
910
|
+
This function also allows passing the following complex arguments:
|
|
911
|
+
|
|
912
|
+
* A list, in which case it is assumed that the ``key`` must be from this list. The type of the first element of the list will be
|
|
913
|
+
used to ``cast()`` values to the target type.
|
|
914
|
+
|
|
915
|
+
* :attr:`cdxcore.config.Int` and :attr:`cdxcore.config.Float` allow defining constrained integers and floating point numbers, respectively.
|
|
916
|
+
|
|
917
|
+
* A tuple of types, in which case any of the types is acceptable.
|
|
918
|
+
A ``None`` here means that the value ``None`` is acceptabl
|
|
919
|
+
(it does not mean that any value is acceptable).
|
|
920
|
+
|
|
921
|
+
* Any callable to validate a parameter.
|
|
922
|
+
|
|
488
923
|
help : str, optional
|
|
489
924
|
If provied adds a help text when self documentation is used.
|
|
490
925
|
help_default : str, optional
|
|
491
926
|
If provided, specifies the default value in plain text.
|
|
492
|
-
If not provided, help_default is equal to the string representation of the default value, if any.
|
|
927
|
+
If not provided, ``help_default`` is equal to the string representation of the ``default`` value, if any.
|
|
493
928
|
Use this for complex default values which are hard to read.
|
|
494
929
|
help_cast : str, optional
|
|
495
930
|
If provided, specifies a description of the cast type.
|
|
496
|
-
If not provided, help_cast is set to the string representation of
|
|
497
|
-
|
|
931
|
+
If not provided, ``help_cast`` is set to the string representation of ``cast``, or
|
|
932
|
+
``None`` if ``cast` is ``None``. Complex casts are supported.
|
|
933
|
+
Use this for cast types which are hard to read.
|
|
498
934
|
mark_done : bool, optional
|
|
499
|
-
If true, marks the respective element as read.
|
|
935
|
+
If true, marks the respective element as read once the function returned successfully.
|
|
500
936
|
record : bool, optional
|
|
501
|
-
If True, records usage of the key and validates that previous usage of the key is consistent with
|
|
937
|
+
If True, records consistency usage of the key and validates that previous usage of the key is consistent with
|
|
502
938
|
the current usage, e.g. that the default values are consistent and that if help was provided it is the same.
|
|
503
939
|
|
|
504
940
|
Returns
|
|
505
941
|
-------
|
|
506
|
-
|
|
942
|
+
Parameter value.
|
|
507
943
|
|
|
508
944
|
Raises
|
|
509
945
|
------
|
|
510
|
-
KeyError
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
For all the help texts empty strings are not compared, ie __call__("x", default=1) will succeed even if a previous call was __call__("x", default=1, help="value for x").
|
|
515
|
-
Note that 'cast' is not validated.
|
|
516
|
-
CastError:
|
|
517
|
-
If an error occcurs casting a provided value.
|
|
518
|
-
ValueError:
|
|
946
|
+
:class:`KeyError`:
|
|
947
|
+
If ``key`` could not be found.
|
|
948
|
+
|
|
949
|
+
:class:`ValueError`:
|
|
519
950
|
For input errors.
|
|
951
|
+
|
|
952
|
+
:class:`cdxcore.config.InconsistencyError`:
|
|
953
|
+
If ``key`` was previously accessed with different ``default``, ``help``, ``help_default`` or
|
|
954
|
+
``help_cast`` values.
|
|
955
|
+
For all the help texts empty strings are not compared, i.e.
|
|
956
|
+
``__call__("x", default=1)`` will succeed even if a previous call was
|
|
957
|
+
``__call__("x", default=1, help="value for x")``.
|
|
958
|
+
|
|
959
|
+
Note that ``cast`` is not validated.
|
|
960
|
+
|
|
961
|
+
:class:`cdxcore.config.CastError`:
|
|
962
|
+
If an error occcurs casting a provided value.
|
|
963
|
+
|
|
520
964
|
"""
|
|
521
|
-
verify( isinstance(key, str), "'key' must be a string
|
|
522
|
-
verify( key.find('.') == -1 , "Error using
|
|
965
|
+
verify( isinstance(key, str), "'key' must be a string but is of type '{typ}'. Key value was '{key}'", typ=type(key), key=key, exception=ValueError )
|
|
966
|
+
verify( key.find('.') == -1 , "Error using Config '{name}': key name cannot contain '.'; found '{key}'", name=self.config_name, key=key, exception=ValueError )
|
|
523
967
|
|
|
524
968
|
# determine raw value
|
|
525
969
|
if not key in self:
|
|
526
970
|
if default == no_default:
|
|
527
|
-
raise KeyError(key, "Error using config '%s': key '%s' not found " % (self.
|
|
971
|
+
raise KeyError(key, "Error using config '%s': key '%s' not found " % (self.config_name, key))
|
|
528
972
|
value = default
|
|
529
973
|
else:
|
|
530
974
|
value = OrderedDict.get(self,key)
|
|
@@ -534,12 +978,15 @@ class Config(OrderedDict):
|
|
|
534
978
|
help = cast
|
|
535
979
|
cast = None
|
|
536
980
|
|
|
537
|
-
#
|
|
538
|
-
caster = _create_caster( cast, self._name, key, none_is_any = True )
|
|
981
|
+
# castdef
|
|
982
|
+
caster = _create_caster( cast=cast, config_name=self._name, key_name=key, none_is_any = True )
|
|
539
983
|
try:
|
|
540
|
-
value = caster( value, self._name, key )
|
|
984
|
+
value = caster( value, config_name=self._name, key_name=key )
|
|
985
|
+
except CastError as e:
|
|
986
|
+
assert False, "Casters should not throw CastError's"
|
|
987
|
+
raise e
|
|
541
988
|
except Exception as e:
|
|
542
|
-
raise CastError( config_name=self._name,
|
|
989
|
+
raise CastError( key=key, config_name=self._name, exception=e )
|
|
543
990
|
|
|
544
991
|
# mark key as read
|
|
545
992
|
if mark_done:
|
|
@@ -555,7 +1002,7 @@ class Config(OrderedDict):
|
|
|
555
1002
|
help_default = str(help_default) if not help_default is None else ""
|
|
556
1003
|
help_default = str(default) if default != no_default and len(help_default) == 0 else help_default
|
|
557
1004
|
help_cast = str(help_cast) if not help_cast is None else str(caster)
|
|
558
|
-
verify( default != no_default or help_default == "", "Config %s setup error for key %s: cannot specify 'help_default' if no default is given", self.
|
|
1005
|
+
verify( default != no_default or help_default == "", "Config %s setup error for key %s: cannot specify 'help_default' if no default is given", self.config_name, key, exception=ValueError )
|
|
559
1006
|
|
|
560
1007
|
raw_use = help == "" and help_cast == "" and help_default == "" # raw_use, e.g. simply get() or []. Including internal use
|
|
561
1008
|
|
|
@@ -591,17 +1038,18 @@ class Config(OrderedDict):
|
|
|
591
1038
|
|
|
592
1039
|
# Both current and past were bona fide recorded uses.
|
|
593
1040
|
# Ensure that their usage is consistent.
|
|
1041
|
+
# Note that we do *not* check consistency of the cast operator.
|
|
594
1042
|
if default != no_default:
|
|
595
1043
|
if 'default' in exst_value:
|
|
596
1044
|
if exst_value['default'] != default:
|
|
597
|
-
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different default values '%s' and '%s'" % ( key, self.
|
|
1045
|
+
raise InconsistencyError(key, self.config_name, "Key '%s' of config '%s' (%s) was read twice with different default values '%s' and '%s'" % ( key, self.config_name, record_key, exst_value['default'], default ))
|
|
598
1046
|
else:
|
|
599
1047
|
exst_value['default'] = default
|
|
600
1048
|
|
|
601
1049
|
if help != "":
|
|
602
1050
|
if exst_value['help'] != "":
|
|
603
1051
|
if exst_value['help'] != help:
|
|
604
|
-
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help' texts '%s' and '%s'" % ( key, self.
|
|
1052
|
+
raise InconsistencyError(key, self.config_name, "Key '%s' of config '%s' (%s) was read twice with different 'help' texts '%s' and '%s'" % ( key, self.config_name, record_key, exst_value['help'], help ) )
|
|
605
1053
|
else:
|
|
606
1054
|
exst_value['help'] = help
|
|
607
1055
|
|
|
@@ -609,14 +1057,14 @@ class Config(OrderedDict):
|
|
|
609
1057
|
if exst_value['help_default'] != "":
|
|
610
1058
|
# we do not insist on the same 'help_default'
|
|
611
1059
|
if exst_value['help_default'] != help_default:
|
|
612
|
-
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help_default' texts '%s' and '%s'" % ( key, self.
|
|
1060
|
+
raise InconsistencyError(key, self.config_name, "Key '%s' of config '%s' (%s) was read twice with different 'help_default' texts '%s' and '%s'" % ( key, self.config_name, record_key, exst_value['help_default'], help_default ) )
|
|
613
1061
|
else:
|
|
614
1062
|
exst_value['help_default'] = help_default
|
|
615
1063
|
|
|
616
1064
|
if help_cast != "" and help_cast != _Simple.STR_NONE_CAST:
|
|
617
1065
|
if exst_value['help_cast'] != "" and exst_value['help_cast'] != _Simple.STR_NONE_CAST:
|
|
618
1066
|
if exst_value['help_cast'] != help_cast:
|
|
619
|
-
raise InconsistencyError(key, "Key '%s' of config '%s' (%s) was read twice with different 'help_cast' texts '%s' and '%s'" % ( key, self.
|
|
1067
|
+
raise InconsistencyError(key, self.config_name, "Key '%s' of config '%s' (%s) was read twice with different 'help_cast' texts '%s' and '%s'" % ( key, self.config_name, record_key, exst_value['help_cast'], help_cast ))
|
|
620
1068
|
else:
|
|
621
1069
|
exst_value['help_cast'] = help_cast
|
|
622
1070
|
# done
|
|
@@ -640,10 +1088,10 @@ class Config(OrderedDict):
|
|
|
640
1088
|
config = Config()
|
|
641
1089
|
config.sub.x = 1 # <-- create 'sub' on the fly
|
|
642
1090
|
"""
|
|
643
|
-
verify( key.find('.') == -1 , "Error using
|
|
1091
|
+
verify( key.find('.') == -1 , "Error using Config '{name}': key name cannot contain '.'; found '{key}", name=self.config_name, key=key, exception=ValueError )
|
|
644
1092
|
if key in self._children:
|
|
645
1093
|
return self._children[key]
|
|
646
|
-
verify( key.find(" ") == -1, "Error using
|
|
1094
|
+
verify( key.find(" ") == -1, "Error using Config '{name}': sub-config names cannot contain spaces. Found '{key}'", name=self.config_name, key=key, exception=ValueError )
|
|
647
1095
|
config = Config()
|
|
648
1096
|
config._name = self._name + "." + key
|
|
649
1097
|
config._recorder = self._recorder
|
|
@@ -652,93 +1100,126 @@ class Config(OrderedDict):
|
|
|
652
1100
|
|
|
653
1101
|
def get(self, *kargs, **kwargs ):
|
|
654
1102
|
"""
|
|
655
|
-
Returns __call__(*kargs, **kwargs)
|
|
1103
|
+
Returns :meth:`cdxcore.config.Config.__call__` ``(*kargs, **kwargs)``.
|
|
656
1104
|
"""
|
|
657
1105
|
return self(*kargs, **kwargs)
|
|
658
1106
|
|
|
659
1107
|
def get_default(self, *kargs, **kwargs ):
|
|
660
1108
|
"""
|
|
661
|
-
Returns __call__(*kargs, **kwargs)
|
|
1109
|
+
Returns :meth:`cdxcore.config.Config.__call__` ``(*kargs, **kwargs)``.
|
|
662
1110
|
"""
|
|
663
1111
|
return self(*kargs, **kwargs)
|
|
664
1112
|
|
|
665
1113
|
def get_raw(self, key : str, default = no_default ):
|
|
666
1114
|
"""
|
|
667
|
-
Reads the
|
|
668
|
-
|
|
1115
|
+
Reads the raw value for ``key`` without any casting,
|
|
1116
|
+
nor marking the element as read, nor recording access to the element.
|
|
1117
|
+
|
|
1118
|
+
Equivalent to using
|
|
1119
|
+
:meth:`cdxcore.config.Config.__call__` ``(key, default, mark_done=False, record=False )``
|
|
1120
|
+
which, without ``default``, is turn itself equivalent to ``self[key]``
|
|
669
1121
|
"""
|
|
670
1122
|
return self(key, default, mark_done=False, record=False)
|
|
671
1123
|
|
|
672
1124
|
def get_recorded(self, key : str ):
|
|
673
1125
|
"""
|
|
674
|
-
Returns the
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
1126
|
+
Returns the casted value returned for ``key`` previously.
|
|
1127
|
+
|
|
1128
|
+
If the parameter ``key`` was provided as part of the input data, this value is returned, subject
|
|
1129
|
+
to casting.
|
|
1130
|
+
|
|
1131
|
+
If ``key`` was not part of the input data, and a ``default`` was provided when the
|
|
1132
|
+
parameter was read with :meth:`cdxcore.config.Config.__call__`, then return this default value, subject
|
|
1133
|
+
to casting.
|
|
1134
|
+
|
|
1135
|
+
Raises
|
|
1136
|
+
------
|
|
1137
|
+
:class:`KeyError`:
|
|
1138
|
+
If the key was not previously read successfully.
|
|
679
1139
|
"""
|
|
680
|
-
verify( key.find('.') == -1 , "Error using
|
|
1140
|
+
verify( key.find('.') == -1 , "Error using Config '{name}': key name cannot contain '.'; found '{key}", name=self.config_name, key=key, exception=ValueError )
|
|
681
1141
|
record_key = self._name + "['" + key + "']" # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
|
|
682
1142
|
record = self._recorder.get(record_key, None)
|
|
683
1143
|
if record is None:
|
|
684
1144
|
raise KeyError(key)
|
|
685
1145
|
return record['value']
|
|
686
1146
|
|
|
687
|
-
def keys(self):
|
|
1147
|
+
def keys(self) -> list:
|
|
688
1148
|
"""
|
|
689
|
-
Returns the keys for the immediate
|
|
690
|
-
This call will *not* return the names of config children
|
|
1149
|
+
Returns the keys for the immediate parameters of this config.
|
|
1150
|
+
This call will *not* return the names of child config; use :attr:`cdxcore.config.Config.children`.
|
|
1151
|
+
|
|
1152
|
+
Use :meth:`cdxcore.config.Config.input_dict` to obtain the full hierarchy of input parameters.
|
|
691
1153
|
"""
|
|
692
1154
|
return OrderedDict.keys(self)
|
|
693
1155
|
|
|
694
1156
|
# Write
|
|
695
1157
|
# -----
|
|
696
1158
|
|
|
697
|
-
def __setattr__(self, key, value):
|
|
1159
|
+
def __setattr__(self, key : str, value):
|
|
698
1160
|
"""
|
|
699
|
-
Assign value using member notation
|
|
700
|
-
|
|
701
|
-
|
|
1161
|
+
Assign value using member notation: ``self.key = value``.
|
|
1162
|
+
|
|
1163
|
+
Identical to ``self[key] = value``.
|
|
1164
|
+
Do not use leading underscores for `config` variables, see below
|
|
702
1165
|
|
|
703
1166
|
Parameters
|
|
704
1167
|
----------
|
|
705
1168
|
key : str
|
|
706
|
-
|
|
707
|
-
|
|
1169
|
+
``key`` to store ``value`` for.
|
|
1170
|
+
|
|
1171
|
+
Key to store. Note that keys starting with underscores are *not* stored as standard
|
|
1172
|
+
parameter values,
|
|
1173
|
+
but become classic members of the object (in ``self.__dict__``).
|
|
1174
|
+
|
|
708
1175
|
value :
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
config
|
|
714
|
-
sub
|
|
715
|
-
config.sub
|
|
716
|
-
a = config.sub("a", 0, int, "Test")
|
|
1176
|
+
The value for ``key``.
|
|
1177
|
+
|
|
1178
|
+
If ``value`` is a ``Config`` object, then a child config is created::
|
|
1179
|
+
|
|
1180
|
+
config = Config()
|
|
1181
|
+
config.sub = Config(a=1)
|
|
1182
|
+
a = config.sub("a", 0, int, "Test")
|
|
717
1183
|
config.done() # <- no error is reported, usage_report() is correct
|
|
1184
|
+
|
|
1185
|
+
*Expert Usage Comment:*
|
|
1186
|
+
this function assumes that if ``value`` is a config it is not used elsewhere.
|
|
1187
|
+
In particular its usage will be reset, and its consistency recorder aligned
|
|
1188
|
+
with ``self``. To avoid side effects for `config`s you wish to re-use elsewhere,
|
|
1189
|
+
call :meth:`cdxcore.config.Config.clean_copy` first.
|
|
718
1190
|
"""
|
|
719
1191
|
self.__setitem__(key,value)
|
|
720
1192
|
|
|
721
|
-
def __setitem__(self, key, value):
|
|
1193
|
+
def __setitem__(self, key : str, value):
|
|
722
1194
|
"""
|
|
723
|
-
Assign value using array notation
|
|
724
|
-
|
|
1195
|
+
Assign a ``value`` to ``key`` using array notation ``self[key] = value``.
|
|
1196
|
+
|
|
1197
|
+
Identical to ``self.key = value``.
|
|
725
1198
|
|
|
726
1199
|
Parameters
|
|
727
1200
|
----------
|
|
728
1201
|
key : str
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
1202
|
+
``key`` to store ``value`` for.
|
|
1203
|
+
|
|
1204
|
+
Key to store. Note that keys starting with underscores are *not* stored as standard
|
|
1205
|
+
parameter values,
|
|
1206
|
+
but become classic members of the object (in ``self.__dict__``).
|
|
1207
|
+
|
|
1208
|
+
``key`` may contain '.' for hierarchical access.
|
|
1209
|
+
|
|
732
1210
|
value :
|
|
733
|
-
If value is a Config object,
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
config
|
|
738
|
-
sub = Config(a=1)
|
|
739
|
-
config.sub = sub
|
|
740
|
-
a = config.sub("a", 0, int, "Test")
|
|
1211
|
+
If ``value`` is a ``Config`` object, then a child config is created::
|
|
1212
|
+
|
|
1213
|
+
config = Config()
|
|
1214
|
+
config.sub = Config(a=1)
|
|
1215
|
+
a = config.sub("a", 0, int, "Test")
|
|
741
1216
|
config.done() # <- no error is reported, usage_report() is correct
|
|
1217
|
+
|
|
1218
|
+
*Expert Usage Comment:*
|
|
1219
|
+
this function assumes that if ``value`` is a config it is not used elsewhere.
|
|
1220
|
+
In particular its usage will be reset, and its consistency recorder aligned
|
|
1221
|
+
with ``self``. To avoid side effects for `config`s you wish to re-use elsewhere,
|
|
1222
|
+
call :meth:`cdxcore.config.Config.clean_copy` first.
|
|
742
1223
|
"""
|
|
743
1224
|
if key[0] == "_" or key in self.__dict__:
|
|
744
1225
|
OrderedDict.__setattr__(self, key, value )
|
|
@@ -765,10 +1246,10 @@ class Config(OrderedDict):
|
|
|
765
1246
|
c = c.__getattr__(key)
|
|
766
1247
|
OrderedDict.__setitem__(c, key, value)
|
|
767
1248
|
|
|
768
|
-
def update( self, other=None, **kwargs ):
|
|
1249
|
+
def update( self, other = None, **kwargs ):
|
|
769
1250
|
"""
|
|
770
1251
|
Overwrite values of 'self' new values.
|
|
771
|
-
Accepts the two main formats
|
|
1252
|
+
Accepts the two main formats::
|
|
772
1253
|
|
|
773
1254
|
update( dictionary )
|
|
774
1255
|
update( config )
|
|
@@ -777,13 +1258,20 @@ class Config(OrderedDict):
|
|
|
777
1258
|
|
|
778
1259
|
Parameters
|
|
779
1260
|
----------
|
|
780
|
-
other : dict, Config
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
If
|
|
1261
|
+
other : dict, Config
|
|
1262
|
+
|
|
1263
|
+
Copy all content of ``other`` into``self``.
|
|
1264
|
+
|
|
1265
|
+
If ``other`` is a config: elements will be clean_copy()ed; ``other`` will not be marked as "read".
|
|
1266
|
+
|
|
1267
|
+
If ``other`` is a dictionary, then '.' notation can be used for hierarchical assignments
|
|
1268
|
+
|
|
785
1269
|
**kwargs
|
|
786
1270
|
Allows assigning specific values.
|
|
1271
|
+
|
|
1272
|
+
Returns
|
|
1273
|
+
-------
|
|
1274
|
+
self : Config
|
|
787
1275
|
"""
|
|
788
1276
|
if not other is None:
|
|
789
1277
|
if isinstance( other, Config ):
|
|
@@ -810,7 +1298,7 @@ class Config(OrderedDict):
|
|
|
810
1298
|
assert key in self
|
|
811
1299
|
assert not key in self._children
|
|
812
1300
|
else:
|
|
813
|
-
verify( isinstance(other, Mapping), "Cannot update
|
|
1301
|
+
verify( isinstance(other, Mapping), "Cannot update a Config with an object of type '{typ}'. Expected 'Mapping' type.", typ=type(other).__name__, exception=ValueError )
|
|
814
1302
|
for key in other:
|
|
815
1303
|
if key[:1] == "_" or key in self.__dict__:
|
|
816
1304
|
continue
|
|
@@ -827,14 +1315,17 @@ class Config(OrderedDict):
|
|
|
827
1315
|
|
|
828
1316
|
if len(kwargs) > 0:
|
|
829
1317
|
self.update( other=kwargs )
|
|
1318
|
+
return self
|
|
830
1319
|
|
|
831
1320
|
# delete
|
|
832
1321
|
# ------
|
|
833
1322
|
|
|
834
1323
|
def delete_children( self, names : list ):
|
|
835
1324
|
"""
|
|
836
|
-
Delete one or several children
|
|
837
|
-
|
|
1325
|
+
Delete one or several children from ``self``.
|
|
1326
|
+
|
|
1327
|
+
This function does not delete recorded consistency information (``defaults`` and ``help``
|
|
1328
|
+
recorded from prior uses of :meth:`cdxcore.config.Config.__call__`).
|
|
838
1329
|
"""
|
|
839
1330
|
if isinstance(names, str):
|
|
840
1331
|
names = [ names ]
|
|
@@ -845,11 +1336,6 @@ class Config(OrderedDict):
|
|
|
845
1336
|
# Usage information & reports
|
|
846
1337
|
# ---------------------------
|
|
847
1338
|
|
|
848
|
-
@property
|
|
849
|
-
def recorder(self) -> SortedDict:
|
|
850
|
-
""" Returns the top level recorder """
|
|
851
|
-
return self._recorder
|
|
852
|
-
|
|
853
1339
|
def usage_report(self, with_values : bool = True,
|
|
854
1340
|
with_help : bool = True,
|
|
855
1341
|
with_defaults: bool = True,
|
|
@@ -874,13 +1360,13 @@ class Config(OrderedDict):
|
|
|
874
1360
|
Whether to print types
|
|
875
1361
|
|
|
876
1362
|
filter_path : str, optional
|
|
877
|
-
If provided, will match all children
|
|
878
|
-
Most useful with filter_path = self.
|
|
1363
|
+
If provided, will match the beginning of the fully qualified path of all children vs this string.
|
|
1364
|
+
Most useful with ``filter_path = self.config_name`` which ensures only children of this (child) config
|
|
1365
|
+
are shown.
|
|
879
1366
|
|
|
880
1367
|
Returns
|
|
881
1368
|
-------
|
|
882
|
-
str
|
|
883
|
-
Report.
|
|
1369
|
+
Report : str
|
|
884
1370
|
"""
|
|
885
1371
|
with_values = bool(with_values)
|
|
886
1372
|
with_help = bool(with_help)
|
|
@@ -925,9 +1411,8 @@ class Config(OrderedDict):
|
|
|
925
1411
|
|
|
926
1412
|
def usage_reproducer(self) -> str:
|
|
927
1413
|
"""
|
|
928
|
-
Returns a string
|
|
929
|
-
|
|
930
|
-
repr() correctly.
|
|
1414
|
+
Returns a string representation of current usage, calling :func:`repr`
|
|
1415
|
+
for each value.
|
|
931
1416
|
"""
|
|
932
1417
|
report = ""
|
|
933
1418
|
for key, record in self._recorder.items():
|
|
@@ -935,16 +1420,28 @@ class Config(OrderedDict):
|
|
|
935
1420
|
report += key + " = " + repr(value) + "\n"
|
|
936
1421
|
return report
|
|
937
1422
|
|
|
938
|
-
def input_report(self) -> str:
|
|
1423
|
+
def input_report(self, max_value_len : int = 100) -> str:
|
|
939
1424
|
"""
|
|
940
|
-
Returns a report of all inputs in a readable format
|
|
941
|
-
|
|
1425
|
+
Returns a report of all inputs in a readable format. Assumes
|
|
1426
|
+
that :func:`str` converts all values into some readable format.
|
|
1427
|
+
|
|
1428
|
+
Parameters
|
|
1429
|
+
----------
|
|
1430
|
+
max_value_len : int
|
|
1431
|
+
Limits the length of :func:`str` for each value to ``max_value_len`` characters.
|
|
1432
|
+
Set to ``None`` to not limit the length.
|
|
1433
|
+
|
|
1434
|
+
Returns
|
|
1435
|
+
-------
|
|
1436
|
+
Report : str
|
|
942
1437
|
"""
|
|
943
1438
|
inputs = []
|
|
1439
|
+
def max_value( s ):
|
|
1440
|
+
return s if max_value_len is None or len(s) < max_value_len else ( s[:max_value_len-3] + "..." )
|
|
944
1441
|
def ireport(self, inputs):
|
|
945
1442
|
for key in self:
|
|
946
1443
|
value = self.get_raw(key)
|
|
947
|
-
report_key = self._name
|
|
1444
|
+
report_key = f"{self._name}[{key}] = {max_value(str(value))}"
|
|
948
1445
|
inputs.append( report_key )
|
|
949
1446
|
for c in self._children.values():
|
|
950
1447
|
ireport(c, inputs)
|
|
@@ -958,7 +1455,15 @@ class Config(OrderedDict):
|
|
|
958
1455
|
|
|
959
1456
|
@property
|
|
960
1457
|
def not_done(self) -> dict:
|
|
961
|
-
"""
|
|
1458
|
+
"""
|
|
1459
|
+
Returns a dictionary of keys which were not read yet.
|
|
1460
|
+
|
|
1461
|
+
Returns
|
|
1462
|
+
-------
|
|
1463
|
+
not_done: dict
|
|
1464
|
+
Dictionary of dictionaries: for value parameters, the respective entry is their ``key`` and ``False``;
|
|
1465
|
+
for children the ``key`` is followed by their ``not_done`` dictionary.
|
|
1466
|
+
"""
|
|
962
1467
|
h = { key : False for key in self if not key in self._done }
|
|
963
1468
|
for k,c in self._children.items():
|
|
964
1469
|
ch = c.not_done
|
|
@@ -966,8 +1471,19 @@ class Config(OrderedDict):
|
|
|
966
1471
|
h[k] = ch
|
|
967
1472
|
return h
|
|
968
1473
|
|
|
969
|
-
|
|
970
|
-
|
|
1474
|
+
@property
|
|
1475
|
+
def recorder(self) -> SortedDict:
|
|
1476
|
+
""" Returns the "recorder", a :class:`sortedcontainers.SortedDict` which contains
|
|
1477
|
+
``key``, ``default``, ``cast``, ``help``, and all other function parameters for
|
|
1478
|
+
all calls of :meth:`cdxcore.config.Config.__call__`. It is used to ensure consistency
|
|
1479
|
+
of parameter calls.
|
|
1480
|
+
|
|
1481
|
+
*Use for debugging only.*
|
|
1482
|
+
"""
|
|
1483
|
+
return self._recorder
|
|
1484
|
+
|
|
1485
|
+
def input_dict(self, ignore_underscore = True ) -> pdct:
|
|
1486
|
+
""" Returns a :class:`cdxcore.pretty.PrettyObject` of all inputs into this config. """
|
|
971
1487
|
inputs = pdct()
|
|
972
1488
|
for key in self:
|
|
973
1489
|
if ignore_underscore and key[:1] == "_":
|
|
@@ -978,63 +1494,231 @@ class Config(OrderedDict):
|
|
|
978
1494
|
continue
|
|
979
1495
|
inputs[k] = c.input_dict()
|
|
980
1496
|
return inputs
|
|
1497
|
+
|
|
1498
|
+
def usage_value_dict( self ) -> SortedDict:
|
|
1499
|
+
"""
|
|
1500
|
+
Return a flat sorted dictionary of both "used" and, where not used, "input" values.
|
|
1501
|
+
|
|
1502
|
+
A "used" value has either been read from user input or was provided as a default. In both cases,
|
|
1503
|
+
it will have been subject to casting.
|
|
981
1504
|
|
|
982
|
-
|
|
1505
|
+
This function will raise a :class:`RuntimeError` in either of the following two cases:
|
|
1506
|
+
|
|
1507
|
+
* A key was marked as "done" (read), but no "value" was recorded at that time. A simple example is when :meth:`cdxcore.config.Config.detach`
|
|
1508
|
+
was called to create a child config, but that config has not yet been read.
|
|
1509
|
+
* A key has not been read yet, but there is a record of a value being returned. An example of this happening is if :meth:`cdxcore.config.Config.reset_done`
|
|
1510
|
+
is called.
|
|
983
1511
|
"""
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
1512
|
+
uvd = SortedDict()
|
|
1513
|
+
for key, record in self._recorder.items():
|
|
1514
|
+
uvd[key] = record['value']
|
|
1515
|
+
|
|
1516
|
+
def add_inputs( config ):
|
|
1517
|
+
for key in config:
|
|
1518
|
+
full_key = config.record_key( key )
|
|
1519
|
+
if key in config._done:
|
|
1520
|
+
verify( full_key in uvd, lambda : f"Error collecting 'usage_value_dict': key '{key}' with full name '{full_key}' is marked as `done` but has no recorder entry. "+\
|
|
1521
|
+
"This typically happens when a sub-config is detached(), and has not been used yet." )
|
|
1522
|
+
else:
|
|
1523
|
+
verify( not full_key in uvd, lambda : f"Error collecting 'usage_value_dict': key '{key}' with full name '{full_key}' is not yet marked as `done` but has a recorder entry" )
|
|
1524
|
+
uvd[full_key] = config[key]
|
|
1525
|
+
for c in config._children.values():
|
|
1526
|
+
add_inputs(c)
|
|
1527
|
+
|
|
1528
|
+
add_inputs(self)
|
|
1529
|
+
return uvd
|
|
1530
|
+
|
|
1531
|
+
# hashing
|
|
1532
|
+
# -------
|
|
1533
|
+
|
|
1534
|
+
def unique_hash(self, *, unique_hash : Callable = None, debug_trace : DebugTrace = None, input_only : bool = True, **unique_hash_parameters ) -> str:
|
|
1535
|
+
r"""
|
|
1536
|
+
Returns a unique hash key for this object - based on its provided inputs and
|
|
1537
|
+
*not* based on its usage.
|
|
1538
|
+
|
|
1539
|
+
This function allows both provision of an existing ``unique_hash`` function or
|
|
1540
|
+
to specify one on the fly using ``unique_hash_parameters``.
|
|
1541
|
+
That means instead of::
|
|
1542
|
+
|
|
1543
|
+
from cdxcore.uniquehash import UniqueHash
|
|
1544
|
+
self.unique_hash( unique_hash=UniqueHash(**p) )
|
|
1545
|
+
|
|
1546
|
+
we can directly call::
|
|
1547
|
+
|
|
1548
|
+
self.unique_hash( **p )
|
|
1549
|
+
|
|
1550
|
+
The purpose of this function is to allow indexing results of heavy computations which were
|
|
1551
|
+
configured with ``Config`` with a simple hash key. A typical application is caching of results
|
|
1552
|
+
based on the relevant user-configuration.
|
|
1553
|
+
|
|
1554
|
+
An example for a simplistic cache::
|
|
1555
|
+
|
|
1556
|
+
from cdxcore.config import Config
|
|
1557
|
+
import tempfile as tempfile
|
|
1558
|
+
import pickle as pickle
|
|
1559
|
+
|
|
1560
|
+
def big_function( cache_dir : str, config : Config = None, **kwargs ):
|
|
1561
|
+
assert not cache_dir[-1] in ["/","\\"], cache_dir
|
|
1562
|
+
config = Config.config_kwargs( config, kwargs )
|
|
1563
|
+
uid = config.unique_hash(length=8)
|
|
1564
|
+
cfile = f"{cache_dir}/{uid}.pck"
|
|
1565
|
+
|
|
1566
|
+
# attempt to read cache
|
|
1567
|
+
try:
|
|
1568
|
+
with open(cfile, "rb") as f:
|
|
1569
|
+
return pickle.load(f)
|
|
1570
|
+
except FileNotFoundError:
|
|
1571
|
+
pass
|
|
1572
|
+
|
|
1573
|
+
# do something big...
|
|
1574
|
+
result = config("a", 0, int, "Value 'a'") * 1000
|
|
1575
|
+
|
|
1576
|
+
# write cache
|
|
1577
|
+
with open(cfile, "wb") as f:
|
|
1578
|
+
pickle.dump(result,f)
|
|
1579
|
+
|
|
1580
|
+
return result
|
|
1581
|
+
|
|
1582
|
+
cache_dir = tempfile.mkdtemp() # for real applications, use a permanent cache_dir.
|
|
1583
|
+
_ = big_function( cache_dir = cache_dir, a=1 )
|
|
1584
|
+
print(_)
|
|
993
1585
|
|
|
1586
|
+
A more sophisticated framework which includes code versioning via :func:`cdxcore.version.version`
|
|
1587
|
+
is implemented with :meth:`cdxcore.subdir.SubDir.cache`.
|
|
1588
|
+
|
|
1589
|
+
**Unique Hash Default Semantics**
|
|
1590
|
+
|
|
1591
|
+
Please consult the documentation for :class:`cdxcore.uniquehash.UniqueHash` before using this functionality;
|
|
1592
|
+
in particular note that by default this function ignores
|
|
1593
|
+
config keys or children with leading underscores; set ``parse_underscore`` to ``"protected"`` or ``"private"`` to change this behaviour.
|
|
1594
|
+
|
|
1595
|
+
**Why is "Usage" not Considered when Computing the Hash (by Default)**
|
|
1596
|
+
|
|
1597
|
+
When using ``Config`` to configure our environment, then we have not only the user's input values
|
|
1598
|
+
but also the realized values in the form of defaults for those values the user has not provided.
|
|
1599
|
+
In most cases, these are the majority of values.
|
|
1600
|
+
|
|
1601
|
+
By only considering actual input values when computing a hash, we stipulate that
|
|
1602
|
+
defaults are not part of the current unique characteristic of the environment.
|
|
1603
|
+
|
|
1604
|
+
That seems inconsistent: consider a program which reads a parameter ``activation`` with default ``relu``.
|
|
1605
|
+
The hash key will be different for the case where the user does not provide a value for ``activation``,
|
|
1606
|
+
and the case where its value is set to ``relu`` by the user. The effective ``activation`` value
|
|
1607
|
+
in both cases is ``relu`` -- why would we not want this to be identified as the same
|
|
1608
|
+
environment configuration.
|
|
1609
|
+
|
|
1610
|
+
The following illustrates this dilemma::
|
|
1611
|
+
|
|
1612
|
+
def big_function( config ):
|
|
1613
|
+
_ = config("activation", "relu", str, "Activation function")
|
|
1614
|
+
config.done()
|
|
1615
|
+
|
|
1616
|
+
config = Config()
|
|
1617
|
+
big_function( config )
|
|
1618
|
+
print( config.unique_hash(length=8) ) # -> 36e9d246
|
|
1619
|
+
|
|
1620
|
+
config = Config(activation="relu")
|
|
1621
|
+
big_function( config )
|
|
1622
|
+
print( config.unique_hash(length=8) ) # -> d715e29c
|
|
1623
|
+
|
|
1624
|
+
*Robustness*
|
|
1625
|
+
|
|
1626
|
+
The key driver of using only input values for hashing is the prevalence of reading (child) configs
|
|
1627
|
+
close to the use of their parameters. That means that often config parameters are only read
|
|
1628
|
+
(and therefore their usage registered) if the respective computation is actually executed:
|
|
1629
|
+
even the ``big_function`` example above shows this issue: the call
|
|
1630
|
+
``config("a", 0, int, "Value 'a'")`` will only be executed if the cache could not be found.
|
|
1631
|
+
|
|
1632
|
+
This can be rectified if it is ensured that all config parameters are read regardless of
|
|
1633
|
+
actual executed code. In this case, set the parameter ``input_only``
|
|
1634
|
+
for ``unique_hash()``
|
|
1635
|
+
to ``False``. Note that when using :meth:`cdxcore.config.Config.detach`
|
|
1636
|
+
you must make sure to have processed all detached configurations
|
|
1637
|
+
before calling ``unique_hash()``.
|
|
1638
|
+
|
|
994
1639
|
Parameters
|
|
995
1640
|
----------
|
|
1641
|
+
unique_hash_parameters : dict
|
|
1642
|
+
|
|
1643
|
+
If ``unique_hash`` is ``None`` these parameters are passed to
|
|
1644
|
+
:meth:`cdxcore.uniquehash.UniqueHash.__call__` to obtain
|
|
1645
|
+
the corrsponding hashing function.
|
|
1646
|
+
|
|
1647
|
+
unique_hash : Callable
|
|
1648
|
+
|
|
1649
|
+
A function to return unique hashes, usally generated using :class:`cdxcore.uniquehash.UniqueHash`.
|
|
1650
|
+
|
|
1651
|
+
debug_trace : :class:`cdxcore.uniquehash.DebugTrace`
|
|
1652
|
+
Allows tracing of hashing activity for debugging purposes.
|
|
1653
|
+
Two implementations of ``DebugTrace`` are currently available:
|
|
1654
|
+
|
|
1655
|
+
* :class:`cdxcore.uniquehash.DebugTraceVerbose` simply prints out hashing activity to stdout.
|
|
1656
|
+
|
|
1657
|
+
* :class:`cdxcore.uniquehash.DebugTraceCollect` collects an array of tracing information.
|
|
1658
|
+
The object itself is an iterable which contains the respective tracing information
|
|
1659
|
+
once the hash function has returned.
|
|
1660
|
+
|
|
1661
|
+
input_only : bool
|
|
1662
|
+
*Expert use only.*
|
|
1663
|
+
|
|
1664
|
+
If True (the default) only user-provided inputs are used to compute the unique hash.
|
|
1665
|
+
If False, then the result of :meth:`cdxcore.config.Config.usage_value_dict` is used
|
|
1666
|
+
to generate the hash. Make sure you read and understand
|
|
1667
|
+
the discussion above on the topic.
|
|
1668
|
+
|
|
996
1669
|
|
|
997
1670
|
Returns
|
|
998
1671
|
-------
|
|
999
|
-
|
|
1672
|
+
Unique hash, str
|
|
1673
|
+
A unique hash of at most the length specified via either ``unique_hash`` or ``unique_hash_parameters``.
|
|
1674
|
+
|
|
1000
1675
|
"""
|
|
1001
|
-
if
|
|
1002
|
-
|
|
1676
|
+
if unique_hash is None:
|
|
1677
|
+
unique_hash = UniqueHash( **unique_hash_parameters )
|
|
1003
1678
|
else:
|
|
1004
|
-
if len(unique_hash_parameters) != 0: raise ValueError("Cannot provide 'unique_hash_parameters' if '
|
|
1679
|
+
if len(unique_hash_parameters) != 0: raise ValueError("Cannot provide 'unique_hash_parameters' if 'unique_hash' is provided")
|
|
1005
1680
|
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
inputs
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1681
|
+
if not input_only:
|
|
1682
|
+
uid = unique_hash( self.usage_value_dict() )
|
|
1683
|
+
|
|
1684
|
+
else:
|
|
1685
|
+
def rec(config):
|
|
1686
|
+
""" Recursive version which returns an empty string for empty sub configs """
|
|
1687
|
+
inputs = {}
|
|
1688
|
+
for key in config:
|
|
1689
|
+
if key[:1] == "_":
|
|
1690
|
+
continue
|
|
1691
|
+
inputs[key] = config.get_raw(key)
|
|
1692
|
+
for c, child in config._children.items():
|
|
1693
|
+
if c[:1] == "_":
|
|
1694
|
+
continue
|
|
1695
|
+
# collect ID for the child
|
|
1696
|
+
child_data = rec(child)
|
|
1697
|
+
# we only register children if they have keys.
|
|
1698
|
+
# this way we do not trigger a change in ID simply due to a failed read access.
|
|
1699
|
+
if child_data != "":
|
|
1700
|
+
inputs[c] = child_data
|
|
1701
|
+
if len(inputs) == 0:
|
|
1702
|
+
return ""
|
|
1703
|
+
return unique_hash(inputs,debug_trace=debug_trace)
|
|
1704
|
+
uid = rec(self)
|
|
1705
|
+
return uid if uid!="" else unique_hash("",debug_trace=debug_trace)
|
|
1027
1706
|
|
|
1028
1707
|
def used_info(self, key : str) -> tuple:
|
|
1029
|
-
"""
|
|
1708
|
+
"""
|
|
1709
|
+
Returns the usage stats for a given key in the form of a tuple ``(done, record)``.
|
|
1710
|
+
|
|
1711
|
+
Here ``done`` is a boolean and ``record`` is a dictionary of consistency
|
|
1712
|
+
information on the key. """
|
|
1030
1713
|
done = key in self._done
|
|
1031
1714
|
record = self._recorder.get( self.record_key(key), None )
|
|
1032
1715
|
return (done, record)
|
|
1033
1716
|
|
|
1034
|
-
def record_key(self, key):
|
|
1717
|
+
def record_key(self, key) -> str:
|
|
1035
1718
|
"""
|
|
1036
|
-
Returns the fully qualified
|
|
1037
|
-
|
|
1719
|
+
Returns the fully qualified string key for ``key``.
|
|
1720
|
+
|
|
1721
|
+
It has the form ``config1.config['entry']``.
|
|
1038
1722
|
"""
|
|
1039
1723
|
return self._name + "['" + key + "']" # using a fully qualified keys allows 'recorders' to be shared accross copy()'d configs.
|
|
1040
1724
|
|
|
@@ -1043,14 +1727,13 @@ class Config(OrderedDict):
|
|
|
1043
1727
|
|
|
1044
1728
|
def __iter__(self):
|
|
1045
1729
|
"""
|
|
1046
|
-
|
|
1047
|
-
using f(**self) call our __getitem__() function.
|
|
1730
|
+
Iterator.
|
|
1048
1731
|
"""
|
|
1049
1732
|
return OrderedDict.__iter__(self)
|
|
1050
1733
|
|
|
1051
1734
|
# pickling
|
|
1052
1735
|
# --------
|
|
1053
|
-
|
|
1736
|
+
|
|
1054
1737
|
def __reduce__(self):
|
|
1055
1738
|
"""
|
|
1056
1739
|
Pickling this object explicitly
|
|
@@ -1076,51 +1759,53 @@ class Config(OrderedDict):
|
|
|
1076
1759
|
keys = state['keys']
|
|
1077
1760
|
for (k,d) in zip(keys,data):
|
|
1078
1761
|
self[k] = d
|
|
1079
|
-
|
|
1762
|
+
|
|
1080
1763
|
# casting
|
|
1081
1764
|
# -------
|
|
1082
1765
|
|
|
1083
1766
|
@staticmethod
|
|
1084
|
-
def
|
|
1085
|
-
"""
|
|
1086
|
-
Makes sure an object is a config, and otherwise tries to convert it into one
|
|
1087
|
-
Classic use case is to transform 'kwargs' to a Config
|
|
1767
|
+
def config_kwargs( config, kwargs : Mapping, config_name : str = "kwargs"):
|
|
1088
1768
|
"""
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
@staticmethod
|
|
1092
|
-
def config_kwargs( config, kwargs : dict, config_name : str = "kwargs"):
|
|
1093
|
-
"""
|
|
1094
|
-
Default implementation for a usage pattern where the user can use both a 'config' and kwargs.
|
|
1095
|
-
This function 'detaches' the current config from 'self' which means done() must be called again.
|
|
1769
|
+
Default implementation for a usage pattern where the user can provide both a :class:`cdxcore.config.Config` parameter and ``** kwargs``.
|
|
1096
1770
|
|
|
1097
|
-
Example
|
|
1098
|
-
-------
|
|
1771
|
+
Example::
|
|
1099
1772
|
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1773
|
+
def f(config, **kwargs):
|
|
1774
|
+
config = Config.config_kwargs( config, kwargs )
|
|
1775
|
+
...
|
|
1776
|
+
x = config("x", 1, ...)
|
|
1777
|
+
config.done() # <-- important to do this here. Remembert that config_kwargs() calls 'detach'
|
|
1105
1778
|
|
|
1106
|
-
and then one can use either
|
|
1107
|
-
|
|
1108
|
-
config = Config()
|
|
1109
|
-
config.x = 1
|
|
1110
|
-
f(config)
|
|
1779
|
+
and then one can use either of the following::
|
|
1111
1780
|
|
|
1112
|
-
|
|
1781
|
+
f(Config(x=1))
|
|
1113
1782
|
f(x=1)
|
|
1783
|
+
|
|
1784
|
+
*Important*: ``config_kwargs`` calls :meth:`cdxcore.config.Config.detach` to obtain a copy of ``config``.
|
|
1785
|
+
This means :meth:`cdxcore.config.Config.done`
|
|
1786
|
+
must be called explicitly for the returned object even if ``done()``
|
|
1787
|
+
will be called elsewhere for the source ``config``.
|
|
1114
1788
|
|
|
1115
1789
|
Parameters
|
|
1116
1790
|
----------
|
|
1117
|
-
config :
|
|
1118
|
-
|
|
1119
|
-
|
|
1791
|
+
config : Config
|
|
1792
|
+
A ``Config`` object or ``None``.
|
|
1793
|
+
|
|
1794
|
+
kwargs : Mapping
|
|
1795
|
+
If ``config`` is provided, the function will call :meth:`cdxcore.config.Config.update` with ``kwargs``.
|
|
1796
|
+
|
|
1797
|
+
config_name : str
|
|
1798
|
+
A declarative name for the config if ``config`` is not proivded.
|
|
1120
1799
|
|
|
1121
1800
|
Returns
|
|
1122
1801
|
-------
|
|
1123
|
-
|
|
1802
|
+
config : Config
|
|
1803
|
+
A new config object. Please note that if ``config`` was provided, then this a copy
|
|
1804
|
+
obtained from calling :meth:`cdxcore.config.Config.detach`, which means that
|
|
1805
|
+
:meth:`cdxcore.config.Config.done` must be called explicitly for this object to
|
|
1806
|
+
ensure no parameters were misspelled (it is not sufficient
|
|
1807
|
+
if :meth:`cdxcore.config.Config.done` is called
|
|
1808
|
+
for ``config``.)
|
|
1124
1809
|
"""
|
|
1125
1810
|
assert isinstance( config_name, str ), "'config_name' must be a string"
|
|
1126
1811
|
if type(config).__name__ == Config.__name__: # we allow for import inconsistencies
|
|
@@ -1131,17 +1816,32 @@ class Config(OrderedDict):
|
|
|
1131
1816
|
config = Config.to_config( kwargs=kwargs, config_name=config_name )
|
|
1132
1817
|
return config
|
|
1133
1818
|
|
|
1819
|
+
@staticmethod
|
|
1820
|
+
def to_config( kwargs : Mapping, config_name : str = "kwargs"):
|
|
1821
|
+
"""
|
|
1822
|
+
Assess whether a parameters is a :class:`cdxcore.config.Config`, and otherwise tries to convert it into one.
|
|
1823
|
+
Classic use case is to transform ``** kwargs`` to a :class:`cdxcore.config.Config` to allow
|
|
1824
|
+
type checking and prevent spelling errors.
|
|
1825
|
+
|
|
1826
|
+
Returns
|
|
1827
|
+
-------
|
|
1828
|
+
config : Config
|
|
1829
|
+
If ``kwargs`` is already a :class:`cdxcore.config.Config` it is returned. Otherwise,
|
|
1830
|
+
create a new :class:`cdxcore.config.Config` from ``kwargs`` named using ``config_name``.
|
|
1831
|
+
"""
|
|
1832
|
+
return kwargs if isinstance(kwargs,Config) else Config( kwargs,config_name=config_name )
|
|
1833
|
+
|
|
1134
1834
|
# for uniqueHash
|
|
1135
1835
|
# --------------
|
|
1136
1836
|
|
|
1137
|
-
def __unique_hash__(self,
|
|
1837
|
+
def __unique_hash__(self, unique_hash : UniqueHash, debug_trace : DebugTrace ) -> str:
|
|
1138
1838
|
"""
|
|
1139
|
-
Returns a unique hash for this object
|
|
1839
|
+
Returns a unique hash for this object.
|
|
1840
|
+
|
|
1140
1841
|
This function is required because by default uniqueHash() ignores members starting with '_', which
|
|
1141
1842
|
in the case of Config means that no children are hashed.
|
|
1142
1843
|
"""
|
|
1143
|
-
return self.
|
|
1144
|
-
|
|
1844
|
+
return self.unique_hash( unique_hash=unique_hash, debug_trace=debug_trace )
|
|
1145
1845
|
|
|
1146
1846
|
# Comparison
|
|
1147
1847
|
# -----------
|
|
@@ -1154,6 +1854,14 @@ class Config(OrderedDict):
|
|
|
1154
1854
|
return False
|
|
1155
1855
|
return OrderedDict.__eq__(self, other)
|
|
1156
1856
|
|
|
1857
|
+
def __neq__(self, other):
|
|
1858
|
+
""" Equality operator comparing 'name' and standard dictionary content """
|
|
1859
|
+
if type(self).__name__ == type(other).__name__: # allow comparison betweenn different imports
|
|
1860
|
+
return False
|
|
1861
|
+
if self._name == other._name:
|
|
1862
|
+
return False
|
|
1863
|
+
return OrderedDict.__neq__(self, other)
|
|
1864
|
+
|
|
1157
1865
|
def __hash__(self):
|
|
1158
1866
|
return hash(self._name) ^ OrderedDict.__hash__(self)
|
|
1159
1867
|
|
|
@@ -1162,115 +1870,68 @@ config_kwargs = Config.config_kwargs
|
|
|
1162
1870
|
Config.no_default = no_default
|
|
1163
1871
|
|
|
1164
1872
|
# ==============================================================================
|
|
1165
|
-
# New in version 0.1.45
|
|
1166
|
-
# Support for conditional types, e.g. we can write
|
|
1167
1873
|
#
|
|
1168
|
-
#
|
|
1874
|
+
# Exceptions
|
|
1875
|
+
#
|
|
1169
1876
|
# ==============================================================================
|
|
1170
1877
|
|
|
1171
|
-
class
|
|
1878
|
+
class NotDoneError( RuntimeError ):
|
|
1172
1879
|
"""
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
a = A(config=w)
|
|
1202
|
-
|
|
1203
|
-
key1, key2 = jax.random.split(jax.random.key(0))
|
|
1204
|
-
x = jnp.zeros((10,10))
|
|
1205
|
-
param = a.init( key1, x )
|
|
1206
|
-
y = a.apply( param, x )
|
|
1207
|
-
|
|
1208
|
-
class A( nn.Module ):
|
|
1209
|
-
config : ConfigField = ConfigField.field()
|
|
1210
|
-
|
|
1211
|
-
@nn.compact
|
|
1212
|
-
def __call__(self, x):
|
|
1213
|
-
a = self.config.x("y", 0.1 ,float)
|
|
1214
|
-
self.config.done()
|
|
1215
|
-
return nn.Dense(1)(x)*a
|
|
1216
|
-
|
|
1217
|
-
print("Config")
|
|
1218
|
-
c = Config()
|
|
1219
|
-
c.x.y = 1.
|
|
1220
|
-
w = ConfigField(c)
|
|
1221
|
-
a = A(config=w)
|
|
1880
|
+
Raised when :meth:`cdxcore.config.Config.done` finds that some config parameters have not been read.
|
|
1881
|
+
|
|
1882
|
+
The set of those arguments is accessible via
|
|
1883
|
+
:attr:`cdxcore.config.NotDoneError.not_done`.
|
|
1884
|
+
"""
|
|
1885
|
+
def __init__(self, not_done : set[str], config_name : str, message : str):
|
|
1886
|
+
#: The oarameter keys which were not read when :meth:`cdxcore.config.Config.done` was called.
|
|
1887
|
+
self.not_done = not_done
|
|
1888
|
+
#: Hierarchical name of the config.
|
|
1889
|
+
self.config_name = config_name
|
|
1890
|
+
RuntimeError.__init__(self, message)
|
|
1891
|
+
|
|
1892
|
+
class InconsistencyError( RuntimeError ):
|
|
1893
|
+
"""
|
|
1894
|
+
Raised when :meth:`cdxcore.config.Config.__call__`
|
|
1895
|
+
used inconsistently between function calls for a given parameter.
|
|
1896
|
+
|
|
1897
|
+
The ``Config`` semantics require that parameters are accessed used with consistent
|
|
1898
|
+
`default` and `help` values between :meth:`cdxcore.config.Config.__call__` calls.
|
|
1899
|
+
|
|
1900
|
+
For *raw* access to any paramters, use ``[]``.
|
|
1901
|
+
"""
|
|
1902
|
+
def __init__(self, key : str, config_name : str, message : str):
|
|
1903
|
+
#: The offending parameter key.
|
|
1904
|
+
self.key = key
|
|
1905
|
+
#: Hierarchical name of the config.
|
|
1906
|
+
self.config_name = config_name
|
|
1907
|
+
RuntimeError.__init__(self, message)
|
|
1222
1908
|
|
|
1223
|
-
|
|
1224
|
-
x = jnp.zeros((10,10))
|
|
1225
|
-
param = a.init( key1, x )
|
|
1226
|
-
y = a.apply( param, x )
|
|
1227
|
-
y = a.apply( param, x )
|
|
1909
|
+
class CastError( RuntimeError ):
|
|
1228
1910
|
"""
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
config = config if type(config).__name__ != type(self).__name__ else config.__config
|
|
1232
|
-
self.__config = Config.config_kwargs( config, kwargs )
|
|
1233
|
-
def __call__(self, *kargs, **kwargs):
|
|
1234
|
-
return self.__config(*kargs,**kwargs)
|
|
1235
|
-
def __getattr__(self, key):
|
|
1236
|
-
if key[:2] == "__":
|
|
1237
|
-
return object.__getattr__(self, key)
|
|
1238
|
-
return getattr(self.__config, key)
|
|
1239
|
-
def __getitem__(self, key):
|
|
1240
|
-
return self.__config[key]
|
|
1241
|
-
def __eq__(self, other):
|
|
1242
|
-
if type(other).__name__ == "Config":
|
|
1243
|
-
return self.__config == other
|
|
1244
|
-
else:
|
|
1245
|
-
return self.__config == other.config
|
|
1246
|
-
def __hash__(self):
|
|
1247
|
-
h = 0
|
|
1248
|
-
for k, v in self.items():
|
|
1249
|
-
h ^= hash(k) ^ hash(v)
|
|
1250
|
-
return h
|
|
1251
|
-
def __unique_hash__(self, *kargs, **kwargs):
|
|
1252
|
-
return self.__config.__unique_hash__(*kargs, **kwargs)
|
|
1253
|
-
def __str__(self):
|
|
1254
|
-
return self.__pdct.__str__()
|
|
1255
|
-
def __repr__(self):
|
|
1256
|
-
return self.__pdct.__repr__()
|
|
1257
|
-
def as_dict(self):
|
|
1258
|
-
return self.__config.as_dict(mark_done=False)
|
|
1259
|
-
def done(self):
|
|
1260
|
-
return self.__config.done()
|
|
1911
|
+
Raised when :meth:`cdxcore.config.Config.__call__` could not cast a
|
|
1912
|
+
value provided by the user to the specified type.
|
|
1261
1913
|
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1914
|
+
Parameters
|
|
1915
|
+
----------
|
|
1916
|
+
key : str
|
|
1917
|
+
Key name of the parameter which failed to cast.
|
|
1918
|
+
config_name : str
|
|
1919
|
+
Name of the ``Config``.
|
|
1920
|
+
|
|
1921
|
+
exception : :class:`Exception`
|
|
1922
|
+
Orginal exception raised by the cast.
|
|
1923
|
+
"""
|
|
1924
|
+
def __init__(self, key : str, config_name : str, exception : Exception):
|
|
1925
|
+
"""
|
|
1926
|
+
:meta private:
|
|
1927
|
+
"""
|
|
1928
|
+
#: Key of the parameter which failed to cast.
|
|
1929
|
+
self.key = key
|
|
1930
|
+
#: Hierarchical name of the config.
|
|
1931
|
+
self.config_name = config_name
|
|
1932
|
+
header = f"Error in cast definition for key '{key}' in config '{config_name}': " if len(config_name) > 0 else f"Error in cast definition for key '{key}': "
|
|
1933
|
+
RuntimeError.__init__(self, header+str(exception))
|
|
1265
1934
|
|
|
1266
|
-
@staticmethod
|
|
1267
|
-
def default():
|
|
1268
|
-
return ConfigField()
|
|
1269
|
-
|
|
1270
|
-
@staticmethod
|
|
1271
|
-
def Field():
|
|
1272
|
-
import dataclasses as dataclasses
|
|
1273
|
-
return dataclasses.field( default_factory=ConfigField )
|
|
1274
1935
|
|
|
1275
1936
|
# ==============================================================================
|
|
1276
1937
|
# New in version 0.1.45
|
|
@@ -1390,7 +2051,7 @@ class _Condition(_Cast):
|
|
|
1390
2051
|
else:
|
|
1391
2052
|
raise RuntimeError("Internal error: unknown operator %s" % str(self.op))
|
|
1392
2053
|
if not ok:
|
|
1393
|
-
raise ValueError( "value for key '
|
|
2054
|
+
raise ValueError( f"value for key '{key_name}' {self.err_str}: found {str(value)[:100]}" )
|
|
1394
2055
|
return self.l_and( value, config_name=config_name, key_name=key_name) if not self.l_and is None else value
|
|
1395
2056
|
|
|
1396
2057
|
def __str__(self) -> str:
|
|
@@ -1453,7 +2114,7 @@ class _CastCond(_Cast): # NOQA
|
|
|
1453
2114
|
return _Condition( self.cast, 'lt', self.cast(other) )
|
|
1454
2115
|
def __call__(self, value, *, config_name : str, key_name : str ):
|
|
1455
2116
|
""" This gets called if the type was used without operators """
|
|
1456
|
-
cast = _Simple(self.cast, config_name=config_name, key_name=key_name, none_is_any=None )
|
|
2117
|
+
cast = _Simple(cast=self.cast, config_name=config_name, key_name=key_name, none_is_any=None )
|
|
1457
2118
|
return cast( value, config_name=config_name, key_name=key_name )
|
|
1458
2119
|
def __str__(self) -> str:
|
|
1459
2120
|
""" This gets called if the type was used without operators """
|
|
@@ -1461,31 +2122,30 @@ class _CastCond(_Cast): # NOQA
|
|
|
1461
2122
|
|
|
1462
2123
|
Float = _CastCond(float)
|
|
1463
2124
|
"""
|
|
1464
|
-
Allows to apply conditions to float
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
In combination with
|
|
1471
|
-
|
|
1472
|
-
probability = config("probability", 0.5, (Float>=0.) & (Float <= 1.), "Probability")
|
|
1473
|
-
```
|
|
2125
|
+
Allows to apply basic range conditions to ``float`` parameters.
|
|
2126
|
+
|
|
2127
|
+
For example::
|
|
2128
|
+
|
|
2129
|
+
timeout = config("timeout", 0.5, Float>=0., "Timeout")
|
|
2130
|
+
|
|
2131
|
+
In combination with ``&`` we can limit a float to a range::
|
|
2132
|
+
|
|
2133
|
+
probability = config("probability", 0.5, (Float>=0.) & (Float <= 1.), "Probability")
|
|
1474
2134
|
"""
|
|
1475
2135
|
|
|
1476
2136
|
|
|
1477
2137
|
Int = _CastCond(int)
|
|
1478
2138
|
"""
|
|
1479
|
-
Allows to apply conditions to int
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
In combination with
|
|
1486
|
-
|
|
1487
|
-
bus_days_per_year = config("bus_days_per_year", 255, (Int > 0) & (Int < 365), "Business days per year")
|
|
1488
|
-
|
|
2139
|
+
Allows to apply basic range conditions to ``int`` parameters.
|
|
2140
|
+
|
|
2141
|
+
For example::
|
|
2142
|
+
|
|
2143
|
+
num_steps = config("num_steps", 1, Int>0., "Number of steps")
|
|
2144
|
+
|
|
2145
|
+
In combination with ``&`` we can limit an int to a range:
|
|
2146
|
+
|
|
2147
|
+
bus_days_per_year = config("bus_days_per_year", 255, (Int > 0) & (Int < 365), "Business days per year")
|
|
2148
|
+
|
|
1489
2149
|
"""
|
|
1490
2150
|
|
|
1491
2151
|
# ================================
|
|
@@ -1511,7 +2171,7 @@ class _Enum(_Cast):
|
|
|
1511
2171
|
if self.enum[0] is None:
|
|
1512
2172
|
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1513
2173
|
f"'cast' for key '{key_name}' is an list, with first element 'None'. Lists are used for enumerator types, and the first element defines their underlying type. Hence you cannot use 'None'. (Did you want to use alternative notation with tuples?)")
|
|
1514
|
-
self.cast = _Simple( type(self.enum[0]), config_name=config_name, key_name=key_name, none_is_any=None )
|
|
2174
|
+
self.cast = _Simple( cast=type(self.enum[0]), config_name=config_name, key_name=key_name, none_is_any=None )
|
|
1515
2175
|
for i in range(1,len(self.enum)):
|
|
1516
2176
|
try:
|
|
1517
2177
|
self.enum[i] = self.cast( self.enum[i], config_name=config_name, key_name=key_name )
|
|
@@ -1526,8 +2186,8 @@ class _Enum(_Cast):
|
|
|
1526
2186
|
Raises a KeyError if the value was not found in our enum
|
|
1527
2187
|
"""
|
|
1528
2188
|
value = self.cast(value, config_name=config_name, key_name=key_name)
|
|
1529
|
-
if not value in self.
|
|
1530
|
-
raise ValueError( f"Value for key '{key_name}' {self.err_str}; found 'str(value)[:
|
|
2189
|
+
if not value in self.enum:
|
|
2190
|
+
raise ValueError( f"Value for key '{key_name}' {self.err_str}; found '{str(value)[:100]}'" )
|
|
1531
2191
|
return value
|
|
1532
2192
|
|
|
1533
2193
|
@property
|
|
@@ -1569,7 +2229,10 @@ class _Alt(_Cast):
|
|
|
1569
2229
|
if len(casts) == 0:
|
|
1570
2230
|
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1571
2231
|
f"'cast' for key '{key_name}' is an empty tuple. Tuples are used for alternative types, hence passing empty tuple is not defined")
|
|
1572
|
-
|
|
2232
|
+
# print("casts####",casts)
|
|
2233
|
+
# if casts==(0,0):
|
|
2234
|
+
# raise RuntimeError()
|
|
2235
|
+
self.casts = [ _create_caster(cast=cast, config_name=config_name, key_name=key_name, none_is_any=False) for cast in casts ]
|
|
1573
2236
|
|
|
1574
2237
|
def __call__( self, value, *, config_name : str, key_name : str ):
|
|
1575
2238
|
""" Cast 'value' to the proper type """
|
|
@@ -1577,7 +2240,7 @@ class _Alt(_Cast):
|
|
|
1577
2240
|
for cast in self.casts:
|
|
1578
2241
|
# None means that value == None is acceptable
|
|
1579
2242
|
try:
|
|
1580
|
-
return cast(value, config_name=config_name,
|
|
2243
|
+
return cast(value, config_name=config_name, key_name=key_name )
|
|
1581
2244
|
except Exception as e:
|
|
1582
2245
|
e0 = e if e0 is None else e0
|
|
1583
2246
|
raise ValueError(f"Error using config '{config_name}': value for key '{key_name}' {self.err_str}. Found '{str(value)}' of type '{type(value).__name__}'")
|
|
@@ -1624,10 +2287,14 @@ def _create_caster( *, cast : type, config_name : str, key_name : str, none_is_a
|
|
|
1624
2287
|
raise ValueError(_cast_err_header(config_name=config_name,key_name=key_name) +\
|
|
1625
2288
|
f"string '{cast}' provided as 'cast'. Strings are not supported. To pass sets of characters, use a list of strings of those single characters.")
|
|
1626
2289
|
if isinstance(cast, list):
|
|
1627
|
-
|
|
2290
|
+
#print("--- create_caster enun", cast)
|
|
2291
|
+
return _Enum( enum=cast, config_name=config_name, key_name=key_name )
|
|
1628
2292
|
elif isinstance(cast, tuple):
|
|
1629
|
-
|
|
2293
|
+
#print("--- create_caster alt", cast)
|
|
2294
|
+
return _Alt( casts=cast, config_name=config_name, key_name=key_name )
|
|
1630
2295
|
elif isinstance(cast,_Cast):
|
|
2296
|
+
#print("--- create_caster-cash", cast)
|
|
1631
2297
|
return cast
|
|
1632
|
-
|
|
2298
|
+
#print("--- create_caster simple", cast)
|
|
2299
|
+
return _Simple( cast=cast, config_name=config_name, key_name=key_name, none_is_any=none_is_any )
|
|
1633
2300
|
|