vortex-nwp 2.0.0b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vortex/__init__.py +135 -0
- vortex/algo/__init__.py +12 -0
- vortex/algo/components.py +2136 -0
- vortex/algo/mpitools.py +1648 -0
- vortex/algo/mpitools_templates/envelope_wrapper_default.tpl +27 -0
- vortex/algo/mpitools_templates/envelope_wrapper_mpiauto.tpl +29 -0
- vortex/algo/mpitools_templates/wrapstd_wrapper_default.tpl +18 -0
- vortex/algo/serversynctools.py +170 -0
- vortex/config.py +115 -0
- vortex/data/__init__.py +13 -0
- vortex/data/abstractstores.py +1572 -0
- vortex/data/containers.py +780 -0
- vortex/data/contents.py +596 -0
- vortex/data/executables.py +284 -0
- vortex/data/flow.py +113 -0
- vortex/data/geometries.ini +2689 -0
- vortex/data/geometries.py +703 -0
- vortex/data/handlers.py +1021 -0
- vortex/data/outflow.py +67 -0
- vortex/data/providers.py +465 -0
- vortex/data/resources.py +201 -0
- vortex/data/stores.py +1271 -0
- vortex/gloves.py +282 -0
- vortex/layout/__init__.py +27 -0
- vortex/layout/appconf.py +109 -0
- vortex/layout/contexts.py +511 -0
- vortex/layout/dataflow.py +1069 -0
- vortex/layout/jobs.py +1276 -0
- vortex/layout/monitor.py +833 -0
- vortex/layout/nodes.py +1424 -0
- vortex/layout/subjobs.py +464 -0
- vortex/nwp/__init__.py +11 -0
- vortex/nwp/algo/__init__.py +12 -0
- vortex/nwp/algo/assim.py +483 -0
- vortex/nwp/algo/clim.py +920 -0
- vortex/nwp/algo/coupling.py +609 -0
- vortex/nwp/algo/eda.py +632 -0
- vortex/nwp/algo/eps.py +613 -0
- vortex/nwp/algo/forecasts.py +745 -0
- vortex/nwp/algo/fpserver.py +927 -0
- vortex/nwp/algo/ifsnaming.py +403 -0
- vortex/nwp/algo/ifsroot.py +311 -0
- vortex/nwp/algo/monitoring.py +202 -0
- vortex/nwp/algo/mpitools.py +554 -0
- vortex/nwp/algo/odbtools.py +974 -0
- vortex/nwp/algo/oopsroot.py +735 -0
- vortex/nwp/algo/oopstests.py +186 -0
- vortex/nwp/algo/request.py +579 -0
- vortex/nwp/algo/stdpost.py +1285 -0
- vortex/nwp/data/__init__.py +12 -0
- vortex/nwp/data/assim.py +392 -0
- vortex/nwp/data/boundaries.py +261 -0
- vortex/nwp/data/climfiles.py +539 -0
- vortex/nwp/data/configfiles.py +149 -0
- vortex/nwp/data/consts.py +929 -0
- vortex/nwp/data/ctpini.py +133 -0
- vortex/nwp/data/diagnostics.py +181 -0
- vortex/nwp/data/eda.py +148 -0
- vortex/nwp/data/eps.py +383 -0
- vortex/nwp/data/executables.py +1039 -0
- vortex/nwp/data/fields.py +96 -0
- vortex/nwp/data/gridfiles.py +308 -0
- vortex/nwp/data/logs.py +551 -0
- vortex/nwp/data/modelstates.py +334 -0
- vortex/nwp/data/monitoring.py +220 -0
- vortex/nwp/data/namelists.py +644 -0
- vortex/nwp/data/obs.py +748 -0
- vortex/nwp/data/oopsexec.py +72 -0
- vortex/nwp/data/providers.py +182 -0
- vortex/nwp/data/query.py +217 -0
- vortex/nwp/data/stores.py +147 -0
- vortex/nwp/data/surfex.py +338 -0
- vortex/nwp/syntax/__init__.py +9 -0
- vortex/nwp/syntax/stdattrs.py +375 -0
- vortex/nwp/tools/__init__.py +10 -0
- vortex/nwp/tools/addons.py +35 -0
- vortex/nwp/tools/agt.py +55 -0
- vortex/nwp/tools/bdap.py +48 -0
- vortex/nwp/tools/bdcp.py +38 -0
- vortex/nwp/tools/bdm.py +21 -0
- vortex/nwp/tools/bdmp.py +49 -0
- vortex/nwp/tools/conftools.py +1311 -0
- vortex/nwp/tools/drhook.py +62 -0
- vortex/nwp/tools/grib.py +268 -0
- vortex/nwp/tools/gribdiff.py +99 -0
- vortex/nwp/tools/ifstools.py +163 -0
- vortex/nwp/tools/igastuff.py +249 -0
- vortex/nwp/tools/mars.py +56 -0
- vortex/nwp/tools/odb.py +548 -0
- vortex/nwp/tools/partitioning.py +234 -0
- vortex/nwp/tools/satrad.py +56 -0
- vortex/nwp/util/__init__.py +6 -0
- vortex/nwp/util/async.py +184 -0
- vortex/nwp/util/beacon.py +40 -0
- vortex/nwp/util/diffpygram.py +359 -0
- vortex/nwp/util/ens.py +198 -0
- vortex/nwp/util/hooks.py +128 -0
- vortex/nwp/util/taskdeco.py +81 -0
- vortex/nwp/util/usepygram.py +591 -0
- vortex/nwp/util/usetnt.py +87 -0
- vortex/proxy.py +6 -0
- vortex/sessions.py +341 -0
- vortex/syntax/__init__.py +9 -0
- vortex/syntax/stdattrs.py +628 -0
- vortex/syntax/stddeco.py +176 -0
- vortex/toolbox.py +982 -0
- vortex/tools/__init__.py +11 -0
- vortex/tools/actions.py +457 -0
- vortex/tools/addons.py +297 -0
- vortex/tools/arm.py +76 -0
- vortex/tools/compression.py +322 -0
- vortex/tools/date.py +20 -0
- vortex/tools/ddhpack.py +10 -0
- vortex/tools/delayedactions.py +672 -0
- vortex/tools/env.py +513 -0
- vortex/tools/folder.py +663 -0
- vortex/tools/grib.py +559 -0
- vortex/tools/lfi.py +746 -0
- vortex/tools/listings.py +354 -0
- vortex/tools/names.py +575 -0
- vortex/tools/net.py +1790 -0
- vortex/tools/odb.py +10 -0
- vortex/tools/parallelism.py +336 -0
- vortex/tools/prestaging.py +186 -0
- vortex/tools/rawfiles.py +10 -0
- vortex/tools/schedulers.py +413 -0
- vortex/tools/services.py +871 -0
- vortex/tools/storage.py +1061 -0
- vortex/tools/surfex.py +61 -0
- vortex/tools/systems.py +3396 -0
- vortex/tools/targets.py +384 -0
- vortex/util/__init__.py +9 -0
- vortex/util/config.py +1071 -0
- vortex/util/empty.py +24 -0
- vortex/util/helpers.py +184 -0
- vortex/util/introspection.py +63 -0
- vortex/util/iosponge.py +76 -0
- vortex/util/roles.py +51 -0
- vortex/util/storefunctions.py +103 -0
- vortex/util/structs.py +26 -0
- vortex/util/worker.py +150 -0
- vortex_nwp-2.0.0b1.dist-info/LICENSE +517 -0
- vortex_nwp-2.0.0b1.dist-info/METADATA +50 -0
- vortex_nwp-2.0.0b1.dist-info/RECORD +146 -0
- vortex_nwp-2.0.0b1.dist-info/WHEEL +5 -0
- vortex_nwp-2.0.0b1.dist-info/top_level.txt +1 -0
vortex/util/config.py
ADDED
|
@@ -0,0 +1,1071 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configuration management through ini and template files.
|
|
3
|
+
|
|
4
|
+
The :func:`load_template` function is the entry-point when looking for template files.
|
|
5
|
+
It returns an object compliant with the interface defined in
|
|
6
|
+
:class:`AbstractTemplatingAdapter`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import abc
|
|
10
|
+
from configparser import NoOptionError, NoSectionError, InterpolationDepthError
|
|
11
|
+
from configparser import ConfigParser
|
|
12
|
+
import contextlib
|
|
13
|
+
import importlib
|
|
14
|
+
import itertools
|
|
15
|
+
import re
|
|
16
|
+
import string
|
|
17
|
+
|
|
18
|
+
import footprints
|
|
19
|
+
from bronx.fancies import loggers
|
|
20
|
+
from bronx.stdtypes import date as bdate
|
|
21
|
+
from bronx.syntax.parsing import StringDecoder, StringDecoderSyntaxError
|
|
22
|
+
from vortex import sessions
|
|
23
|
+
|
|
24
|
+
__all__ = []
|
|
25
|
+
|
|
26
|
+
logger = loggers.getLogger(__name__)
|
|
27
|
+
|
|
28
|
+
_RE_AUTO_TPL = re.compile(r'^@(([^/].*)\.tpl)$')
|
|
29
|
+
|
|
30
|
+
_RE_ENCODING = re.compile(r"^\s*#.*?coding[=:]\s*([-\w.]+)")
|
|
31
|
+
|
|
32
|
+
_RE_TEMPLATING = re.compile(r"^\s*#\s*vortex-templating\s*[=:]\s*([-\w.]+)$")
|
|
33
|
+
|
|
34
|
+
_DEFAULT_CONFIG_PARSER = ConfigParser
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class AbstractTemplatingAdapter(metaclass=abc.ABCMeta):
|
|
38
|
+
"""Interface to any templating system.
|
|
39
|
+
|
|
40
|
+
To render the template, just call the object with a list of named arguments
|
|
41
|
+
that should be used during template rendering.
|
|
42
|
+
"""
|
|
43
|
+
|
|
44
|
+
KIND = None
|
|
45
|
+
|
|
46
|
+
def __init__(self, tpl_str, tpl_file, tpl_encoding, tpl_dirs):
|
|
47
|
+
"""
|
|
48
|
+
:param tpl_str: The template (as a string)
|
|
49
|
+
:param tpl_file: The template filename (when read from disk)
|
|
50
|
+
:param tpl_encoding: The template encoding (when read from disk)
|
|
51
|
+
:param tpl_dirs: The lookup directories for additional templates
|
|
52
|
+
"""
|
|
53
|
+
self._tpl_file = tpl_file
|
|
54
|
+
self._tpl_encoding = tpl_encoding
|
|
55
|
+
self._tpl_dirs = tpl_dirs
|
|
56
|
+
self._tpl_obj = self._rendering_tool_init(tpl_str)
|
|
57
|
+
|
|
58
|
+
@property
|
|
59
|
+
def srcfile(self):
|
|
60
|
+
"""The template filename (when read from disk)."""
|
|
61
|
+
return self._tpl_file
|
|
62
|
+
|
|
63
|
+
@abc.abstractmethod
|
|
64
|
+
def _rendering_tool_init(self, tpl_str):
|
|
65
|
+
pass
|
|
66
|
+
|
|
67
|
+
def substitute(self, *kargs, **kwargs):
|
|
68
|
+
"""Render the template using the kargs and kwargs dictionaries."""
|
|
69
|
+
todo = dict()
|
|
70
|
+
for m in kargs:
|
|
71
|
+
todo.update(m)
|
|
72
|
+
todo.update(kwargs)
|
|
73
|
+
return self(** todo)
|
|
74
|
+
|
|
75
|
+
safe_substitute = substitute
|
|
76
|
+
|
|
77
|
+
@abc.abstractmethod
|
|
78
|
+
def __call__(self, **kwargs):
|
|
79
|
+
"""Render the template using the kwargs dictionary."""
|
|
80
|
+
pass
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class LegacyTemplatingAdapter(AbstractTemplatingAdapter):
|
|
84
|
+
"""Just use :class:`string.Template` for the rendering.
|
|
85
|
+
|
|
86
|
+
See :class:`AbstractTemplatingAdapter` for more details on this class usage.
|
|
87
|
+
"""
|
|
88
|
+
|
|
89
|
+
KIND = 'legacy'
|
|
90
|
+
|
|
91
|
+
def _rendering_tool_init(self, tpl_str):
|
|
92
|
+
return string.Template(tpl_str)
|
|
93
|
+
|
|
94
|
+
def safe_substitute(self, *kargs, **kwargs):
|
|
95
|
+
"""Render the template using the kargs and kwargs dictionaries."""
|
|
96
|
+
return self._tpl_obj.safe_substitute(*kargs, **kwargs)
|
|
97
|
+
|
|
98
|
+
def __call__(self, **kwargs):
|
|
99
|
+
"""Render the template using the kwargs dictionary."""
|
|
100
|
+
return self._tpl_obj.substitute(kwargs)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
class TwoPassLegacyTemplatingAdapter(AbstractTemplatingAdapter):
|
|
104
|
+
"""Just use :class:`string.Template`, but render the template two times.
|
|
105
|
+
|
|
106
|
+
(it allows for two level of nesting in the variable to be rendered).
|
|
107
|
+
|
|
108
|
+
See :class:`AbstractTemplatingAdapter` for more details on this class usage.
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
KIND = 'twopasslegacy'
|
|
112
|
+
|
|
113
|
+
def _rendering_tool_init(self, tpl_str):
|
|
114
|
+
return string.Template(tpl_str)
|
|
115
|
+
|
|
116
|
+
def safe_substitute(self, *kargs, **kwargs):
|
|
117
|
+
"""Render the template using the kargs and kwargs dictionaries."""
|
|
118
|
+
return string.Template(
|
|
119
|
+
self._tpl_obj.safe_substitute(*kargs, **kwargs)
|
|
120
|
+
).safe_substitute(*kargs, **kwargs)
|
|
121
|
+
|
|
122
|
+
def __call__(self, **kwargs):
|
|
123
|
+
"""Render the template using the kwargs dictionary."""
|
|
124
|
+
return string.Template(self._tpl_obj.substitute(kwargs)).substitute(kwargs)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class Jinja2TemplatingAdapter(AbstractTemplatingAdapter):
|
|
128
|
+
"""Use the jinja2 templating engine to render the template.
|
|
129
|
+
|
|
130
|
+
It requires the, external, :mod:`jinja2` package. Please refer to the
|
|
131
|
+
jinja2 documentation for more details on the jinja2 templating language.
|
|
132
|
+
|
|
133
|
+
See :class:`AbstractTemplatingAdapter` for more details on this class usage.
|
|
134
|
+
"""
|
|
135
|
+
|
|
136
|
+
KIND = 'jinja2'
|
|
137
|
+
|
|
138
|
+
@contextlib.contextmanager
|
|
139
|
+
def _elaborate_on_jinja2_error(self):
|
|
140
|
+
import jinja2
|
|
141
|
+
try:
|
|
142
|
+
yield
|
|
143
|
+
except jinja2.exceptions.TemplateError as e:
|
|
144
|
+
if isinstance(e, jinja2.exceptions.TemplateSyntaxError):
|
|
145
|
+
logger.error("%s exception while processing Jinja2 templates:\n" +
|
|
146
|
+
" Toplevel template file: %s\n"
|
|
147
|
+
" Jinja2 template info : %s (line %d)\n" +
|
|
148
|
+
" Jinja2 error message : %s",
|
|
149
|
+
e.__class__, self._tpl_file, e.name, e.lineno, e.message)
|
|
150
|
+
else:
|
|
151
|
+
logger.error("%s exception while processing Jinja2 templates:\n" +
|
|
152
|
+
" Toplevel template file: %s",
|
|
153
|
+
e.__class__, self._tpl_file)
|
|
154
|
+
raise
|
|
155
|
+
|
|
156
|
+
def _rendering_tool_init(self, tpl_str):
|
|
157
|
+
import jinja2
|
|
158
|
+
loader = jinja2.FileSystemLoader(
|
|
159
|
+
self._tpl_dirs,
|
|
160
|
+
encoding=self._tpl_encoding,
|
|
161
|
+
followlinks=True) if self._tpl_dirs else None
|
|
162
|
+
j_env = jinja2.Environment(loader=loader, autoescape=False)
|
|
163
|
+
with self._elaborate_on_jinja2_error():
|
|
164
|
+
return j_env.from_string(tpl_str)
|
|
165
|
+
|
|
166
|
+
def __call__(self, **kwargs):
|
|
167
|
+
"""Render the template using the kwargs dictionary."""
|
|
168
|
+
with self._elaborate_on_jinja2_error():
|
|
169
|
+
return self._tpl_obj.render(kwargs)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def load_template(t, tplfile, encoding=None, version=None, default_templating='legacy'):
|
|
173
|
+
"""Load a template according to *tplfile*.
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
:param vortex.sessions.Ticket t: The Vortex' session to be used
|
|
177
|
+
:param str tplfile: The name of the desired template file
|
|
178
|
+
:param str encoding: The characters encoding of the template file
|
|
179
|
+
:param int version: Find a template file with version >= to version
|
|
180
|
+
:param str default_templating: The default templating engine that will be used.
|
|
181
|
+
The content of the template file is always searched
|
|
182
|
+
in order to detect a "# vortex-templating:" comment
|
|
183
|
+
that will override this default.
|
|
184
|
+
:return: A :class:`AbstractTemplatingAdapter` object
|
|
185
|
+
|
|
186
|
+
*tplfile* can be a relative or absolute filename. However, most of the time,
|
|
187
|
+
it is a string like ``@foo.tpl``. In such a case, a file named ``foo.tpl`` will
|
|
188
|
+
be looked for in the ``~/.vortexrc/templates`` and in the ``templates``
|
|
189
|
+
sub-directory of the Vortex source code distribution.
|
|
190
|
+
|
|
191
|
+
The characters encoding of the template file may be specified. If *encoding*
|
|
192
|
+
equals ``script``, a line looking like ``# encoding:special-encoding`` will be
|
|
193
|
+
searched for in the first ten lines of the template file. If it exists, the
|
|
194
|
+
``special-encoding`` will be used as an encoding and the
|
|
195
|
+
``# encoding:special-encoding`` line will be stripped from the template.
|
|
196
|
+
|
|
197
|
+
Different templating engine may be used to render the template file. It
|
|
198
|
+
defaults to ``legacy`` that is compatible with Python's :class:`string.Template`
|
|
199
|
+
class. However, another default may be provided using the
|
|
200
|
+
*default_templating* argument. In any case, a line looking like
|
|
201
|
+
``# vortex-templating:kind`` will be searched for in the first ten lines
|
|
202
|
+
of the template file. If it exists, the ``kind`` templating engine will be
|
|
203
|
+
used and the ``# vortex-templating:kind`` line will be stripped.
|
|
204
|
+
|
|
205
|
+
Currently, only few templating engines are supported:
|
|
206
|
+
|
|
207
|
+
* ``legacy``: see :class:`LegacyTemplatingAdapter`
|
|
208
|
+
* ``twopasslegacy``: see :class:`TwoPassLegacyTemplatingAdapter`
|
|
209
|
+
* ``jinja2``: see :class:`Jinja2TemplatingAdapter`
|
|
210
|
+
"""
|
|
211
|
+
persodir = t.sh.path.join(t.glove.configrc, 'templates')
|
|
212
|
+
sitedir = t.sh.path.join(t.glove.siteroot, 'templates')
|
|
213
|
+
with importlib.resources.as_file(
|
|
214
|
+
importlib.resources.files("vortex.algo")
|
|
215
|
+
) as path:
|
|
216
|
+
pkgdir = path / "mpitools_templates"
|
|
217
|
+
searchdirs = list()
|
|
218
|
+
autofile = _RE_AUTO_TPL.match(tplfile)
|
|
219
|
+
if autofile is None:
|
|
220
|
+
if t.sh.path.exists(tplfile):
|
|
221
|
+
tplfile = t.sh.path.abspath(tplfile)
|
|
222
|
+
else:
|
|
223
|
+
raise ValueError('Template file not found: <{}>'.format(tplfile))
|
|
224
|
+
else:
|
|
225
|
+
searchdirs = (persodir, pkgdir, sitedir)
|
|
226
|
+
new_tplfile = None
|
|
227
|
+
if version is None:
|
|
228
|
+
autofile = autofile.group(1)
|
|
229
|
+
for dirname in searchdirs:
|
|
230
|
+
filename = t.sh.path.join(dirname, autofile)
|
|
231
|
+
if t.sh.path.exists(filename):
|
|
232
|
+
new_tplfile = filename
|
|
233
|
+
break
|
|
234
|
+
else:
|
|
235
|
+
autofile = autofile.group(2)
|
|
236
|
+
autodir = t.sh.path.dirname(autofile)
|
|
237
|
+
if autodir:
|
|
238
|
+
persodir = t.sh.path.join(persodir, autodir)
|
|
239
|
+
sitedir = t.sh.path.join(sitedir, sitedir)
|
|
240
|
+
autofile = t.sh.path.basename(autofile)
|
|
241
|
+
allowedre = re.compile(autofile + r'-v(\d+).tpl')
|
|
242
|
+
alloweditems = dict()
|
|
243
|
+
for inputdir in (sitedir, persodir):
|
|
244
|
+
if not t.sh.path.exists(inputdir):
|
|
245
|
+
continue
|
|
246
|
+
for fs_item in t.sh.listdir(inputdir):
|
|
247
|
+
fs_match = allowedre.match(fs_item)
|
|
248
|
+
if fs_match:
|
|
249
|
+
alloweditems[int(fs_match.group(1))] = t.sh.path.join(inputdir, fs_item)
|
|
250
|
+
for item_version in sorted(alloweditems.keys(), reverse=True):
|
|
251
|
+
if item_version <= version:
|
|
252
|
+
new_tplfile = alloweditems[item_version]
|
|
253
|
+
break
|
|
254
|
+
if not new_tplfile:
|
|
255
|
+
raise ValueError('Template file not found: <{}> with version >= {!s}.'
|
|
256
|
+
.format(tplfile, version))
|
|
257
|
+
else:
|
|
258
|
+
tplfile = new_tplfile
|
|
259
|
+
try:
|
|
260
|
+
ignored_lines = set()
|
|
261
|
+
actual_encoding = None if encoding == 'script' else encoding
|
|
262
|
+
actual_templating = default_templating
|
|
263
|
+
# To determine the encoding & templating open the file with the default
|
|
264
|
+
# encoding (ignoring decoding errors) and look for comments
|
|
265
|
+
with open(tplfile, errors='replace') as tpfld_tmp:
|
|
266
|
+
if encoding is None:
|
|
267
|
+
actual_encoding = tpfld_tmp.encoding
|
|
268
|
+
# Only inspect the first 10 lines
|
|
269
|
+
for iline, line in enumerate(itertools.islice(tpfld_tmp, 10)):
|
|
270
|
+
# Encoding
|
|
271
|
+
if encoding == 'script':
|
|
272
|
+
encoding_match = _RE_ENCODING.match(line)
|
|
273
|
+
if encoding_match:
|
|
274
|
+
ignored_lines.add(iline)
|
|
275
|
+
actual_encoding = encoding_match.group(1)
|
|
276
|
+
# Templating
|
|
277
|
+
templating_match = _RE_TEMPLATING.match(line)
|
|
278
|
+
if templating_match:
|
|
279
|
+
ignored_lines.add(iline)
|
|
280
|
+
actual_templating = templating_match.group(1)
|
|
281
|
+
# Read the template and delete the encoding line if present
|
|
282
|
+
logger.debug('Opening %s with encoding %s', tplfile, str(actual_encoding))
|
|
283
|
+
with open(tplfile, encoding=actual_encoding) as tpfld:
|
|
284
|
+
tpl_txt = "".join([l for (i, l) in enumerate(tpfld)
|
|
285
|
+
if i not in ignored_lines])
|
|
286
|
+
|
|
287
|
+
template_rendering_classes = {cls.KIND: cls for cls in globals().values()
|
|
288
|
+
if (isinstance(cls, type) and
|
|
289
|
+
issubclass(cls, AbstractTemplatingAdapter) and
|
|
290
|
+
cls.KIND)}
|
|
291
|
+
try:
|
|
292
|
+
template_rendering_cls = template_rendering_classes[actual_templating]
|
|
293
|
+
except KeyError:
|
|
294
|
+
raise ValueError('Unknown templating system < {:s} >'.format(actual_templating))
|
|
295
|
+
tpl = template_rendering_cls(tpl_txt, tplfile, actual_encoding, searchdirs)
|
|
296
|
+
except Exception as pb:
|
|
297
|
+
logger.error('Could not read template <%s>', str(pb))
|
|
298
|
+
raise
|
|
299
|
+
return tpl
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class GenericReadOnlyConfigParser:
|
|
303
|
+
"""A Basic ReadOnly configuration file parser.
|
|
304
|
+
|
|
305
|
+
It relies on a :class:`ConfigParser.ConfigParser` parser (or another class
|
|
306
|
+
that satisfies the interface) to access the configuration data.
|
|
307
|
+
|
|
308
|
+
:param str inifile: Path to a configuration file or a configuration file name
|
|
309
|
+
(see the :meth:`setfile` method for more details)
|
|
310
|
+
:param ConfigParser.ConfigParser parser: an existing configuration parser
|
|
311
|
+
object the will be used to access the configuration
|
|
312
|
+
:param bool mkforce: If the configuration file doesn't exists. Create an empty
|
|
313
|
+
one in ``~/.vortexrc``
|
|
314
|
+
:param type clsparser: The class that will be used to create a parser object
|
|
315
|
+
(if needed)
|
|
316
|
+
:param str encoding: The configuration file encoding
|
|
317
|
+
:param str defaultinifile: The name of a default ini file (read before,
|
|
318
|
+
and possibly overwritten by **inifile**)
|
|
319
|
+
|
|
320
|
+
:note: Some of the parser's methods are directly accessible because ``__getattr__``
|
|
321
|
+
is implemented. For this ReadOnly class, only methods ``defaults``,
|
|
322
|
+
``sections``, ``options``, ``items``, ``has_section`` and ``has_option``
|
|
323
|
+
are accessible. The user will refer to the Python's ConfigParser module
|
|
324
|
+
documentation for more details.
|
|
325
|
+
"""
|
|
326
|
+
|
|
327
|
+
_RE_AUTO_SETFILE = re.compile(r'^@([^/]+\.ini)$')
|
|
328
|
+
|
|
329
|
+
def __init__(self, inifile=None, parser=None, mkforce=False,
|
|
330
|
+
clsparser=_DEFAULT_CONFIG_PARSER, encoding=None, defaultinifile=None):
|
|
331
|
+
self.parser = parser
|
|
332
|
+
self.mkforce = mkforce
|
|
333
|
+
self.clsparser = clsparser
|
|
334
|
+
self.defaultencoding = encoding
|
|
335
|
+
self.defaultinifile = defaultinifile
|
|
336
|
+
if inifile:
|
|
337
|
+
self.setfile(inifile, encoding=None)
|
|
338
|
+
else:
|
|
339
|
+
self.file = None
|
|
340
|
+
|
|
341
|
+
def __deepcopy__(self, memo):
|
|
342
|
+
"""Warning: deepcopy of any item of the class is... itself!"""
|
|
343
|
+
memo[id(self)] = self
|
|
344
|
+
return self
|
|
345
|
+
|
|
346
|
+
def as_dump(self):
|
|
347
|
+
"""Return a nicely formated class name for dump in footprint."""
|
|
348
|
+
return 'file={!s}'.format(self.file)
|
|
349
|
+
|
|
350
|
+
def setfile(self, inifile, encoding=None):
|
|
351
|
+
"""Read the specified **inifile** as new configuration.
|
|
352
|
+
|
|
353
|
+
**inifile** may be:
|
|
354
|
+
|
|
355
|
+
* A File like object
|
|
356
|
+
* A path to a file
|
|
357
|
+
* A file name preceded by '@'
|
|
358
|
+
|
|
359
|
+
In the latter case, the configuration file is looked for both in
|
|
360
|
+
``~/.vortexrc`` and in the ``conf`` directory of the vortex installation.
|
|
361
|
+
If a section/option is defined in ``~/.vortexrc`` it takes precedence
|
|
362
|
+
over the one defined in ``conf``.
|
|
363
|
+
|
|
364
|
+
:example:
|
|
365
|
+
|
|
366
|
+
Let's consider the following declaration in ``conf``::
|
|
367
|
+
|
|
368
|
+
[mysection]
|
|
369
|
+
var1=Toto
|
|
370
|
+
var2=Titi
|
|
371
|
+
|
|
372
|
+
Let's consider the following declaration in ``~/.vortexrc``::
|
|
373
|
+
|
|
374
|
+
[mysection]
|
|
375
|
+
var1=Personalised
|
|
376
|
+
|
|
377
|
+
A call to ``get('mysection', 'var1')`` will return ``Personalised`` and a
|
|
378
|
+
call to ``get('mysection', 'var2')`` will return ``Titi``.
|
|
379
|
+
"""
|
|
380
|
+
if self.parser is None:
|
|
381
|
+
self.parser = self.clsparser()
|
|
382
|
+
if encoding is None:
|
|
383
|
+
encoding = self.defaultencoding
|
|
384
|
+
self.file = None
|
|
385
|
+
filestack = list()
|
|
386
|
+
local = sessions.system()
|
|
387
|
+
glove = sessions.current().glove
|
|
388
|
+
if not isinstance(inifile, str):
|
|
389
|
+
if self.defaultinifile:
|
|
390
|
+
sitedefaultinifile = glove.siteconf + '/' + self.defaultinifile
|
|
391
|
+
if local.path.exists(sitedefaultinifile):
|
|
392
|
+
with open(sitedefaultinifile, encoding=encoding) as a_fh:
|
|
393
|
+
self.parser.read_file(a_fh)
|
|
394
|
+
else:
|
|
395
|
+
raise ValueError('Configuration file ' + sitedefaultinifile + ' not found')
|
|
396
|
+
# Assume it's an IO descriptor
|
|
397
|
+
inifile.seek(0)
|
|
398
|
+
self.parser.read_file(inifile)
|
|
399
|
+
self.file = repr(inifile)
|
|
400
|
+
if self.defaultinifile:
|
|
401
|
+
self.file = sitedefaultinifile + "," + self.file
|
|
402
|
+
else:
|
|
403
|
+
# Let's continue as usual
|
|
404
|
+
autofile = self._RE_AUTO_SETFILE.match(inifile)
|
|
405
|
+
if not autofile:
|
|
406
|
+
if local.path.exists(inifile):
|
|
407
|
+
filestack.append(local.path.abspath(inifile))
|
|
408
|
+
else:
|
|
409
|
+
raise ValueError('Configuration file ' + inifile + ' not found')
|
|
410
|
+
else:
|
|
411
|
+
autofile = autofile.group(1)
|
|
412
|
+
sitefile = glove.siteconf + '/' + autofile
|
|
413
|
+
persofile = glove.configrc + '/' + autofile
|
|
414
|
+
if local.path.exists(sitefile):
|
|
415
|
+
filestack.append(sitefile)
|
|
416
|
+
if local.path.exists(persofile):
|
|
417
|
+
filestack.append(persofile)
|
|
418
|
+
if not filestack:
|
|
419
|
+
if self.mkforce:
|
|
420
|
+
filestack.append(persofile)
|
|
421
|
+
local.filecocoon(persofile)
|
|
422
|
+
local.touch(persofile)
|
|
423
|
+
else:
|
|
424
|
+
raise ValueError('Configuration file ' + inifile + ' not found')
|
|
425
|
+
if self.defaultinifile:
|
|
426
|
+
sitedefaultinifile = glove.siteconf + '/' + self.defaultinifile
|
|
427
|
+
if local.path.exists(sitedefaultinifile):
|
|
428
|
+
# Insert at the beginning (i.e. smallest priority)
|
|
429
|
+
filestack.insert(0, local.path.abspath(sitedefaultinifile))
|
|
430
|
+
else:
|
|
431
|
+
raise ValueError('Configuration file ' + sitedefaultinifile + ' not found')
|
|
432
|
+
self.file = ",".join(filestack)
|
|
433
|
+
for a_file in filestack:
|
|
434
|
+
with open(a_file, encoding=encoding) as a_fh:
|
|
435
|
+
self.parser.read_file(a_fh)
|
|
436
|
+
|
|
437
|
+
def as_dict(self, merged=True):
|
|
438
|
+
"""Export the configuration file as a dictionary."""
|
|
439
|
+
if merged:
|
|
440
|
+
dico = dict()
|
|
441
|
+
else:
|
|
442
|
+
dico = dict(defaults=dict(self.defaults()))
|
|
443
|
+
for section in self.sections():
|
|
444
|
+
if merged:
|
|
445
|
+
dico[section] = dict(self.items(section))
|
|
446
|
+
else:
|
|
447
|
+
dico[section] = {k: v for k, v in self.items(section)
|
|
448
|
+
if k in self.parser._sections[section]}
|
|
449
|
+
return dico
|
|
450
|
+
|
|
451
|
+
def __getattr__(self, attr):
|
|
452
|
+
# Give access to a very limited set of methods
|
|
453
|
+
if attr.startswith('get') or attr in ('defaults', 'sections', 'options', 'items',
|
|
454
|
+
'has_section', 'has_option'):
|
|
455
|
+
return getattr(self.parser, attr)
|
|
456
|
+
else:
|
|
457
|
+
raise AttributeError(self.__class__.__name__ + " instance has no attribute '" +
|
|
458
|
+
str(attr) + "'")
|
|
459
|
+
|
|
460
|
+
def footprint_export(self):
|
|
461
|
+
return self.file
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
class ExtendedReadOnlyConfigParser(GenericReadOnlyConfigParser):
|
|
465
|
+
"""A ReadOnly configuration file parser with a nice inheritance feature.
|
|
466
|
+
|
|
467
|
+
Using this readonly configuration parser, a section can inherit from one or
|
|
468
|
+
several other sections. The basic interpolation (with the usual ``%(varname)s``
|
|
469
|
+
syntax) is available.
|
|
470
|
+
|
|
471
|
+
It relies on a :class:`ConfigParser.ConfigParser` parser (or another class
|
|
472
|
+
that satisfies the interface) to access the configuration data.
|
|
473
|
+
|
|
474
|
+
:param str inifile: Path to a configuration file or a configuration file name
|
|
475
|
+
:param ConfigParser.ConfigParser parser: an existing configuration parser
|
|
476
|
+
object the will be used to access the configuration
|
|
477
|
+
:param bool mkforce: If the configuration file doesn't exists. Create an empty
|
|
478
|
+
one in ``~/.vortexrc``
|
|
479
|
+
:param type clsparser: The class that will be used to create a parser object
|
|
480
|
+
(if needed)
|
|
481
|
+
|
|
482
|
+
:example: Here is an example using the inheritance mechanism. Let's consider
|
|
483
|
+
the following section declaration::
|
|
484
|
+
|
|
485
|
+
[newsection:base1:base2]
|
|
486
|
+
var1=...
|
|
487
|
+
|
|
488
|
+
``newsection`` will inherit the variables contained in sections ``base1``
|
|
489
|
+
and ``base2``. In case of a conflict, ``base1`` takes precedence over ``base2``.
|
|
490
|
+
"""
|
|
491
|
+
|
|
492
|
+
_RE_VALIDATE = re.compile(r'([\w-]+)[ \t]*:?')
|
|
493
|
+
_RE_KEYC = re.compile(r"%\(([^)]+)\)s")
|
|
494
|
+
|
|
495
|
+
_max_interpolation_depth = 20
|
|
496
|
+
|
|
497
|
+
def _get_section_list(self, zend_section):
|
|
498
|
+
"""
|
|
499
|
+
Return the stack of sections that will be used to look for a given
|
|
500
|
+
variable. Somehow, it is close to python's MRO.
|
|
501
|
+
"""
|
|
502
|
+
found_sections = []
|
|
503
|
+
if self.parser.has_section(zend_section):
|
|
504
|
+
found_sections.append(zend_section)
|
|
505
|
+
for section in self.parser.sections():
|
|
506
|
+
pieces = re.split(r'[ \t]*:[ \t]*', section)
|
|
507
|
+
if len(pieces) >= 2 and pieces[0] == zend_section:
|
|
508
|
+
found_sections.append(section)
|
|
509
|
+
for inherited in pieces[1:]:
|
|
510
|
+
found_sections.extend(self._get_section_list(inherited))
|
|
511
|
+
break
|
|
512
|
+
return found_sections
|
|
513
|
+
|
|
514
|
+
def _interpolate(self, section, rawval):
|
|
515
|
+
"""Performs the basic interpolation."""
|
|
516
|
+
value = rawval
|
|
517
|
+
depth = self._max_interpolation_depth
|
|
518
|
+
|
|
519
|
+
def _interpolation_replace(match):
|
|
520
|
+
s = match.group(1)
|
|
521
|
+
return self.get(section, self.parser.optionxform(s), raw=False)
|
|
522
|
+
|
|
523
|
+
while depth: # Loop through this until it's done
|
|
524
|
+
depth -= 1
|
|
525
|
+
if value and self._RE_KEYC.match(value):
|
|
526
|
+
value = self._RE_KEYC.sub(_interpolation_replace, value)
|
|
527
|
+
else:
|
|
528
|
+
break
|
|
529
|
+
if value and self._RE_KEYC.match(value):
|
|
530
|
+
raise InterpolationDepthError(self.options(section), section, rawval)
|
|
531
|
+
return value
|
|
532
|
+
|
|
533
|
+
def get(self, section, option, raw=False, myvars=None):
|
|
534
|
+
"""Behaves like the GenericConfigParser's ``get`` method."""
|
|
535
|
+
expanded = [s for s in self._get_section_list(section) if s is not None]
|
|
536
|
+
if not expanded:
|
|
537
|
+
raise NoSectionError(section)
|
|
538
|
+
expanded.reverse()
|
|
539
|
+
acc_result = None
|
|
540
|
+
acc_except = None
|
|
541
|
+
mydefault = self.defaults().get(option, None)
|
|
542
|
+
for isection in expanded:
|
|
543
|
+
try:
|
|
544
|
+
tmp_result = self.parser.get(isection, option, raw=True, vars=myvars)
|
|
545
|
+
if tmp_result is not mydefault:
|
|
546
|
+
acc_result = tmp_result
|
|
547
|
+
except NoOptionError as err:
|
|
548
|
+
acc_except = err
|
|
549
|
+
if acc_result is None and mydefault is not None:
|
|
550
|
+
acc_result = mydefault
|
|
551
|
+
if acc_result is not None:
|
|
552
|
+
if not raw:
|
|
553
|
+
acc_result = self._interpolate(section, acc_result)
|
|
554
|
+
return acc_result
|
|
555
|
+
else:
|
|
556
|
+
raise acc_except
|
|
557
|
+
|
|
558
|
+
def sections(self):
|
|
559
|
+
"""Behaves like the Python ConfigParser's ``section`` method."""
|
|
560
|
+
seen = set()
|
|
561
|
+
for section_m in [self._RE_VALIDATE.match(s) for s in self.parser.sections()]:
|
|
562
|
+
if section_m is not None:
|
|
563
|
+
seen.add(section_m.group(1))
|
|
564
|
+
return list(seen)
|
|
565
|
+
|
|
566
|
+
def has_section(self, section):
|
|
567
|
+
"""Return whether a section exists or not."""
|
|
568
|
+
return section in self.sections()
|
|
569
|
+
|
|
570
|
+
def options(self, section):
|
|
571
|
+
"""Behaves like the Python ConfigParser's ``options`` method."""
|
|
572
|
+
expanded = self._get_section_list(section)
|
|
573
|
+
if not expanded:
|
|
574
|
+
return self.parser.options(section) # A realistic exception will be thrown !
|
|
575
|
+
options = set()
|
|
576
|
+
for isection in [s for s in expanded]:
|
|
577
|
+
options.update(set(self.parser.options(isection)))
|
|
578
|
+
return list(options)
|
|
579
|
+
|
|
580
|
+
def has_option(self, section, option):
|
|
581
|
+
"""Return whether an option exists or not."""
|
|
582
|
+
return option in self.options(section)
|
|
583
|
+
|
|
584
|
+
def items(self, section, raw=False, myvars=None):
|
|
585
|
+
"""Behaves like the Python ConfigParser's ``items`` method."""
|
|
586
|
+
return [(o, self.get(section, o, raw, myvars)) for o in self.options(section)]
|
|
587
|
+
|
|
588
|
+
def __getattr__(self, attr):
|
|
589
|
+
# Give access to a very limited set of methods
|
|
590
|
+
if attr in ('defaults',):
|
|
591
|
+
return getattr(self.parser, attr)
|
|
592
|
+
else:
|
|
593
|
+
raise AttributeError(self.__class__.__name__ + " instance has no attribute '" +
|
|
594
|
+
str(attr) + "'")
|
|
595
|
+
|
|
596
|
+
def as_dict(self, merged=True):
|
|
597
|
+
"""Export the configuration file as a dictionary."""
|
|
598
|
+
if not merged:
|
|
599
|
+
raise ValueError("merged=False is not allowed with ExtendedReadOnlyConfigParser.")
|
|
600
|
+
return super().as_dict(merged=True)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
class GenericConfigParser(GenericReadOnlyConfigParser):
|
|
604
|
+
"""A Basic Read/Write configuration file parser.
|
|
605
|
+
|
|
606
|
+
It relies on a :class:`ConfigParser.ConfigParser` parser (or another class
|
|
607
|
+
that satisfies the interface) to access the configuration data.
|
|
608
|
+
|
|
609
|
+
:param str inifile: Path to a configuration file or a configuration file name
|
|
610
|
+
:param ConfigParser.ConfigParser parser: an existing configuration parser
|
|
611
|
+
object the will be used to access the configuration
|
|
612
|
+
:param bool mkforce: If the configuration file doesn't exists. Create an empty
|
|
613
|
+
one in ``~/.vortexrc``
|
|
614
|
+
:param type clsparser: The class that will be used to create a parser object
|
|
615
|
+
(if needed)
|
|
616
|
+
:param str encoding: The configuration file encoding
|
|
617
|
+
:param str defaultinifile: The name of a default ini file (read before,
|
|
618
|
+
and possibly overwritten by **inifile**)
|
|
619
|
+
|
|
620
|
+
:note: All of the parser's methods are directly accessible because ``__getattr__``
|
|
621
|
+
is implemented. The user will refer to the Python's ConfigParser module
|
|
622
|
+
documentation for more details.
|
|
623
|
+
"""
|
|
624
|
+
|
|
625
|
+
def __init__(self, inifile=None, parser=None, mkforce=False,
|
|
626
|
+
clsparser=_DEFAULT_CONFIG_PARSER, encoding=None, defaultinifile=None):
|
|
627
|
+
super().__init__(inifile, parser, mkforce, clsparser, encoding, defaultinifile)
|
|
628
|
+
self.updates = list()
|
|
629
|
+
|
|
630
|
+
def setall(self, kw):
|
|
631
|
+
"""Define in all sections the couples of ( key, values ) given as dictionary argument."""
|
|
632
|
+
self.updates.append(kw)
|
|
633
|
+
for section in self.sections():
|
|
634
|
+
for key, value in kw.items():
|
|
635
|
+
self.set(section, key, str(value))
|
|
636
|
+
|
|
637
|
+
def save(self):
|
|
638
|
+
"""Write the current state of the configuration in the inital file."""
|
|
639
|
+
with open(self.file.split(",").pop(), 'wb') as configfile:
|
|
640
|
+
self.write(configfile)
|
|
641
|
+
|
|
642
|
+
@property
|
|
643
|
+
def updated(self):
|
|
644
|
+
"""Return if this configuration has been updated or not."""
|
|
645
|
+
return bool(self.updates)
|
|
646
|
+
|
|
647
|
+
def history(self):
|
|
648
|
+
"""Return a list of the description for each update performed."""
|
|
649
|
+
return self.updates[:]
|
|
650
|
+
|
|
651
|
+
def __getattr__(self, attr):
|
|
652
|
+
# Give access to all of the parser's methods
|
|
653
|
+
if attr.startswith('__'):
|
|
654
|
+
raise AttributeError(self.__class__.__name__ + " instance has no attribute '" +
|
|
655
|
+
str(attr) + "'")
|
|
656
|
+
return getattr(self.parser, attr)
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
class DelayedConfigParser(GenericConfigParser):
|
|
660
|
+
"""Configuration file parser with possible delayed loading.
|
|
661
|
+
|
|
662
|
+
:param str inifile: Path to a configuration file or a configuration file name
|
|
663
|
+
|
|
664
|
+
:note: All of the parser's methods are directly accessible because ``__getattr__``
|
|
665
|
+
is implemented. The user will refer to the Python's ConfigParser module
|
|
666
|
+
documentation for more details.
|
|
667
|
+
"""
|
|
668
|
+
|
|
669
|
+
def __init__(self, inifile=None):
|
|
670
|
+
GenericConfigParser.__init__(self)
|
|
671
|
+
self.delay = inifile
|
|
672
|
+
|
|
673
|
+
def refresh(self):
|
|
674
|
+
"""Load the delayed inifile."""
|
|
675
|
+
if self.delay:
|
|
676
|
+
self.setfile(self.delay)
|
|
677
|
+
self.delay = None
|
|
678
|
+
|
|
679
|
+
def __getattribute__(self, attr):
|
|
680
|
+
try:
|
|
681
|
+
logger.debug('Getattr %s < %s >', attr, self)
|
|
682
|
+
if attr in filter(lambda x: not x.startswith('_'),
|
|
683
|
+
dir(_DEFAULT_CONFIG_PARSER) + ['setall', 'save']):
|
|
684
|
+
object.__getattribute__(self, 'refresh')()
|
|
685
|
+
except Exception:
|
|
686
|
+
logger.critical('Trouble getattr %s < %s >', attr, self)
|
|
687
|
+
return object.__getattribute__(self, attr)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
class JacketConfigParser(GenericConfigParser):
|
|
691
|
+
"""Configuration parser for Jacket files.
|
|
692
|
+
|
|
693
|
+
:param str inifile: Path to a configuration file or a configuration file name
|
|
694
|
+
:param ConfigParser.ConfigParser parser: an existing configuration parser
|
|
695
|
+
object the will be used to access the configuration
|
|
696
|
+
:param bool mkforce: If the configuration file doesn't exists. Create an empty
|
|
697
|
+
one in ``~/.vortexrc``
|
|
698
|
+
:param type clsparser: The class that will be used to create a parser object
|
|
699
|
+
(if needed)
|
|
700
|
+
|
|
701
|
+
:note: All of the parser's methods are directly accessible because ``__getattr__``
|
|
702
|
+
is implemented. The user will refer to the Python's ConfigParser module
|
|
703
|
+
documentation for more details.
|
|
704
|
+
"""
|
|
705
|
+
|
|
706
|
+
def get(self, section, option):
|
|
707
|
+
"""
|
|
708
|
+
Return for the specified ``option`` in the ``section`` a sequence of values
|
|
709
|
+
build on the basis of a comma separated list.
|
|
710
|
+
"""
|
|
711
|
+
s = _DEFAULT_CONFIG_PARSER.get(self, section, option)
|
|
712
|
+
tmplist = s.replace(' ', '').split(',')
|
|
713
|
+
if len(tmplist) > 1:
|
|
714
|
+
return tmplist
|
|
715
|
+
else:
|
|
716
|
+
return tmplist[0]
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
class AppConfigStringDecoder(StringDecoder):
|
|
720
|
+
"""Convert a string from a configuration file into a proper Python's object.
|
|
721
|
+
|
|
722
|
+
See the :class:`StringDecoder` class documentation for a complete description
|
|
723
|
+
the configuration string's syntax.
|
|
724
|
+
|
|
725
|
+
This class extends the :class:`StringDecoder` as follow:
|
|
726
|
+
|
|
727
|
+
* It's possible to convert (i.e. remap) configuration lines to Vortex's
|
|
728
|
+
geometries: ``geometry(geo_tagname)``
|
|
729
|
+
* The :func:`footprints.util.rangex` can be called to generate a list:
|
|
730
|
+
``dict(production:rangex(0-6-1) assim:rangex(0-3-1))`` will generate
|
|
731
|
+
the following object ``{u'assim': [0, 1, 2, 3], u'production': [0, 1, 2, 3, 4, 5, 6]}``
|
|
732
|
+
* It is possible to create an object using the *iniconf* footprint's collector:
|
|
733
|
+
``'iniconf(family:pollutants kind:elements version:std)'`` will generate
|
|
734
|
+
the following object ``<intairpol.data.elements.PollutantsElementsTable at 0x...>``
|
|
735
|
+
(provided that the :mod:`intairpol` package has been imported).
|
|
736
|
+
* It is possible to create an object using the *conftools* footprint's collector
|
|
737
|
+
(following the previous example's syntax).
|
|
738
|
+
|
|
739
|
+
"""
|
|
740
|
+
|
|
741
|
+
BUILDERS = StringDecoder.BUILDERS + ['geometry', 'date', 'time',
|
|
742
|
+
'rangex', 'daterangex',
|
|
743
|
+
'iniconf', 'conftool']
|
|
744
|
+
|
|
745
|
+
def remap_geometry(self, value):
|
|
746
|
+
"""Convert all values to Geometry objects."""
|
|
747
|
+
from vortex.data import geometries
|
|
748
|
+
try:
|
|
749
|
+
value = geometries.get(tag=value)
|
|
750
|
+
except ValueError:
|
|
751
|
+
pass
|
|
752
|
+
return value
|
|
753
|
+
|
|
754
|
+
def remap_date(self, value):
|
|
755
|
+
"""Convert all values to bronx' Date objects."""
|
|
756
|
+
try:
|
|
757
|
+
value = bdate.Date(value)
|
|
758
|
+
except (ValueError, TypeError):
|
|
759
|
+
pass
|
|
760
|
+
return value
|
|
761
|
+
|
|
762
|
+
def remap_time(self, value):
|
|
763
|
+
"""Convert all values to bronx' Time objects."""
|
|
764
|
+
try:
|
|
765
|
+
value = bdate.Time(value)
|
|
766
|
+
except (ValueError, TypeError):
|
|
767
|
+
pass
|
|
768
|
+
return value
|
|
769
|
+
|
|
770
|
+
def _build_geometry(self, value, remap, subs):
|
|
771
|
+
val = self._value_expand(value, remap, subs)
|
|
772
|
+
from vortex.data import geometries
|
|
773
|
+
return geometries.get(tag=val)
|
|
774
|
+
|
|
775
|
+
def _build_date(self, value, remap, subs):
|
|
776
|
+
val = self._value_expand(value, remap, subs)
|
|
777
|
+
return bdate.Date(val)
|
|
778
|
+
|
|
779
|
+
def _build_time(self, value, remap, subs):
|
|
780
|
+
val = self._value_expand(value, remap, subs)
|
|
781
|
+
return bdate.Time(val)
|
|
782
|
+
|
|
783
|
+
def _build_generic_rangex(self, cb, value, remap, subs):
|
|
784
|
+
"""Build a rangex or daterangex from the **value** string."""
|
|
785
|
+
# Try to read names arguments
|
|
786
|
+
try:
|
|
787
|
+
values = self._sparser(value, itemsep=' ', keysep=':')
|
|
788
|
+
if all([k in ('start', 'end', 'step', 'shift', 'fmt', 'prefix')
|
|
789
|
+
for k in values.keys()]):
|
|
790
|
+
return cb(**{k: self._value_expand(v, remap, subs)
|
|
791
|
+
for k, v in values.items()})
|
|
792
|
+
except StringDecoderSyntaxError:
|
|
793
|
+
pass
|
|
794
|
+
# The usual case...
|
|
795
|
+
return cb([self._value_expand(v, remap, subs)
|
|
796
|
+
for v in self._sparser(value, itemsep=',')])
|
|
797
|
+
|
|
798
|
+
def _build_rangex(self, value, remap, subs):
|
|
799
|
+
"""Build a rangex from the **value** string."""
|
|
800
|
+
return self._build_generic_rangex(bdate.timeintrangex, value, remap, subs)
|
|
801
|
+
|
|
802
|
+
def _build_daterangex(self, value, remap, subs):
|
|
803
|
+
"""Build a daterangex from the **value** string."""
|
|
804
|
+
return self._build_generic_rangex(bdate.daterangex, value, remap, subs)
|
|
805
|
+
|
|
806
|
+
def _build_fpgeneric(self, value, remap, subs, collector):
|
|
807
|
+
fp = {k: self._value_expand(v, remap, subs)
|
|
808
|
+
for k, v in self._sparser(value, itemsep=' ', keysep=':').items()}
|
|
809
|
+
obj = footprints.collectors.get(tag=collector).load(**fp)
|
|
810
|
+
if obj is None:
|
|
811
|
+
raise StringDecoderSyntaxError(value,
|
|
812
|
+
'No object could be created from the {} collector'.
|
|
813
|
+
format(collector))
|
|
814
|
+
return obj
|
|
815
|
+
|
|
816
|
+
def _build_iniconf(self, value, remap, subs):
|
|
817
|
+
return self._build_fpgeneric(value, remap, subs, 'iniconf')
|
|
818
|
+
|
|
819
|
+
def _build_conftool(self, value, remap, subs):
|
|
820
|
+
return self._build_fpgeneric(value, remap, subs, 'conftool')
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
class IniConf(footprints.FootprintBase):
|
|
824
|
+
"""
|
|
825
|
+
Generic Python configuration file.
|
|
826
|
+
"""
|
|
827
|
+
_collector = ('iniconf',)
|
|
828
|
+
_abstract = True
|
|
829
|
+
_footprint = dict(
|
|
830
|
+
info='Abstract Python Inifile',
|
|
831
|
+
attr=dict(
|
|
832
|
+
kind = dict(
|
|
833
|
+
info = "The configuration object kind.",
|
|
834
|
+
values = ['generic', ],
|
|
835
|
+
),
|
|
836
|
+
clsconfig = dict(
|
|
837
|
+
type = GenericReadOnlyConfigParser,
|
|
838
|
+
isclass = True,
|
|
839
|
+
optional = True,
|
|
840
|
+
default = GenericReadOnlyConfigParser,
|
|
841
|
+
doc_visibility = footprints.doc.visibility.ADVANCED,
|
|
842
|
+
),
|
|
843
|
+
inifile = dict(
|
|
844
|
+
kind = 'The configuration file to look for.',
|
|
845
|
+
optional = True,
|
|
846
|
+
default = '@[kind].ini',
|
|
847
|
+
),
|
|
848
|
+
)
|
|
849
|
+
)
|
|
850
|
+
|
|
851
|
+
def __init__(self, *args, **kw):
|
|
852
|
+
logger.debug('Ini Conf %s', self.__class__)
|
|
853
|
+
super().__init__(*args, **kw)
|
|
854
|
+
self._config = self.clsconfig(inifile=self.inifile)
|
|
855
|
+
|
|
856
|
+
@property
|
|
857
|
+
def config(self):
|
|
858
|
+
return self._config
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
class ConfigurationTable(IniConf):
|
|
862
|
+
"""
|
|
863
|
+
A specialised version of :class:`IniConf` that automatically create a list of
|
|
864
|
+
items (instantiated from the tableitem footprint's collector) from a given
|
|
865
|
+
configuration file.
|
|
866
|
+
"""
|
|
867
|
+
_abstract = True
|
|
868
|
+
_footprint = dict(
|
|
869
|
+
info = 'Abstract configuration tables',
|
|
870
|
+
attr = dict(
|
|
871
|
+
kind = dict(
|
|
872
|
+
info = "The configuration's table kind.",
|
|
873
|
+
),
|
|
874
|
+
family = dict(
|
|
875
|
+
info = "The configuration's table family.",
|
|
876
|
+
),
|
|
877
|
+
version = dict(
|
|
878
|
+
info = "The configuration's table version.",
|
|
879
|
+
optional = True,
|
|
880
|
+
default = 'std',
|
|
881
|
+
),
|
|
882
|
+
searchkeys = dict(
|
|
883
|
+
info = "Item's attributes used to perform the lookup in the find method.",
|
|
884
|
+
type = footprints.FPTuple,
|
|
885
|
+
optional = True,
|
|
886
|
+
default = footprints.FPTuple(),
|
|
887
|
+
),
|
|
888
|
+
groupname = dict(
|
|
889
|
+
info = "The class attribute matching the configuration file groupname",
|
|
890
|
+
optional = True,
|
|
891
|
+
default = 'family',
|
|
892
|
+
),
|
|
893
|
+
inifile = dict(
|
|
894
|
+
optional = True,
|
|
895
|
+
default = '@[family]-[kind]-[version].ini',
|
|
896
|
+
),
|
|
897
|
+
clsconfig = dict(
|
|
898
|
+
default = ExtendedReadOnlyConfigParser,
|
|
899
|
+
),
|
|
900
|
+
language = dict(
|
|
901
|
+
info = "The default language for the translator property.",
|
|
902
|
+
optional = True,
|
|
903
|
+
default = 'en',
|
|
904
|
+
),
|
|
905
|
+
)
|
|
906
|
+
)
|
|
907
|
+
|
|
908
|
+
@property
|
|
909
|
+
def realkind(self):
|
|
910
|
+
return 'configuration-table'
|
|
911
|
+
|
|
912
|
+
def groups(self):
|
|
913
|
+
"""Actual list of items groups described in the current iniconf."""
|
|
914
|
+
return [x for x in self.config.parser.sections()
|
|
915
|
+
if ':' not in x and not x.startswith('lang_')]
|
|
916
|
+
|
|
917
|
+
def keys(self):
|
|
918
|
+
"""Actual list of different items in the current iniconf."""
|
|
919
|
+
return [x for x in self.config.sections()
|
|
920
|
+
if x not in self.groups() and not x.startswith('lang_')]
|
|
921
|
+
|
|
922
|
+
@property
|
|
923
|
+
def translator(self):
|
|
924
|
+
"""The special section of the iniconf dedicated to translation, as a dict."""
|
|
925
|
+
if not hasattr(self, '_translator'):
|
|
926
|
+
if self.config.has_section('lang_' + self.language):
|
|
927
|
+
self._translator = self.config.as_dict()['lang_' + self.language]
|
|
928
|
+
else:
|
|
929
|
+
self._translator = None
|
|
930
|
+
return self._translator
|
|
931
|
+
|
|
932
|
+
@property
|
|
933
|
+
def tablelist(self):
|
|
934
|
+
"""List of unique instances of items described in the current iniconf."""
|
|
935
|
+
if not hasattr(self, '_tablelist'):
|
|
936
|
+
self._tablelist = list()
|
|
937
|
+
d = self.config.as_dict()
|
|
938
|
+
for item, group in [x.split(':') for x in self.config.parser.sections() if ':' in x]:
|
|
939
|
+
try:
|
|
940
|
+
for k, v in d[item].items():
|
|
941
|
+
# Can occur in case of a redundant entry in the config file
|
|
942
|
+
if isinstance(v, str) and v:
|
|
943
|
+
if re.match('none$', v, re.IGNORECASE):
|
|
944
|
+
d[item][k] = None
|
|
945
|
+
if re.search('[a-z]_[a-z]', v, re.IGNORECASE):
|
|
946
|
+
d[item][k] = v.replace('_', "'")
|
|
947
|
+
d[item][self.searchkeys[0]] = item
|
|
948
|
+
d[item][self.groupname] = group
|
|
949
|
+
d[item]['translator'] = self.translator
|
|
950
|
+
itemobj = footprints.proxy.tableitem(**d[item])
|
|
951
|
+
if itemobj is not None:
|
|
952
|
+
self._tablelist.append(itemobj)
|
|
953
|
+
else:
|
|
954
|
+
logger.error("Unable to create the %s item object. Check the footprint !", item)
|
|
955
|
+
except (KeyError, IndexError):
|
|
956
|
+
logger.warning('Some item description could not match')
|
|
957
|
+
return self._tablelist
|
|
958
|
+
|
|
959
|
+
def get(self, item):
|
|
960
|
+
"""Return the item with main key exactly matching the given argument."""
|
|
961
|
+
candidates = [x for x in self.tablelist
|
|
962
|
+
if x.footprint_getattr(self.searchkeys[0]) == item]
|
|
963
|
+
if candidates:
|
|
964
|
+
return candidates[0]
|
|
965
|
+
else:
|
|
966
|
+
return None
|
|
967
|
+
|
|
968
|
+
def match(self, item):
|
|
969
|
+
"""Return the item with main key matching the given argument without case consideration."""
|
|
970
|
+
candidates = [x for x in self.tablelist
|
|
971
|
+
if x.footprint_getattr(self.searchkeys[0]).lower().startswith(item.lower())]
|
|
972
|
+
if candidates:
|
|
973
|
+
return candidates[0]
|
|
974
|
+
else:
|
|
975
|
+
return None
|
|
976
|
+
|
|
977
|
+
def grep(self, item):
|
|
978
|
+
"""Return a list of items with main key loosely matching the given argument."""
|
|
979
|
+
return [x for x in self.tablelist
|
|
980
|
+
if re.search(item, x.footprint_getattr(self.searchkeys[0]), re.IGNORECASE)]
|
|
981
|
+
|
|
982
|
+
def find(self, item):
|
|
983
|
+
"""Return a list of items with main key or name loosely matching the given argument."""
|
|
984
|
+
return [x for x in self.tablelist
|
|
985
|
+
if any([re.search(item, x.footprint_getattr(thiskey), re.IGNORECASE)
|
|
986
|
+
for thiskey in self.searchkeys])]
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
class TableItem(footprints.FootprintBase):
|
|
990
|
+
"""
|
|
991
|
+
Abstract configuration table's item.
|
|
992
|
+
"""
|
|
993
|
+
|
|
994
|
+
#: Attribute describing the item's name during RST exports
|
|
995
|
+
_RST_NAME = ''
|
|
996
|
+
#: Attributes that will appear on the top line of RST exports
|
|
997
|
+
_RST_HOTKEYS = []
|
|
998
|
+
|
|
999
|
+
_abstract = True
|
|
1000
|
+
_collector = ('tableitem',)
|
|
1001
|
+
_footprint = dict(
|
|
1002
|
+
info = "Abstract configuration table's item.",
|
|
1003
|
+
attr = dict(
|
|
1004
|
+
# Define your own...
|
|
1005
|
+
translator = dict(
|
|
1006
|
+
optional = True,
|
|
1007
|
+
type = footprints.FPDict,
|
|
1008
|
+
default = None,
|
|
1009
|
+
),
|
|
1010
|
+
)
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
@property
|
|
1014
|
+
def realkind(self):
|
|
1015
|
+
return 'tableitem'
|
|
1016
|
+
|
|
1017
|
+
def _translated_items(self, mkshort=True):
|
|
1018
|
+
"""Returns a list of 3-elements tuples describing the item attributes.
|
|
1019
|
+
|
|
1020
|
+
[(translated_key, value, original_key), ...]
|
|
1021
|
+
"""
|
|
1022
|
+
output_stack = list()
|
|
1023
|
+
if self.translator:
|
|
1024
|
+
for k in self.translator.get('ordered_dump', '').split(','):
|
|
1025
|
+
if not mkshort or self.footprint_getattr(k) is not None:
|
|
1026
|
+
output_stack.append((self.translator.get(k, k.replace('_', ' ').title()),
|
|
1027
|
+
str(self.footprint_getattr(k)), k))
|
|
1028
|
+
else:
|
|
1029
|
+
for k in self.footprint_attributes:
|
|
1030
|
+
if ((not mkshort or self.footprint_getattr(k) is not None) and k != 'translator'):
|
|
1031
|
+
output_stack.append((k, str(self.footprint_getattr(k)), k))
|
|
1032
|
+
return output_stack
|
|
1033
|
+
|
|
1034
|
+
def nice_str(self, mkshort=True):
|
|
1035
|
+
"""Produces a nice ordered representation of the item attributes."""
|
|
1036
|
+
output_stack = self._translated_items(mkshort=mkshort)
|
|
1037
|
+
output_list = []
|
|
1038
|
+
if output_stack:
|
|
1039
|
+
max_keylen = max([len(i[0]) for i in output_stack])
|
|
1040
|
+
print_fmt = '{0:' + str(max_keylen) + 's} : {1:s}'
|
|
1041
|
+
for item in output_stack:
|
|
1042
|
+
output_list.append(print_fmt.format(*item))
|
|
1043
|
+
return '\n'.join(output_list)
|
|
1044
|
+
|
|
1045
|
+
def __str__(self):
|
|
1046
|
+
return self.nice_str()
|
|
1047
|
+
|
|
1048
|
+
def nice_print(self, mkshort=True):
|
|
1049
|
+
"""Print a nice ordered output of the item attributes."""
|
|
1050
|
+
print(self.nice_str(mkshort=mkshort))
|
|
1051
|
+
|
|
1052
|
+
def nice_rst(self, mkshort=True):
|
|
1053
|
+
"""Produces a nice ordered RST output of the item attributes."""
|
|
1054
|
+
assert self._RST_NAME, "Please override _RST_NAME"
|
|
1055
|
+
output_stack = self._translated_items(mkshort=mkshort)
|
|
1056
|
+
i_name = '????'
|
|
1057
|
+
i_hot = []
|
|
1058
|
+
i_other = []
|
|
1059
|
+
for item in output_stack:
|
|
1060
|
+
if item[2] == self._RST_NAME:
|
|
1061
|
+
i_name = item
|
|
1062
|
+
elif item[2] in self._RST_HOTKEYS:
|
|
1063
|
+
i_hot.append(item)
|
|
1064
|
+
else:
|
|
1065
|
+
i_other.append(item)
|
|
1066
|
+
return '**{}** : `{}`\n\n{}\n\n'.format(i_name[1],
|
|
1067
|
+
', '.join(['{:s}={:s}'.format(*i)
|
|
1068
|
+
for i in i_hot]),
|
|
1069
|
+
'\n'.join([' * {:s}: {:s}'.format(*i)
|
|
1070
|
+
for i in i_other])
|
|
1071
|
+
)
|