fastlisaresponse 1.1.14__cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fastlisaresponse/__init__.py +93 -0
- fastlisaresponse/_version.py +34 -0
- fastlisaresponse/cutils/__init__.py +141 -0
- fastlisaresponse/git_version.py +7 -0
- fastlisaresponse/git_version.py.in +7 -0
- fastlisaresponse/response.py +813 -0
- fastlisaresponse/utils/__init__.py +1 -0
- fastlisaresponse/utils/citations.py +356 -0
- fastlisaresponse/utils/config.py +793 -0
- fastlisaresponse/utils/exceptions.py +95 -0
- fastlisaresponse/utils/parallelbase.py +11 -0
- fastlisaresponse/utils/utility.py +82 -0
- fastlisaresponse-1.1.14.dist-info/METADATA +166 -0
- fastlisaresponse-1.1.14.dist-info/RECORD +17 -0
- fastlisaresponse-1.1.14.dist-info/WHEEL +6 -0
- fastlisaresponse_backend_cpu/git_version.py +7 -0
- fastlisaresponse_backend_cpu/responselisa.cpython-311-x86_64-linux-gnu.so +0 -0
|
@@ -0,0 +1,793 @@
|
|
|
1
|
+
"""Implementation of a centralized configuration management for GBGPU."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import abc
|
|
6
|
+
import argparse
|
|
7
|
+
import dataclasses
|
|
8
|
+
import enum
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
import pathlib
|
|
12
|
+
from typing import (
|
|
13
|
+
Any,
|
|
14
|
+
Callable,
|
|
15
|
+
Dict,
|
|
16
|
+
Generic,
|
|
17
|
+
List,
|
|
18
|
+
Mapping,
|
|
19
|
+
Optional,
|
|
20
|
+
Sequence,
|
|
21
|
+
Tuple,
|
|
22
|
+
TypeVar,
|
|
23
|
+
Union,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
from ..cutils import KNOWN_BACKENDS
|
|
27
|
+
from . import exceptions
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class ConfigSource(enum.Enum):
|
|
31
|
+
"""Enumeration of config option sources."""
|
|
32
|
+
|
|
33
|
+
DEFAULT = "default"
|
|
34
|
+
"""Config value comes from its default value"""
|
|
35
|
+
|
|
36
|
+
CFGFILE = "config_file"
|
|
37
|
+
"""Config value comes from the configuration file"""
|
|
38
|
+
|
|
39
|
+
ENVVAR = "environment_var"
|
|
40
|
+
"""Config value comes from environment variable"""
|
|
41
|
+
|
|
42
|
+
CLIOPT = "command_line"
|
|
43
|
+
"""Config value comes from command line parameter"""
|
|
44
|
+
|
|
45
|
+
SETTER = "setter"
|
|
46
|
+
"""Config value set by config setter after importing GBGPU"""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
T = TypeVar("T")
|
|
50
|
+
|
|
51
|
+
ENVVAR_PREFIX: str = "GBGPU_"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclasses.dataclass
|
|
55
|
+
class ConfigEntry(Generic[T]):
|
|
56
|
+
"""Description of a configuration entry."""
|
|
57
|
+
|
|
58
|
+
label: str
|
|
59
|
+
"""How the entry is referred to in Python code (config.get("label"))"""
|
|
60
|
+
description: str
|
|
61
|
+
"""Description of the configuration entry"""
|
|
62
|
+
|
|
63
|
+
type: TypeVar = T
|
|
64
|
+
"""Type of the value"""
|
|
65
|
+
|
|
66
|
+
default: Optional[T] = None
|
|
67
|
+
"""Default value"""
|
|
68
|
+
|
|
69
|
+
cfg_entry: Optional[str] = None
|
|
70
|
+
"""Name of the entry in a config file"""
|
|
71
|
+
|
|
72
|
+
env_var: Optional[str] = None
|
|
73
|
+
"""Entry corresponding env var (uppercase, without GBGPU_ prefix)"""
|
|
74
|
+
|
|
75
|
+
cli_flags: Optional[Union[str, List[str]]] = None
|
|
76
|
+
"""Flag(s) for CLI arguments of this entry (e.g. "-f")"""
|
|
77
|
+
|
|
78
|
+
cli_kwargs: Optional[Dict[str, Any]] = None
|
|
79
|
+
"""Supplementary arguments to argparse add_argument method for CLI options"""
|
|
80
|
+
|
|
81
|
+
convert: Callable[[str], T] = None
|
|
82
|
+
"""Method used to convert a user input to expected type"""
|
|
83
|
+
|
|
84
|
+
validate: Callable[[T], bool] = lambda _: True
|
|
85
|
+
"""Method used to validate the provided option value"""
|
|
86
|
+
|
|
87
|
+
overwrite: Callable[[T, T], T] = lambda _, new: new
|
|
88
|
+
"""Method used to update the value if given by multiple means"""
|
|
89
|
+
|
|
90
|
+
def __post_init__(self):
|
|
91
|
+
if self.convert is None:
|
|
92
|
+
self.convert = lambda v: self.type(v)
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def compatibility_isinstance(obj, cls) -> bool:
|
|
96
|
+
import sys
|
|
97
|
+
import typing
|
|
98
|
+
|
|
99
|
+
if (sys.version_info >= (3, 10)) or (typing.get_origin(cls) is None):
|
|
100
|
+
try:
|
|
101
|
+
return isinstance(obj, cls)
|
|
102
|
+
except TypeError:
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
if typing.get_origin(cls) is typing.Union:
|
|
106
|
+
for arg in typing.get_args(cls):
|
|
107
|
+
if compatibility_isinstance(obj, arg):
|
|
108
|
+
return True
|
|
109
|
+
return False
|
|
110
|
+
|
|
111
|
+
if typing.get_origin(cls) is list:
|
|
112
|
+
if not isinstance(obj, list):
|
|
113
|
+
return False
|
|
114
|
+
for item in obj:
|
|
115
|
+
if not compatibility_isinstance(item, typing.get_args(cls)[0]):
|
|
116
|
+
return False
|
|
117
|
+
return True
|
|
118
|
+
|
|
119
|
+
import collections.abc
|
|
120
|
+
|
|
121
|
+
if typing.get_origin(cls) is collections.abc.Sequence:
|
|
122
|
+
if not hasattr(obj, "__iter__"):
|
|
123
|
+
return False
|
|
124
|
+
for item in obj:
|
|
125
|
+
if not compatibility_isinstance(item, typing.get_args(cls)[0]):
|
|
126
|
+
return False
|
|
127
|
+
return True
|
|
128
|
+
|
|
129
|
+
raise NotImplementedError(
|
|
130
|
+
"Compatiblity wrapper for isinstance on Python 3.9 does not support given type."
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclasses.dataclass
|
|
135
|
+
class ConfigItem(Generic[T]):
|
|
136
|
+
"""Actual configuration entry with its run-time properties (value, source, ...)"""
|
|
137
|
+
|
|
138
|
+
value: T # Item value
|
|
139
|
+
source: ConfigSource # Source of the item current value
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class ConfigConsumer(abc.ABC):
|
|
143
|
+
"""
|
|
144
|
+
Base class for actual configs.
|
|
145
|
+
|
|
146
|
+
This class handles building config values from default, file, environment and CLI options.
|
|
147
|
+
It keeps a list of unused parameters for an other consumer.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
_entries: Dict[str, ConfigEntry]
|
|
151
|
+
_items: Dict[str, ConfigItem]
|
|
152
|
+
|
|
153
|
+
_extra_file: Dict[str, str]
|
|
154
|
+
_extra_env: Dict[str, str]
|
|
155
|
+
_extra_cli: List[str]
|
|
156
|
+
|
|
157
|
+
@classmethod
|
|
158
|
+
@abc.abstractmethod
|
|
159
|
+
def config_entries(cls) -> List[ConfigEntry]:
|
|
160
|
+
"""Return the list of the class config entries"""
|
|
161
|
+
raise NotImplementedError(
|
|
162
|
+
"A ConfigConsumer must implement 'config_entries' method."
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
def __init__(
|
|
166
|
+
self,
|
|
167
|
+
config_file: Union[os.PathLike, Mapping[str, str], None] = None,
|
|
168
|
+
env_vars: Optional[Mapping[str, str]] = None,
|
|
169
|
+
cli_args: Optional[Sequence[str]] = None,
|
|
170
|
+
set_args: Optional[Dict[str, Any]] = None,
|
|
171
|
+
):
|
|
172
|
+
"""Initialize the items list and extra parameters."""
|
|
173
|
+
config_entries = self.config_entries()
|
|
174
|
+
|
|
175
|
+
# Build the entries mapping
|
|
176
|
+
self._entries = {entry.label: entry for entry in config_entries}
|
|
177
|
+
|
|
178
|
+
# Build default items
|
|
179
|
+
default_items = ConfigConsumer._build_items_from_default(config_entries)
|
|
180
|
+
|
|
181
|
+
# Retrieve option from sources
|
|
182
|
+
opt_from_file = ConfigConsumer._get_from_config_file(config_file)
|
|
183
|
+
opt_from_env = ConfigConsumer._get_from_envvars(env_vars)
|
|
184
|
+
opt_from_cli = ConfigConsumer._get_from_cli_args(cli_args)
|
|
185
|
+
|
|
186
|
+
# Consume options to build other item lists
|
|
187
|
+
file_items, self._extra_file = ConfigConsumer._build_items_from_file(
|
|
188
|
+
config_entries, opt_from_file
|
|
189
|
+
)
|
|
190
|
+
env_items, self._extra_env = ConfigConsumer._build_items_from_env(
|
|
191
|
+
config_entries, opt_from_env
|
|
192
|
+
)
|
|
193
|
+
cli_items, self._extra_cli = ConfigConsumer._build_items_from_cli(
|
|
194
|
+
config_entries, opt_from_cli
|
|
195
|
+
)
|
|
196
|
+
set_items = ConfigConsumer._build_items_from_set(config_entries, set_args)
|
|
197
|
+
|
|
198
|
+
# Build final item mapping
|
|
199
|
+
self._items = self._overwrite(
|
|
200
|
+
default_items, file_items, env_items, cli_items, set_items
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
# Validate items:
|
|
204
|
+
errors: List[Exception] = []
|
|
205
|
+
for label, entry in self._entries.items():
|
|
206
|
+
if label not in self._items:
|
|
207
|
+
errors.append(
|
|
208
|
+
exceptions.ConfigurationMissing(
|
|
209
|
+
"Configuration entry '{}' is missing.".format(label)
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
continue
|
|
213
|
+
item = self._items[label]
|
|
214
|
+
if not entry.validate(item.value):
|
|
215
|
+
errors.append(
|
|
216
|
+
exceptions.ConfigurationValidationError(
|
|
217
|
+
"Configuration entry '{}' has invalid value '{}'".format(
|
|
218
|
+
label, item.value
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
if errors:
|
|
224
|
+
raise (
|
|
225
|
+
errors[0]
|
|
226
|
+
if len(errors) == 1
|
|
227
|
+
else exceptions.ExceptionGroup(
|
|
228
|
+
"Invalid configuration due to previous issues.", errors
|
|
229
|
+
)
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
def _overwrite(
|
|
233
|
+
self,
|
|
234
|
+
old_items: Dict[str, ConfigItem],
|
|
235
|
+
new_items: Dict[str, ConfigItem],
|
|
236
|
+
*other_items,
|
|
237
|
+
) -> Dict[str, ConfigItem]:
|
|
238
|
+
merged_items = {}
|
|
239
|
+
for label, old_item in old_items.items():
|
|
240
|
+
if label not in new_items:
|
|
241
|
+
merged_items[label] = old_item
|
|
242
|
+
|
|
243
|
+
for label, new_item in new_items.items():
|
|
244
|
+
if label not in old_items:
|
|
245
|
+
merged_items[label] = new_item
|
|
246
|
+
continue
|
|
247
|
+
old_item = old_items[label]
|
|
248
|
+
entry = self._entries[label]
|
|
249
|
+
merged_items[label] = ConfigItem(
|
|
250
|
+
value=entry.overwrite(old_item.value, new_item.value),
|
|
251
|
+
source=new_item.source,
|
|
252
|
+
)
|
|
253
|
+
if len(other_items) > 0:
|
|
254
|
+
return self._overwrite(merged_items, *other_items)
|
|
255
|
+
return merged_items
|
|
256
|
+
|
|
257
|
+
def __getitem__(self, key: str) -> Any:
|
|
258
|
+
"""Get the value of a config entry."""
|
|
259
|
+
return self._items[key].value
|
|
260
|
+
|
|
261
|
+
def __getattr__(self, attr: str) -> Any:
|
|
262
|
+
"""Get the value of a config entry via attributes."""
|
|
263
|
+
if attr in self._items:
|
|
264
|
+
return self._items[attr].value
|
|
265
|
+
return self.__getattribute__(attr)
|
|
266
|
+
|
|
267
|
+
def get_item(self, key: str) -> Tuple[ConfigItem, ConfigEntry]:
|
|
268
|
+
return self._items[key], self._entries[key]
|
|
269
|
+
|
|
270
|
+
def get_items(self) -> List[Tuple[ConfigItem, ConfigEntry]]:
|
|
271
|
+
return [(self._items[key], entry) for key, entry in self._entries.items()]
|
|
272
|
+
|
|
273
|
+
def get_extras(self) -> Tuple[Mapping[str, str], Mapping[str, str], Sequence[str]]:
|
|
274
|
+
"""Return extra file, env and cli entries for other consumer."""
|
|
275
|
+
return self._extra_file, self._extra_env, self._extra_cli
|
|
276
|
+
|
|
277
|
+
@staticmethod
|
|
278
|
+
def _get_from_config_file(
|
|
279
|
+
config_file: Union[os.PathLike, Mapping[str, str], None],
|
|
280
|
+
) -> Dict[str, str]:
|
|
281
|
+
"""Read a config file and return its items as a dictionary."""
|
|
282
|
+
if config_file is None:
|
|
283
|
+
return {}
|
|
284
|
+
if isinstance(config_file, Mapping):
|
|
285
|
+
return {key: value for key, value in config_file.items()}
|
|
286
|
+
import configparser
|
|
287
|
+
|
|
288
|
+
config = configparser.ConfigParser()
|
|
289
|
+
config.read(config_file)
|
|
290
|
+
if "gbgpu" not in config:
|
|
291
|
+
return {}
|
|
292
|
+
gbgpu_section = config["gbgpu"]
|
|
293
|
+
return {key: gbgpu_section[key] for key in gbgpu_section}
|
|
294
|
+
|
|
295
|
+
@staticmethod
|
|
296
|
+
def _get_from_envvars(env_vars: Optional[Mapping[str, str]]) -> Dict[str, str]:
|
|
297
|
+
"""Filter-out environment variables not matching a given prefix."""
|
|
298
|
+
return (
|
|
299
|
+
{
|
|
300
|
+
key: value
|
|
301
|
+
for key, value in env_vars.items()
|
|
302
|
+
if key.startswith(ENVVAR_PREFIX)
|
|
303
|
+
}
|
|
304
|
+
if env_vars is not None
|
|
305
|
+
else {}
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
@staticmethod
|
|
309
|
+
def _get_from_cli_args(cli_args: Optional[Sequence[str]]) -> List[str]:
|
|
310
|
+
"""Build list of CLI arguments."""
|
|
311
|
+
return [arg for arg in cli_args] if cli_args is not None else []
|
|
312
|
+
|
|
313
|
+
@staticmethod
|
|
314
|
+
def _build_items_from_default(
|
|
315
|
+
config_entries: Sequence[ConfigEntry],
|
|
316
|
+
) -> Dict[str, ConfigItem]:
|
|
317
|
+
"""Build a list of ConfigItem built from default-valued ConfigEntries."""
|
|
318
|
+
return {
|
|
319
|
+
entry.label: ConfigItem(value=entry.default, source=ConfigSource.DEFAULT)
|
|
320
|
+
for entry in config_entries
|
|
321
|
+
if compatibility_isinstance(entry.default, entry.type)
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
@staticmethod
|
|
325
|
+
def _build_items_from_file(
|
|
326
|
+
config_entries: Sequence[ConfigEntry], opt_from_file: Mapping[str, str]
|
|
327
|
+
) -> Tuple[Dict[str, ConfigItem], Dict[str, str]]:
|
|
328
|
+
"""Extract configuration items from file option and build dict of unconsumed extra items."""
|
|
329
|
+
extras_from_file = {**opt_from_file}
|
|
330
|
+
items_from_file = {
|
|
331
|
+
entry.label: ConfigItem(
|
|
332
|
+
value=entry.convert(extras_from_file.pop(entry.cfg_entry)),
|
|
333
|
+
source=ConfigSource.CFGFILE,
|
|
334
|
+
)
|
|
335
|
+
for entry in config_entries
|
|
336
|
+
if entry.cfg_entry in extras_from_file
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
return items_from_file, extras_from_file
|
|
340
|
+
|
|
341
|
+
@staticmethod
|
|
342
|
+
def _build_items_from_env(
|
|
343
|
+
config_entries: Sequence[ConfigEntry], opt_from_env: Mapping[str, str]
|
|
344
|
+
) -> Tuple[Dict[str, ConfigItem], Dict[str, str]]:
|
|
345
|
+
"""Extract configuration items from file option and build dict of unconsumed extra items."""
|
|
346
|
+
extras_from_env = {**opt_from_env}
|
|
347
|
+
items_from_env = {
|
|
348
|
+
entry.label: ConfigItem(
|
|
349
|
+
value=entry.convert(extras_from_env.pop(ENVVAR_PREFIX + entry.env_var)),
|
|
350
|
+
source=ConfigSource.ENVVAR,
|
|
351
|
+
)
|
|
352
|
+
for entry in config_entries
|
|
353
|
+
if entry.env_var is not None
|
|
354
|
+
and ENVVAR_PREFIX + entry.env_var in extras_from_env
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
return items_from_env, extras_from_env
|
|
358
|
+
|
|
359
|
+
@staticmethod
|
|
360
|
+
def _build_parser(
|
|
361
|
+
config_entries: Sequence[ConfigEntry],
|
|
362
|
+
parent_parsers: Optional[Sequence[argparse.ArgumentParser]] = None,
|
|
363
|
+
) -> argparse.ArgumentParser:
|
|
364
|
+
parser = argparse.ArgumentParser(
|
|
365
|
+
add_help=False,
|
|
366
|
+
argument_default=argparse.SUPPRESS,
|
|
367
|
+
parents=[] if parent_parsers is None else parent_parsers,
|
|
368
|
+
)
|
|
369
|
+
for config_entry in config_entries:
|
|
370
|
+
if config_entry.cli_flags:
|
|
371
|
+
cli_options = {
|
|
372
|
+
"help": config_entry.description,
|
|
373
|
+
"dest": config_entry.label,
|
|
374
|
+
}
|
|
375
|
+
if config_entry.cli_kwargs is not None:
|
|
376
|
+
cli_options = cli_options | config_entry.cli_kwargs
|
|
377
|
+
flags = (
|
|
378
|
+
[config_entry.cli_flags]
|
|
379
|
+
if isinstance(config_entry.cli_flags, str)
|
|
380
|
+
else config_entry.cli_flags
|
|
381
|
+
)
|
|
382
|
+
parser.add_argument(*flags, **cli_options)
|
|
383
|
+
return parser
|
|
384
|
+
|
|
385
|
+
@staticmethod
|
|
386
|
+
def _build_items_from_cli(
|
|
387
|
+
config_entries: Sequence[ConfigEntry], opt_from_cli: Sequence[str]
|
|
388
|
+
) -> Tuple[Dict[str, ConfigItem], List[str]]:
|
|
389
|
+
"""Extract configuration items from file option and build dict of unconsumed extra items."""
|
|
390
|
+
parser = ConfigConsumer._build_parser(config_entries)
|
|
391
|
+
parsed_options, extras_from_cli = parser.parse_known_args(opt_from_cli)
|
|
392
|
+
|
|
393
|
+
parsed_dict = vars(parsed_options)
|
|
394
|
+
|
|
395
|
+
items_from_cli = {}
|
|
396
|
+
|
|
397
|
+
for entry in config_entries:
|
|
398
|
+
if entry.label not in parsed_dict:
|
|
399
|
+
continue
|
|
400
|
+
parsed_value = parsed_dict[entry.label]
|
|
401
|
+
if compatibility_isinstance(parsed_dict[entry.label], entry.type):
|
|
402
|
+
value = parsed_value
|
|
403
|
+
else:
|
|
404
|
+
value = entry.convert(parsed_value)
|
|
405
|
+
|
|
406
|
+
items_from_cli[entry.label] = ConfigItem(
|
|
407
|
+
value=value, source=ConfigSource.CLIOPT
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
return items_from_cli, extras_from_cli
|
|
411
|
+
|
|
412
|
+
@staticmethod
|
|
413
|
+
def _build_items_from_set(
|
|
414
|
+
config_entries: Sequence[ConfigEntry], set_values: Optional[Dict[str, Any]]
|
|
415
|
+
) -> Dict[str, ConfigItem]:
|
|
416
|
+
"""Check that provided items match the entries."""
|
|
417
|
+
set_items = {}
|
|
418
|
+
|
|
419
|
+
if set_values is None:
|
|
420
|
+
return set_items
|
|
421
|
+
|
|
422
|
+
for config_entry in config_entries:
|
|
423
|
+
if (label := config_entry.label) in set_values:
|
|
424
|
+
set_value = set_values[label]
|
|
425
|
+
if not config_entry.validate(set_value):
|
|
426
|
+
raise exceptions.ConfigurationValidationError(
|
|
427
|
+
"Configuration entry '{}' has invalid value '{}'".format(
|
|
428
|
+
label, set_value
|
|
429
|
+
)
|
|
430
|
+
)
|
|
431
|
+
set_items[label] = ConfigItem(set_value, ConfigSource.SETTER)
|
|
432
|
+
|
|
433
|
+
return set_items
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def userstr_to_bool(user_str: str) -> Optional[bool]:
|
|
437
|
+
"""Convert a yes/no, on/off or true/false to bool."""
|
|
438
|
+
if user_str.lower().startswith(("y", "t", "on", "1")):
|
|
439
|
+
return True
|
|
440
|
+
if user_str.lower().startswith(("n", "f", "off", "0")):
|
|
441
|
+
return False
|
|
442
|
+
return None
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def userinput_to_pathlist(user_input) -> List[pathlib.Path]:
|
|
446
|
+
"""Convert a user input to a list of paths"""
|
|
447
|
+
if user_input is None:
|
|
448
|
+
return []
|
|
449
|
+
if isinstance(user_input, str):
|
|
450
|
+
return userinput_to_pathlist(user_input.split(";"))
|
|
451
|
+
if compatibility_isinstance(user_input, Sequence[str]):
|
|
452
|
+
return [pathlib.Path(path_str) for path_str in user_input]
|
|
453
|
+
if compatibility_isinstance(user_input, Sequence[pathlib.Path]):
|
|
454
|
+
return user_input
|
|
455
|
+
raise ValueError(
|
|
456
|
+
"User input '{}' of type '{}' is not convertible to a list of paths".format(
|
|
457
|
+
user_input, type(user_input)
|
|
458
|
+
)
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def userinput_to_strlist(user_input) -> List[str]:
|
|
463
|
+
"""Convert a user input to a list of paths"""
|
|
464
|
+
if user_input is None:
|
|
465
|
+
return []
|
|
466
|
+
if isinstance(user_input, str):
|
|
467
|
+
return user_input.replace(" ", ";").split(";")
|
|
468
|
+
if compatibility_isinstance(user_input, List[str]):
|
|
469
|
+
return user_input
|
|
470
|
+
if compatibility_isinstance(user_input, Sequence[str]):
|
|
471
|
+
return [input for input in user_input]
|
|
472
|
+
raise ValueError(
|
|
473
|
+
"User input '{}' of type '{}' is not convertible to a list of strings".format(
|
|
474
|
+
user_input, type(user_input)
|
|
475
|
+
)
|
|
476
|
+
)
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
class InitialConfigConsumer(ConfigConsumer):
|
|
480
|
+
"""
|
|
481
|
+
Class implementing first-pass config consumer.
|
|
482
|
+
|
|
483
|
+
On first pass, we only detect if there are CLI arguments which disable
|
|
484
|
+
config file or environment variables.
|
|
485
|
+
"""
|
|
486
|
+
|
|
487
|
+
ignore_cfg: bool
|
|
488
|
+
ignore_env: bool
|
|
489
|
+
config_file: Optional[pathlib.Path]
|
|
490
|
+
|
|
491
|
+
@staticmethod
|
|
492
|
+
def config_entries() -> List[ConfigEntry]:
|
|
493
|
+
return [
|
|
494
|
+
ConfigEntry(
|
|
495
|
+
label="ignore_cfg",
|
|
496
|
+
description="Whether to ignore config file options",
|
|
497
|
+
type=bool,
|
|
498
|
+
default=False,
|
|
499
|
+
env_var="IGNORE_CFG_FILE",
|
|
500
|
+
cli_flags="--ignore-config-file",
|
|
501
|
+
cli_kwargs={"action": "store_const", "const": True},
|
|
502
|
+
convert=userstr_to_bool,
|
|
503
|
+
validate=lambda x: isinstance(x, bool),
|
|
504
|
+
),
|
|
505
|
+
ConfigEntry(
|
|
506
|
+
label="ignore_env",
|
|
507
|
+
description="Whether to ignore environment variables",
|
|
508
|
+
type=bool,
|
|
509
|
+
default=False,
|
|
510
|
+
cli_flags="--ignore-env",
|
|
511
|
+
cli_kwargs={"action": "store_const", "const": True},
|
|
512
|
+
convert=userstr_to_bool,
|
|
513
|
+
validate=lambda x: isinstance(x, bool),
|
|
514
|
+
),
|
|
515
|
+
ConfigEntry(
|
|
516
|
+
label="config_file",
|
|
517
|
+
description="Path to GBGPU configuration file",
|
|
518
|
+
type=Optional[pathlib.Path],
|
|
519
|
+
default=None,
|
|
520
|
+
cli_flags=["-C", "--config-file"],
|
|
521
|
+
env_var="CONFIG_FILE",
|
|
522
|
+
convert=lambda p: None if p is None else pathlib.Path(p),
|
|
523
|
+
validate=lambda p: True if p is None else p.is_file(),
|
|
524
|
+
),
|
|
525
|
+
]
|
|
526
|
+
|
|
527
|
+
def __init__(
|
|
528
|
+
self,
|
|
529
|
+
env_vars: Optional[Mapping[str, str]] = None,
|
|
530
|
+
cli_args: Optional[Sequence[str]] = None,
|
|
531
|
+
):
|
|
532
|
+
super().__init__(config_file=None, env_vars=env_vars, cli_args=cli_args)
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
def get_package_basepath() -> pathlib.Path:
|
|
536
|
+
import gbgpu
|
|
537
|
+
|
|
538
|
+
return pathlib.Path(gbgpu.__file__).parent
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
class Configuration(ConfigConsumer):
|
|
542
|
+
"""
|
|
543
|
+
Class implementing GBGPU complete configuration for the library.
|
|
544
|
+
"""
|
|
545
|
+
|
|
546
|
+
log_level: int
|
|
547
|
+
log_format: str
|
|
548
|
+
file_registry_path: Optional[pathlib.Path]
|
|
549
|
+
file_storage_path: Optional[pathlib.Path]
|
|
550
|
+
file_download_path: Optional[pathlib.Path]
|
|
551
|
+
file_allow_download: bool
|
|
552
|
+
file_integrity_check: str
|
|
553
|
+
file_extra_paths: List[pathlib.Path]
|
|
554
|
+
file_disabled_tags: Optional[List[str]]
|
|
555
|
+
enabled_backends: Optional[List[str]]
|
|
556
|
+
|
|
557
|
+
@staticmethod
|
|
558
|
+
def config_entries() -> List[ConfigEntry]:
|
|
559
|
+
from gbgpu import _is_editable as is_editable_mode
|
|
560
|
+
|
|
561
|
+
return [
|
|
562
|
+
ConfigEntry(
|
|
563
|
+
label="log_level",
|
|
564
|
+
description="Application log level",
|
|
565
|
+
type=int,
|
|
566
|
+
default=logging.WARN,
|
|
567
|
+
cli_flags=["--log-level"],
|
|
568
|
+
env_var="LOG_LEVEL",
|
|
569
|
+
cfg_entry="log-level",
|
|
570
|
+
convert=Configuration._str_to_logging_level,
|
|
571
|
+
),
|
|
572
|
+
ConfigEntry(
|
|
573
|
+
label="log_format",
|
|
574
|
+
description="Application log format",
|
|
575
|
+
type=Optional[str],
|
|
576
|
+
default=None,
|
|
577
|
+
cli_flags=["--log-format"],
|
|
578
|
+
env_var="LOG_FORMAT",
|
|
579
|
+
cfg_entry="log-format",
|
|
580
|
+
convert=lambda input: input,
|
|
581
|
+
validate=lambda input: input is None or isinstance(input, str),
|
|
582
|
+
),
|
|
583
|
+
# ConfigEntry(
|
|
584
|
+
# label="file_registry_path",
|
|
585
|
+
# description="File Registry path",
|
|
586
|
+
# type=Optional[pathlib.Path],
|
|
587
|
+
# default=None,
|
|
588
|
+
# cli_flags=["--file-registry"],
|
|
589
|
+
# env_var="FILE_REGISTRY",
|
|
590
|
+
# cfg_entry="file-registry",
|
|
591
|
+
# convert=lambda p: None if p is None else pathlib.Path(p),
|
|
592
|
+
# validate=lambda p: True if p is None else p.is_file(),
|
|
593
|
+
# ),
|
|
594
|
+
# ConfigEntry(
|
|
595
|
+
# label="file_storage_path",
|
|
596
|
+
# description="File Manager storage directory (absolute or relative to current working directory, must exist)",
|
|
597
|
+
# type=Optional[pathlib.Path],
|
|
598
|
+
# default=(get_package_basepath() / "data") if is_editable_mode else None,
|
|
599
|
+
# cli_flags=["--storage-dir"],
|
|
600
|
+
# env_var="FILE_STORAGE_DIR",
|
|
601
|
+
# cfg_entry="file-storage-dir",
|
|
602
|
+
# convert=lambda p: None if p is None else pathlib.Path(p).absolute(),
|
|
603
|
+
# validate=lambda p: True if p is None else p.is_dir(),
|
|
604
|
+
# ),
|
|
605
|
+
# ConfigEntry(
|
|
606
|
+
# label="file_download_path",
|
|
607
|
+
# description="File Manager download directory (absolute, or relative to storage_path)",
|
|
608
|
+
# type=Optional[pathlib.Path],
|
|
609
|
+
# default=(get_package_basepath() / "data") if is_editable_mode else None,
|
|
610
|
+
# cli_flags=["--download-dir"],
|
|
611
|
+
# env_var="FILE_DOWNLOAD_DIR",
|
|
612
|
+
# cfg_entry="file-download-dir",
|
|
613
|
+
# convert=lambda p: None if p is None else pathlib.Path(p),
|
|
614
|
+
# validate=lambda p: True
|
|
615
|
+
# if p is None
|
|
616
|
+
# else (p.is_dir() if p.is_absolute() else True),
|
|
617
|
+
# ),
|
|
618
|
+
# ConfigEntry(
|
|
619
|
+
# label="file_allow_download",
|
|
620
|
+
# description="Whether file manager can download missing files from internet",
|
|
621
|
+
# type=bool,
|
|
622
|
+
# default=True,
|
|
623
|
+
# cli_flags="--file-download",
|
|
624
|
+
# cli_kwargs={"action": argparse.BooleanOptionalAction},
|
|
625
|
+
# env_var="FILE_ALLOW_DOWNLOAD",
|
|
626
|
+
# cfg_entry="file-allow-download",
|
|
627
|
+
# convert=lambda x: userstr_to_bool(x) if isinstance(x, str) else bool(x),
|
|
628
|
+
# ),
|
|
629
|
+
# ConfigEntry(
|
|
630
|
+
# label="file_integrity_check",
|
|
631
|
+
# description="When should th file manager perform integrity checks (never, once, always)",
|
|
632
|
+
# type=str,
|
|
633
|
+
# default="once",
|
|
634
|
+
# cli_flags="--file-integrity-check",
|
|
635
|
+
# cli_kwargs={"choices": ("never", "once", "always")},
|
|
636
|
+
# env_var="FILE_INTEGRITY_CHECK",
|
|
637
|
+
# cfg_entry="file-integrity-check",
|
|
638
|
+
# validate=lambda x: x in ("never", "once", "always"),
|
|
639
|
+
# ),
|
|
640
|
+
# ConfigEntry(
|
|
641
|
+
# label="file_extra_paths",
|
|
642
|
+
# description="Supplementary paths in which GBGPU will search for files",
|
|
643
|
+
# type=Optional[List[pathlib.Path]],
|
|
644
|
+
# default=[get_package_basepath() / "data"],
|
|
645
|
+
# cli_flags=["--extra-path"],
|
|
646
|
+
# cli_kwargs={"action": "append"},
|
|
647
|
+
# env_var="FILE_EXTRA_PATHS",
|
|
648
|
+
# cfg_entry="file-extra-paths",
|
|
649
|
+
# convert=userinput_to_pathlist,
|
|
650
|
+
# overwrite=lambda old, new: old + new
|
|
651
|
+
# if old is not None
|
|
652
|
+
# else new, # concatenate extra path lists
|
|
653
|
+
# ),
|
|
654
|
+
# ConfigEntry(
|
|
655
|
+
# label="file_disabled_tags",
|
|
656
|
+
# description="Tags for which access to associated files is disabled",
|
|
657
|
+
# type=Optional[List[str]],
|
|
658
|
+
# default=None,
|
|
659
|
+
# env_var="DISABLED_TAGS",
|
|
660
|
+
# convert=userinput_to_strlist,
|
|
661
|
+
# overwrite=lambda old, new: old + new
|
|
662
|
+
# if old is not None
|
|
663
|
+
# else new, # concatenate tag lists
|
|
664
|
+
# ),
|
|
665
|
+
ConfigEntry(
|
|
666
|
+
label="enabled_backends",
|
|
667
|
+
description="List of backends that must be enabled",
|
|
668
|
+
type=Optional[List[str]],
|
|
669
|
+
default=None,
|
|
670
|
+
cli_flags="--enable-backend",
|
|
671
|
+
cli_kwargs={"action": "append"},
|
|
672
|
+
env_var="ENABLED_BACKENDS",
|
|
673
|
+
cfg_entry="enabled-backends",
|
|
674
|
+
convert=lambda x: [v.lower() for v in userinput_to_strlist(x)],
|
|
675
|
+
validate=lambda x: all(v in KNOWN_BACKENDS for v in x)
|
|
676
|
+
if x is not None
|
|
677
|
+
else True,
|
|
678
|
+
),
|
|
679
|
+
]
|
|
680
|
+
|
|
681
|
+
def __init__(
|
|
682
|
+
self,
|
|
683
|
+
config_file: Union[os.PathLike, Mapping[str, str], None] = None,
|
|
684
|
+
env_vars: Optional[Mapping[str, str]] = None,
|
|
685
|
+
cli_args: Optional[Sequence[str]] = None,
|
|
686
|
+
set_args: Optional[Dict[str, Any]] = None,
|
|
687
|
+
):
|
|
688
|
+
super().__init__(
|
|
689
|
+
config_file=config_file,
|
|
690
|
+
env_vars=env_vars,
|
|
691
|
+
cli_args=cli_args,
|
|
692
|
+
set_args=set_args,
|
|
693
|
+
)
|
|
694
|
+
|
|
695
|
+
# Post-init task: read -v and -q options
|
|
696
|
+
self._handle_verbosity()
|
|
697
|
+
|
|
698
|
+
@staticmethod
|
|
699
|
+
def _str_to_logging_level(input: Union[str, int]) -> int:
|
|
700
|
+
if isinstance(input, int):
|
|
701
|
+
return input
|
|
702
|
+
as_int_level = logging.getLevelName(input.upper())
|
|
703
|
+
if isinstance(as_int_level, int):
|
|
704
|
+
return as_int_level
|
|
705
|
+
raise exceptions.ConfigurationValidationError(
|
|
706
|
+
"'{}' is not a valid log level.".format(input)
|
|
707
|
+
)
|
|
708
|
+
|
|
709
|
+
def _apply_verbosity(self, level):
|
|
710
|
+
self._items["log_level"] = ConfigItem(value=level, source=ConfigSource.CLIOPT)
|
|
711
|
+
|
|
712
|
+
@staticmethod
|
|
713
|
+
def _build_verbosity_parser() -> argparse.ArgumentParser:
|
|
714
|
+
"""Build a parser to handle -v and -Q options"""
|
|
715
|
+
parser = argparse.ArgumentParser(add_help=False)
|
|
716
|
+
exclusive_groups = parser.add_mutually_exclusive_group()
|
|
717
|
+
exclusive_groups.add_argument(
|
|
718
|
+
"-v",
|
|
719
|
+
"--verbose",
|
|
720
|
+
dest="verbose_count",
|
|
721
|
+
action="count",
|
|
722
|
+
default=0,
|
|
723
|
+
help="Increase the verbosity (can be used multiple times)",
|
|
724
|
+
)
|
|
725
|
+
exclusive_groups.add_argument(
|
|
726
|
+
"-Q",
|
|
727
|
+
"--quiet",
|
|
728
|
+
dest="quiet",
|
|
729
|
+
action="store_true",
|
|
730
|
+
help="Disable all logging outputs",
|
|
731
|
+
)
|
|
732
|
+
return parser
|
|
733
|
+
|
|
734
|
+
def _handle_verbosity(self):
|
|
735
|
+
parser = self._build_verbosity_parser()
|
|
736
|
+
parsed_options, self._extra_cli = parser.parse_known_args(self._extra_cli)
|
|
737
|
+
|
|
738
|
+
from .globals import get_logger
|
|
739
|
+
|
|
740
|
+
if parsed_options.quiet:
|
|
741
|
+
get_logger().debug(
|
|
742
|
+
"Logger level set to CRITICAL since quiet mode is requested."
|
|
743
|
+
)
|
|
744
|
+
self._apply_verbosity(logging.CRITICAL)
|
|
745
|
+
else:
|
|
746
|
+
if (count := parsed_options.verbose_count) == 0:
|
|
747
|
+
return
|
|
748
|
+
old_level = self.log_level
|
|
749
|
+
new_level = old_level - count * 10
|
|
750
|
+
get_logger().debug(
|
|
751
|
+
"Logger level is decreased from {} to {} since verbose flag was set {} times".format(
|
|
752
|
+
old_level, new_level, count
|
|
753
|
+
)
|
|
754
|
+
)
|
|
755
|
+
self._apply_verbosity(
|
|
756
|
+
new_level if new_level > logging.DEBUG else logging.DEBUG
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
def build_cli_parent_parsers(self) -> List[argparse.ArgumentParser]:
|
|
760
|
+
"""Build a Parser that can be used as parent parser from CLI-specific parsers"""
|
|
761
|
+
init_parser = ConfigConsumer._build_parser(
|
|
762
|
+
InitialConfigConsumer.config_entries()
|
|
763
|
+
)
|
|
764
|
+
verbosity_parser = self._build_verbosity_parser()
|
|
765
|
+
complete_parser = ConfigConsumer._build_parser(
|
|
766
|
+
Configuration.config_entries(),
|
|
767
|
+
parent_parsers=[init_parser, verbosity_parser],
|
|
768
|
+
)
|
|
769
|
+
return [complete_parser]
|
|
770
|
+
|
|
771
|
+
|
|
772
|
+
def detect_cfg_file() -> Optional[pathlib.Path]:
|
|
773
|
+
"""Test common path locations for config and return highest-priority existing one (if any)."""
|
|
774
|
+
import platformdirs
|
|
775
|
+
|
|
776
|
+
from .. import __version_tuple__
|
|
777
|
+
|
|
778
|
+
LOCATIONS = [
|
|
779
|
+
pathlib.Path.cwd() / "gbgpu.ini",
|
|
780
|
+
platformdirs.user_config_path() / "gbgpu.ini",
|
|
781
|
+
platformdirs.site_config_path()
|
|
782
|
+
/ "gbgpu"
|
|
783
|
+
/ "v{}.{}".format(__version_tuple__[0], __version_tuple__[1])
|
|
784
|
+
/ "gbgpu.ini",
|
|
785
|
+
]
|
|
786
|
+
from .globals import get_logger
|
|
787
|
+
|
|
788
|
+
for location in LOCATIONS:
|
|
789
|
+
if location.is_file():
|
|
790
|
+
get_logger().debug("Configuration file located in '{}'".format(location))
|
|
791
|
+
return location
|
|
792
|
+
get_logger().debug("Configuration file not found in '{}'".format(location))
|
|
793
|
+
return None
|