esgvoc 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of esgvoc might be problematic. Click here for more details.
- esgvoc/__init__.py +3 -1
- esgvoc/api/__init__.py +23 -34
- esgvoc/api/_utils.py +28 -14
- esgvoc/api/data_descriptors/__init__.py +18 -12
- esgvoc/api/data_descriptors/activity.py +8 -45
- esgvoc/api/data_descriptors/area_label.py +6 -0
- esgvoc/api/data_descriptors/branded_suffix.py +5 -0
- esgvoc/api/data_descriptors/branded_variable.py +5 -0
- esgvoc/api/data_descriptors/consortium.py +16 -56
- esgvoc/api/data_descriptors/data_descriptor.py +106 -0
- esgvoc/api/data_descriptors/date.py +3 -46
- esgvoc/api/data_descriptors/directory_date.py +3 -46
- esgvoc/api/data_descriptors/experiment.py +19 -54
- esgvoc/api/data_descriptors/forcing_index.py +3 -45
- esgvoc/api/data_descriptors/frequency.py +6 -43
- esgvoc/api/data_descriptors/grid_label.py +6 -44
- esgvoc/api/data_descriptors/horizontal_label.py +6 -0
- esgvoc/api/data_descriptors/initialisation_index.py +3 -44
- esgvoc/api/data_descriptors/institution.py +11 -54
- esgvoc/api/data_descriptors/license.py +4 -44
- esgvoc/api/data_descriptors/mip_era.py +6 -44
- esgvoc/api/data_descriptors/model_component.py +7 -45
- esgvoc/api/data_descriptors/organisation.py +3 -40
- esgvoc/api/data_descriptors/physic_index.py +3 -45
- esgvoc/api/data_descriptors/product.py +4 -43
- esgvoc/api/data_descriptors/realisation_index.py +3 -44
- esgvoc/api/data_descriptors/realm.py +4 -42
- esgvoc/api/data_descriptors/resolution.py +6 -44
- esgvoc/api/data_descriptors/source.py +18 -53
- esgvoc/api/data_descriptors/source_type.py +3 -41
- esgvoc/api/data_descriptors/sub_experiment.py +3 -41
- esgvoc/api/data_descriptors/table.py +6 -48
- esgvoc/api/data_descriptors/temporal_label.py +6 -0
- esgvoc/api/data_descriptors/time_range.py +3 -27
- esgvoc/api/data_descriptors/variable.py +13 -71
- esgvoc/api/data_descriptors/variant_label.py +3 -47
- esgvoc/api/data_descriptors/vertical_label.py +5 -0
- esgvoc/api/projects.py +187 -171
- esgvoc/api/report.py +21 -12
- esgvoc/api/search.py +3 -1
- esgvoc/api/universe.py +44 -34
- esgvoc/apps/__init__.py +3 -4
- esgvoc/apps/drs/generator.py +166 -161
- esgvoc/apps/drs/report.py +222 -131
- esgvoc/apps/drs/validator.py +103 -105
- esgvoc/cli/drs.py +29 -19
- esgvoc/cli/get.py +26 -25
- esgvoc/cli/install.py +11 -8
- esgvoc/cli/main.py +0 -2
- esgvoc/cli/status.py +5 -5
- esgvoc/cli/valid.py +40 -40
- esgvoc/core/db/models/universe.py +3 -3
- esgvoc/core/db/project_ingestion.py +1 -1
- esgvoc/core/db/universe_ingestion.py +6 -5
- esgvoc/core/logging_handler.py +1 -1
- esgvoc/core/repo_fetcher.py +4 -3
- esgvoc/core/service/__init__.py +37 -5
- esgvoc/core/service/configuration/config_manager.py +188 -0
- esgvoc/core/service/configuration/setting.py +88 -0
- esgvoc/core/service/state.py +49 -32
- {esgvoc-0.2.1.dist-info → esgvoc-0.3.0.dist-info}/METADATA +34 -3
- esgvoc-0.3.0.dist-info/RECORD +78 -0
- esgvoc/cli/config.py +0 -82
- esgvoc/core/service/settings.py +0 -73
- esgvoc/core/service/settings.toml +0 -17
- esgvoc/core/service/settings_default.toml +0 -17
- esgvoc-0.2.1.dist-info/RECORD +0 -73
- {esgvoc-0.2.1.dist-info → esgvoc-0.3.0.dist-info}/WHEEL +0 -0
- {esgvoc-0.2.1.dist-info → esgvoc-0.3.0.dist-info}/entry_points.txt +0 -0
- {esgvoc-0.2.1.dist-info → esgvoc-0.3.0.dist-info}/licenses/LICENSE.txt +0 -0
esgvoc/apps/drs/validator.py
CHANGED
|
@@ -1,26 +1,18 @@
|
|
|
1
1
|
from typing import cast
|
|
2
|
-
|
|
3
|
-
DrsType,
|
|
4
|
-
DrsPart,
|
|
5
|
-
DrsSpecification,
|
|
6
|
-
DrsPartKind,
|
|
7
|
-
DrsCollection,
|
|
8
|
-
DrsConstant)
|
|
2
|
+
|
|
9
3
|
import esgvoc.api.projects as projects
|
|
10
4
|
import esgvoc.apps.drs.constants as constants
|
|
11
|
-
from esgvoc.
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
ExtraSeparator,
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
MissingToken,
|
|
23
|
-
FileNameExtensionIssue)
|
|
5
|
+
from esgvoc.api import APIException
|
|
6
|
+
from esgvoc.api.project_specs import (DrsCollection, DrsConstant, DrsPart,
|
|
7
|
+
DrsPartKind, DrsSpecification, DrsType,
|
|
8
|
+
ProjectSpecs)
|
|
9
|
+
from esgvoc.apps.drs.report import (BlankTerm, ComplianceIssue, DrsIssue,
|
|
10
|
+
DrsValidationReport, ExtraChar,
|
|
11
|
+
ExtraSeparator, ExtraTerm,
|
|
12
|
+
FileNameExtensionIssue, InvalidTerm,
|
|
13
|
+
MissingTerm, ParsingIssue, Space,
|
|
14
|
+
Unparsable, ValidationError,
|
|
15
|
+
ValidationWarning)
|
|
24
16
|
|
|
25
17
|
|
|
26
18
|
class DrsApplication:
|
|
@@ -33,7 +25,9 @@ class DrsApplication:
|
|
|
33
25
|
"""The project id."""
|
|
34
26
|
self.pedantic: bool = pedantic
|
|
35
27
|
"""Same as the option of GCC: turn warnings into errors. Default False."""
|
|
36
|
-
project_specs: ProjectSpecs = projects.
|
|
28
|
+
project_specs: ProjectSpecs|None = projects.find_project(project_id)
|
|
29
|
+
if not project_specs:
|
|
30
|
+
raise APIException(f'unable to find project {project_id}')
|
|
37
31
|
for specs in project_specs.drs_specs:
|
|
38
32
|
match specs.type:
|
|
39
33
|
case DrsType.DIRECTORY:
|
|
@@ -46,7 +40,7 @@ class DrsApplication:
|
|
|
46
40
|
self.dataset_id_specs: DrsSpecification = specs
|
|
47
41
|
"""The DRS dataset id specs of the project."""
|
|
48
42
|
case _:
|
|
49
|
-
raise
|
|
43
|
+
raise RuntimeError(f'unsupported DRS specs type {specs.type}')
|
|
50
44
|
|
|
51
45
|
def _get_full_file_name_extension(self) -> str:
|
|
52
46
|
"""
|
|
@@ -61,38 +55,33 @@ class DrsApplication:
|
|
|
61
55
|
full_extension = specs.properties[constants.FILE_NAME_EXTENSION_SEPARATOR_KEY] + \
|
|
62
56
|
specs.properties[constants.FILE_NAME_EXTENSION_KEY]
|
|
63
57
|
else:
|
|
64
|
-
raise
|
|
65
|
-
|
|
58
|
+
raise RuntimeError('missing properties in the DRS file name specifications of the ' +
|
|
59
|
+
f'project {self.project_id}')
|
|
66
60
|
return full_extension
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
match drs_type:
|
|
70
|
-
case DrsType.DIRECTORY:
|
|
71
|
-
specs = self.directory_specs
|
|
72
|
-
case DrsType.FILE_NAME:
|
|
73
|
-
specs = self.file_name_specs
|
|
74
|
-
case DrsType.DATASET_ID:
|
|
75
|
-
specs = self.dataset_id_specs
|
|
76
|
-
case _:
|
|
77
|
-
raise ValueError(f'unsupported DRS type {drs_type}')
|
|
78
|
-
return specs
|
|
79
|
-
|
|
61
|
+
|
|
62
|
+
|
|
80
63
|
class DrsValidator(DrsApplication):
|
|
81
64
|
"""
|
|
82
65
|
Valid a DRS directory, dataset id and file name expression against a project.
|
|
83
66
|
"""
|
|
84
|
-
|
|
85
|
-
def validate_directory(self, drs_expression: str
|
|
67
|
+
|
|
68
|
+
def validate_directory(self, drs_expression: str,
|
|
69
|
+
prefix: str|None = None) -> DrsValidationReport:
|
|
86
70
|
"""
|
|
87
71
|
Validate a DRS directory expression.
|
|
88
72
|
|
|
89
73
|
:param drs_expression: A DRS directory expression.
|
|
90
74
|
:type drs_expression: str
|
|
75
|
+
:param prefix: A directory prefix to be removed from the directory expression.
|
|
76
|
+
:type prefix: str|None
|
|
91
77
|
:returns: A validation report.
|
|
92
78
|
:rtype: DrsValidationReport
|
|
93
79
|
"""
|
|
80
|
+
if prefix:
|
|
81
|
+
# Remove prefix if present. Always returns a copy.
|
|
82
|
+
drs_expression = drs_expression.removeprefix(prefix)
|
|
94
83
|
return self._validate(drs_expression, self.directory_specs)
|
|
95
|
-
|
|
84
|
+
|
|
96
85
|
def validate_dataset_id(self, drs_expression: str) -> DrsValidationReport:
|
|
97
86
|
"""
|
|
98
87
|
Validate a DRS dataset id expression.
|
|
@@ -134,13 +123,20 @@ class DrsValidator(DrsApplication):
|
|
|
134
123
|
:returns: A validation report.
|
|
135
124
|
:rtype: DrsValidationReport
|
|
136
125
|
"""
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
126
|
+
match drs_type:
|
|
127
|
+
case DrsType.DIRECTORY:
|
|
128
|
+
return self.validate_directory(drs_expression=drs_expression)
|
|
129
|
+
case DrsType.FILE_NAME:
|
|
130
|
+
return self.validate_file_name(drs_expression=drs_expression)
|
|
131
|
+
case DrsType.DATASET_ID:
|
|
132
|
+
return self.validate_dataset_id(drs_expression=drs_expression)
|
|
133
|
+
case _:
|
|
134
|
+
raise RuntimeError(f'unsupported drs type {drs_type}')
|
|
135
|
+
|
|
140
136
|
def _parse(self,
|
|
141
137
|
drs_expression: str,
|
|
142
138
|
separator: str,
|
|
143
|
-
drs_type: DrsType) -> tuple[list[str]|None, #
|
|
139
|
+
drs_type: DrsType) -> tuple[list[str]|None, # terms
|
|
144
140
|
list[DrsIssue], # Errors
|
|
145
141
|
list[DrsIssue]]: # Warnings
|
|
146
142
|
errors: list[DrsIssue] = list()
|
|
@@ -150,7 +146,7 @@ class DrsValidator(DrsApplication):
|
|
|
150
146
|
start_with_space = drs_expression[0].isspace()
|
|
151
147
|
end_with_space = drs_expression[-1].isspace()
|
|
152
148
|
if start_with_space or end_with_space:
|
|
153
|
-
issue:
|
|
149
|
+
issue: ParsingIssue = Space()
|
|
154
150
|
if self.pedantic:
|
|
155
151
|
errors.append(issue)
|
|
156
152
|
else:
|
|
@@ -161,78 +157,78 @@ class DrsValidator(DrsApplication):
|
|
|
161
157
|
cursor_offset = previous_len - len(drs_expression)
|
|
162
158
|
if end_with_space:
|
|
163
159
|
drs_expression = drs_expression.rstrip()
|
|
164
|
-
|
|
165
|
-
if len(
|
|
160
|
+
terms = drs_expression.split(separator)
|
|
161
|
+
if len(terms) < 2:
|
|
166
162
|
errors.append(Unparsable(expected_drs_type=drs_type))
|
|
167
163
|
return None, errors, warnings # Early exit
|
|
168
|
-
|
|
164
|
+
max_term_index = len(terms)
|
|
169
165
|
cursor_position = initial_cursor_position = len(drs_expression) + 1
|
|
170
|
-
|
|
171
|
-
for index in range(
|
|
172
|
-
|
|
173
|
-
if (
|
|
174
|
-
|
|
175
|
-
cursor_position -= len(
|
|
176
|
-
del
|
|
166
|
+
has_white_term = False
|
|
167
|
+
for index in range(max_term_index-1, -1, -1):
|
|
168
|
+
term = terms[index]
|
|
169
|
+
if (is_white_term := term.isspace()) or (not term):
|
|
170
|
+
has_white_term = has_white_term or is_white_term
|
|
171
|
+
cursor_position -= len(term) + 1
|
|
172
|
+
del terms[index]
|
|
177
173
|
continue
|
|
178
174
|
else:
|
|
179
175
|
break
|
|
180
176
|
if cursor_position != initial_cursor_position:
|
|
181
|
-
|
|
177
|
+
max_term_index = len(terms)
|
|
182
178
|
column = cursor_position+cursor_offset
|
|
183
|
-
if (drs_type == DrsType.DIRECTORY) and (not
|
|
179
|
+
if (drs_type == DrsType.DIRECTORY) and (not has_white_term):
|
|
184
180
|
issue = ExtraSeparator(column=column)
|
|
185
181
|
warnings.append(issue)
|
|
186
182
|
else:
|
|
187
183
|
issue = ExtraChar(column=column)
|
|
188
184
|
errors.append(issue)
|
|
189
|
-
for index in range(
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
if not
|
|
185
|
+
for index in range(max_term_index-1, -1, -1):
|
|
186
|
+
term = terms[index]
|
|
187
|
+
len_term = len(term)
|
|
188
|
+
if not term:
|
|
193
189
|
column = cursor_position + cursor_offset
|
|
194
190
|
issue = ExtraSeparator(column=column)
|
|
195
191
|
if (drs_type != DrsType.DIRECTORY) or self.pedantic or (index == 0):
|
|
196
192
|
errors.append(issue)
|
|
197
193
|
else:
|
|
198
194
|
warnings.append(issue)
|
|
199
|
-
del
|
|
200
|
-
if
|
|
201
|
-
column = cursor_position + cursor_offset -
|
|
202
|
-
issue =
|
|
195
|
+
del terms[index]
|
|
196
|
+
if term.isspace():
|
|
197
|
+
column = cursor_position + cursor_offset - len_term
|
|
198
|
+
issue = BlankTerm(column=column)
|
|
203
199
|
errors.append(issue)
|
|
204
|
-
del
|
|
205
|
-
cursor_position -=
|
|
206
|
-
|
|
207
|
-
# Mypy doesn't understand that
|
|
200
|
+
del terms[index]
|
|
201
|
+
cursor_position -= len_term + 1
|
|
202
|
+
|
|
203
|
+
# Mypy doesn't understand that ParsingIssues are DrsIssues...
|
|
208
204
|
sorted_errors = DrsValidator._sort_parser_issues(errors) # type: ignore
|
|
209
205
|
sorted_warnings = DrsValidator._sort_parser_issues(warnings) # type: ignore
|
|
210
|
-
return
|
|
211
|
-
|
|
206
|
+
return terms, sorted_errors, sorted_warnings # type: ignore
|
|
207
|
+
|
|
212
208
|
@staticmethod
|
|
213
|
-
def _sort_parser_issues(issues: list[
|
|
209
|
+
def _sort_parser_issues(issues: list[ParsingIssue]) -> list[ParsingIssue]:
|
|
214
210
|
return sorted(issues, key=lambda issue: issue.column if issue.column else 0)
|
|
215
211
|
|
|
216
|
-
def
|
|
212
|
+
def _validate_term(self, term: str, part: DrsPart) -> bool:
|
|
217
213
|
match part.kind:
|
|
218
214
|
case DrsPartKind.COLLECTION:
|
|
219
215
|
casted_part: DrsCollection = cast(DrsCollection, part)
|
|
220
216
|
try:
|
|
221
|
-
matching_terms = projects.valid_term_in_collection(
|
|
217
|
+
matching_terms = projects.valid_term_in_collection(term,
|
|
222
218
|
self.project_id,
|
|
223
219
|
casted_part.collection_id)
|
|
224
220
|
except Exception as e:
|
|
225
|
-
msg = f'problem while validating
|
|
226
|
-
raise
|
|
221
|
+
msg = f'problem while validating term: {e}.Abort.'
|
|
222
|
+
raise APIException(msg) from e
|
|
227
223
|
if len(matching_terms) > 0:
|
|
228
224
|
return True
|
|
229
225
|
else:
|
|
230
226
|
return False
|
|
231
227
|
case DrsPartKind.CONSTANT:
|
|
232
228
|
part_casted: DrsConstant = cast(DrsConstant, part)
|
|
233
|
-
return part_casted.value !=
|
|
229
|
+
return part_casted.value != term
|
|
234
230
|
case _:
|
|
235
|
-
raise
|
|
231
|
+
raise RuntimeError(f'unsupported DRS specs part type {part.kind}')
|
|
236
232
|
|
|
237
233
|
def _create_report(self,
|
|
238
234
|
type: DrsType,
|
|
@@ -240,67 +236,69 @@ class DrsValidator(DrsApplication):
|
|
|
240
236
|
errors: list[DrsIssue],
|
|
241
237
|
warnings: list[DrsIssue]) -> DrsValidationReport:
|
|
242
238
|
return DrsValidationReport(project_id=self.project_id, type=type,
|
|
243
|
-
expression=drs_expression,
|
|
239
|
+
expression=drs_expression,
|
|
240
|
+
errors=cast(list[ValidationError], errors),
|
|
241
|
+
warnings=cast(list[ValidationWarning], warnings))
|
|
244
242
|
|
|
245
243
|
def _validate(self,
|
|
246
244
|
drs_expression: str,
|
|
247
245
|
specs: DrsSpecification) -> DrsValidationReport:
|
|
248
|
-
|
|
249
|
-
if not
|
|
246
|
+
terms, errors, warnings = self._parse(drs_expression, specs.separator, specs.type)
|
|
247
|
+
if not terms:
|
|
250
248
|
return self._create_report(specs.type, drs_expression, errors, warnings) # Early exit.
|
|
251
|
-
|
|
252
|
-
|
|
249
|
+
term_index = 0
|
|
250
|
+
term_max_index = len(terms)
|
|
253
251
|
part_index = 0
|
|
254
252
|
part_max_index = len(specs.parts)
|
|
255
253
|
matching_code_mapping = dict()
|
|
256
254
|
while part_index < part_max_index:
|
|
257
|
-
|
|
255
|
+
term = terms[term_index]
|
|
258
256
|
part = specs.parts[part_index]
|
|
259
|
-
if self.
|
|
260
|
-
|
|
257
|
+
if self._validate_term(term, part):
|
|
258
|
+
term_index += 1
|
|
261
259
|
part_index += 1
|
|
262
260
|
matching_code_mapping[part.__str__()] = 0
|
|
263
261
|
elif part.kind == DrsPartKind.CONSTANT or \
|
|
264
262
|
cast(DrsCollection, part).is_required:
|
|
265
|
-
issue:
|
|
266
|
-
|
|
263
|
+
issue: ComplianceIssue = InvalidTerm(term=term,
|
|
264
|
+
term_position=term_index+1,
|
|
267
265
|
collection_id_or_constant_value=str(part))
|
|
268
266
|
errors.append(issue)
|
|
269
267
|
matching_code_mapping[part.__str__()] = 1
|
|
270
|
-
|
|
268
|
+
term_index += 1
|
|
271
269
|
part_index += 1
|
|
272
|
-
else: # The part is not required so try to match the
|
|
270
|
+
else: # The part is not required so try to match the term with the next part.
|
|
273
271
|
part_index += 1
|
|
274
272
|
matching_code_mapping[part.__str__()] = -1
|
|
275
|
-
if
|
|
273
|
+
if term_index == term_max_index:
|
|
276
274
|
break
|
|
277
275
|
# Cases:
|
|
278
|
-
# - All
|
|
279
|
-
# - Not enough
|
|
280
|
-
# - Extra
|
|
281
|
-
# + The last collections are required => report extra
|
|
282
|
-
# + The last collections are not required and these
|
|
276
|
+
# - All terms and collections have been processed.
|
|
277
|
+
# - Not enough term to process all collections.
|
|
278
|
+
# - Extra terms left whereas all collections have been processed:
|
|
279
|
+
# + The last collections are required => report extra terms.
|
|
280
|
+
# + The last collections are not required and these terms were not validated by them.
|
|
283
281
|
# => Should report error even if the collections are not required.
|
|
284
|
-
if part_index < part_max_index: # Missing
|
|
282
|
+
if part_index < part_max_index: # Missing terms.
|
|
285
283
|
for index in range(part_index, part_max_index):
|
|
286
284
|
part = specs.parts[index]
|
|
287
|
-
issue =
|
|
285
|
+
issue = MissingTerm(collection_id=str(part), collection_position=index+1)
|
|
288
286
|
if part.kind == DrsPartKind.CONSTANT or \
|
|
289
287
|
cast(DrsCollection, part).is_required:
|
|
290
288
|
errors.append(issue)
|
|
291
289
|
else:
|
|
292
290
|
warnings.append(issue)
|
|
293
|
-
elif
|
|
294
|
-
part_index -=
|
|
295
|
-
for index in range(
|
|
296
|
-
|
|
291
|
+
elif term_index < term_max_index: # Extra terms.
|
|
292
|
+
part_index -= term_max_index - term_index
|
|
293
|
+
for index in range(term_index, term_max_index):
|
|
294
|
+
term = terms[index]
|
|
297
295
|
part = specs.parts[part_index]
|
|
298
296
|
if part.kind != DrsPartKind.CONSTANT and \
|
|
299
297
|
(not cast(DrsCollection, part).is_required) and \
|
|
300
298
|
matching_code_mapping[part.__str__()] < 0:
|
|
301
|
-
issue =
|
|
299
|
+
issue = ExtraTerm(term=term, term_position=index, collection_id=str(part))
|
|
302
300
|
else:
|
|
303
|
-
issue =
|
|
301
|
+
issue = ExtraTerm(term=term, term_position=index, collection_id=None)
|
|
304
302
|
errors.append(issue)
|
|
305
303
|
part_index += 1
|
|
306
304
|
return self._create_report(specs.type, drs_expression, errors, warnings)
|
|
@@ -329,4 +327,4 @@ if __name__ == "__main__":
|
|
|
329
327
|
for warning in report.warnings:
|
|
330
328
|
print(warning)
|
|
331
329
|
else:
|
|
332
|
-
print('warning(s): 0')
|
|
330
|
+
print('warning(s): 0')
|
esgvoc/cli/drs.py
CHANGED
|
@@ -1,13 +1,15 @@
|
|
|
1
|
-
|
|
2
|
-
from esgvoc.apps.drs.report import DrsValidationReport, DrsGeneratorReport
|
|
3
|
-
from esgvoc.apps.drs.validator import DrsValidator
|
|
1
|
+
import shlex
|
|
4
2
|
import sys
|
|
3
|
+
from typing import List, Optional
|
|
4
|
+
|
|
5
5
|
import typer
|
|
6
6
|
from rich.console import Console
|
|
7
7
|
from rich.table import Table
|
|
8
|
-
|
|
8
|
+
|
|
9
9
|
import esgvoc.api as ev
|
|
10
|
-
import
|
|
10
|
+
from esgvoc.apps.drs.generator import DrsGenerator
|
|
11
|
+
from esgvoc.apps.drs.report import DrsGenerationReport, DrsValidationReport
|
|
12
|
+
from esgvoc.apps.drs.validator import DrsValidator
|
|
11
13
|
|
|
12
14
|
app = typer.Typer()
|
|
13
15
|
console = Console()
|
|
@@ -35,6 +37,10 @@ def drsvalid(
|
|
|
35
37
|
file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS validation inputs, one per line in the form <project> <drstype> <string>"),
|
|
36
38
|
verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed validation results"),
|
|
37
39
|
output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the DRS entries validation"),
|
|
40
|
+
rm_prefix: Optional[str] = typer.Option(None,"-p","--prefix", help="Remove given prefix from all checked directory"),
|
|
41
|
+
pedantic: Optional[bool] = typer.Option(False,"-e","--enforce", help="Enable pedantic mode, enforcing strict compliance, mean that warnings are now errors.")
|
|
42
|
+
|
|
43
|
+
|
|
38
44
|
|
|
39
45
|
) -> List[DrsValidationReport]:
|
|
40
46
|
"""
|
|
@@ -88,13 +94,17 @@ def drsvalid(
|
|
|
88
94
|
|
|
89
95
|
string = entries[i]
|
|
90
96
|
i += 1
|
|
91
|
-
validator = DrsValidator(current_project)
|
|
97
|
+
validator = DrsValidator(current_project, pedantic=pedantic)
|
|
92
98
|
report = None
|
|
93
99
|
match current_drs_type:
|
|
94
100
|
case "filename":
|
|
95
101
|
report = validator.validate_file_name(string)
|
|
96
102
|
case "directory":
|
|
97
|
-
|
|
103
|
+
if rm_prefix:
|
|
104
|
+
prefix = rm_prefix+"/" if rm_prefix[-1]!="/" else ""
|
|
105
|
+
else:
|
|
106
|
+
prefix=None
|
|
107
|
+
report = validator.validate_directory(string, prefix)
|
|
98
108
|
case "dataset":
|
|
99
109
|
report = validator.validate_dataset_id(string)
|
|
100
110
|
case _:
|
|
@@ -111,7 +121,7 @@ def drsvalid(
|
|
|
111
121
|
|
|
112
122
|
for report in reports:
|
|
113
123
|
entry = str(report.expression)
|
|
114
|
-
proj_and_type = str(report.project_id) + " " + report.type + " "
|
|
124
|
+
proj_and_type = str(report.project_id) + " " + report.type + " "
|
|
115
125
|
warnings = "\n".join(["⚠️ " + str(warning) for warning in report.warnings])
|
|
116
126
|
errors = "\n".join(["⚠️ " + str(error) for error in report.errors])
|
|
117
127
|
valid = "✅ Valid" if report else "❌ Invalid"
|
|
@@ -136,16 +146,16 @@ def drsvalid(
|
|
|
136
146
|
|
|
137
147
|
@app.command()
|
|
138
148
|
def drsgen(
|
|
139
|
-
drs_entries: Optional[List[str]] = typer.Argument(None, help="List of inputs to generate DRS in the form <project> <drstype> <
|
|
140
|
-
file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS generation inputs, one per line in the form <project> <drstype> <
|
|
149
|
+
drs_entries: Optional[List[str]] = typer.Argument(None, help="List of inputs to generate DRS in the form <project> <drstype> <bag_of_terms>"),
|
|
150
|
+
file: Optional[typer.FileText] = typer.Option(None, "--file", "-f", help="File containing DRS generation inputs, one per line in the form <project> <drstype> <bag_of_terms>"),
|
|
141
151
|
verbose: bool = typer.Option(False, "-v", "--verbose", help="Provide detailed generation results"),
|
|
142
152
|
output: Optional[str] = typer.Option(None, "-o", "--output", help="File to save the generated DRS entries"),
|
|
143
|
-
) -> List[
|
|
153
|
+
) -> List[DrsGenerationReport]:
|
|
144
154
|
"""
|
|
145
|
-
Generates DRS strings for a specific project and type based on input bag of
|
|
155
|
+
Generates DRS strings for a specific project and type based on input bag of terms.
|
|
146
156
|
|
|
147
157
|
Args:
|
|
148
|
-
drs_entries (Optional[List[str]]): A list of inputs in the form <project> <drstype> <
|
|
158
|
+
drs_entries (Optional[List[str]]): A list of inputs in the form <project> <drstype> <bag_of_terms>.
|
|
149
159
|
file (Optional[typer.FileText]): File containing DRS generation inputs, one per line.
|
|
150
160
|
verbose (bool): If true, prints detailed generation results.
|
|
151
161
|
output (Optional[str]): File path to save the generated DRS entries.
|
|
@@ -189,19 +199,19 @@ def drsgen(
|
|
|
189
199
|
if current_drs_type is None:
|
|
190
200
|
raise typer.BadParameter(f"Invalid drs_type: {entries[i]}")
|
|
191
201
|
|
|
192
|
-
|
|
193
|
-
|
|
202
|
+
bag_of_terms = entries[i]
|
|
203
|
+
bag_of_terms = set(entries[i].split(" "))
|
|
194
204
|
i += 1
|
|
195
205
|
|
|
196
206
|
generator = DrsGenerator(current_project)
|
|
197
207
|
report = None
|
|
198
208
|
match current_drs_type:
|
|
199
209
|
case "filename":
|
|
200
|
-
report = generator.
|
|
210
|
+
report = generator.generate_file_name_from_bag_of_terms(bag_of_terms)
|
|
201
211
|
case "directory":
|
|
202
|
-
report = generator.
|
|
212
|
+
report = generator.generate_directory_from_bag_of_terms(bag_of_terms)
|
|
203
213
|
case "dataset":
|
|
204
|
-
report = generator.
|
|
214
|
+
report = generator.generate_dataset_id_from_bag_of_terms(bag_of_terms)
|
|
205
215
|
case _:
|
|
206
216
|
raise RuntimeError("drstype is not known")
|
|
207
217
|
generated_reports.append(report)
|
|
@@ -216,7 +226,7 @@ def drsgen(
|
|
|
216
226
|
entry = str(report.mapping_used)
|
|
217
227
|
warnings = "\n".join(["⚠️ " + str(warning) for warning in report.warnings])
|
|
218
228
|
errors = "\n".join([f"🔍 {error}" for error in report.errors])
|
|
219
|
-
result = report.
|
|
229
|
+
result = report.generated_drs_expression
|
|
220
230
|
table.add_row(entry, warnings, errors, result)
|
|
221
231
|
table.add_row("----", "----", "----", "----")
|
|
222
232
|
if table.columns[3].width is not None and len(result) > table.columns[3].width:
|
esgvoc/cli/get.py
CHANGED
|
@@ -19,7 +19,7 @@ def validate_key_format(key: str):
|
|
|
19
19
|
"""
|
|
20
20
|
Validate if the key matches the XXXX:YYYY:ZZZZ format.
|
|
21
21
|
"""
|
|
22
|
-
if not re.match(r"^[a-zA-Z0-9\/_]*:[a-zA-Z0-9\/_]*:[a-zA-Z0-9\/_]*$", key):
|
|
22
|
+
if not re.match(r"^[a-zA-Z0-9\/_]*:[a-zA-Z0-9\/_]*:[a-zA-Z0-9\/_.]*$", key):
|
|
23
23
|
raise typer.BadParameter(f"Invalid key format: {key}. Must be XXXX:YYYY:ZZZZ.")
|
|
24
24
|
return key.split(":")
|
|
25
25
|
|
|
@@ -96,30 +96,31 @@ def display(data:Any):
|
|
|
96
96
|
@app.command()
|
|
97
97
|
def get(keys: list[str] = typer.Argument(..., help="List of keys in XXXX:YYYY:ZZZZ format")):
|
|
98
98
|
"""
|
|
99
|
-
Retrieve a specific value from the database system
|
|
100
|
-
This command allows you to fetch a value by specifying the universe/project, data_descriptor/collection,
|
|
101
|
-
and term in a structured format
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
<
|
|
109
|
-
<
|
|
110
|
-
|
|
99
|
+
Retrieve a specific value from the database system.\n
|
|
100
|
+
This command allows you to fetch a value by specifying the universe/project, data_descriptor/collection,
|
|
101
|
+
and term in a structured format.\n
|
|
102
|
+
\n
|
|
103
|
+
|
|
104
|
+
Usage:\n
|
|
105
|
+
`get <project>:<collection>:<term>`\n
|
|
106
|
+
\n
|
|
107
|
+
Arguments:\n
|
|
108
|
+
<project>\tThe project id to query. like `cmip6plus`\n
|
|
109
|
+
<collection>\tThe collection id in the specified database.\n
|
|
110
|
+
<term>\t\tThe term id within the specified collection.\n
|
|
111
|
+
\n
|
|
111
112
|
Example:
|
|
112
|
-
To retrieve the value from the "cmip6plus" project, under the "institution_id" column,
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
Notes
|
|
119
|
-
- Ensure data exist in your system before using this command (use status command to see whats available)
|
|
120
|
-
- Use a colon (`:`) to separate the parts of the argument.
|
|
121
|
-
- if more than one argument is given i.e get X:Y:Z A:B:C the 2 results are appended.
|
|
122
|
-
|
|
113
|
+
To retrieve the value from the "cmip6plus" project, under the "institution_id" column, the term with the identifier "ipsl", you would use: \n
|
|
114
|
+
`get cmip6plus:institution_id:ipsl`\n
|
|
115
|
+
The default project is the universe CV : the argument would be like `universe:institution:ipsl` or `:institution:ipsl` \n
|
|
116
|
+
- to get list of available term from universe institution `:institution:` \n
|
|
117
|
+
\n
|
|
118
|
+
\n
|
|
119
|
+
Notes:\n
|
|
120
|
+
- Ensure data exist in your system before using this command (use `esgvoc status` command to see whats available).\n
|
|
121
|
+
- Use a colon (`:`) to separate the parts of the argument. \n
|
|
122
|
+
- if more than one argument is given i.e get X:Y:Z A:B:C the 2 results are appended. \n
|
|
123
|
+
\n
|
|
123
124
|
"""
|
|
124
125
|
known_projects = get_all_projects()
|
|
125
126
|
|
|
@@ -133,7 +134,7 @@ def get(keys: list[str] = typer.Argument(..., help="List of keys in XXXX:YYYY:ZZ
|
|
|
133
134
|
if where == "" or where=="universe":
|
|
134
135
|
res = handle_universe(what,who)
|
|
135
136
|
elif where in known_projects:
|
|
136
|
-
res = handle_project(where,what,who,
|
|
137
|
+
res = handle_project(where,what,who,None)
|
|
137
138
|
else:
|
|
138
139
|
res = handle_unknown(where,what,who)
|
|
139
140
|
|
esgvoc/cli/install.py
CHANGED
|
@@ -1,14 +1,17 @@
|
|
|
1
1
|
import typer
|
|
2
|
-
from esgvoc.core.service import
|
|
2
|
+
from esgvoc.core.service import current_state
|
|
3
3
|
|
|
4
4
|
app = typer.Typer()
|
|
5
5
|
|
|
6
6
|
@app.command()
|
|
7
7
|
def install():
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
"""Initialize default config and apply settings"""
|
|
9
|
+
try:
|
|
10
|
+
typer.echo("Initialized default configuration")
|
|
11
|
+
current_state.synchronize_all()
|
|
12
|
+
except Exception as e:
|
|
13
|
+
typer.echo(f"Error during installation: {str(e)}", err=True)
|
|
14
|
+
raise typer.Exit(1)
|
|
15
|
+
|
|
16
|
+
if __name__ == "__main__":
|
|
17
|
+
app()
|
esgvoc/cli/main.py
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import typer
|
|
2
|
-
from esgvoc.cli.config import app as config_app
|
|
3
2
|
from esgvoc.cli.get import app as get_app
|
|
4
3
|
from esgvoc.cli.status import app as status_app
|
|
5
4
|
from esgvoc.cli.valid import app as valid_app
|
|
@@ -8,7 +7,6 @@ from esgvoc.cli.drs import app as drs_app
|
|
|
8
7
|
app = typer.Typer()
|
|
9
8
|
|
|
10
9
|
# Register the subcommands
|
|
11
|
-
app.add_typer(config_app)
|
|
12
10
|
app.add_typer(get_app)
|
|
13
11
|
app.add_typer(status_app)
|
|
14
12
|
app.add_typer(valid_app)
|
esgvoc/cli/status.py
CHANGED
|
@@ -20,17 +20,17 @@ def status():
|
|
|
20
20
|
i.e summary of version of usable ressources (between remote/cached)
|
|
21
21
|
|
|
22
22
|
"""
|
|
23
|
-
|
|
24
|
-
service.
|
|
23
|
+
assert(service.current_state is not None)
|
|
24
|
+
service.current_state.get_state_summary()
|
|
25
25
|
#display(service.state_service.table())
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
table = Table(show_header=False, show_lines=True)
|
|
29
29
|
|
|
30
30
|
table.add_row("","Remote github repo","Local repository","Cache Database", style = "bright_green")
|
|
31
|
-
table.add_row("Universe path",service.
|
|
32
|
-
table.add_row("Version",service.
|
|
33
|
-
for proj_name,proj in service.
|
|
31
|
+
table.add_row("Universe path",service.current_state.universe.github_repo,service.current_state.universe.local_path,service.current_state.universe.db_path, style = "white")
|
|
32
|
+
table.add_row("Version",service.current_state.universe.github_version,service.current_state.universe.local_version,service.current_state.universe.db_version, style="bright_blue")
|
|
33
|
+
for proj_name,proj in service.current_state.projects.items():
|
|
34
34
|
table.add_row(f"{proj_name} path",proj.github_repo,proj.local_path,proj.db_path, style="white")
|
|
35
35
|
table.add_row("Version",proj.github_version,proj.local_version,proj.db_version,style ="bright_blue")
|
|
36
36
|
display(table)
|