rdetoolkit 1.5.1__cp314-cp314-win_amd64.whl → 1.5.2__cp314-cp314-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rdetoolkit/__init__.py +19 -1
- rdetoolkit/cli/validate.py +84 -55
- rdetoolkit/cmd/validate.py +18 -15
- rdetoolkit/config.py +211 -23
- rdetoolkit/core.cp314-win_amd64.pyd +0 -0
- rdetoolkit/exceptions.py +94 -0
- rdetoolkit/exceptions.pyi +21 -0
- rdetoolkit/models/metadata.py +97 -2
- rdetoolkit/models/metadata.pyi +22 -1
- rdetoolkit/validation.py +94 -8
- rdetoolkit/validation.pyi +5 -0
- rdetoolkit/workflows.py +8 -2
- {rdetoolkit-1.5.1.dist-info → rdetoolkit-1.5.2.dist-info}/METADATA +17 -9
- {rdetoolkit-1.5.1.dist-info → rdetoolkit-1.5.2.dist-info}/RECORD +17 -17
- {rdetoolkit-1.5.1.dist-info → rdetoolkit-1.5.2.dist-info}/WHEEL +0 -0
- {rdetoolkit-1.5.1.dist-info → rdetoolkit-1.5.2.dist-info}/entry_points.txt +0 -0
- {rdetoolkit-1.5.1.dist-info → rdetoolkit-1.5.2.dist-info}/licenses/LICENSE +0 -0
rdetoolkit/__init__.py
CHANGED
|
@@ -1,9 +1,21 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import sys
|
|
4
|
+
import warnings
|
|
5
|
+
|
|
6
|
+
# Python 3.9 deprecation warning (Issue #360)
|
|
7
|
+
if sys.version_info < (3, 10):
|
|
8
|
+
warnings.warn(
|
|
9
|
+
"Python 3.9 support is deprecated and will be removed in rdetoolkit v2.0. "
|
|
10
|
+
"Please upgrade to Python 3.10 or later.",
|
|
11
|
+
DeprecationWarning,
|
|
12
|
+
stacklevel=2,
|
|
13
|
+
)
|
|
14
|
+
|
|
3
15
|
from importlib import import_module
|
|
4
16
|
from typing import Any
|
|
5
17
|
|
|
6
|
-
__version__ = "1.5.
|
|
18
|
+
__version__ = "1.5.2"
|
|
7
19
|
|
|
8
20
|
_LAZY_ATTRS: dict[str, tuple[str, str]] = {
|
|
9
21
|
"DirectoryOps": ("rdetoolkit.core", "DirectoryOps"),
|
|
@@ -26,6 +38,9 @@ _LAZY_MODULES: dict[str, str] = {
|
|
|
26
38
|
"rde2util": "rdetoolkit.rde2util",
|
|
27
39
|
"rdelogger": "rdetoolkit.rdelogger",
|
|
28
40
|
"workflows": "rdetoolkit.workflows",
|
|
41
|
+
"processing": "rdetoolkit.processing",
|
|
42
|
+
"storage": "rdetoolkit.storage",
|
|
43
|
+
"traceback": "rdetoolkit.traceback",
|
|
29
44
|
"config": "rdetoolkit.models.config",
|
|
30
45
|
"invoice": "rdetoolkit.models.invoice",
|
|
31
46
|
"invoice_schema": "rdetoolkit.models.invoice_schema",
|
|
@@ -55,6 +70,9 @@ __all__ = [
|
|
|
55
70
|
"rde2util",
|
|
56
71
|
"rdelogger",
|
|
57
72
|
"workflows",
|
|
73
|
+
"processing",
|
|
74
|
+
"storage",
|
|
75
|
+
"traceback",
|
|
58
76
|
"config",
|
|
59
77
|
"invoice",
|
|
60
78
|
"invoice_schema",
|
rdetoolkit/cli/validate.py
CHANGED
|
@@ -13,7 +13,13 @@ if TYPE_CHECKING:
|
|
|
13
13
|
|
|
14
14
|
app = typer.Typer(
|
|
15
15
|
name="validate",
|
|
16
|
-
help="Validate RDE schema and data files
|
|
16
|
+
help="""Validate RDE schema and data files.
|
|
17
|
+
|
|
18
|
+
All validate commands use standardized exit codes:
|
|
19
|
+
0 = Success (validation passed)
|
|
20
|
+
1 = Validation failure (data/schema issues)
|
|
21
|
+
2 = Usage error (invalid arguments, missing files)
|
|
22
|
+
""",
|
|
17
23
|
no_args_is_help=True,
|
|
18
24
|
)
|
|
19
25
|
|
|
@@ -87,10 +93,11 @@ def handle_validation_error(error: Exception, error_type: str) -> None:
|
|
|
87
93
|
typer.echo(str(error), err=True)
|
|
88
94
|
raise typer.Exit(code=1) from error
|
|
89
95
|
|
|
90
|
-
#
|
|
91
|
-
|
|
96
|
+
# All other errors (including internal errors) map to exit code 2
|
|
97
|
+
# This includes unexpected exceptions that indicate configuration or usage issues
|
|
98
|
+
msg = f"Error during {error_type}: {error}"
|
|
92
99
|
typer.echo(msg, err=True)
|
|
93
|
-
raise typer.Exit(code=
|
|
100
|
+
raise typer.Exit(code=2) from error
|
|
94
101
|
|
|
95
102
|
|
|
96
103
|
# Common options for validation commands
|
|
@@ -141,6 +148,11 @@ def invoice_schema(
|
|
|
141
148
|
Validates that the invoice schema file conforms to the expected structure
|
|
142
149
|
and contains valid field definitions.
|
|
143
150
|
|
|
151
|
+
Exit Codes:
|
|
152
|
+
0: Success - validation passed
|
|
153
|
+
1: Validation failure - schema structure issues found
|
|
154
|
+
2: Usage error - file not found or invalid arguments
|
|
155
|
+
|
|
144
156
|
Examples:
|
|
145
157
|
rdetoolkit validate invoice-schema tasksupport/invoice.schema.json
|
|
146
158
|
rdetoolkit validate invoice-schema schema.json --format json
|
|
@@ -151,6 +163,9 @@ def invoice_schema(
|
|
|
151
163
|
command = InvoiceSchemaCommand(schema_path)
|
|
152
164
|
result = command.execute()
|
|
153
165
|
handle_validation_result(result, format_type, strict, quiet)
|
|
166
|
+
except typer.Exit:
|
|
167
|
+
# Re-raise typer.Exit to let Typer handle it properly
|
|
168
|
+
raise
|
|
154
169
|
except FileNotFoundError:
|
|
155
170
|
handle_file_not_found(schema_path, "Schema")
|
|
156
171
|
except Exception as e:
|
|
@@ -177,6 +192,11 @@ def metadata_def(
|
|
|
177
192
|
Validates that the metadata definition file conforms to the expected
|
|
178
193
|
structure defined by the MetadataItem schema.
|
|
179
194
|
|
|
195
|
+
Exit Codes:
|
|
196
|
+
0: Success - validation passed
|
|
197
|
+
1: Validation failure - definition structure issues found
|
|
198
|
+
2: Usage error - file not found or invalid arguments
|
|
199
|
+
|
|
180
200
|
Examples:
|
|
181
201
|
rdetoolkit validate metadata-def tasksupport/metadata_def.json
|
|
182
202
|
rdetoolkit validate metadata-def metadata_def.json --format json
|
|
@@ -187,6 +207,9 @@ def metadata_def(
|
|
|
187
207
|
command = MetadataDefCommand(metadata_def_path)
|
|
188
208
|
result = command.execute()
|
|
189
209
|
handle_validation_result(result, format_type, strict, quiet)
|
|
210
|
+
except typer.Exit:
|
|
211
|
+
# Re-raise typer.Exit to let Typer handle it properly
|
|
212
|
+
raise
|
|
190
213
|
except FileNotFoundError:
|
|
191
214
|
handle_file_not_found(metadata_def_path, "Metadata definition")
|
|
192
215
|
except Exception as e:
|
|
@@ -195,26 +218,22 @@ def metadata_def(
|
|
|
195
218
|
|
|
196
219
|
@app.command("invoice")
|
|
197
220
|
def invoice(
|
|
198
|
-
invoice_path:
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
dir_okay=False,
|
|
215
|
-
resolve_path=True,
|
|
216
|
-
),
|
|
217
|
-
],
|
|
221
|
+
invoice_path: Path = typer.Argument(
|
|
222
|
+
...,
|
|
223
|
+
help="Path to invoice.json file",
|
|
224
|
+
exists=True,
|
|
225
|
+
dir_okay=False,
|
|
226
|
+
resolve_path=True,
|
|
227
|
+
),
|
|
228
|
+
schema_path: Path = typer.Option(
|
|
229
|
+
...,
|
|
230
|
+
"--schema",
|
|
231
|
+
"-s",
|
|
232
|
+
help="Path to invoice.schema.json file",
|
|
233
|
+
exists=True,
|
|
234
|
+
dir_okay=False,
|
|
235
|
+
resolve_path=True,
|
|
236
|
+
),
|
|
218
237
|
format_type: FormatOption = OutputFormat.TEXT,
|
|
219
238
|
strict: StrictOption = False,
|
|
220
239
|
quiet: QuietOption = False,
|
|
@@ -224,6 +243,11 @@ def invoice(
|
|
|
224
243
|
Validates that the invoice.json file conforms to the structure and
|
|
225
244
|
constraints defined in the invoice.schema.json file.
|
|
226
245
|
|
|
246
|
+
Exit Codes:
|
|
247
|
+
0: Success - validation passed
|
|
248
|
+
1: Validation failure - data violates schema constraints
|
|
249
|
+
2: Usage error - files not found or invalid arguments
|
|
250
|
+
|
|
227
251
|
Examples:
|
|
228
252
|
rdetoolkit validate invoice raw/invoice.json --schema tasksupport/invoice.schema.json
|
|
229
253
|
rdetoolkit validate invoice invoice.json -s schema.json --format json
|
|
@@ -251,26 +275,22 @@ def invoice(
|
|
|
251
275
|
|
|
252
276
|
@app.command("metadata")
|
|
253
277
|
def metadata(
|
|
254
|
-
metadata_path:
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
dir_okay=False,
|
|
271
|
-
resolve_path=True,
|
|
272
|
-
),
|
|
273
|
-
],
|
|
278
|
+
metadata_path: Path = typer.Argument(
|
|
279
|
+
...,
|
|
280
|
+
help="Path to metadata.json file",
|
|
281
|
+
exists=True,
|
|
282
|
+
dir_okay=False,
|
|
283
|
+
resolve_path=True,
|
|
284
|
+
),
|
|
285
|
+
schema_path: Path = typer.Option(
|
|
286
|
+
...,
|
|
287
|
+
"--schema",
|
|
288
|
+
"-s",
|
|
289
|
+
help="Path to metadata definition JSON file",
|
|
290
|
+
exists=True,
|
|
291
|
+
dir_okay=False,
|
|
292
|
+
resolve_path=True,
|
|
293
|
+
),
|
|
274
294
|
format_type: FormatOption = OutputFormat.TEXT,
|
|
275
295
|
strict: StrictOption = False,
|
|
276
296
|
quiet: QuietOption = False,
|
|
@@ -280,6 +300,11 @@ def metadata(
|
|
|
280
300
|
Validates that the metadata.json file conforms to the structure
|
|
281
301
|
defined in the metadata definition file.
|
|
282
302
|
|
|
303
|
+
Exit Codes:
|
|
304
|
+
0: Success - validation passed
|
|
305
|
+
1: Validation failure - data violates definition constraints
|
|
306
|
+
2: Usage error - files not found or invalid arguments
|
|
307
|
+
|
|
283
308
|
Examples:
|
|
284
309
|
rdetoolkit validate metadata raw/metadata.json --schema tasksupport/metadata_def.json
|
|
285
310
|
rdetoolkit validate metadata metadata.json -s metadata_def.json --format json
|
|
@@ -290,6 +315,9 @@ def metadata(
|
|
|
290
315
|
command = MetadataCommand(metadata_path, schema_path)
|
|
291
316
|
result = command.execute()
|
|
292
317
|
handle_validation_result(result, format_type, strict, quiet)
|
|
318
|
+
except typer.Exit:
|
|
319
|
+
# Re-raise typer.Exit to let Typer handle it properly
|
|
320
|
+
raise
|
|
293
321
|
except FileNotFoundError:
|
|
294
322
|
# Determine which file is missing
|
|
295
323
|
if not metadata_path.exists():
|
|
@@ -304,16 +332,14 @@ def metadata(
|
|
|
304
332
|
|
|
305
333
|
@app.command("all")
|
|
306
334
|
def validate_all(
|
|
307
|
-
project_dir:
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
),
|
|
316
|
-
] = None,
|
|
335
|
+
project_dir: Optional[Path] = typer.Argument(
|
|
336
|
+
None,
|
|
337
|
+
help="Root directory of RDE project (defaults to current directory)",
|
|
338
|
+
exists=True,
|
|
339
|
+
file_okay=False,
|
|
340
|
+
dir_okay=True,
|
|
341
|
+
resolve_path=True,
|
|
342
|
+
),
|
|
317
343
|
format_type: FormatOption = OutputFormat.TEXT,
|
|
318
344
|
strict: StrictOption = False,
|
|
319
345
|
quiet: QuietOption = False,
|
|
@@ -329,7 +355,10 @@ def validate_all(
|
|
|
329
355
|
- input/invoice/invoice.json (with schema)
|
|
330
356
|
- input/metadata/metadata.json (if exists, with schema)
|
|
331
357
|
|
|
332
|
-
Exit
|
|
358
|
+
Exit Codes:
|
|
359
|
+
0: Success - all validations passed
|
|
360
|
+
1: Validation failure - one or more validations failed
|
|
361
|
+
2: Usage error - project directory not found or invalid arguments
|
|
333
362
|
|
|
334
363
|
Examples:
|
|
335
364
|
rdetoolkit validate all
|
rdetoolkit/cmd/validate.py
CHANGED
|
@@ -8,7 +8,7 @@ from dataclasses import dataclass
|
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from typing import Optional, Union
|
|
10
10
|
|
|
11
|
-
from rdetoolkit.validation import InvoiceValidator, MetadataValidator
|
|
11
|
+
from rdetoolkit.validation import InvoiceValidator, MetadataDefinitionValidator, MetadataValidator
|
|
12
12
|
|
|
13
13
|
|
|
14
14
|
@dataclass
|
|
@@ -181,8 +181,7 @@ def determine_exit_code(result: ValidationResult, strict: bool = False) -> int:
|
|
|
181
181
|
Exit codes:
|
|
182
182
|
- 0: Validation passed
|
|
183
183
|
- 1: Validation failed (errors or warnings in strict mode)
|
|
184
|
-
- 2: Usage/argument errors (handled by
|
|
185
|
-
- 3: Internal errors (handled by exception handlers)
|
|
184
|
+
- 2: Usage/argument errors (handled by CLI layer)
|
|
186
185
|
|
|
187
186
|
Args:
|
|
188
187
|
result: Validation result
|
|
@@ -428,7 +427,11 @@ class MetadataDefCommand(_ValidationErrorParser):
|
|
|
428
427
|
"""Command to validate metadata definition JSON files.
|
|
429
428
|
|
|
430
429
|
This command validates metadata definition files using the
|
|
431
|
-
|
|
430
|
+
MetadataDefinitionValidator (for metadata-def.json structure).
|
|
431
|
+
|
|
432
|
+
Note:
|
|
433
|
+
This is separate from MetadataCommand which validates metadata.json
|
|
434
|
+
data files against metadata definition schemas.
|
|
432
435
|
"""
|
|
433
436
|
|
|
434
437
|
def __init__(self, metadata_def_path: str | Path) -> None:
|
|
@@ -460,7 +463,8 @@ class MetadataDefCommand(_ValidationErrorParser):
|
|
|
460
463
|
warnings: list[ValidationWarning] = []
|
|
461
464
|
|
|
462
465
|
try:
|
|
463
|
-
|
|
466
|
+
# Use MetadataDefinitionValidator for metadata-def.json
|
|
467
|
+
validator = MetadataDefinitionValidator()
|
|
464
468
|
# validate() with path parameter validates the file
|
|
465
469
|
_ = validator.validate(path=self.metadata_def_path)
|
|
466
470
|
|
|
@@ -520,16 +524,15 @@ class MetadataCommand(_ValidationErrorParser):
|
|
|
520
524
|
warnings: list[ValidationWarning] = []
|
|
521
525
|
|
|
522
526
|
try:
|
|
523
|
-
# First validate the schema/definition
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
#
|
|
529
|
-
#
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
_ = validator.validate(path=self.metadata_path)
|
|
527
|
+
# First validate the schema/definition file (metadata-def.json)
|
|
528
|
+
# Uses MetadataDefinitionValidator for dict[str, MetadataDefEntry] structure
|
|
529
|
+
def_validator = MetadataDefinitionValidator()
|
|
530
|
+
_ = def_validator.validate(path=self.schema_path)
|
|
531
|
+
|
|
532
|
+
# Then validate the metadata data file (metadata.json)
|
|
533
|
+
# Uses MetadataValidator for MetadataItem structure (constant/variable)
|
|
534
|
+
data_validator = MetadataValidator()
|
|
535
|
+
_ = data_validator.validate(path=self.metadata_path)
|
|
533
536
|
|
|
534
537
|
except Exception as e:
|
|
535
538
|
# Parse validation errors from exception message
|
rdetoolkit/config.py
CHANGED
|
@@ -6,8 +6,11 @@ from typing import Any, Final
|
|
|
6
6
|
|
|
7
7
|
import yaml
|
|
8
8
|
from pydantic import ValidationError
|
|
9
|
+
from tomlkit.exceptions import TOMLKitError
|
|
9
10
|
from tomlkit.toml_file import TOMLFile
|
|
11
|
+
from yaml import YAMLError
|
|
10
12
|
|
|
13
|
+
from rdetoolkit.exceptions import ConfigError
|
|
11
14
|
from rdetoolkit.models.config import Config, TracebackSettings, MultiDataTileSettings, SystemSettings, SmartTableSettings
|
|
12
15
|
from rdetoolkit.models.rde2types import RdeFsPath
|
|
13
16
|
|
|
@@ -16,6 +19,69 @@ PYPROJECT_CONFIG_FILES: Final = ["pyproject.toml"]
|
|
|
16
19
|
CONFIG_FILES = CONFIG_FILE + PYPROJECT_CONFIG_FILES
|
|
17
20
|
|
|
18
21
|
|
|
22
|
+
def _format_validation_error(
|
|
23
|
+
validation_error: ValidationError,
|
|
24
|
+
file_path: str,
|
|
25
|
+
) -> ConfigError:
|
|
26
|
+
"""Format pydantic ValidationError into user-friendly ConfigError.
|
|
27
|
+
|
|
28
|
+
Args:
|
|
29
|
+
validation_error: The pydantic ValidationError
|
|
30
|
+
file_path: Path to the configuration file
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
ConfigError with detailed field-level information
|
|
34
|
+
"""
|
|
35
|
+
errors = validation_error.errors()
|
|
36
|
+
|
|
37
|
+
if not errors:
|
|
38
|
+
return ConfigError(
|
|
39
|
+
"Configuration validation failed",
|
|
40
|
+
file_path=file_path,
|
|
41
|
+
error_type="validation_error",
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
# Take the first error for the main message
|
|
45
|
+
first_error = errors[0]
|
|
46
|
+
field_path = ".".join(str(loc) for loc in first_error["loc"])
|
|
47
|
+
error_msg = first_error["msg"]
|
|
48
|
+
error_type_detail = first_error["type"]
|
|
49
|
+
|
|
50
|
+
# Build detailed message
|
|
51
|
+
message_parts = [f"Invalid configuration in '{file_path}'"]
|
|
52
|
+
|
|
53
|
+
if field_path:
|
|
54
|
+
message_parts.append(f"Field '{field_path}' validation failed: {error_msg}")
|
|
55
|
+
else:
|
|
56
|
+
message_parts.append(f"Validation failed: {error_msg}")
|
|
57
|
+
|
|
58
|
+
# Add information about expected values if available
|
|
59
|
+
if "input" in first_error:
|
|
60
|
+
input_value = first_error["input"]
|
|
61
|
+
message_parts.append(f"Provided value: {input_value!r}")
|
|
62
|
+
|
|
63
|
+
# For extended_mode, provide specific guidance
|
|
64
|
+
if "extended_mode" in field_path and "enum" in error_type_detail.lower():
|
|
65
|
+
message_parts.append(
|
|
66
|
+
"Valid values for 'extended_mode': ['rdeformat', 'MultiDataTile']",
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
# Add validation error context if multiple errors exist
|
|
70
|
+
if len(errors) > 1:
|
|
71
|
+
message_parts.append(
|
|
72
|
+
f"Note: {len(errors)} validation error(s) found. Showing the first one.",
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
full_message = "\n".join(message_parts)
|
|
76
|
+
|
|
77
|
+
return ConfigError(
|
|
78
|
+
full_message,
|
|
79
|
+
file_path=file_path,
|
|
80
|
+
error_type="validation_error",
|
|
81
|
+
field_name=field_path,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
19
85
|
def parse_config_file(*, path: str | None = None) -> Config:
|
|
20
86
|
"""Parse the configuration file and return a Config object.
|
|
21
87
|
|
|
@@ -26,7 +92,7 @@ def parse_config_file(*, path: str | None = None) -> Config:
|
|
|
26
92
|
Config: The parsed configuration object.
|
|
27
93
|
|
|
28
94
|
Raises:
|
|
29
|
-
|
|
95
|
+
ConfigError: If the specified configuration file does not exist or cannot be parsed.
|
|
30
96
|
|
|
31
97
|
File Loading Priority:
|
|
32
98
|
1. If `path` is provided and the file extension is ".toml", the function will attempt to read the file as a TOML file.
|
|
@@ -39,10 +105,14 @@ def parse_config_file(*, path: str | None = None) -> Config:
|
|
|
39
105
|
- "pyproject.toml"
|
|
40
106
|
|
|
41
107
|
Note:
|
|
42
|
-
- If the specified
|
|
108
|
+
- If the specified file is not a recognized config file name (not in CONFIG_FILES),
|
|
109
|
+
an empty Config object will be returned.
|
|
110
|
+
- If the specified file does not exist, ConfigError is raised.
|
|
111
|
+
- If the file contains invalid YAML/TOML syntax, ConfigError is raised with line/column info.
|
|
112
|
+
- If the configuration fails validation, ConfigError is raised with field details.
|
|
43
113
|
|
|
44
114
|
Example:
|
|
45
|
-
parse_config_file(path="
|
|
115
|
+
parse_config_file(path="rdeconfig.yaml")
|
|
46
116
|
|
|
47
117
|
"""
|
|
48
118
|
config_data: dict[str, Any] = {
|
|
@@ -50,14 +120,57 @@ def parse_config_file(*, path: str | None = None) -> Config:
|
|
|
50
120
|
"multidata_tile": MultiDataTileSettings().model_dump(),
|
|
51
121
|
"smarttable": SmartTableSettings().model_dump(),
|
|
52
122
|
}
|
|
123
|
+
|
|
124
|
+
# Check file existence when path is provided
|
|
125
|
+
if path is not None:
|
|
126
|
+
path_obj = Path(path)
|
|
127
|
+
if not path_obj.exists():
|
|
128
|
+
msg = (
|
|
129
|
+
f"Configuration file not found: '{path}'. "
|
|
130
|
+
f"Create a configuration file or use 'rdetoolkit gen-config' to generate one."
|
|
131
|
+
)
|
|
132
|
+
raise ConfigError(
|
|
133
|
+
msg,
|
|
134
|
+
file_path=path,
|
|
135
|
+
error_type="file_not_found",
|
|
136
|
+
)
|
|
137
|
+
|
|
53
138
|
if path is not None and Path(path).name not in CONFIG_FILES:
|
|
54
139
|
return Config(system=SystemSettings(), multidata_tile=MultiDataTileSettings(), smarttable=SmartTableSettings())
|
|
55
140
|
|
|
56
141
|
if path is not None and is_toml(path):
|
|
57
142
|
config_data = __read_pyproject_toml(path)
|
|
58
143
|
elif path is not None and is_yaml(path):
|
|
59
|
-
|
|
60
|
-
|
|
144
|
+
try:
|
|
145
|
+
with open(path, encoding="utf-8") as f:
|
|
146
|
+
config_data = yaml.safe_load(f)
|
|
147
|
+
except YAMLError as e:
|
|
148
|
+
# Extract line and column information from YAMLError
|
|
149
|
+
line_number = None
|
|
150
|
+
column_number = None
|
|
151
|
+
|
|
152
|
+
if hasattr(e, "problem_mark") and e.problem_mark is not None:
|
|
153
|
+
line_number = e.problem_mark.line + 1 # YAML uses 0-indexed lines
|
|
154
|
+
column_number = e.problem_mark.column + 1
|
|
155
|
+
|
|
156
|
+
error_msg = "Failed to parse YAML file: invalid syntax"
|
|
157
|
+
if hasattr(e, "problem"):
|
|
158
|
+
error_msg = f"Failed to parse YAML file: {e.problem}"
|
|
159
|
+
|
|
160
|
+
raise ConfigError(
|
|
161
|
+
error_msg,
|
|
162
|
+
file_path=path,
|
|
163
|
+
error_type="parse_error",
|
|
164
|
+
line_number=line_number,
|
|
165
|
+
column_number=column_number,
|
|
166
|
+
) from e
|
|
167
|
+
except OSError as e:
|
|
168
|
+
msg = f"Failed to read YAML file: {e}"
|
|
169
|
+
raise ConfigError(
|
|
170
|
+
msg,
|
|
171
|
+
file_path=path,
|
|
172
|
+
error_type="io_error",
|
|
173
|
+
) from e
|
|
61
174
|
elif path is None:
|
|
62
175
|
project_path = Path.cwd()
|
|
63
176
|
pyproject_toml = project_path.joinpath(PYPROJECT_CONFIG_FILES[0])
|
|
@@ -68,19 +181,81 @@ def parse_config_file(*, path: str | None = None) -> Config:
|
|
|
68
181
|
if config_data is None:
|
|
69
182
|
return Config(system=SystemSettings(), multidata_tile=MultiDataTileSettings(), smarttable=SmartTableSettings())
|
|
70
183
|
|
|
71
|
-
|
|
184
|
+
try:
|
|
185
|
+
return Config(**config_data)
|
|
186
|
+
except ValidationError as e:
|
|
187
|
+
# Use helper to format validation error
|
|
188
|
+
if path:
|
|
189
|
+
raise _format_validation_error(e, path) from e
|
|
190
|
+
# Fallback for when path is None
|
|
191
|
+
msg = f"Configuration validation failed: {str(e)}"
|
|
192
|
+
raise ConfigError(
|
|
193
|
+
msg,
|
|
194
|
+
error_type="validation_error",
|
|
195
|
+
) from e
|
|
196
|
+
except TypeError as e:
|
|
197
|
+
# This occurs when config_data is not a mapping (e.g., list or string),
|
|
198
|
+
# which makes Config(**config_data) invalid. Normalize it to ConfigError.
|
|
199
|
+
base_msg = "Configuration data must be a mapping (key-value structure)"
|
|
200
|
+
msg = f"{base_msg} in file '{path}'" if path else base_msg
|
|
201
|
+
raise ConfigError(
|
|
202
|
+
msg,
|
|
203
|
+
file_path=path,
|
|
204
|
+
error_type="validation_error",
|
|
205
|
+
) from e
|
|
72
206
|
|
|
73
207
|
|
|
74
208
|
def __read_pyproject_toml(path: str) -> dict[str, Any]:
|
|
75
209
|
"""Read the pyproject.toml file and return the contents as a dictionary.
|
|
76
210
|
|
|
211
|
+
Args:
|
|
212
|
+
path: Path to the pyproject.toml file
|
|
213
|
+
|
|
77
214
|
Returns:
|
|
78
215
|
dict[str, Any]: The contents of the pyproject.toml file.
|
|
216
|
+
|
|
217
|
+
Raises:
|
|
218
|
+
ConfigError: If the file does not exist or cannot be parsed
|
|
79
219
|
"""
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
220
|
+
# Check file existence first
|
|
221
|
+
path_obj = Path(path)
|
|
222
|
+
if not path_obj.exists():
|
|
223
|
+
msg = (
|
|
224
|
+
f"Configuration file not found: '{path}'. "
|
|
225
|
+
f"Create a pyproject.toml file with [tool.rdetoolkit] section."
|
|
226
|
+
)
|
|
227
|
+
raise ConfigError(
|
|
228
|
+
msg,
|
|
229
|
+
file_path=path,
|
|
230
|
+
error_type="file_not_found",
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
try:
|
|
234
|
+
toml = TOMLFile(path)
|
|
235
|
+
obj = toml.read()
|
|
236
|
+
_obj = obj.unwrap()
|
|
237
|
+
return _obj.get("tool", {}).get("rdetoolkit", {})
|
|
238
|
+
except TOMLKitError as e:
|
|
239
|
+
# Extract line information if available
|
|
240
|
+
line_number = None
|
|
241
|
+
if hasattr(e, "line"):
|
|
242
|
+
line_number = e.line
|
|
243
|
+
|
|
244
|
+
error_msg = f"Failed to parse TOML file: {str(e)}"
|
|
245
|
+
|
|
246
|
+
raise ConfigError(
|
|
247
|
+
error_msg,
|
|
248
|
+
file_path=path,
|
|
249
|
+
error_type="parse_error",
|
|
250
|
+
line_number=line_number,
|
|
251
|
+
) from e
|
|
252
|
+
except OSError as e:
|
|
253
|
+
msg = f"Failed to read TOML file: {e}"
|
|
254
|
+
raise ConfigError(
|
|
255
|
+
msg,
|
|
256
|
+
file_path=path,
|
|
257
|
+
error_type="io_error",
|
|
258
|
+
) from e
|
|
84
259
|
|
|
85
260
|
|
|
86
261
|
def is_toml(filename: str) -> bool:
|
|
@@ -153,27 +328,32 @@ def get_config(target_dir_path: RdeFsPath) -> Config | None:
|
|
|
153
328
|
|
|
154
329
|
Returns:
|
|
155
330
|
Optional[Config]: The first valid configuration found, or None if no valid configuration is found.
|
|
331
|
+
|
|
332
|
+
Raises:
|
|
333
|
+
ConfigError: If the target directory does not exist.
|
|
156
334
|
"""
|
|
157
335
|
if isinstance(target_dir_path, str):
|
|
158
336
|
target_dir_path = Path(target_dir_path)
|
|
159
337
|
if not target_dir_path.exists():
|
|
160
|
-
|
|
338
|
+
msg = (
|
|
339
|
+
f"Configuration directory not found: '{target_dir_path}'. "
|
|
340
|
+
f"Ensure the directory exists and contains a valid configuration file."
|
|
341
|
+
)
|
|
342
|
+
raise ConfigError(
|
|
343
|
+
msg,
|
|
344
|
+
file_path=str(target_dir_path),
|
|
345
|
+
error_type="directory_not_found",
|
|
346
|
+
)
|
|
161
347
|
for cfg_file in find_config_files(target_dir_path):
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
except ValidationError as e:
|
|
165
|
-
emsg = f"Invalid configuration file: {cfg_file}"
|
|
166
|
-
raise ValueError(emsg) from e
|
|
348
|
+
# parse_config_file now converts ValidationError to ConfigError internally
|
|
349
|
+
__config = parse_config_file(path=cfg_file)
|
|
167
350
|
if __config is not None:
|
|
168
351
|
return __config
|
|
169
352
|
|
|
170
353
|
pyproject_toml_path = get_pyproject_toml()
|
|
171
354
|
if pyproject_toml_path is not None:
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
except ValidationError as e:
|
|
175
|
-
emsg = f"Invalid configuration file: {pyproject_toml_path}"
|
|
176
|
-
raise ValueError(emsg) from e
|
|
355
|
+
# parse_config_file now converts ValidationError to ConfigError internally
|
|
356
|
+
__config = parse_config_file(path=str(pyproject_toml_path))
|
|
177
357
|
if __config is not None:
|
|
178
358
|
return __config
|
|
179
359
|
return None
|
|
@@ -194,8 +374,16 @@ def load_config(tasksupport_path: RdeFsPath, *, config: Config | None = None) ->
|
|
|
194
374
|
if config is not None:
|
|
195
375
|
__config = config
|
|
196
376
|
else:
|
|
197
|
-
|
|
198
|
-
|
|
377
|
+
try:
|
|
378
|
+
__rtn_config = get_config(tasksupport_path)
|
|
379
|
+
__config = Config() if __rtn_config is None else __rtn_config
|
|
380
|
+
except ConfigError as e:
|
|
381
|
+
# Only swallow directory_not_found errors for backward compatibility
|
|
382
|
+
# Re-raise other ConfigError types (parse_error, validation_error, etc.)
|
|
383
|
+
if getattr(e, "error_type", None) == "directory_not_found":
|
|
384
|
+
__config = Config()
|
|
385
|
+
else:
|
|
386
|
+
raise
|
|
199
387
|
return __config
|
|
200
388
|
|
|
201
389
|
|
|
Binary file
|
rdetoolkit/exceptions.py
CHANGED
|
@@ -176,3 +176,97 @@ class InvalidSearchParametersError(Exception):
|
|
|
176
176
|
def __init__(self, message: str = "Invalid search term") -> None:
|
|
177
177
|
self.message = message
|
|
178
178
|
super().__init__(self.message)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class ConfigError(Exception):
|
|
182
|
+
"""Exception raised for configuration file loading errors.
|
|
183
|
+
|
|
184
|
+
This exception provides structured, informative error messages for configuration
|
|
185
|
+
file failures, including file paths, error types, line/column information for
|
|
186
|
+
parse errors, and documentation links.
|
|
187
|
+
|
|
188
|
+
Attributes:
|
|
189
|
+
message: The error message describing what went wrong.
|
|
190
|
+
file_path: Path to the configuration file that failed to load.
|
|
191
|
+
error_type: Type of error (e.g., 'file_not_found', 'parse_error', 'validation_error').
|
|
192
|
+
line_number: Line number where error occurred (for parse errors).
|
|
193
|
+
column_number: Column number where error occurred (for parse errors).
|
|
194
|
+
field_name: Field name that failed validation (for validation errors).
|
|
195
|
+
doc_url: Documentation URL for help and troubleshooting.
|
|
196
|
+
|
|
197
|
+
Examples:
|
|
198
|
+
File not found error:
|
|
199
|
+
>>> raise ConfigError(
|
|
200
|
+
... "Configuration file not found",
|
|
201
|
+
... file_path="config.yaml",
|
|
202
|
+
... error_type="file_not_found"
|
|
203
|
+
... )
|
|
204
|
+
|
|
205
|
+
Parse error with line information:
|
|
206
|
+
>>> raise ConfigError(
|
|
207
|
+
... "Invalid YAML syntax: expected <block end>",
|
|
208
|
+
... file_path="config.yaml",
|
|
209
|
+
... error_type="parse_error",
|
|
210
|
+
... line_number=10,
|
|
211
|
+
... column_number=5
|
|
212
|
+
... )
|
|
213
|
+
|
|
214
|
+
Validation error with field information:
|
|
215
|
+
>>> raise ConfigError(
|
|
216
|
+
... "Invalid value for field",
|
|
217
|
+
... file_path="config.yaml",
|
|
218
|
+
... error_type="validation_error",
|
|
219
|
+
... field_name="system.extended_mode"
|
|
220
|
+
... )
|
|
221
|
+
"""
|
|
222
|
+
|
|
223
|
+
def __init__(
|
|
224
|
+
self,
|
|
225
|
+
message: str,
|
|
226
|
+
*,
|
|
227
|
+
file_path: str | None = None,
|
|
228
|
+
error_type: str = "unknown",
|
|
229
|
+
line_number: int | None = None,
|
|
230
|
+
column_number: int | None = None,
|
|
231
|
+
field_name: str | None = None,
|
|
232
|
+
doc_url: str = "https://nims-mdpf.github.io/rdetoolkit/usage/config/config/",
|
|
233
|
+
) -> None:
|
|
234
|
+
"""Initialize ConfigError with detailed information.
|
|
235
|
+
|
|
236
|
+
Args:
|
|
237
|
+
message: The error message describing what went wrong.
|
|
238
|
+
file_path: Path to the configuration file that failed.
|
|
239
|
+
error_type: Type of error (e.g., 'file_not_found', 'parse_error', 'validation_error').
|
|
240
|
+
line_number: Line number where error occurred (for parse errors).
|
|
241
|
+
column_number: Column number where error occurred (for parse errors).
|
|
242
|
+
field_name: Field name that failed validation (for validation errors).
|
|
243
|
+
doc_url: Documentation URL for help and troubleshooting.
|
|
244
|
+
"""
|
|
245
|
+
self.message = message
|
|
246
|
+
self.file_path = file_path
|
|
247
|
+
self.error_type = error_type
|
|
248
|
+
self.line_number = line_number
|
|
249
|
+
self.column_number = column_number
|
|
250
|
+
self.field_name = field_name
|
|
251
|
+
self.doc_url = doc_url
|
|
252
|
+
|
|
253
|
+
# Build comprehensive error message
|
|
254
|
+
parts = []
|
|
255
|
+
if file_path:
|
|
256
|
+
parts.append(f"Configuration file: '{file_path}'")
|
|
257
|
+
|
|
258
|
+
parts.append(message)
|
|
259
|
+
|
|
260
|
+
if line_number is not None:
|
|
261
|
+
location = f"line {line_number}"
|
|
262
|
+
if column_number is not None:
|
|
263
|
+
location += f", column {column_number}"
|
|
264
|
+
parts.append(f"Location: {location}")
|
|
265
|
+
|
|
266
|
+
if field_name:
|
|
267
|
+
parts.append(f"Field: {field_name}")
|
|
268
|
+
|
|
269
|
+
parts.append(f"See: {doc_url}")
|
|
270
|
+
|
|
271
|
+
full_message = "\n".join(parts)
|
|
272
|
+
super().__init__(full_message)
|
rdetoolkit/exceptions.pyi
CHANGED
|
@@ -57,3 +57,24 @@ class NoResultsFoundError(Exception):
|
|
|
57
57
|
class InvalidSearchParametersError(Exception):
|
|
58
58
|
message: Incomplete
|
|
59
59
|
def __init__(self, message: str = 'Invalid search term') -> None: ...
|
|
60
|
+
|
|
61
|
+
class ConfigError(Exception):
|
|
62
|
+
message: str
|
|
63
|
+
file_path: str | None
|
|
64
|
+
error_type: str
|
|
65
|
+
line_number: int | None
|
|
66
|
+
column_number: int | None
|
|
67
|
+
field_name: str | None
|
|
68
|
+
doc_url: str
|
|
69
|
+
|
|
70
|
+
def __init__(
|
|
71
|
+
self,
|
|
72
|
+
message: str,
|
|
73
|
+
*,
|
|
74
|
+
file_path: str | None = None,
|
|
75
|
+
error_type: str = 'unknown',
|
|
76
|
+
line_number: int | None = None,
|
|
77
|
+
column_number: int | None = None,
|
|
78
|
+
field_name: str | None = None,
|
|
79
|
+
doc_url: str = 'https://nims-mdpf.github.io/rdetoolkit/usage/config/config/',
|
|
80
|
+
) -> None: ...
|
rdetoolkit/models/metadata.py
CHANGED
|
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
|
2
2
|
|
|
3
3
|
from typing import Any, Final
|
|
4
4
|
|
|
5
|
-
from pydantic import BaseModel, RootModel, field_validator
|
|
5
|
+
from pydantic import AnyUrl, BaseModel, ConfigDict, Field, RootModel, field_validator
|
|
6
6
|
|
|
7
7
|
MAX_VALUE_SIZE: Final[int] = 1024
|
|
8
8
|
|
|
@@ -71,7 +71,7 @@ class ValidableItems(RootModel):
|
|
|
71
71
|
|
|
72
72
|
|
|
73
73
|
class MetadataItem(BaseModel):
|
|
74
|
-
"""metadata
|
|
74
|
+
"""metadata.json class.
|
|
75
75
|
|
|
76
76
|
Stores metadata extracted by the data structuring process.
|
|
77
77
|
|
|
@@ -82,3 +82,98 @@ class MetadataItem(BaseModel):
|
|
|
82
82
|
|
|
83
83
|
constant: dict[str, MetaValue]
|
|
84
84
|
variable: ValidableItems
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class NameField(BaseModel):
|
|
88
|
+
"""Multilingual name field for metadata definition.
|
|
89
|
+
|
|
90
|
+
Attributes:
|
|
91
|
+
ja: Japanese name
|
|
92
|
+
en: English name
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
ja: str
|
|
96
|
+
en: str
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class SchemaField(BaseModel):
|
|
100
|
+
"""Schema field for metadata definition.
|
|
101
|
+
|
|
102
|
+
Attributes:
|
|
103
|
+
type: Type of the metadata value. One of "array", "boolean", "integer", "number", "string"
|
|
104
|
+
format: Optional format specifier. One of "date-time" or "duration"
|
|
105
|
+
"""
|
|
106
|
+
|
|
107
|
+
type: str # "array", "boolean", "integer", "number", "string"
|
|
108
|
+
format: str | None = None # "date-time", "duration"
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class MetadataDefEntry(BaseModel):
|
|
112
|
+
"""Single metadata definition entry in metadata-def.json.
|
|
113
|
+
|
|
114
|
+
Represents one metadata item definition. This is used for metadata-def.json,
|
|
115
|
+
not for metadata.json (which uses MetadataItem instead).
|
|
116
|
+
|
|
117
|
+
Attributes:
|
|
118
|
+
name: Multilingual name (ja/en required)
|
|
119
|
+
schema_field: Type and format definition (type required, serialized as "schema")
|
|
120
|
+
unit: Optional unit for the metadata value
|
|
121
|
+
description: Optional description
|
|
122
|
+
uri: Optional URI/URL for the metadata key
|
|
123
|
+
mode: Optional measurement mode
|
|
124
|
+
order: Optional display order
|
|
125
|
+
original_name: Optional original name (serialized as "originalName")
|
|
126
|
+
|
|
127
|
+
Example:
|
|
128
|
+
```json
|
|
129
|
+
{
|
|
130
|
+
"temperature": {
|
|
131
|
+
"name": {"ja": "温度", "en": "Temperature"},
|
|
132
|
+
"schema": {"type": "number"},
|
|
133
|
+
"unit": "K"
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
```
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
name: NameField
|
|
140
|
+
schema_field: SchemaField = Field(alias="schema")
|
|
141
|
+
unit: str | None = None
|
|
142
|
+
description: str | None = None
|
|
143
|
+
uri: AnyUrl | None = None
|
|
144
|
+
mode: str | None = None
|
|
145
|
+
order: int | None = None
|
|
146
|
+
original_name: str | None = Field(default=None, alias="originalName")
|
|
147
|
+
|
|
148
|
+
model_config = ConfigDict(
|
|
149
|
+
# Allow undefined fields (e.g., "variable" field is ignored per docs)
|
|
150
|
+
extra="allow",
|
|
151
|
+
# Enable alias for JSON parsing and serialization
|
|
152
|
+
populate_by_name=True,
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class MetadataDefinition(RootModel):
|
|
157
|
+
"""metadata-def.json root model.
|
|
158
|
+
|
|
159
|
+
Represents the entire metadata definition file as a dictionary
|
|
160
|
+
mapping metadata keys to their definitions. This is used for
|
|
161
|
+
metadata-def.json, not for metadata.json (which uses MetadataItem instead).
|
|
162
|
+
|
|
163
|
+
Example:
|
|
164
|
+
```json
|
|
165
|
+
{
|
|
166
|
+
"temperature": {
|
|
167
|
+
"name": {"ja": "温度", "en": "Temperature"},
|
|
168
|
+
"schema": {"type": "number"},
|
|
169
|
+
"unit": "K"
|
|
170
|
+
},
|
|
171
|
+
"operator": {
|
|
172
|
+
"name": {"ja": "測定者", "en": "Operator"},
|
|
173
|
+
"schema": {"type": "string"}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
```
|
|
177
|
+
"""
|
|
178
|
+
|
|
179
|
+
root: dict[str, MetadataDefEntry]
|
rdetoolkit/models/metadata.pyi
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from pydantic import BaseModel, RootModel
|
|
1
|
+
from pydantic import AnyUrl, BaseModel, RootModel
|
|
2
2
|
from typing import Any, Final
|
|
3
3
|
|
|
4
4
|
MAX_VALUE_SIZE: Final[int]
|
|
@@ -20,3 +20,24 @@ class ValidableItems(RootModel):
|
|
|
20
20
|
class MetadataItem(BaseModel):
|
|
21
21
|
constant: dict[str, MetaValue]
|
|
22
22
|
variable: ValidableItems
|
|
23
|
+
|
|
24
|
+
class NameField(BaseModel):
|
|
25
|
+
ja: str
|
|
26
|
+
en: str
|
|
27
|
+
|
|
28
|
+
class SchemaField(BaseModel):
|
|
29
|
+
type: str
|
|
30
|
+
format: str | None
|
|
31
|
+
|
|
32
|
+
class MetadataDefEntry(BaseModel):
|
|
33
|
+
name: NameField
|
|
34
|
+
schema_field: SchemaField
|
|
35
|
+
unit: str | None
|
|
36
|
+
description: str | None
|
|
37
|
+
uri: AnyUrl | None
|
|
38
|
+
mode: str | None
|
|
39
|
+
order: int | None
|
|
40
|
+
original_name: str | None
|
|
41
|
+
|
|
42
|
+
class MetadataDefinition(RootModel):
|
|
43
|
+
root: dict[str, MetadataDefEntry]
|
rdetoolkit/validation.py
CHANGED
|
@@ -29,7 +29,19 @@ def _pydantic_validation_error() -> type[PydanticValidationError]:
|
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class MetadataValidator:
|
|
32
|
+
"""Validator for metadata files (metadata.json).
|
|
33
|
+
|
|
34
|
+
This validator checks metadata.json files against the
|
|
35
|
+
MetadataItem Pydantic model, ensuring proper structure
|
|
36
|
+
for actual metadata data.
|
|
37
|
+
|
|
38
|
+
Note:
|
|
39
|
+
This is separate from MetadataDefinitionValidator which validates
|
|
40
|
+
metadata-def.json files (metadata definitions).
|
|
41
|
+
"""
|
|
42
|
+
|
|
32
43
|
def __init__(self) -> None:
|
|
44
|
+
"""Initialize metadata validator with schema."""
|
|
33
45
|
from rdetoolkit.models.metadata import MetadataItem
|
|
34
46
|
|
|
35
47
|
self.schema = MetadataItem
|
|
@@ -59,28 +71,102 @@ class MetadataValidator:
|
|
|
59
71
|
|
|
60
72
|
if path is not None:
|
|
61
73
|
__data = readf_json(path)
|
|
62
|
-
elif json_obj is not None:
|
|
63
|
-
__data = json_obj
|
|
64
74
|
else:
|
|
65
|
-
|
|
66
|
-
raise ValueError(emsg)
|
|
75
|
+
__data = json_obj
|
|
67
76
|
|
|
68
77
|
self.schema(**__data)
|
|
69
78
|
return __data
|
|
70
79
|
|
|
71
80
|
|
|
81
|
+
class MetadataDefinitionValidator:
|
|
82
|
+
"""Validator for metadata definition files (metadata-def.json).
|
|
83
|
+
|
|
84
|
+
This validator checks metadata-def.json files against the
|
|
85
|
+
MetadataDefinition Pydantic model, ensuring proper structure
|
|
86
|
+
for metadata definitions.
|
|
87
|
+
|
|
88
|
+
Note:
|
|
89
|
+
This is separate from MetadataValidator which validates
|
|
90
|
+
metadata.json files (actual metadata data).
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(self) -> None:
|
|
94
|
+
"""Initialize metadata definition validator with schema."""
|
|
95
|
+
from rdetoolkit.models.metadata import MetadataDefinition
|
|
96
|
+
|
|
97
|
+
self.schema = MetadataDefinition
|
|
98
|
+
|
|
99
|
+
def validate(
|
|
100
|
+
self,
|
|
101
|
+
*,
|
|
102
|
+
path: str | Path | None = None,
|
|
103
|
+
json_obj: dict[str, Any] | None = None,
|
|
104
|
+
) -> dict[str, Any]:
|
|
105
|
+
"""Validate metadata definition JSON against schema.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
path: Path to metadata-def.json file to validate
|
|
109
|
+
json_obj: JSON object to validate (alternative to path)
|
|
110
|
+
|
|
111
|
+
Returns:
|
|
112
|
+
Validated JSON data as dict
|
|
113
|
+
|
|
114
|
+
Raises:
|
|
115
|
+
ValueError: If neither path nor json_obj provided, or both provided
|
|
116
|
+
MetadataValidationError: If validation fails with detailed error info
|
|
117
|
+
|
|
118
|
+
Examples:
|
|
119
|
+
>>> validator = MetadataDefinitionValidator()
|
|
120
|
+
>>> data = validator.validate(path="metadata-def.json")
|
|
121
|
+
"""
|
|
122
|
+
# Input validation
|
|
123
|
+
if path is None and json_obj is None:
|
|
124
|
+
emsg = "At least one of 'path' or 'json_obj' must be provided"
|
|
125
|
+
raise ValueError(emsg)
|
|
126
|
+
if path is not None and json_obj is not None:
|
|
127
|
+
emsg = "Both 'path' and 'json_obj' cannot be provided at the same time"
|
|
128
|
+
raise ValueError(emsg)
|
|
129
|
+
|
|
130
|
+
# Load data
|
|
131
|
+
if path is not None:
|
|
132
|
+
__data = readf_json(path)
|
|
133
|
+
else:
|
|
134
|
+
__data = json_obj
|
|
135
|
+
|
|
136
|
+
# Validate with Pydantic model
|
|
137
|
+
try:
|
|
138
|
+
self.schema(__data)
|
|
139
|
+
except _pydantic_validation_error() as validation_error:
|
|
140
|
+
# Format error message for metadata-def.json
|
|
141
|
+
emsg = "Validation Errors in metadata-def.json. Please correct the following fields\n"
|
|
142
|
+
for idx, error in enumerate(validation_error.errors(), start=1):
|
|
143
|
+
# Extract field path (e.g., ['key', 'name', 'ja'])
|
|
144
|
+
field_path = ".".join([str(e) for e in error["loc"]])
|
|
145
|
+
emsg += f"{idx}. Field: {field_path}\n"
|
|
146
|
+
emsg += f" Type: {error['type']}\n"
|
|
147
|
+
emsg += f" Context: {error['msg']}\n"
|
|
148
|
+
raise MetadataValidationError(emsg) from validation_error
|
|
149
|
+
|
|
150
|
+
return __data
|
|
151
|
+
|
|
152
|
+
|
|
72
153
|
def metadata_validate(path: str | Path) -> None:
|
|
73
154
|
"""Validate metadata.json file.
|
|
74
155
|
|
|
75
|
-
This function validates the metadata
|
|
76
|
-
It checks if the file exists and then uses
|
|
156
|
+
This function validates the metadata.json file (actual metadata data)
|
|
157
|
+
specified by the given path. It checks if the file exists and then uses
|
|
158
|
+
MetadataValidator to validate the file against the MetadataItem schema.
|
|
159
|
+
|
|
160
|
+
Note:
|
|
161
|
+
This function is for metadata.json files. For metadata-def.json
|
|
162
|
+
files, use MetadataDefinitionValidator instead.
|
|
77
163
|
|
|
78
164
|
Args:
|
|
79
|
-
path (Union[str, Path]): The path to the metadata
|
|
165
|
+
path (Union[str, Path]): The path to the metadata.json file.
|
|
80
166
|
|
|
81
167
|
Raises:
|
|
82
168
|
FileNotFoundError: If the schema and path do not exist.
|
|
83
|
-
MetadataValidationError: If there is an error in validating the metadata
|
|
169
|
+
MetadataValidationError: If there is an error in validating the metadata file.
|
|
84
170
|
"""
|
|
85
171
|
if isinstance(path, str):
|
|
86
172
|
path = Path(path)
|
rdetoolkit/validation.pyi
CHANGED
|
@@ -8,6 +8,11 @@ class MetadataValidator:
|
|
|
8
8
|
def __init__(self) -> None: ...
|
|
9
9
|
def validate(self, *, path: str | Path | None = None, json_obj: dict[str, Any] | None = None) -> dict[str, Any]: ...
|
|
10
10
|
|
|
11
|
+
class MetadataDefinitionValidator:
|
|
12
|
+
schema: Incomplete
|
|
13
|
+
def __init__(self) -> None: ...
|
|
14
|
+
def validate(self, *, path: str | Path | None = None, json_obj: dict[str, Any] | None = None) -> dict[str, Any]: ...
|
|
15
|
+
|
|
11
16
|
def metadata_validate(path: str | Path) -> None: ...
|
|
12
17
|
|
|
13
18
|
class InvoiceValidator:
|
rdetoolkit/workflows.py
CHANGED
|
@@ -419,8 +419,14 @@ def run(*, custom_dataset_function: DatasetCallback | None = None, config: Confi
|
|
|
419
419
|
Exception: If a generic error occurs during the process.
|
|
420
420
|
|
|
421
421
|
Note:
|
|
422
|
-
|
|
423
|
-
|
|
422
|
+
Execution mode is selected in the following order:
|
|
423
|
+
|
|
424
|
+
1. SmartTable CSV is present (`smarttable_file` is not None) -> `SmartTableInvoice` mode.
|
|
425
|
+
2. Excel invoice bundle is provided (`excel_invoice_files` is not None) -> `Excelinvoice` mode.
|
|
426
|
+
3. `extended_mode` matches (case-insensitive) `rdeformat` or `MultiDataTile` -> the corresponding extended mode.
|
|
427
|
+
4. Otherwise -> `Invoice` mode.
|
|
428
|
+
|
|
429
|
+
The mode name recorded in logs/results matches the branch that executed. No `excelinvoice` value is accepted in `extended_mode`.
|
|
424
430
|
|
|
425
431
|
Example:
|
|
426
432
|
```python
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rdetoolkit
|
|
3
|
-
Version: 1.5.
|
|
3
|
+
Version: 1.5.2
|
|
4
4
|
Classifier: Development Status :: 3 - Alpha
|
|
5
5
|
Classifier: Programming Language :: Python
|
|
6
6
|
Classifier: Programming Language :: Python :: 3
|
|
@@ -36,6 +36,7 @@ Requires-Dist: pyarrow>=19.0.0
|
|
|
36
36
|
Requires-Dist: pip>=24.3.1
|
|
37
37
|
Requires-Dist: rpds-py>=0.26
|
|
38
38
|
Requires-Dist: markdown>=3.7
|
|
39
|
+
Requires-Dist: pytz>=2024.1
|
|
39
40
|
Requires-Dist: types-pytz>=2025.2.0.20250326
|
|
40
41
|
Requires-Dist: matplotlib>=3.9.4
|
|
41
42
|
Requires-Dist: minio>=7.2.15 ; extra == 'minio'
|
|
@@ -51,11 +52,11 @@ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
|
51
52
|
Project-URL: Bug Tracker, https://github.com/nims-dpfc/rdetoolkit
|
|
52
53
|
Project-URL: Homepage, https://github.com/nims-dpfc/rdetoolkit
|
|
53
54
|
|
|
54
|
-

|
|
55
56
|
[](https://www.python.org/downloads/release/python-3917/)
|
|
56
|
-
[](https://github.com/nims-
|
|
57
|
-
[](https://github.com/nims-
|
|
58
|
-
](https://github.com/nims-mdpf/rdetoolkit/blob/main/LICENSE)
|
|
58
|
+
[](https://github.com/nims-mdpf/rdetoolkit/issues)
|
|
59
|
+

|
|
59
60
|

|
|
60
61
|
|
|
61
62
|
> [日本語ドキュメント](docs/README_ja.md)
|
|
@@ -76,6 +77,13 @@ If you wish to make changes, please read the following document first:
|
|
|
76
77
|
|
|
77
78
|
- [CONTRIBUTING.md](https://github.com/nims-mdpf/rdetoolkit/blob/main/CONTRIBUTING.md)
|
|
78
79
|
|
|
80
|
+
## Requirements
|
|
81
|
+
|
|
82
|
+
- **Python**: 3.9 or higher (Python 3.9 support will be removed in v2.0; upgrade to Python 3.10+ recommended)
|
|
83
|
+
|
|
84
|
+
!!! warning "Python 3.9 Deprecation"
|
|
85
|
+
Python 3.9 support is deprecated and will be removed in rdetoolkit v2.0. While Python 3.9 continues to work in rdetoolkit 1.x, users will see a `DeprecationWarning` on import. Please plan to upgrade to Python 3.10 or later before the v2.0 release.
|
|
86
|
+
|
|
79
87
|
## Install
|
|
80
88
|
|
|
81
89
|
To install, run the following command:
|
|
@@ -143,20 +151,20 @@ def dataset(paths: RdeDatasetPaths) -> None:
|
|
|
143
151
|
...
|
|
144
152
|
```
|
|
145
153
|
|
|
146
|
-
In this example, we define a dummy function `
|
|
154
|
+
In this example, we define a dummy function `display_message()` under `modules` to demonstrate how to implement custom structuring processing. Create a file named `modules/modules.py` as follows:
|
|
147
155
|
|
|
148
156
|
```python
|
|
149
157
|
# modules/modules.py
|
|
150
158
|
from rdetoolkit.models.rde2types import RdeDatasetPaths
|
|
151
159
|
|
|
152
160
|
|
|
153
|
-
def
|
|
161
|
+
def display_message(path):
|
|
154
162
|
print(f"Test Message!: {path}")
|
|
155
163
|
|
|
156
164
|
|
|
157
165
|
def dataset(paths: RdeDatasetPaths) -> None:
|
|
158
|
-
|
|
159
|
-
|
|
166
|
+
display_message(paths.inputdata)
|
|
167
|
+
display_message(paths.struct)
|
|
160
168
|
```
|
|
161
169
|
|
|
162
170
|
### About the Entry Point
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
rdetoolkit\__init__.py,sha256=
|
|
1
|
+
rdetoolkit\__init__.py,sha256=JHyHP569oIiGeWRopO7rd0KelZFxsHzIFri753zqaGo,3306
|
|
2
2
|
rdetoolkit\__init__.pyi,sha256=2ILXdK0bu7extORHJnwcCw2GHmlSGrN6-hDV0SV7CF8,472
|
|
3
3
|
rdetoolkit\__main__.py,sha256=RpkjdhIR-v6U17ZAnT3DAIcDwADDqQabSkQoNFiCXQk,126
|
|
4
4
|
rdetoolkit\__main__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -9,7 +9,7 @@ rdetoolkit\artifact\report.pyi,sha256=yTvSNg6JDqobsS5uhU1-RX_5qwabmOwBXqPNKOzgiR
|
|
|
9
9
|
rdetoolkit\cli\__init__.py,sha256=ZJ5C5YbpaRC3rnavu1_zG30WJ6sePrTcuvmVwAla9sc,1832
|
|
10
10
|
rdetoolkit\cli\app.py,sha256=GPhLbnYs3ufGVu4qUgHtx4ZP5jHk4uR7jocQHffuIgY,19281
|
|
11
11
|
rdetoolkit\cli\app.pyi,sha256=EyyA59hDGfQ7oo0MysCtmoIIotyj7yZUsCT-VRR7cPo,2131
|
|
12
|
-
rdetoolkit\cli\validate.py,sha256=
|
|
12
|
+
rdetoolkit\cli\validate.py,sha256=BhFn_N8fveCnJ9LWSW4TJnWhPtlemvLjlB_-8IoMG_4,14695
|
|
13
13
|
rdetoolkit\cli\validate.pyi,sha256=3o8rsuY_l018FBUs0mXUOXDyMmE1TE244f54Eiaa2Wg,1734
|
|
14
14
|
rdetoolkit\cli.pyi,sha256=eJk2lAG9DVnEi9woNVwVfqpFtzCV40EaOyjuSo77m84,1648
|
|
15
15
|
rdetoolkit\cmd\__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -26,16 +26,16 @@ rdetoolkit\cmd\gen_config.py,sha256=4l8R1uMW7HbZi23K4yGolXSEOgKrT0efEoIdBn_qXN0,
|
|
|
26
26
|
rdetoolkit\cmd\gen_config.pyi,sha256=zBY4XATVx_dDf_Vlkc9wqpgOSWNcTudKXueu_hTHW8I,1120
|
|
27
27
|
rdetoolkit\cmd\gen_excelinvoice.py,sha256=4EiV3PUkYUzn29fDutYsZ14POOtaA0RL0GLN8swM69Q,2136
|
|
28
28
|
rdetoolkit\cmd\gen_excelinvoice.pyi,sha256=GrywtydtlXyYeHBY2fHCh3BQOfGG90807rIMOFo5sig,407
|
|
29
|
-
rdetoolkit\cmd\validate.py,sha256=
|
|
29
|
+
rdetoolkit\cmd\validate.py,sha256=wFWbOEKvzpq3zur7wYMjvLfwxDg3zxM42PJefG4RgXY,28099
|
|
30
30
|
rdetoolkit\cmd\validate.pyi,sha256=75ca4HTq93Z1ZFtlo7hGFiMVUoswvzCOjdWaaHvnBME,2691
|
|
31
|
-
rdetoolkit\config.py,sha256=
|
|
31
|
+
rdetoolkit\config.py,sha256=7sblHYkOzMPg7cdkBNO4ynnxko896fW6AH6QY8GULfs,15301
|
|
32
32
|
rdetoolkit\config.pyi,sha256=MUZCZXOJtcIGWGNNtKToAfWnf7llO9Bdy4HxULNIYXQ,969
|
|
33
|
-
rdetoolkit\core.cp314-win_amd64.pyd,sha256=
|
|
33
|
+
rdetoolkit\core.cp314-win_amd64.pyd,sha256=vpcpWeBq7cDOz0KpS3cmB59P7V0RYjlOL3qT4pdU8vI,6771712
|
|
34
34
|
rdetoolkit\core.pyi,sha256=VxUXk6kEkwsA5a8t0FtfTPt5VtfHjMy7yMz9UzG6r4U,3154
|
|
35
35
|
rdetoolkit\errors.py,sha256=j2tHMViF3WsLXcsyTtC2uq0bOMagdB3Dg5J58-jnF5k,13966
|
|
36
36
|
rdetoolkit\errors.pyi,sha256=XRnfjCkS3k6NIncUUnvDEK9_9Yfe5dghnN1jXZr_9mE,1445
|
|
37
|
-
rdetoolkit\exceptions.py,sha256=
|
|
38
|
-
rdetoolkit\exceptions.pyi,sha256=
|
|
37
|
+
rdetoolkit\exceptions.py,sha256=W8JICpzsNfmN2T68poI5bbGGykuyUmG6yC5JEM3uSl4,11126
|
|
38
|
+
rdetoolkit\exceptions.pyi,sha256=zJrKfbT-4yCXWOUPN36yqzppV5sH3ijDQwNdk00IU4c,2764
|
|
39
39
|
rdetoolkit\fileops.py,sha256=UgiEWIMD5JCwPYbX6bloj1J9PJSA3iyRERNG49yNIPc,2041
|
|
40
40
|
rdetoolkit\fileops.pyi,sha256=_UP6GbYdPODUby6TSV7maKnU2xD4_ttUg-bKmjbnNEY,325
|
|
41
41
|
rdetoolkit\graph\__init__.py,sha256=4txp3FJ-rroWXIDv8mqF8ZxtiBUh8qs2vsy6Y96epLU,3106
|
|
@@ -121,8 +121,8 @@ rdetoolkit\models\invoice.py,sha256=3J1f7_is7fdFkmFwXmgSJ79ydh0dL8AvV2-DZ_suHRE,
|
|
|
121
121
|
rdetoolkit\models\invoice.pyi,sha256=t5SOOq1ZwjfJ0LyfuL2N8ChnuWW7CMuvN2-E-imEhDk,2983
|
|
122
122
|
rdetoolkit\models\invoice_schema.py,sha256=57S8yjLpH2fzvLYn7oSZ85b92-C8Txlf1ifRyLr-Vsc,16936
|
|
123
123
|
rdetoolkit\models\invoice_schema.pyi,sha256=QlDeARuGa4KzOr44iFkjUQgp-RJkaSfBqcucS7Us0vc,3731
|
|
124
|
-
rdetoolkit\models\metadata.py,sha256=
|
|
125
|
-
rdetoolkit\models\metadata.pyi,sha256=
|
|
124
|
+
rdetoolkit\models\metadata.py,sha256=4IYN1FKenqejR_wpt3dTIfuv6YxL0QCrU68YeqTmZ5M,5562
|
|
125
|
+
rdetoolkit\models\metadata.pyi,sha256=ObHQUf8I4dCdCYC33paBYHfn7V7cw62sVeZ-k4lMu2Y,1024
|
|
126
126
|
rdetoolkit\models\rde2types.py,sha256=i2U-qpl2ynj-sSKdTY1FRxkP_hj--aIXTVr2pdlcY-I,32345
|
|
127
127
|
rdetoolkit\models\rde2types.pyi,sha256=KiUjEDdZM2Nkmq5mZI2LU9fgpY8DzVym3MiqafOswcg,7644
|
|
128
128
|
rdetoolkit\models\reports.py,sha256=KWjskAMJAEgwuciOOGVmnYIasEjZtX2q9vXc9JkjvWA,422
|
|
@@ -180,12 +180,12 @@ rdetoolkit\traceback\formatter.py,sha256=WnJmQkRvQsIY-nKrh13GjkTvL2PY_1iVw5gKn_M
|
|
|
180
180
|
rdetoolkit\traceback\formatter.pyi,sha256=Ry2FHNLfOZXDHSxsTR0tPktyZ0GYKXKsOFiO-opqScE,398
|
|
181
181
|
rdetoolkit\traceback\masking.py,sha256=9CjqmbPmRBhszgktIgp9x60KKumELIuf9ILFUeCO438,4678
|
|
182
182
|
rdetoolkit\traceback\masking.pyi,sha256=cbejAwTmUr7nasHfJzb8wJYciBaBCws5KPYe_TW0PQU,652
|
|
183
|
-
rdetoolkit\validation.py,sha256=
|
|
184
|
-
rdetoolkit\validation.pyi,sha256=
|
|
185
|
-
rdetoolkit\workflows.py,sha256=
|
|
183
|
+
rdetoolkit\validation.py,sha256=ZIdULGWAzQHv8VdfbBZuKSD5eZHtjsMsBhPDD3y4uCA,18416
|
|
184
|
+
rdetoolkit\validation.pyi,sha256=916wLyRHzqtr9UNL9p8_vQC9ya-OpLDMOZkKukfKuyw,999
|
|
185
|
+
rdetoolkit\workflows.py,sha256=jVz_CHbNAVc2FLVyfV-Dhan-EWvoZDKWhMqXZVdwI38,22429
|
|
186
186
|
rdetoolkit\workflows.pyi,sha256=i2QN0RP0xQYCjs1Ctp1y6zoRBqeB0cCw7mYlDAS2jiQ,2199
|
|
187
|
-
rdetoolkit-1.5.
|
|
188
|
-
rdetoolkit-1.5.
|
|
189
|
-
rdetoolkit-1.5.
|
|
190
|
-
rdetoolkit-1.5.
|
|
191
|
-
rdetoolkit-1.5.
|
|
187
|
+
rdetoolkit-1.5.2.dist-info\METADATA,sha256=LyLSFsBegG4HM6cBzcfqacmfOnkKT30TBcLjiC7sQPU,9120
|
|
188
|
+
rdetoolkit-1.5.2.dist-info\WHEEL,sha256=TASrtxyeL-Pi7odwPBMCgR1YebCHdBFZvgqiADG_4b0,97
|
|
189
|
+
rdetoolkit-1.5.2.dist-info\entry_points.txt,sha256=WMEB0vLFWxdgPfIxgGPzK2djc-aVoGclcqvZlXxpxu4,52
|
|
190
|
+
rdetoolkit-1.5.2.dist-info\licenses\LICENSE,sha256=Xh2kqHbF9jlxeCxl_60qCRAAuO8DYe2hkv0bBAjxcko,1103
|
|
191
|
+
rdetoolkit-1.5.2.dist-info\RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|