pysdmx 1.9.0__py3-none-any.whl → 1.10.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pysdmx/__extras_check.py +1 -1
- pysdmx/__init__.py +1 -1
- pysdmx/io/input_processor.py +9 -6
- pysdmx/io/xml/header.py +41 -34
- pysdmx/toolkit/vtl/__init__.py +10 -1
- pysdmx/toolkit/vtl/_validations.py +8 -12
- pysdmx/toolkit/vtl/convert.py +333 -0
- pysdmx/toolkit/vtl/script_generation.py +1 -1
- {pysdmx-1.9.0.dist-info → pysdmx-1.10.0rc1.dist-info}/METADATA +3 -3
- {pysdmx-1.9.0.dist-info → pysdmx-1.10.0rc1.dist-info}/RECORD +12 -11
- {pysdmx-1.9.0.dist-info → pysdmx-1.10.0rc1.dist-info}/WHEEL +0 -0
- {pysdmx-1.9.0.dist-info → pysdmx-1.10.0rc1.dist-info}/licenses/LICENSE +0 -0
pysdmx/__extras_check.py
CHANGED
pysdmx/__init__.py
CHANGED
pysdmx/io/input_processor.py
CHANGED
|
@@ -29,16 +29,19 @@ def __check_xml(input_str: str) -> bool:
|
|
|
29
29
|
|
|
30
30
|
def __check_csv(input_str: str) -> bool:
|
|
31
31
|
try:
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
lines = input_str.splitlines()
|
|
33
|
+
|
|
34
|
+
# Use the first N complete lines
|
|
35
|
+
# (1 should be enough)
|
|
36
|
+
max_lines = 1
|
|
37
|
+
sample = "\n".join(lines[:max_lines])
|
|
38
|
+
|
|
39
|
+
dialect = csv.Sniffer().sniff(sample)
|
|
34
40
|
control_csv_format = (
|
|
35
41
|
dialect.delimiter == "," and dialect.quotechar == '"'
|
|
36
42
|
)
|
|
37
43
|
# Check we can access the data and it is not empty
|
|
38
|
-
if (
|
|
39
|
-
len(input_str.splitlines()) > 1
|
|
40
|
-
or input_str.splitlines()[0].count(",") > 1
|
|
41
|
-
) and control_csv_format:
|
|
44
|
+
if (len(lines) > 1 or lines[0].count(",") > 1) and control_csv_format:
|
|
42
45
|
return True
|
|
43
46
|
except Exception:
|
|
44
47
|
return False
|
pysdmx/io/xml/header.py
CHANGED
|
@@ -31,6 +31,7 @@ from pysdmx.io.xml.__tokens import (
|
|
|
31
31
|
URN,
|
|
32
32
|
VERSION,
|
|
33
33
|
)
|
|
34
|
+
from pysdmx.io.xml.utils import add_list
|
|
34
35
|
from pysdmx.model import Organisation, Reference
|
|
35
36
|
from pysdmx.model.dataset import ActionType
|
|
36
37
|
from pysdmx.model.message import Header
|
|
@@ -87,43 +88,49 @@ def __parse_sender_receiver(
|
|
|
87
88
|
def __parse_structure(
|
|
88
89
|
structure: Union[Dict[str, Any], None],
|
|
89
90
|
) -> Union[Dict[str, str], None]:
|
|
90
|
-
"""Parses the structure of the SDMX header."""
|
|
91
|
+
"""Parses the structure/s of the SDMX header."""
|
|
91
92
|
if structure is None:
|
|
92
93
|
return None
|
|
93
94
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
95
|
+
result = {}
|
|
96
|
+
|
|
97
|
+
for struct in add_list(structure):
|
|
98
|
+
dim_at_obs = struct.get(DIM_OBS, "AllDimensions")
|
|
99
|
+
|
|
100
|
+
if STRUCTURE in struct:
|
|
101
|
+
structure_info = struct[STRUCTURE]
|
|
102
|
+
sdmx_type = DSD
|
|
103
|
+
elif STR_USAGE in struct:
|
|
104
|
+
structure_info = struct[STR_USAGE]
|
|
105
|
+
sdmx_type = DFW
|
|
106
|
+
elif PROV_AGREEMENT in struct:
|
|
107
|
+
structure_info = struct[PROV_AGREEMENT]
|
|
108
|
+
sdmx_type = PROV_AGREEMENT
|
|
109
|
+
else:
|
|
110
|
+
# Provision Agrement is a typo in the SDMX 2.1 schema,
|
|
111
|
+
# and it is later solved in SDMX 3.0
|
|
112
|
+
structure_info = struct[PROV_AGREMENT]
|
|
113
|
+
sdmx_type = PROV_AGREMENT
|
|
114
|
+
|
|
115
|
+
if REF in structure_info:
|
|
116
|
+
reference = structure_info[REF]
|
|
117
|
+
agency_id = reference[AGENCY_ID]
|
|
118
|
+
structure_id = reference[ID]
|
|
119
|
+
version = reference[VERSION]
|
|
120
|
+
ref_obj = Reference(
|
|
121
|
+
sdmx_type=sdmx_type,
|
|
122
|
+
agency=agency_id,
|
|
123
|
+
id=structure_id,
|
|
124
|
+
version=version,
|
|
125
|
+
)
|
|
126
|
+
elif URN in structure_info:
|
|
127
|
+
ref_obj = parse_maintainable_urn(structure_info[URN])
|
|
128
|
+
else:
|
|
129
|
+
ref_obj = parse_maintainable_urn(structure_info)
|
|
130
|
+
|
|
131
|
+
result[str(ref_obj)] = dim_at_obs
|
|
132
|
+
|
|
133
|
+
return result
|
|
127
134
|
|
|
128
135
|
|
|
129
136
|
def __parse_source(source: Optional[Dict[str, Any]]) -> Optional[str]:
|
pysdmx/toolkit/vtl/__init__.py
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
"""VTL toolkit for PySDMX."""
|
|
2
2
|
|
|
3
|
+
from pysdmx.toolkit.vtl.convert import (
|
|
4
|
+
convert_dataset_to_sdmx,
|
|
5
|
+
convert_dataset_to_vtl,
|
|
6
|
+
)
|
|
3
7
|
from pysdmx.toolkit.vtl.script_generation import generate_vtl_script
|
|
4
8
|
from pysdmx.toolkit.vtl.validation import model_validations
|
|
5
9
|
|
|
6
|
-
__all__ = [
|
|
10
|
+
__all__ = [
|
|
11
|
+
"model_validations",
|
|
12
|
+
"generate_vtl_script",
|
|
13
|
+
"convert_dataset_to_vtl",
|
|
14
|
+
"convert_dataset_to_sdmx",
|
|
15
|
+
]
|
|
@@ -1,13 +1,9 @@
|
|
|
1
1
|
"""Private module for VTL validation functions."""
|
|
2
2
|
|
|
3
|
-
from vtlengine.API import create_ast
|
|
4
|
-
from vtlengine.AST import
|
|
5
|
-
DPRuleset as ASTDPRuleset,
|
|
6
|
-
)
|
|
3
|
+
from vtlengine.API import create_ast
|
|
4
|
+
from vtlengine.AST import DPRuleset as ASTDPRuleset
|
|
7
5
|
from vtlengine.AST import HRuleset as ASTHRuleset
|
|
8
|
-
from vtlengine.AST import
|
|
9
|
-
Operator as ASTOperator,
|
|
10
|
-
)
|
|
6
|
+
from vtlengine.AST import Operator as ASTOperator
|
|
11
7
|
|
|
12
8
|
from pysdmx.errors import Invalid
|
|
13
9
|
from pysdmx.model import Reference
|
|
@@ -37,14 +33,14 @@ def _ruleset_validation(ruleset: Ruleset) -> None:
|
|
|
37
33
|
ast.children[0], ASTDPRuleset
|
|
38
34
|
):
|
|
39
35
|
raise Invalid("Ruleset type does not match the definition")
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
36
|
+
|
|
37
|
+
child = ast.children[0]
|
|
38
|
+
signature_type = getattr(child, "signature_type", None)
|
|
39
|
+
if ruleset.ruleset_scope == "variable" and signature_type != "variable":
|
|
44
40
|
raise Invalid("Ruleset scope does not match the definition")
|
|
45
41
|
if (
|
|
46
42
|
ruleset.ruleset_scope == "valuedomain"
|
|
47
|
-
and
|
|
43
|
+
and signature_type != "valuedomain"
|
|
48
44
|
):
|
|
49
45
|
raise Invalid("Ruleset scope does not match the definition")
|
|
50
46
|
|
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
"""Conversions between pysdmx PandasDataset and vtlengine Dataset."""
|
|
2
|
+
|
|
3
|
+
from typing import Dict, Optional, Type, Union
|
|
4
|
+
|
|
5
|
+
from vtlengine.API import load_datasets # type: ignore[attr-defined]
|
|
6
|
+
from vtlengine.API._InternalApi import to_vtl_json
|
|
7
|
+
from vtlengine.DataTypes import (
|
|
8
|
+
Boolean,
|
|
9
|
+
Date,
|
|
10
|
+
Duration,
|
|
11
|
+
Integer,
|
|
12
|
+
Number,
|
|
13
|
+
ScalarType,
|
|
14
|
+
String,
|
|
15
|
+
TimeInterval,
|
|
16
|
+
TimePeriod,
|
|
17
|
+
)
|
|
18
|
+
from vtlengine.Model import Dataset as VTLengineDataset
|
|
19
|
+
from vtlengine.Model import Role as VTLRole
|
|
20
|
+
|
|
21
|
+
from pysdmx.errors import Invalid
|
|
22
|
+
from pysdmx.io.pd import PandasDataset
|
|
23
|
+
from pysdmx.model import Component, Components, Concept, Reference
|
|
24
|
+
from pysdmx.model.concept import DataType
|
|
25
|
+
from pysdmx.model.dataflow import Role, Schema
|
|
26
|
+
|
|
27
|
+
# VTL to SDMX type mapping
|
|
28
|
+
VTL_TO_SDMX_TYPE_MAP: Dict[Type[ScalarType], DataType] = {
|
|
29
|
+
String: DataType.STRING,
|
|
30
|
+
Integer: DataType.INTEGER,
|
|
31
|
+
Number: DataType.DOUBLE,
|
|
32
|
+
Boolean: DataType.BOOLEAN,
|
|
33
|
+
Date: DataType.DATE,
|
|
34
|
+
TimePeriod: DataType.PERIOD,
|
|
35
|
+
TimeInterval: DataType.TIME,
|
|
36
|
+
Duration: DataType.DURATION,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
# Role mapping
|
|
40
|
+
# ViralAttribute is not yet supported as a separate role in VTL 1.2.2,
|
|
41
|
+
# so it is mapped to Attribute following vtlengine's behavior
|
|
42
|
+
VTL_TO_SDMX_ROLE_MAP: Dict[VTLRole, Role] = {
|
|
43
|
+
VTLRole.IDENTIFIER: Role.DIMENSION,
|
|
44
|
+
VTLRole.MEASURE: Role.MEASURE,
|
|
45
|
+
VTLRole.ATTRIBUTE: Role.ATTRIBUTE,
|
|
46
|
+
"ViralAttribute": Role.ATTRIBUTE, # type: ignore[dict-item]
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
VALID_SDMX_TYPES = {"DataStructure", "Dataflow", "ProvisionAgreement"}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def convert_dataset_to_vtl(
|
|
53
|
+
dataset: PandasDataset, vtl_dataset_name: str
|
|
54
|
+
) -> VTLengineDataset:
|
|
55
|
+
"""Convert a PandasDataset to a vtlengine Dataset.
|
|
56
|
+
|
|
57
|
+
This function converts a PandasDataset, which contains both data and
|
|
58
|
+
structure (Schema), into a vtlengine Dataset. It uses vtlengine's
|
|
59
|
+
conversion functions to handle the Schema to VTL structure mapping.
|
|
60
|
+
|
|
61
|
+
It raises an Invalid exception if the dataset structure is not a
|
|
62
|
+
Schema object.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
dataset: The PandasDataset to convert.
|
|
66
|
+
vtl_dataset_name: The name for the vtlengine Dataset.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
A vtlengine Dataset with the data and structure from the
|
|
70
|
+
PandasDataset.
|
|
71
|
+
|
|
72
|
+
Raises:
|
|
73
|
+
Invalid: If the dataset structure is not a Schema object or if
|
|
74
|
+
component types cannot be mapped.
|
|
75
|
+
"""
|
|
76
|
+
if not isinstance(dataset.structure, Schema):
|
|
77
|
+
raise Invalid(
|
|
78
|
+
"Validation Error",
|
|
79
|
+
"Dataset structure must be a Schema object for conversion to VTL",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
schema = dataset.structure
|
|
83
|
+
pd_dataset = dataset.data
|
|
84
|
+
|
|
85
|
+
# Use vtlengine's built-in conversion function to convert Schema to VTL
|
|
86
|
+
vtl_json = to_vtl_json(schema, vtl_dataset_name)
|
|
87
|
+
|
|
88
|
+
# Load the dataset structure using vtlengine's API
|
|
89
|
+
datasets, scalars = load_datasets(vtl_json)
|
|
90
|
+
vtl_dataset = datasets[vtl_dataset_name]
|
|
91
|
+
|
|
92
|
+
# Assign the pandas DataFrame to the VTL dataset
|
|
93
|
+
vtl_dataset.data = pd_dataset
|
|
94
|
+
|
|
95
|
+
return vtl_dataset
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def convert_dataset_to_sdmx(
|
|
99
|
+
dataset: VTLengineDataset,
|
|
100
|
+
reference: Optional[Reference] = None,
|
|
101
|
+
schema: Optional[Schema] = None,
|
|
102
|
+
) -> PandasDataset:
|
|
103
|
+
"""Convert a VTLengine Dataset to a PandasDataset.
|
|
104
|
+
|
|
105
|
+
This function converts a `vtlengine.Model.Dataset` into
|
|
106
|
+
a `PandasDataset` by:
|
|
107
|
+
|
|
108
|
+
* Using a provided `Schema` for direct validation and conversion.
|
|
109
|
+
* Generating a new SDMX-compatible `Schema` from the dataset components,
|
|
110
|
+
using metadata from a provided `Reference`.
|
|
111
|
+
|
|
112
|
+
When a `schema` is supplied, the dataset is first validated against it and,
|
|
113
|
+
if validation passes, the data is wrapped in a `PandasDataset` with that
|
|
114
|
+
schema. If no `schema` is provided, a `reference` must be given so a new
|
|
115
|
+
SDMX structure (with components, roles, and data types mapped from the
|
|
116
|
+
VTL dataset) can be created.
|
|
117
|
+
|
|
118
|
+
Invalid is raised in the following cases:
|
|
119
|
+
* If neither `schema` nor `reference` is provided.
|
|
120
|
+
* If the `reference` has an unsupported `sdmx_type`.
|
|
121
|
+
* If the `dataset` contains no data.
|
|
122
|
+
* If component types or roles cannot be mapped to SDMX equivalents.
|
|
123
|
+
* If validation fails when a `schema` is provided.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
dataset: The VTLengine dataset to convert.
|
|
127
|
+
Must include components and associated data.
|
|
128
|
+
reference: Optional reference to the SDMX structure
|
|
129
|
+
(DataStructure, Dataflow, or ProvisionAgreement).
|
|
130
|
+
Required only when no `schema` is provided.
|
|
131
|
+
Used to build a schema and supply contextual identifiers.
|
|
132
|
+
schema: Optional schema describing the SDMX structure.
|
|
133
|
+
If provided, the dataset is validated against it
|
|
134
|
+
and the same schema is used directly in the output.
|
|
135
|
+
|
|
136
|
+
Returns:
|
|
137
|
+
A `PandasDataset` containing the converted data and the associated SDMX
|
|
138
|
+
structure (either the provided schema or a generated one).
|
|
139
|
+
|
|
140
|
+
Raises:
|
|
141
|
+
Invalid: If the reference sdmx_type is not valid, if component types
|
|
142
|
+
cannot be mapped, or if validation fails when schema is provided.
|
|
143
|
+
"""
|
|
144
|
+
# If schema is provided
|
|
145
|
+
if schema is not None:
|
|
146
|
+
_validate_vtl_dataset_against_schema(dataset, schema)
|
|
147
|
+
|
|
148
|
+
data = dataset.data
|
|
149
|
+
if data is None:
|
|
150
|
+
raise Invalid(
|
|
151
|
+
"Validation Error",
|
|
152
|
+
"VTL dataset has no data for conversion to SDMX",
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
pandas_dataset = PandasDataset(
|
|
156
|
+
structure=schema,
|
|
157
|
+
data=data,
|
|
158
|
+
)
|
|
159
|
+
return pandas_dataset
|
|
160
|
+
|
|
161
|
+
# If schema is not provided, reference must be provided
|
|
162
|
+
if reference is None:
|
|
163
|
+
raise Invalid(
|
|
164
|
+
"Validation Error",
|
|
165
|
+
"Either schema or reference must be provided",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
# Validate reference.sdmx_type
|
|
169
|
+
if reference.sdmx_type not in VALID_SDMX_TYPES:
|
|
170
|
+
raise Invalid(
|
|
171
|
+
"Validation Error",
|
|
172
|
+
f"Reference sdmx_type must be one of {VALID_SDMX_TYPES}, "
|
|
173
|
+
f"but got '{reference.sdmx_type}'",
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
data = dataset.data
|
|
177
|
+
if data is None:
|
|
178
|
+
raise Invalid(
|
|
179
|
+
"Validation Error",
|
|
180
|
+
"VTL dataset has no data for conversion to SDMX",
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
# Generate a new Schema from VTL Dataset components
|
|
184
|
+
sdmx_components = []
|
|
185
|
+
|
|
186
|
+
for comp_name, vtl_comp in dataset.components.items():
|
|
187
|
+
# Map VTL data type to SDMX data type
|
|
188
|
+
sdmx_dtype = _map_vtl_dtype_to_sdmx(vtl_comp.data_type)
|
|
189
|
+
|
|
190
|
+
# Map VTL role to SDMX role
|
|
191
|
+
sdmx_role = _map_vtl_role_to_sdmx(vtl_comp.role)
|
|
192
|
+
|
|
193
|
+
# Determine attachment_level for attributes
|
|
194
|
+
attachment_level = "O" if sdmx_role == Role.ATTRIBUTE else None
|
|
195
|
+
|
|
196
|
+
# Create SDMX Component
|
|
197
|
+
sdmx_comp = Component(
|
|
198
|
+
id=comp_name,
|
|
199
|
+
required=not vtl_comp.nullable,
|
|
200
|
+
role=sdmx_role,
|
|
201
|
+
concept=Concept(comp_name, dtype=sdmx_dtype),
|
|
202
|
+
attachment_level=attachment_level,
|
|
203
|
+
)
|
|
204
|
+
sdmx_components.append(sdmx_comp)
|
|
205
|
+
|
|
206
|
+
# Create Schema using reference information
|
|
207
|
+
generated_schema = Schema(
|
|
208
|
+
context=reference.sdmx_type.lower(), # type: ignore[arg-type]
|
|
209
|
+
agency=reference.agency,
|
|
210
|
+
id=reference.id,
|
|
211
|
+
version=reference.version,
|
|
212
|
+
components=Components(sdmx_components),
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
pandas_dataset = PandasDataset(
|
|
216
|
+
structure=generated_schema,
|
|
217
|
+
data=data,
|
|
218
|
+
)
|
|
219
|
+
return pandas_dataset
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _map_vtl_dtype_to_sdmx(
|
|
223
|
+
vtl_dtype_value: Union[ScalarType, Type[ScalarType]],
|
|
224
|
+
) -> DataType:
|
|
225
|
+
"""Return the SDMX DataType for a given VTL scalar.
|
|
226
|
+
|
|
227
|
+
Args:
|
|
228
|
+
vtl_dtype_value: The VTL scalar type or instance to map.
|
|
229
|
+
|
|
230
|
+
Returns:
|
|
231
|
+
The corresponding SDMX DataType.
|
|
232
|
+
|
|
233
|
+
Raises:
|
|
234
|
+
Invalid: If the VTL DataType cannot be mapped to an SDMX DataType.
|
|
235
|
+
"""
|
|
236
|
+
if isinstance(vtl_dtype_value, type):
|
|
237
|
+
vtl_dtype_class: type[ScalarType] = vtl_dtype_value
|
|
238
|
+
else:
|
|
239
|
+
vtl_dtype_class = type(vtl_dtype_value)
|
|
240
|
+
|
|
241
|
+
if vtl_dtype_class not in VTL_TO_SDMX_TYPE_MAP:
|
|
242
|
+
supported = ", ".join(str(t.__name__) for t in VTL_TO_SDMX_TYPE_MAP)
|
|
243
|
+
raise Invalid(
|
|
244
|
+
"Validation Error",
|
|
245
|
+
f"VTL DataType '{vtl_dtype_class.__name__}' cannot be "
|
|
246
|
+
f"mapped to an SDMX type. Supported types are: {supported}",
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
return VTL_TO_SDMX_TYPE_MAP[vtl_dtype_class]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _map_vtl_role_to_sdmx(vtl_role: VTLRole) -> Role:
|
|
253
|
+
"""Return the SDMX Role for a given VTL Role.
|
|
254
|
+
|
|
255
|
+
Args:
|
|
256
|
+
vtl_role: The VTLRole to map.
|
|
257
|
+
|
|
258
|
+
Returns:
|
|
259
|
+
The corresponding SDMX Role.
|
|
260
|
+
|
|
261
|
+
Raises:
|
|
262
|
+
Invalid: If the VTL Role cannot be mapped to an SDMX Role.
|
|
263
|
+
"""
|
|
264
|
+
if vtl_role not in VTL_TO_SDMX_ROLE_MAP:
|
|
265
|
+
raise Invalid(
|
|
266
|
+
"Validation Error",
|
|
267
|
+
f"VTL Role '{vtl_role}' cannot be mapped to an SDMX Role",
|
|
268
|
+
)
|
|
269
|
+
return VTL_TO_SDMX_ROLE_MAP[vtl_role]
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _validate_vtl_dataset_against_schema(
|
|
273
|
+
dataset: VTLengineDataset,
|
|
274
|
+
schema: Schema,
|
|
275
|
+
) -> None:
|
|
276
|
+
"""Validate VTLengine Dataset against SDMX Schema.
|
|
277
|
+
|
|
278
|
+
Args:
|
|
279
|
+
dataset: The VTLengineDataset instance whose components, roles, and
|
|
280
|
+
data types will be validated.
|
|
281
|
+
schema: The SDMX Schema that defines the expected components,
|
|
282
|
+
SDMX data types, and SDMX roles for validation.
|
|
283
|
+
|
|
284
|
+
Raises:
|
|
285
|
+
Invalid: If component names differ, if types or roles cannot be mapped,
|
|
286
|
+
or if any mismatch is detected between the Dataset and Schema.
|
|
287
|
+
"""
|
|
288
|
+
# Validate that schema components match VTL dataset components
|
|
289
|
+
vtl_component_names = set(dataset.components.keys())
|
|
290
|
+
schema_component_names = {comp.id for comp in schema.components}
|
|
291
|
+
|
|
292
|
+
if vtl_component_names != schema_component_names:
|
|
293
|
+
missing_in_schema = vtl_component_names - schema_component_names
|
|
294
|
+
missing_in_vtl = schema_component_names - vtl_component_names
|
|
295
|
+
error_parts = []
|
|
296
|
+
if missing_in_schema:
|
|
297
|
+
error_parts.append(
|
|
298
|
+
f"VTL components not in Schema: {missing_in_schema}"
|
|
299
|
+
)
|
|
300
|
+
if missing_in_vtl:
|
|
301
|
+
error_parts.append(
|
|
302
|
+
f"Schema components not in VTL: {missing_in_vtl}"
|
|
303
|
+
)
|
|
304
|
+
raise Invalid(
|
|
305
|
+
"Validation Error",
|
|
306
|
+
"Component mismatch between VTL Dataset and Schema. "
|
|
307
|
+
f"{'; '.join(error_parts)}",
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
# Validate that component types and roles match
|
|
311
|
+
for component in schema.components:
|
|
312
|
+
comp_id = str(component.id)
|
|
313
|
+
vtl_comp = dataset.components[comp_id]
|
|
314
|
+
|
|
315
|
+
# Validate data type using helper
|
|
316
|
+
expected_sdmx_dtype = _map_vtl_dtype_to_sdmx(vtl_comp.data_type)
|
|
317
|
+
if component.dtype != expected_sdmx_dtype:
|
|
318
|
+
raise Invalid(
|
|
319
|
+
"Validation Error",
|
|
320
|
+
"Component mismatch between VTL Dataset and Schema. "
|
|
321
|
+
f"Component '{comp_id}' has type {expected_sdmx_dtype} "
|
|
322
|
+
f"in VTL but {component.dtype} in Schema",
|
|
323
|
+
)
|
|
324
|
+
|
|
325
|
+
# Validate role using helper
|
|
326
|
+
expected_sdmx_role = _map_vtl_role_to_sdmx(vtl_comp.role)
|
|
327
|
+
if component.role != expected_sdmx_role:
|
|
328
|
+
raise Invalid(
|
|
329
|
+
"Validation Error",
|
|
330
|
+
"Component mismatch between VTL Dataset and Schema. "
|
|
331
|
+
f"Component '{comp_id}' has role {expected_sdmx_role} "
|
|
332
|
+
f"in VTL but {component.role} in Schema",
|
|
333
|
+
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pysdmx
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.10.0rc1
|
|
4
4
|
Summary: Your opinionated Python SDMX library
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
@@ -32,8 +32,8 @@ Requires-Dist: python-dateutil (>=2.8.2) ; extra == "dc"
|
|
|
32
32
|
Requires-Dist: sdmxschemas (>=1.0.0) ; extra == "all"
|
|
33
33
|
Requires-Dist: sdmxschemas (>=1.0.0) ; extra == "json"
|
|
34
34
|
Requires-Dist: sdmxschemas (>=1.0.0) ; extra == "xml"
|
|
35
|
-
Requires-Dist: vtlengine (>=1.
|
|
36
|
-
Requires-Dist: vtlengine (>=1.
|
|
35
|
+
Requires-Dist: vtlengine (>=1.2,<2.0) ; extra == "all"
|
|
36
|
+
Requires-Dist: vtlengine (>=1.2,<2.0) ; extra == "vtl"
|
|
37
37
|
Requires-Dist: xmltodict (>=0.13) ; extra == "all"
|
|
38
38
|
Requires-Dist: xmltodict (>=0.13) ; extra == "xml"
|
|
39
39
|
Project-URL: Bug Tracker, https://bis-med-it.github.io/pysdmx/issues
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
pysdmx/__extras_check.py,sha256=
|
|
2
|
-
pysdmx/__init__.py,sha256=
|
|
1
|
+
pysdmx/__extras_check.py,sha256=Tmluui2OuJVyJB6a1Jl0PlrRjpsswhtCjAqtRLOSero,2059
|
|
2
|
+
pysdmx/__init__.py,sha256=00h-DwPfv9b8RnK5FXzBTiZQOciFTXqXk7axxd3VD28,71
|
|
3
3
|
pysdmx/api/__init__.py,sha256=8lRaF6kEO51ehl0fmW_pHLvkN_34TtEhqhr3oKo6E6g,26
|
|
4
4
|
pysdmx/api/dc/__init__.py,sha256=oPU32X8CRZy4T1to9mO5KMqMwxQsVI424dPqai-I8zI,121
|
|
5
5
|
pysdmx/api/dc/_api.py,sha256=poy1FYFXnF6maBGy5lpOodf32-7QQjH8PCBNDkuOXxQ,7747
|
|
@@ -38,7 +38,7 @@ pysdmx/io/csv/sdmx21/__init__.py,sha256=I3_dwi4A4if62_mwEjqbOa-F7mhoIMf0D6szpDf3
|
|
|
38
38
|
pysdmx/io/csv/sdmx21/reader/__init__.py,sha256=J1cCkZh3klgZZWjdQ_U1zkfzT_DVzQmdreGZhN33SLs,2866
|
|
39
39
|
pysdmx/io/csv/sdmx21/writer/__init__.py,sha256=CH8Nm7hqvXyN6XM_D2nJRmbKj6CJV-X1QzSF0WJrs0E,2484
|
|
40
40
|
pysdmx/io/format.py,sha256=EO-PyYpiU0WswvEGA5UHokladxPezcwBUo1AJTqxp1Q,5250
|
|
41
|
-
pysdmx/io/input_processor.py,sha256=
|
|
41
|
+
pysdmx/io/input_processor.py,sha256=P1_jKegrOyV7EaZLjLrq8fX2u1EI7gPBJoKvlBCNkP0,6967
|
|
42
42
|
pysdmx/io/json/fusion/messages/__init__.py,sha256=ac2jWfjGGBcfoSutiKy68LzqwNp_clt2RzmJOaYCxL0,2142
|
|
43
43
|
pysdmx/io/json/fusion/messages/category.py,sha256=2NnYTptQ6nAdeYBIkAysyriTFF6dW8fjb9AvRWtTOao,5606
|
|
44
44
|
pysdmx/io/json/fusion/messages/code.py,sha256=YayxSKiHa124WTr_lK8HTfXHomh_e0oDZvuydd29XKg,8300
|
|
@@ -107,7 +107,7 @@ pysdmx/io/xml/__write_data_aux.py,sha256=Kfwxi8XHXj4W94D41cTdPig6GYw_3wTUzfDucNW
|
|
|
107
107
|
pysdmx/io/xml/__write_structure_specific_aux.py,sha256=reRDVw4Xwag0ODyMzm9uOk9WJ_e1ELxAPYHSMUUDJBQ,8919
|
|
108
108
|
pysdmx/io/xml/config.py,sha256=R24cczVkzkhjVLXpv-qfEm88W3_QTqVt2Qofi8IvJ5Y,93
|
|
109
109
|
pysdmx/io/xml/doc_validation.py,sha256=WXDhte96VEAeZMMHJ0Y68WW8HEoOhEiOYEnbGP5Zwjw,1795
|
|
110
|
-
pysdmx/io/xml/header.py,sha256=
|
|
110
|
+
pysdmx/io/xml/header.py,sha256=My03uhWD3AkfTwfUqiblmLIZuqd7uvIEYsOial6TClg,5971
|
|
111
111
|
pysdmx/io/xml/sdmx21/__init__.py,sha256=_rh_dz1d-LmUw4iVbsZQkTXANTJeR8ov8z_4jNRGhtg,38
|
|
112
112
|
pysdmx/io/xml/sdmx21/reader/__init__.py,sha256=2jMsySFbbq-So3ymqzKVi5gOPRwO0qasBBwntD03rTw,34
|
|
113
113
|
pysdmx/io/xml/sdmx21/reader/error.py,sha256=ySY3dBPcOaF_lm8yX1Khc92R5XgyD1u-djjiOrgZJP8,885
|
|
@@ -153,15 +153,16 @@ pysdmx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
|
153
153
|
pysdmx/toolkit/__init__.py,sha256=yBRw4FXQH_RUJgF6vwZscNwCVBt-FMwslJEfDKa7eXo,26
|
|
154
154
|
pysdmx/toolkit/pd/__init__.py,sha256=3JoCZHamJR4mZfLt9GmaznLlG_YilmY0hAhuW5COQC8,3138
|
|
155
155
|
pysdmx/toolkit/pd/_data_utils.py,sha256=fzXl_a1M8NI7HHI23tRmLlS4PBZ9N4Tgtqh-zxg19qk,3265
|
|
156
|
-
pysdmx/toolkit/vtl/__init__.py,sha256=
|
|
157
|
-
pysdmx/toolkit/vtl/_validations.py,sha256=
|
|
158
|
-
pysdmx/toolkit/vtl/
|
|
156
|
+
pysdmx/toolkit/vtl/__init__.py,sha256=jYWScnAomHlkvTtl07ur82UEqALCveiAEmMbqdWhT1A,388
|
|
157
|
+
pysdmx/toolkit/vtl/_validations.py,sha256=k0le8fDUOGNl97P-QbNglrNyr5fCh5dLPMiTk1yu1jw,5590
|
|
158
|
+
pysdmx/toolkit/vtl/convert.py,sha256=FNToVTG4Zu9n7h_Fre-LEGihx8RMGSJb7G6-FFyPg8Y,11355
|
|
159
|
+
pysdmx/toolkit/vtl/script_generation.py,sha256=pIDKd12uQrmbiU21DbDm8UbYMW2PYldFV_OXCzyqU_8,3203
|
|
159
160
|
pysdmx/toolkit/vtl/validation.py,sha256=UieJUYwxkw9McHZwixKFQdgYKFsgFwFo5XCLIIDcr7Q,3594
|
|
160
161
|
pysdmx/util/__init__.py,sha256=m_XWRAmVJ7F6ai4Ckrj_YuPbhg3cJZAXeZrEThrL88k,3997
|
|
161
162
|
pysdmx/util/_date_pattern_map.py,sha256=IS1qONwVHbTBNIFCT0Rqbijj2a9mYvs7onXSK6GeQAQ,1620
|
|
162
163
|
pysdmx/util/_model_utils.py,sha256=nQ1yWBt-tZYDios9xvRvJ7tHq6A8_RoGdY1wi7WGz2w,3793
|
|
163
164
|
pysdmx/util/_net_utils.py,sha256=nOTz_x3FgFrwKh42_J70IqYXz9duQkMFJWtejZXcLHs,1326
|
|
164
|
-
pysdmx-1.
|
|
165
|
-
pysdmx-1.
|
|
166
|
-
pysdmx-1.
|
|
167
|
-
pysdmx-1.
|
|
165
|
+
pysdmx-1.10.0rc1.dist-info/METADATA,sha256=kPs3kQG4EX60hnWKin3iojI-3tHGysb93P5Xc7uy56k,4852
|
|
166
|
+
pysdmx-1.10.0rc1.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
|
|
167
|
+
pysdmx-1.10.0rc1.dist-info/licenses/LICENSE,sha256=3XTNDPtv2RxDUNkQzn9MIWit2u7_Ob5daMLEq-4pBJs,649
|
|
168
|
+
pysdmx-1.10.0rc1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|