pysdmx 1.10.0rc2__py3-none-any.whl → 1.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pysdmx/__init__.py +1 -1
- pysdmx/api/fmr/__init__.py +3 -2
- pysdmx/api/fmr/maintenance.py +10 -5
- pysdmx/api/qb/data.py +2 -0
- pysdmx/io/csv/__csv_aux_writer.py +0 -23
- pysdmx/io/csv/sdmx10/reader/__init__.py +1 -1
- pysdmx/io/csv/sdmx10/writer/__init__.py +9 -15
- pysdmx/io/csv/sdmx20/reader/__init__.py +1 -1
- pysdmx/io/csv/sdmx20/writer/__init__.py +1 -1
- pysdmx/io/csv/sdmx21/reader/__init__.py +1 -1
- pysdmx/io/csv/sdmx21/writer/__init__.py +1 -1
- pysdmx/io/input_processor.py +4 -0
- pysdmx/io/json/fusion/messages/dsd.py +27 -16
- pysdmx/io/json/fusion/messages/msd.py +6 -9
- pysdmx/io/json/sdmxjson2/messages/core.py +12 -5
- pysdmx/io/json/sdmxjson2/messages/dsd.py +46 -31
- pysdmx/io/json/sdmxjson2/messages/msd.py +2 -5
- pysdmx/io/json/sdmxjson2/messages/report.py +7 -3
- pysdmx/io/json/sdmxjson2/messages/structure.py +7 -3
- pysdmx/io/json/sdmxjson2/reader/doc_validation.py +4 -0
- pysdmx/io/json/sdmxjson2/reader/metadata.py +3 -3
- pysdmx/io/json/sdmxjson2/reader/structure.py +3 -3
- pysdmx/io/json/sdmxjson2/writer/_helper.py +118 -0
- pysdmx/io/json/sdmxjson2/writer/v2_0/__init__.py +1 -0
- pysdmx/io/json/sdmxjson2/writer/v2_0/metadata.py +33 -0
- pysdmx/io/json/sdmxjson2/writer/v2_0/structure.py +33 -0
- pysdmx/io/json/sdmxjson2/writer/v2_1/__init__.py +1 -0
- pysdmx/io/json/sdmxjson2/writer/v2_1/metadata.py +31 -0
- pysdmx/io/json/sdmxjson2/writer/v2_1/structure.py +33 -0
- pysdmx/io/reader.py +12 -3
- pysdmx/io/writer.py +13 -3
- pysdmx/io/xml/__ss_aux_reader.py +39 -17
- pysdmx/io/xml/__structure_aux_reader.py +221 -33
- pysdmx/io/xml/__structure_aux_writer.py +304 -5
- pysdmx/io/xml/__tokens.py +12 -0
- pysdmx/io/xml/__write_aux.py +9 -0
- pysdmx/io/xml/__write_data_aux.py +9 -20
- pysdmx/io/xml/__write_structure_specific_aux.py +54 -71
- pysdmx/io/xml/sdmx21/writer/generic.py +20 -32
- pysdmx/model/concept.py +0 -16
- pysdmx/model/dataflow.py +11 -8
- pysdmx/toolkit/pd/_data_utils.py +1 -1
- {pysdmx-1.10.0rc2.dist-info → pysdmx-1.11.0.dist-info}/METADATA +7 -1
- {pysdmx-1.10.0rc2.dist-info → pysdmx-1.11.0.dist-info}/RECORD +46 -42
- {pysdmx-1.10.0rc2.dist-info → pysdmx-1.11.0.dist-info}/WHEEL +1 -1
- pysdmx/io/_pd_utils.py +0 -83
- pysdmx/io/json/sdmxjson2/writer/metadata.py +0 -60
- pysdmx/io/json/sdmxjson2/writer/structure.py +0 -61
- {pysdmx-1.10.0rc2.dist-info → pysdmx-1.11.0.dist-info}/licenses/LICENSE +0 -0
pysdmx/__init__.py
CHANGED
pysdmx/api/fmr/__init__.py
CHANGED
|
@@ -39,6 +39,7 @@ from pysdmx.model import (
|
|
|
39
39
|
Dataflow,
|
|
40
40
|
DataflowInfo,
|
|
41
41
|
DataProvider,
|
|
42
|
+
DataStructureDefinition,
|
|
42
43
|
Hierarchy,
|
|
43
44
|
HierarchyAssociation,
|
|
44
45
|
Metadataflow,
|
|
@@ -679,7 +680,7 @@ class RegistryClient(__BaseRegistryClient):
|
|
|
679
680
|
agency: str = "*",
|
|
680
681
|
id: str = "*",
|
|
681
682
|
version: str = "+",
|
|
682
|
-
) -> Sequence[
|
|
683
|
+
) -> Sequence[DataStructureDefinition]:
|
|
683
684
|
"""Get the data structures(s) matching the supplied parameters.
|
|
684
685
|
|
|
685
686
|
Args:
|
|
@@ -1253,7 +1254,7 @@ class AsyncRegistryClient(__BaseRegistryClient):
|
|
|
1253
1254
|
agency: str = "*",
|
|
1254
1255
|
id: str = "*",
|
|
1255
1256
|
version: str = "+",
|
|
1256
|
-
) -> Sequence[
|
|
1257
|
+
) -> Sequence[DataStructureDefinition]:
|
|
1257
1258
|
"""Get the data structures(s) matching the supplied parameters.
|
|
1258
1259
|
|
|
1259
1260
|
Args:
|
pysdmx/api/fmr/maintenance.py
CHANGED
|
@@ -64,9 +64,7 @@ class RegistryMaintenanceClient:
|
|
|
64
64
|
timeout: The maximum number of seconds to wait before considering
|
|
65
65
|
that a request timed out. Defaults to 10 seconds.
|
|
66
66
|
"""
|
|
67
|
-
|
|
68
|
-
api_endpoint = api_endpoint[0:-1]
|
|
69
|
-
self._api_endpoint = f"{api_endpoint}"
|
|
67
|
+
self._api_endpoint = self.__sanitize_endpoint(api_endpoint)
|
|
70
68
|
self._user = user
|
|
71
69
|
self._password = password
|
|
72
70
|
self._timeout = timeout
|
|
@@ -87,7 +85,6 @@ class RegistryMaintenanceClient:
|
|
|
87
85
|
) -> None:
|
|
88
86
|
with httpx.Client(verify=self._ssl_context) as client:
|
|
89
87
|
try:
|
|
90
|
-
url = f"{endpoint}"
|
|
91
88
|
auth = httpx.BasicAuth(self._user, self._password)
|
|
92
89
|
headers = {
|
|
93
90
|
"Content-Type": "application/text",
|
|
@@ -99,7 +96,7 @@ class RegistryMaintenanceClient:
|
|
|
99
96
|
serializer = serializers.structure_message
|
|
100
97
|
bodyjs = self._encoder.encode(serializer.from_model(message))
|
|
101
98
|
r = client.post(
|
|
102
|
-
|
|
99
|
+
endpoint,
|
|
103
100
|
headers=headers,
|
|
104
101
|
content=bodyjs,
|
|
105
102
|
timeout=self._timeout,
|
|
@@ -156,3 +153,11 @@ class RegistryMaintenanceClient:
|
|
|
156
153
|
message = MetadataMessage(header=header, reports=reports)
|
|
157
154
|
endpoint = f"{self._api_endpoint}/ws/secure/sdmx/v2/metadata"
|
|
158
155
|
return self.__post(message, action, endpoint)
|
|
156
|
+
|
|
157
|
+
def __sanitize_endpoint(self, endpoint: str) -> str:
|
|
158
|
+
if endpoint.endswith("/"):
|
|
159
|
+
endpoint = endpoint[0:-1]
|
|
160
|
+
endpoint = endpoint.replace("/ws/secure/sdmx/v2/metadata", "")
|
|
161
|
+
endpoint = endpoint.replace("/sdmx/v2", "")
|
|
162
|
+
endpoint = endpoint.replace("/ws/secure/sdmxapi/rest", "")
|
|
163
|
+
return endpoint
|
pysdmx/api/qb/data.py
CHANGED
|
@@ -329,6 +329,8 @@ class DataQuery(_CoreDataQuery, frozen=True, omit_defaults=True):
|
|
|
329
329
|
|
|
330
330
|
def __get_short_v2_qs(self, api_version: ApiVersion) -> str:
|
|
331
331
|
qs = ""
|
|
332
|
+
if self.components:
|
|
333
|
+
qs += self._create_component_filters(self.components)
|
|
332
334
|
if self.updated_after:
|
|
333
335
|
qs = super()._append_qs_param(
|
|
334
336
|
qs,
|
|
@@ -3,8 +3,6 @@ from typing import List, Literal, Optional, Sequence
|
|
|
3
3
|
|
|
4
4
|
import pandas as pd
|
|
5
5
|
|
|
6
|
-
from pysdmx.errors import Invalid
|
|
7
|
-
from pysdmx.io._pd_utils import _fill_na_values
|
|
8
6
|
from pysdmx.io.pd import PandasDataset
|
|
9
7
|
from pysdmx.model import Schema
|
|
10
8
|
from pysdmx.model.dataset import ActionType
|
|
@@ -18,25 +16,6 @@ SDMX_CSV_ACTION_MAPPER = {
|
|
|
18
16
|
}
|
|
19
17
|
|
|
20
18
|
|
|
21
|
-
def _validate_schema_exists(dataset: PandasDataset) -> Schema:
|
|
22
|
-
"""Validates that the dataset has a Schema defined.
|
|
23
|
-
|
|
24
|
-
Args:
|
|
25
|
-
dataset: The dataset to validate.
|
|
26
|
-
|
|
27
|
-
Returns:
|
|
28
|
-
The `Schema` from the dataset.
|
|
29
|
-
|
|
30
|
-
Raises:
|
|
31
|
-
Invalid: If the structure is not a `Schema`.
|
|
32
|
-
"""
|
|
33
|
-
if not isinstance(dataset.structure, Schema):
|
|
34
|
-
raise Invalid(
|
|
35
|
-
"Dataset Structure is not a Schema. Cannot perform operation."
|
|
36
|
-
)
|
|
37
|
-
return dataset.structure
|
|
38
|
-
|
|
39
|
-
|
|
40
19
|
def __write_time_period(df: pd.DataFrame, time_format: str) -> None:
|
|
41
20
|
# TODO: Correct handle of normalized time format
|
|
42
21
|
raise NotImplementedError("Normalized time format is not implemented yet.")
|
|
@@ -91,10 +70,8 @@ def _write_csv_2_aux(
|
|
|
91
70
|
) -> List[pd.DataFrame]:
|
|
92
71
|
dataframes = []
|
|
93
72
|
for dataset in datasets:
|
|
94
|
-
schema = _validate_schema_exists(dataset)
|
|
95
73
|
# Create a copy of the dataset
|
|
96
74
|
df: pd.DataFrame = copy(dataset.data)
|
|
97
|
-
df = _fill_na_values(df, schema)
|
|
98
75
|
structure_ref, unique_id = dataset.short_urn.split("=", maxsplit=1)
|
|
99
76
|
|
|
100
77
|
# Add additional attributes to the dataset
|
|
@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
|
|
|
24
24
|
"""
|
|
25
25
|
# Get Dataframe from CSV file
|
|
26
26
|
df_csv = pd.read_csv(
|
|
27
|
-
StringIO(input_str), keep_default_na=False, na_values=[]
|
|
27
|
+
StringIO(input_str), keep_default_na=False, na_values=[""]
|
|
28
28
|
)
|
|
29
29
|
# Drop empty columns
|
|
30
30
|
df_csv = df_csv.dropna(axis=1, how="all")
|
|
@@ -6,12 +6,9 @@ from typing import Literal, Optional, Sequence, Union
|
|
|
6
6
|
|
|
7
7
|
import pandas as pd
|
|
8
8
|
|
|
9
|
-
from pysdmx.io.
|
|
10
|
-
from pysdmx.io.csv.__csv_aux_writer import (
|
|
11
|
-
__write_time_period,
|
|
12
|
-
_validate_schema_exists,
|
|
13
|
-
)
|
|
9
|
+
from pysdmx.io.csv.__csv_aux_writer import __write_time_period
|
|
14
10
|
from pysdmx.io.pd import PandasDataset
|
|
11
|
+
from pysdmx.model import Schema
|
|
15
12
|
from pysdmx.toolkit.pd._data_utils import format_labels
|
|
16
13
|
|
|
17
14
|
|
|
@@ -47,26 +44,22 @@ def write(
|
|
|
47
44
|
# Create a copy of the dataset
|
|
48
45
|
dataframes = []
|
|
49
46
|
for dataset in datasets:
|
|
50
|
-
# Validate that dataset has a proper Schema
|
|
51
|
-
schema = _validate_schema_exists(dataset)
|
|
52
|
-
|
|
53
47
|
df: pd.DataFrame = copy(dataset.data)
|
|
54
48
|
|
|
55
|
-
# Fill missing values
|
|
56
|
-
df = _fill_na_values(df, schema)
|
|
57
|
-
|
|
58
49
|
# Add additional attributes to the dataset
|
|
59
50
|
for k, v in dataset.attributes.items():
|
|
60
51
|
df[k] = v
|
|
61
52
|
structure_id = dataset.short_urn.split("=")[1]
|
|
62
53
|
if time_format is not None and time_format != "original":
|
|
63
54
|
__write_time_period(df, time_format)
|
|
64
|
-
if labels is not None:
|
|
65
|
-
format_labels(df, labels,
|
|
55
|
+
if labels is not None and isinstance(dataset.structure, Schema):
|
|
56
|
+
format_labels(df, labels, dataset.structure.components)
|
|
66
57
|
if labels == "id":
|
|
67
58
|
df.insert(0, "DATAFLOW", structure_id)
|
|
68
59
|
else:
|
|
69
|
-
df.insert(
|
|
60
|
+
df.insert(
|
|
61
|
+
0, "DATAFLOW", f"{structure_id}:{dataset.structure.name}"
|
|
62
|
+
)
|
|
70
63
|
else:
|
|
71
64
|
df.insert(0, "DATAFLOW", structure_id)
|
|
72
65
|
|
|
@@ -75,7 +68,8 @@ def write(
|
|
|
75
68
|
# Concatenate the dataframes
|
|
76
69
|
all_data = pd.concat(dataframes, ignore_index=True, axis=0)
|
|
77
70
|
|
|
78
|
-
|
|
71
|
+
# Ensure null values are represented as empty strings
|
|
72
|
+
all_data = all_data.astype(str).replace({"nan": "", "<NA>": ""})
|
|
79
73
|
# If the output path is an empty string we use None
|
|
80
74
|
output_path = (
|
|
81
75
|
None
|
|
@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
|
|
|
24
24
|
"""
|
|
25
25
|
# Get Dataframe from CSV file
|
|
26
26
|
df_csv = pd.read_csv(
|
|
27
|
-
StringIO(input_str), keep_default_na=False, na_values=[]
|
|
27
|
+
StringIO(input_str), keep_default_na=False, na_values=[""]
|
|
28
28
|
)
|
|
29
29
|
# Drop empty columns
|
|
30
30
|
df_csv = df_csv.dropna(axis=1, how="all")
|
|
@@ -24,7 +24,7 @@ def read(input_str: str) -> Sequence[PandasDataset]:
|
|
|
24
24
|
"""
|
|
25
25
|
# Get Dataframe from CSV file
|
|
26
26
|
df_csv = pd.read_csv(
|
|
27
|
-
StringIO(input_str), keep_default_na=False, na_values=[]
|
|
27
|
+
StringIO(input_str), keep_default_na=False, na_values=[""]
|
|
28
28
|
)
|
|
29
29
|
# Drop empty columns
|
|
30
30
|
df_csv = df_csv.dropna(axis=1, how="all")
|
pysdmx/io/input_processor.py
CHANGED
|
@@ -97,6 +97,10 @@ def __get_sdmx_json_flavour(input_str: str) -> Tuple[str, Format]:
|
|
|
97
97
|
return input_str, Format.STRUCTURE_SDMX_JSON_2_0_0
|
|
98
98
|
elif "2.0.0/sdmx-json-metadata-schema.json" in flavour_check:
|
|
99
99
|
return input_str, Format.REFMETA_SDMX_JSON_2_0_0
|
|
100
|
+
elif "2.1/sdmx-json-structure-schema.json" in flavour_check:
|
|
101
|
+
return input_str, Format.STRUCTURE_SDMX_JSON_2_1_0
|
|
102
|
+
elif "2.1/sdmx-json-metadata-schema.json" in flavour_check:
|
|
103
|
+
return input_str, Format.REFMETA_SDMX_JSON_2_1_0
|
|
100
104
|
elif "sdmx-json" in flavour_check:
|
|
101
105
|
raise NotImplemented(
|
|
102
106
|
"Unsupported format", "This flavour of SDMX-JSON is not supported."
|
|
@@ -20,6 +20,7 @@ from pysdmx.model import (
|
|
|
20
20
|
Codelist,
|
|
21
21
|
Component,
|
|
22
22
|
Components,
|
|
23
|
+
Concept,
|
|
23
24
|
DataStructureDefinition,
|
|
24
25
|
DataType,
|
|
25
26
|
Facets,
|
|
@@ -32,14 +33,15 @@ from pysdmx.util import parse_item_urn
|
|
|
32
33
|
def _find_concept(
|
|
33
34
|
cs: Sequence[FusionConceptScheme],
|
|
34
35
|
urn: str,
|
|
35
|
-
) -> FusionConcept:
|
|
36
|
+
) -> Optional[FusionConcept]:
|
|
36
37
|
r = parse_item_urn(urn)
|
|
37
38
|
f = [
|
|
38
39
|
m
|
|
39
40
|
for m in cs
|
|
40
41
|
if (m.agency == r.agency and m.id == r.id and m.version == r.version)
|
|
41
42
|
]
|
|
42
|
-
|
|
43
|
+
m = [c for c in f[0].items if c.id == r.item_id]
|
|
44
|
+
return m[0] if m else None
|
|
43
45
|
|
|
44
46
|
|
|
45
47
|
def _get_representation(
|
|
@@ -80,8 +82,14 @@ class FusionAttribute(Struct, frozen=True):
|
|
|
80
82
|
measureReferences: Optional[Sequence[str]] = None
|
|
81
83
|
|
|
82
84
|
def __derive_level(self, groups: Sequence[FusionGroup]) -> str:
|
|
83
|
-
if self.
|
|
84
|
-
|
|
85
|
+
if self.measureReferences:
|
|
86
|
+
if (
|
|
87
|
+
len(self.measureReferences) == 1
|
|
88
|
+
and self.measureReferences[0] == "OBS_VALUE"
|
|
89
|
+
):
|
|
90
|
+
return "O"
|
|
91
|
+
else:
|
|
92
|
+
return ",".join(self.measureReferences)
|
|
85
93
|
elif self.attachmentLevel == "DATA_SET":
|
|
86
94
|
return "D"
|
|
87
95
|
elif self.attachmentLevel == "GROUP":
|
|
@@ -106,12 +114,13 @@ class FusionAttribute(Struct, frozen=True):
|
|
|
106
114
|
groups: Sequence[FusionGroup],
|
|
107
115
|
) -> Component:
|
|
108
116
|
"""Returns an attribute."""
|
|
109
|
-
|
|
117
|
+
m = _find_concept(cs, self.concept) if cs else None
|
|
118
|
+
c = m.to_model(cls) if m else parse_item_urn(self.concept)
|
|
110
119
|
dt, facets, codes, ab = _get_representation(
|
|
111
120
|
self.id, self.representation, cls, cons
|
|
112
121
|
)
|
|
113
122
|
lvl = self.__derive_level(groups)
|
|
114
|
-
desc = c.
|
|
123
|
+
desc = c.description if isinstance(c, Concept) else None
|
|
115
124
|
if self.representation and self.representation.representation:
|
|
116
125
|
local_enum_ref = self.representation.representation
|
|
117
126
|
else:
|
|
@@ -120,10 +129,10 @@ class FusionAttribute(Struct, frozen=True):
|
|
|
120
129
|
id=self.id,
|
|
121
130
|
required=self.mandatory,
|
|
122
131
|
role=Role.ATTRIBUTE,
|
|
123
|
-
concept=c
|
|
132
|
+
concept=c,
|
|
124
133
|
local_dtype=dt,
|
|
125
134
|
local_facets=facets,
|
|
126
|
-
name=c.
|
|
135
|
+
name=c.name if isinstance(c, Concept) else None,
|
|
127
136
|
description=desc,
|
|
128
137
|
local_codes=codes,
|
|
129
138
|
attachment_level=lvl,
|
|
@@ -162,11 +171,12 @@ class FusionDimension(Struct, frozen=True):
|
|
|
162
171
|
cons: Dict[str, Sequence[str]],
|
|
163
172
|
) -> Component:
|
|
164
173
|
"""Returns a dimension."""
|
|
165
|
-
|
|
174
|
+
m = _find_concept(cs, self.concept) if cs else None
|
|
175
|
+
c = m.to_model(cls) if m else parse_item_urn(self.concept)
|
|
166
176
|
dt, facets, codes, ab = _get_representation(
|
|
167
177
|
self.id, self.representation, cls, cons
|
|
168
178
|
)
|
|
169
|
-
desc = c.
|
|
179
|
+
desc = c.description if isinstance(c, Concept) else None
|
|
170
180
|
if self.representation and self.representation.representation:
|
|
171
181
|
local_enum_ref = self.representation.representation
|
|
172
182
|
else:
|
|
@@ -175,10 +185,10 @@ class FusionDimension(Struct, frozen=True):
|
|
|
175
185
|
id=self.id,
|
|
176
186
|
required=True,
|
|
177
187
|
role=Role.DIMENSION,
|
|
178
|
-
concept=c
|
|
188
|
+
concept=c,
|
|
179
189
|
local_dtype=dt,
|
|
180
190
|
local_facets=facets,
|
|
181
|
-
name=c.
|
|
191
|
+
name=c.name if isinstance(c, Concept) else None,
|
|
182
192
|
description=desc,
|
|
183
193
|
local_codes=codes,
|
|
184
194
|
array_def=ab,
|
|
@@ -216,11 +226,12 @@ class FusionMeasure(Struct, frozen=True):
|
|
|
216
226
|
cons: Dict[str, Sequence[str]],
|
|
217
227
|
) -> Component:
|
|
218
228
|
"""Returns a measure."""
|
|
219
|
-
|
|
229
|
+
m = _find_concept(cs, self.concept) if cs else None
|
|
230
|
+
c = m.to_model(cls) if m else parse_item_urn(self.concept)
|
|
220
231
|
dt, facets, codes, ab = _get_representation(
|
|
221
232
|
self.id, self.representation, cls, cons
|
|
222
233
|
)
|
|
223
|
-
desc = c.
|
|
234
|
+
desc = c.description if isinstance(c, Concept) else None
|
|
224
235
|
if self.representation and self.representation.representation:
|
|
225
236
|
local_enum_ref = self.representation.representation
|
|
226
237
|
else:
|
|
@@ -229,10 +240,10 @@ class FusionMeasure(Struct, frozen=True):
|
|
|
229
240
|
id=self.id,
|
|
230
241
|
required=self.mandatory,
|
|
231
242
|
role=Role.MEASURE,
|
|
232
|
-
concept=c
|
|
243
|
+
concept=c,
|
|
233
244
|
local_dtype=dt,
|
|
234
245
|
local_facets=facets,
|
|
235
|
-
name=c.
|
|
246
|
+
name=c.name if isinstance(c, Concept) else None,
|
|
236
247
|
description=desc,
|
|
237
248
|
local_codes=codes,
|
|
238
249
|
array_def=ab,
|
|
@@ -14,13 +14,9 @@ from pysdmx.io.json.fusion.messages.dsd import (
|
|
|
14
14
|
_find_concept,
|
|
15
15
|
_get_representation,
|
|
16
16
|
)
|
|
17
|
-
from pysdmx.model import
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
)
|
|
21
|
-
from pysdmx.model import (
|
|
22
|
-
MetadataStructure as MSD,
|
|
23
|
-
)
|
|
17
|
+
from pysdmx.model import ArrayBoundaries, MetadataComponent
|
|
18
|
+
from pysdmx.model import MetadataStructure as MSD
|
|
19
|
+
from pysdmx.util import parse_item_urn
|
|
24
20
|
|
|
25
21
|
|
|
26
22
|
class FusionMetadataAttribute(Struct, frozen=True):
|
|
@@ -40,7 +36,8 @@ class FusionMetadataAttribute(Struct, frozen=True):
|
|
|
40
36
|
cls: Sequence[FusionCodelist],
|
|
41
37
|
) -> MetadataComponent:
|
|
42
38
|
"""Returns an attribute."""
|
|
43
|
-
|
|
39
|
+
m = _find_concept(cs, self.concept) if cs else None
|
|
40
|
+
c = m.to_model(cls) if m else parse_item_urn(self.concept)
|
|
44
41
|
dt, facets, codes, _ = _get_representation(
|
|
45
42
|
self.id, self.representation, cls, {}
|
|
46
43
|
)
|
|
@@ -60,7 +57,7 @@ class FusionMetadataAttribute(Struct, frozen=True):
|
|
|
60
57
|
return MetadataComponent(
|
|
61
58
|
self.id,
|
|
62
59
|
is_presentational=self.presentational, # type: ignore[arg-type]
|
|
63
|
-
concept=c
|
|
60
|
+
concept=c,
|
|
64
61
|
local_dtype=dt,
|
|
65
62
|
local_facets=facets,
|
|
66
63
|
local_codes=codes,
|
|
@@ -307,17 +307,24 @@ class JsonHeader(msgspec.Struct, frozen=True, omit_defaults=True):
|
|
|
307
307
|
self,
|
|
308
308
|
header: Header,
|
|
309
309
|
msg_type: Literal["structure", "metadata"] = "structure",
|
|
310
|
+
msg_version: Literal["2.0.0", "2.1"] = "2.0.0",
|
|
310
311
|
) -> "JsonHeader":
|
|
311
312
|
"""Create an SDMX-JSON header from a pysdmx Header."""
|
|
313
|
+
if msg_version == "2.0.0":
|
|
314
|
+
schema = (
|
|
315
|
+
"https://raw.githubusercontent.com/sdmx-twg/sdmx-json/"
|
|
316
|
+
f"develop/structure-message/tools/schemas/{msg_version}/"
|
|
317
|
+
f"sdmx-json-{msg_type}-schema.json"
|
|
318
|
+
)
|
|
319
|
+
else:
|
|
320
|
+
schema = (
|
|
321
|
+
f"https://json.sdmx.org/2.1/sdmx-json-{msg_type}-schema.json"
|
|
322
|
+
)
|
|
312
323
|
return JsonHeader(
|
|
313
324
|
header.id,
|
|
314
325
|
header.prepared,
|
|
315
326
|
header.sender,
|
|
316
327
|
header.test,
|
|
317
328
|
receivers=header.receiver if header.receiver else None,
|
|
318
|
-
schema=
|
|
319
|
-
"https://raw.githubusercontent.com/sdmx-twg/sdmx-json/"
|
|
320
|
-
"develop/structure-message/tools/schemas/2.0.0/"
|
|
321
|
-
f"sdmx-json-{msg_type}-schema.json"
|
|
322
|
-
),
|
|
329
|
+
schema=schema,
|
|
323
330
|
)
|
|
@@ -34,14 +34,17 @@ from pysdmx.model.dataflow import Group
|
|
|
34
34
|
from pysdmx.util import parse_item_urn
|
|
35
35
|
|
|
36
36
|
|
|
37
|
-
def _find_concept(
|
|
37
|
+
def _find_concept(
|
|
38
|
+
cs: Sequence[JsonConceptScheme], urn: str
|
|
39
|
+
) -> Optional[JsonConcept]:
|
|
38
40
|
r = parse_item_urn(urn)
|
|
39
41
|
f = [
|
|
40
42
|
m
|
|
41
43
|
for m in cs
|
|
42
44
|
if (m.agency == r.agency and m.id == r.id and m.version == r.version)
|
|
43
45
|
]
|
|
44
|
-
|
|
46
|
+
m = [c for c in f[0].concepts if c.id == r.item_id]
|
|
47
|
+
return m[0] if m else None
|
|
45
48
|
|
|
46
49
|
|
|
47
50
|
def _get_type(repr_: JsonRepresentation) -> Optional[str]:
|
|
@@ -126,7 +129,10 @@ class JsonAttributeRelationship(Struct, frozen=True, omit_defaults=True):
|
|
|
126
129
|
) -> str:
|
|
127
130
|
"""Returns the attachment level."""
|
|
128
131
|
if measures:
|
|
129
|
-
|
|
132
|
+
if len(measures) == 1 and measures[0] == "OBS_VALUE":
|
|
133
|
+
return "O"
|
|
134
|
+
else:
|
|
135
|
+
return ",".join(measures)
|
|
130
136
|
elif self.dimensions:
|
|
131
137
|
return ",".join(self.dimensions)
|
|
132
138
|
elif self.group:
|
|
@@ -136,15 +142,17 @@ class JsonAttributeRelationship(Struct, frozen=True, omit_defaults=True):
|
|
|
136
142
|
return "D"
|
|
137
143
|
|
|
138
144
|
@classmethod
|
|
139
|
-
def from_model(
|
|
145
|
+
def from_model(
|
|
146
|
+
self, rel: str, has_measure_rel: bool = False
|
|
147
|
+
) -> "JsonAttributeRelationship":
|
|
140
148
|
"""Converts a pysdmx attribute relationship to an SDMX-JSON one."""
|
|
141
149
|
if rel == "D":
|
|
142
150
|
return JsonAttributeRelationship(dataflow={})
|
|
143
|
-
elif rel == "O":
|
|
151
|
+
elif rel == "O" or has_measure_rel:
|
|
144
152
|
return JsonAttributeRelationship(observation={})
|
|
145
153
|
else:
|
|
146
|
-
|
|
147
|
-
return JsonAttributeRelationship(dimensions=
|
|
154
|
+
comps = rel.split(",")
|
|
155
|
+
return JsonAttributeRelationship(dimensions=comps)
|
|
148
156
|
|
|
149
157
|
|
|
150
158
|
class JsonDimension(Struct, frozen=True, omit_defaults=True):
|
|
@@ -163,11 +171,8 @@ class JsonDimension(Struct, frozen=True, omit_defaults=True):
|
|
|
163
171
|
cons: Dict[str, Sequence[str]],
|
|
164
172
|
) -> Component:
|
|
165
173
|
"""Returns a component."""
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
if cs
|
|
169
|
-
else parse_item_urn(self.conceptIdentity)
|
|
170
|
-
)
|
|
174
|
+
m = _find_concept(cs, self.conceptIdentity) if cs else None
|
|
175
|
+
c = m.to_model(cls) if m else parse_item_urn(self.conceptIdentity)
|
|
171
176
|
name = c.name if isinstance(c, Concept) else None
|
|
172
177
|
desc = c.description if isinstance(c, Concept) else None
|
|
173
178
|
dt, facets, codes, ab = _get_representation(
|
|
@@ -222,11 +227,8 @@ class JsonAttribute(Struct, frozen=True, omit_defaults=True):
|
|
|
222
227
|
groups: Sequence[JsonGroup],
|
|
223
228
|
) -> Component:
|
|
224
229
|
"""Returns a component."""
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
if cs
|
|
228
|
-
else parse_item_urn(self.conceptIdentity)
|
|
229
|
-
)
|
|
230
|
+
m = _find_concept(cs, self.conceptIdentity) if cs else None
|
|
231
|
+
c = m.to_model(cls) if m else parse_item_urn(self.conceptIdentity)
|
|
230
232
|
name = c.name if isinstance(c, Concept) else None
|
|
231
233
|
desc = c.description if isinstance(c, Concept) else None
|
|
232
234
|
dt, facets, codes, ab = _get_representation(
|
|
@@ -257,17 +259,29 @@ class JsonAttribute(Struct, frozen=True, omit_defaults=True):
|
|
|
257
259
|
)
|
|
258
260
|
|
|
259
261
|
@classmethod
|
|
260
|
-
def from_model(
|
|
262
|
+
def from_model(
|
|
263
|
+
self, attribute: Component, measures: Sequence[Component]
|
|
264
|
+
) -> "JsonAttribute":
|
|
261
265
|
"""Converts a pysdmx attribute to an SDMX-JSON one."""
|
|
262
266
|
concept = _get_concept_reference(attribute)
|
|
263
267
|
usage = "mandatory" if attribute.required else "optional"
|
|
268
|
+
repr = _get_json_representation(attribute)
|
|
269
|
+
|
|
270
|
+
ids = attribute.attachment_level.split(",") # type: ignore[union-attr]
|
|
271
|
+
comps = set(ids)
|
|
272
|
+
mids = {m.id for m in measures}
|
|
273
|
+
has_measure_rel = len(comps.intersection(mids)) > 0
|
|
264
274
|
level = JsonAttributeRelationship.from_model(
|
|
265
|
-
attribute.attachment_level # type: ignore[arg-type]
|
|
275
|
+
attribute.attachment_level, # type: ignore[arg-type]
|
|
276
|
+
has_measure_rel,
|
|
266
277
|
)
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
278
|
+
|
|
279
|
+
if attribute.attachment_level == "O":
|
|
280
|
+
mr = ["OBS_VALUE"]
|
|
281
|
+
elif has_measure_rel:
|
|
282
|
+
mr = ids
|
|
283
|
+
else:
|
|
284
|
+
mr = None
|
|
271
285
|
|
|
272
286
|
return JsonAttribute(
|
|
273
287
|
id=attribute.id,
|
|
@@ -295,11 +309,8 @@ class JsonMeasure(Struct, frozen=True, omit_defaults=True):
|
|
|
295
309
|
cons: Dict[str, Sequence[str]],
|
|
296
310
|
) -> Component:
|
|
297
311
|
"""Returns a component."""
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
if cs
|
|
301
|
-
else parse_item_urn(self.conceptIdentity)
|
|
302
|
-
)
|
|
312
|
+
m = _find_concept(cs, self.conceptIdentity) if cs else None
|
|
313
|
+
c = m.to_model(cls) if m else parse_item_urn(self.conceptIdentity)
|
|
303
314
|
name = c.name if isinstance(c, Concept) else None
|
|
304
315
|
desc = c.description if isinstance(c, Concept) else None
|
|
305
316
|
dt, facets, codes, ab = _get_representation(
|
|
@@ -356,12 +367,14 @@ class JsonAttributes(Struct, frozen=True, omit_defaults=True):
|
|
|
356
367
|
|
|
357
368
|
@classmethod
|
|
358
369
|
def from_model(
|
|
359
|
-
self, attributes: Sequence[Component]
|
|
370
|
+
self, attributes: Sequence[Component], measures: Sequence[Component]
|
|
360
371
|
) -> Optional["JsonAttributes"]:
|
|
361
372
|
"""Converts a pysdmx list of attributes to an SDMX-JSON one."""
|
|
362
373
|
if len(attributes) > 0:
|
|
363
374
|
return JsonAttributes(
|
|
364
|
-
attributes=[
|
|
375
|
+
attributes=[
|
|
376
|
+
JsonAttribute.from_model(a, measures) for a in attributes
|
|
377
|
+
]
|
|
365
378
|
)
|
|
366
379
|
else:
|
|
367
380
|
return None
|
|
@@ -502,7 +515,9 @@ class JsonComponents(Struct, frozen=True, omit_defaults=True):
|
|
|
502
515
|
) -> "JsonComponents":
|
|
503
516
|
"""Converts a pysdmx components list to an SDMX-JSON one."""
|
|
504
517
|
dimensions = JsonDimensions.from_model(components.dimensions)
|
|
505
|
-
attributes = JsonAttributes.from_model(
|
|
518
|
+
attributes = JsonAttributes.from_model(
|
|
519
|
+
components.attributes, components.measures
|
|
520
|
+
)
|
|
506
521
|
measures = JsonMeasures.from_model(components.measures)
|
|
507
522
|
if grps is None:
|
|
508
523
|
groups = []
|
|
@@ -49,11 +49,8 @@ class JsonMetadataAttribute(Struct, frozen=True, omit_defaults=True):
|
|
|
49
49
|
self, cs: Sequence[JsonConceptScheme], cls: Sequence[Codelist]
|
|
50
50
|
) -> MetadataComponent:
|
|
51
51
|
"""Returns a metadata component."""
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
if cs
|
|
55
|
-
else parse_item_urn(self.conceptIdentity)
|
|
56
|
-
)
|
|
52
|
+
m = _find_concept(cs, self.conceptIdentity) if cs else None
|
|
53
|
+
c = m.to_model(cls) if m else parse_item_urn(self.conceptIdentity)
|
|
57
54
|
dt, facets, codes, _ = _get_representation(
|
|
58
55
|
self.id, self.localRepresentation, cls, {}
|
|
59
56
|
)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Collection of SDMX-JSON schemas for reference metadata queries."""
|
|
2
2
|
|
|
3
|
-
from typing import Any, Optional, Sequence
|
|
3
|
+
from typing import Any, Literal, Optional, Sequence
|
|
4
4
|
|
|
5
5
|
from msgspec import Struct
|
|
6
6
|
|
|
@@ -162,7 +162,11 @@ class JsonMetadataMessage(Struct, frozen=True, omit_defaults=True):
|
|
|
162
162
|
return MetadataMessage(header, reports)
|
|
163
163
|
|
|
164
164
|
@classmethod
|
|
165
|
-
def from_model(
|
|
165
|
+
def from_model(
|
|
166
|
+
self,
|
|
167
|
+
msg: MetadataMessage,
|
|
168
|
+
msg_version: Literal["2.0.0", "2.1"] = "2.0.0",
|
|
169
|
+
) -> "JsonMetadataMessage":
|
|
166
170
|
"""Converts a pysdmx metadata message to an SDMX-JSON one."""
|
|
167
171
|
if not msg.header:
|
|
168
172
|
raise errors.Invalid(
|
|
@@ -175,6 +179,6 @@ class JsonMetadataMessage(Struct, frozen=True, omit_defaults=True):
|
|
|
175
179
|
"SDMX-JSON metadata messages must have metadata reports.",
|
|
176
180
|
)
|
|
177
181
|
|
|
178
|
-
header = JsonHeader.from_model(msg.header, "metadata")
|
|
182
|
+
header = JsonHeader.from_model(msg.header, "metadata", msg_version)
|
|
179
183
|
reports = [JsonMetadataReport.from_model(r) for r in msg.reports]
|
|
180
184
|
return JsonMetadataMessage(header, JsonMetadataSets(reports))
|