pysdmx 1.5.2__py3-none-any.whl → 1.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pysdmx/__init__.py +1 -1
- pysdmx/api/fmr/__init__.py +8 -3
- pysdmx/api/fmr/maintenance.py +158 -0
- pysdmx/api/qb/structure.py +1 -0
- pysdmx/api/qb/util.py +1 -0
- pysdmx/io/csv/__csv_aux_reader.py +99 -0
- pysdmx/io/csv/__csv_aux_writer.py +118 -0
- pysdmx/io/csv/sdmx10/reader/__init__.py +9 -14
- pysdmx/io/csv/sdmx10/writer/__init__.py +28 -2
- pysdmx/io/csv/sdmx20/__init__.py +0 -9
- pysdmx/io/csv/sdmx20/reader/__init__.py +8 -61
- pysdmx/io/csv/sdmx20/writer/__init__.py +32 -25
- pysdmx/io/csv/sdmx21/__init__.py +1 -0
- pysdmx/io/csv/sdmx21/reader/__init__.py +86 -0
- pysdmx/io/csv/sdmx21/writer/__init__.py +70 -0
- pysdmx/io/format.py +8 -0
- pysdmx/io/input_processor.py +20 -6
- pysdmx/io/json/fusion/messages/code.py +21 -4
- pysdmx/io/json/fusion/messages/concept.py +10 -8
- pysdmx/io/json/fusion/messages/dataflow.py +8 -1
- pysdmx/io/json/fusion/messages/dsd.py +15 -0
- pysdmx/io/json/fusion/messages/schema.py +8 -1
- pysdmx/io/json/sdmxjson2/messages/agency.py +43 -7
- pysdmx/io/json/sdmxjson2/messages/category.py +92 -7
- pysdmx/io/json/sdmxjson2/messages/code.py +265 -22
- pysdmx/io/json/sdmxjson2/messages/concept.py +75 -13
- pysdmx/io/json/sdmxjson2/messages/constraint.py +5 -5
- pysdmx/io/json/sdmxjson2/messages/core.py +121 -14
- pysdmx/io/json/sdmxjson2/messages/dataflow.py +63 -8
- pysdmx/io/json/sdmxjson2/messages/dsd.py +215 -20
- pysdmx/io/json/sdmxjson2/messages/map.py +200 -24
- pysdmx/io/json/sdmxjson2/messages/pa.py +36 -5
- pysdmx/io/json/sdmxjson2/messages/provider.py +35 -7
- pysdmx/io/json/sdmxjson2/messages/report.py +85 -7
- pysdmx/io/json/sdmxjson2/messages/schema.py +11 -12
- pysdmx/io/json/sdmxjson2/messages/structure.py +150 -2
- pysdmx/io/json/sdmxjson2/messages/vtl.py +547 -17
- pysdmx/io/json/sdmxjson2/reader/metadata.py +32 -0
- pysdmx/io/json/sdmxjson2/reader/structure.py +32 -0
- pysdmx/io/json/sdmxjson2/writer/__init__.py +9 -0
- pysdmx/io/json/sdmxjson2/writer/metadata.py +60 -0
- pysdmx/io/json/sdmxjson2/writer/structure.py +61 -0
- pysdmx/io/reader.py +28 -9
- pysdmx/io/serde.py +17 -0
- pysdmx/io/writer.py +45 -9
- pysdmx/io/xml/__ss_aux_reader.py +1 -2
- pysdmx/io/xml/__structure_aux_reader.py +15 -10
- pysdmx/io/xml/__structure_aux_writer.py +15 -13
- pysdmx/io/xml/__write_data_aux.py +6 -57
- pysdmx/io/xml/__write_structure_specific_aux.py +7 -3
- pysdmx/io/xml/doc_validation.py +1 -3
- pysdmx/io/xml/sdmx21/writer/generic.py +6 -4
- pysdmx/model/__init__.py +1 -3
- pysdmx/model/code.py +11 -1
- pysdmx/model/dataflow.py +23 -0
- pysdmx/model/map.py +19 -13
- pysdmx/model/message.py +10 -5
- pysdmx/toolkit/pd/_data_utils.py +99 -0
- pysdmx/toolkit/vtl/_validations.py +2 -3
- {pysdmx-1.5.2.dist-info → pysdmx-1.7.0.dist-info}/METADATA +4 -3
- {pysdmx-1.5.2.dist-info → pysdmx-1.7.0.dist-info}/RECORD +63 -51
- {pysdmx-1.5.2.dist-info → pysdmx-1.7.0.dist-info}/WHEEL +1 -1
- {pysdmx-1.5.2.dist-info → pysdmx-1.7.0.dist-info/licenses}/LICENSE +0 -0
|
@@ -3,14 +3,25 @@
|
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
from typing import Dict, Sequence, Set
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
import msgspec
|
|
7
7
|
|
|
8
|
-
from pysdmx.io.json.sdmxjson2.messages.core import
|
|
8
|
+
from pysdmx.io.json.sdmxjson2.messages.core import (
|
|
9
|
+
ItemSchemeType,
|
|
10
|
+
JsonAnnotation,
|
|
11
|
+
)
|
|
9
12
|
from pysdmx.io.json.sdmxjson2.messages.dataflow import JsonDataflow
|
|
10
13
|
from pysdmx.model import Agency, AgencyScheme, DataflowRef
|
|
11
14
|
|
|
12
15
|
|
|
13
|
-
|
|
16
|
+
def _sanitize_agency(agency: Agency, is_sdmx_scheme: bool) -> Agency:
|
|
17
|
+
if is_sdmx_scheme:
|
|
18
|
+
nid = agency.id
|
|
19
|
+
else:
|
|
20
|
+
nid = agency.id[agency.id.rindex(".") + 1 :]
|
|
21
|
+
return msgspec.structs.replace(agency, id=nid, dataflows=())
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class JsonAgencyScheme(ItemSchemeType, frozen=True, omit_defaults=True):
|
|
14
25
|
"""SDMX-JSON payload for an agency scheme."""
|
|
15
26
|
|
|
16
27
|
agencies: Sequence[Agency] = ()
|
|
@@ -26,7 +37,7 @@ class JsonAgencyScheme(ItemSchemeType, frozen=True):
|
|
|
26
37
|
description=a.description,
|
|
27
38
|
contacts=a.contacts,
|
|
28
39
|
dataflows=flows,
|
|
29
|
-
annotations=[a.to_model() for a in self.annotations],
|
|
40
|
+
annotations=tuple([a.to_model() for a in self.annotations]),
|
|
30
41
|
)
|
|
31
42
|
|
|
32
43
|
def to_model(self, dataflows: Sequence[JsonDataflow]) -> AgencyScheme:
|
|
@@ -43,15 +54,40 @@ class JsonAgencyScheme(ItemSchemeType, frozen=True):
|
|
|
43
54
|
description=self.description,
|
|
44
55
|
agency=self.agency,
|
|
45
56
|
items=agencies,
|
|
46
|
-
annotations=[a.to_model() for a in self.annotations],
|
|
57
|
+
annotations=tuple([a.to_model() for a in self.annotations]),
|
|
47
58
|
is_external_reference=self.isExternalReference,
|
|
48
59
|
is_partial=self.isPartial,
|
|
49
60
|
valid_from=self.validFrom,
|
|
50
61
|
valid_to=self.validTo,
|
|
51
62
|
)
|
|
52
63
|
|
|
64
|
+
@classmethod
|
|
65
|
+
def from_model(self, asc: AgencyScheme) -> "JsonAgencyScheme":
|
|
66
|
+
"""Converts a pysdmx agency scheme to an SDMX-JSON one."""
|
|
67
|
+
agency = (
|
|
68
|
+
asc.agency.id if isinstance(asc.agency, Agency) else asc.agency
|
|
69
|
+
)
|
|
70
|
+
is_sdmx_scheme = agency == "SDMX"
|
|
71
|
+
children = [_sanitize_agency(a, is_sdmx_scheme) for a in asc.items]
|
|
72
|
+
|
|
73
|
+
return JsonAgencyScheme(
|
|
74
|
+
id="AGENCIES",
|
|
75
|
+
name="AGENCIES",
|
|
76
|
+
agency=agency,
|
|
77
|
+
description=asc.description,
|
|
78
|
+
version="1.0",
|
|
79
|
+
agencies=children,
|
|
80
|
+
annotations=tuple(
|
|
81
|
+
[JsonAnnotation.from_model(a) for a in asc.annotations]
|
|
82
|
+
),
|
|
83
|
+
isExternalReference=asc.is_external_reference,
|
|
84
|
+
isPartial=asc.is_partial,
|
|
85
|
+
validFrom=asc.valid_from,
|
|
86
|
+
validTo=asc.valid_to,
|
|
87
|
+
)
|
|
88
|
+
|
|
53
89
|
|
|
54
|
-
class JsonAgencySchemes(Struct, frozen=True):
|
|
90
|
+
class JsonAgencySchemes(msgspec.Struct, frozen=True, omit_defaults=True):
|
|
55
91
|
"""SDMX-JSON payload for the list of agency schemes."""
|
|
56
92
|
|
|
57
93
|
agencySchemes: Sequence[JsonAgencyScheme]
|
|
@@ -62,7 +98,7 @@ class JsonAgencySchemes(Struct, frozen=True):
|
|
|
62
98
|
return [a.to_model(self.dataflows) for a in self.agencySchemes]
|
|
63
99
|
|
|
64
100
|
|
|
65
|
-
class JsonAgencyMessage(Struct, frozen=True):
|
|
101
|
+
class JsonAgencyMessage(msgspec.Struct, frozen=True, omit_defaults=True):
|
|
66
102
|
"""SDMX-JSON payload for /agencyscheme queries."""
|
|
67
103
|
|
|
68
104
|
data: JsonAgencySchemes
|
|
@@ -5,8 +5,10 @@ from typing import Dict, Sequence
|
|
|
5
5
|
|
|
6
6
|
from msgspec import Struct
|
|
7
7
|
|
|
8
|
+
from pysdmx import errors
|
|
8
9
|
from pysdmx.io.json.sdmxjson2.messages.core import (
|
|
9
10
|
ItemSchemeType,
|
|
11
|
+
JsonAnnotation,
|
|
10
12
|
MaintainableType,
|
|
11
13
|
NameableType,
|
|
12
14
|
)
|
|
@@ -23,7 +25,10 @@ from pysdmx.util import find_by_urn
|
|
|
23
25
|
|
|
24
26
|
|
|
25
27
|
class JsonCategorisation(
|
|
26
|
-
MaintainableType,
|
|
28
|
+
MaintainableType,
|
|
29
|
+
frozen=True,
|
|
30
|
+
rename={"agency": "agencyID"},
|
|
31
|
+
omit_defaults=True,
|
|
27
32
|
):
|
|
28
33
|
"""SDMX-JSON payload for a categorisation."""
|
|
29
34
|
|
|
@@ -46,8 +51,35 @@ class JsonCategorisation(
|
|
|
46
51
|
annotations=[a.to_model() for a in self.annotations],
|
|
47
52
|
)
|
|
48
53
|
|
|
54
|
+
@classmethod
|
|
55
|
+
def from_model(self, cat: Categorisation) -> "JsonCategorisation":
|
|
56
|
+
"""Converts a pysdmx categorisation to an SDMX-JSON one."""
|
|
57
|
+
if not cat.name:
|
|
58
|
+
raise errors.Invalid(
|
|
59
|
+
"Invalid input",
|
|
60
|
+
"SDMX-JSON categorisations must have a name",
|
|
61
|
+
{"categorisation": cat.id},
|
|
62
|
+
)
|
|
63
|
+
return JsonCategorisation(
|
|
64
|
+
agency=(
|
|
65
|
+
cat.agency.id if isinstance(cat.agency, Agency) else cat.agency
|
|
66
|
+
),
|
|
67
|
+
id=cat.id,
|
|
68
|
+
name=cat.name,
|
|
69
|
+
version=cat.version,
|
|
70
|
+
isExternalReference=cat.is_external_reference,
|
|
71
|
+
validFrom=cat.valid_from,
|
|
72
|
+
validTo=cat.valid_to,
|
|
73
|
+
description=cat.description,
|
|
74
|
+
annotations=tuple(
|
|
75
|
+
[JsonAnnotation.from_model(a) for a in cat.annotations]
|
|
76
|
+
),
|
|
77
|
+
source=cat.source,
|
|
78
|
+
target=cat.target,
|
|
79
|
+
)
|
|
80
|
+
|
|
49
81
|
|
|
50
|
-
class JsonCategory(NameableType, frozen=True):
|
|
82
|
+
class JsonCategory(NameableType, frozen=True, omit_defaults=True):
|
|
51
83
|
"""SDMX-JSON payload for a category."""
|
|
52
84
|
|
|
53
85
|
categories: Sequence["JsonCategory"] = ()
|
|
@@ -62,9 +94,33 @@ class JsonCategory(NameableType, frozen=True):
|
|
|
62
94
|
annotations=[a.to_model() for a in self.annotations],
|
|
63
95
|
)
|
|
64
96
|
|
|
97
|
+
@classmethod
|
|
98
|
+
def from_model(self, cat: Category) -> "JsonCategory":
|
|
99
|
+
"""Converts a pysdmx category to an SDMX-JSON one."""
|
|
100
|
+
if not cat.name:
|
|
101
|
+
raise errors.Invalid(
|
|
102
|
+
"Invalid input",
|
|
103
|
+
"SDMX-JSON category must have a name",
|
|
104
|
+
{"category": cat.id},
|
|
105
|
+
)
|
|
106
|
+
return JsonCategory(
|
|
107
|
+
id=cat.id,
|
|
108
|
+
name=cat.name,
|
|
109
|
+
description=cat.description,
|
|
110
|
+
annotations=tuple(
|
|
111
|
+
[JsonAnnotation.from_model(a) for a in cat.annotations]
|
|
112
|
+
),
|
|
113
|
+
categories=tuple(
|
|
114
|
+
[JsonCategory.from_model(c) for c in cat.categories]
|
|
115
|
+
),
|
|
116
|
+
)
|
|
117
|
+
|
|
65
118
|
|
|
66
119
|
class JsonCategoryScheme(
|
|
67
|
-
ItemSchemeType,
|
|
120
|
+
ItemSchemeType,
|
|
121
|
+
frozen=True,
|
|
122
|
+
rename={"agency": "agencyID"},
|
|
123
|
+
omit_defaults=True,
|
|
68
124
|
):
|
|
69
125
|
"""SDMX-JSON payload for a category scheme."""
|
|
70
126
|
|
|
@@ -86,8 +142,37 @@ class JsonCategoryScheme(
|
|
|
86
142
|
annotations=[a.to_model() for a in self.annotations],
|
|
87
143
|
)
|
|
88
144
|
|
|
145
|
+
@classmethod
|
|
146
|
+
def from_model(self, cs: CategoryScheme) -> "JsonCategoryScheme":
|
|
147
|
+
"""Converts a pysdmx category scheme to an SDMX-JSON one."""
|
|
148
|
+
if not cs.name:
|
|
149
|
+
raise errors.Invalid(
|
|
150
|
+
"Invalid input",
|
|
151
|
+
"SDMX-JSON category schemes must have a name",
|
|
152
|
+
{"category_scheme": cs.id},
|
|
153
|
+
)
|
|
154
|
+
return JsonCategoryScheme(
|
|
155
|
+
agency=(
|
|
156
|
+
cs.agency.id if isinstance(cs.agency, Agency) else cs.agency
|
|
157
|
+
),
|
|
158
|
+
id=cs.id,
|
|
159
|
+
name=cs.name,
|
|
160
|
+
version=cs.version,
|
|
161
|
+
isExternalReference=cs.is_external_reference,
|
|
162
|
+
validFrom=cs.valid_from,
|
|
163
|
+
validTo=cs.valid_to,
|
|
164
|
+
description=cs.description,
|
|
165
|
+
annotations=tuple(
|
|
166
|
+
[JsonAnnotation.from_model(a) for a in cs.annotations]
|
|
167
|
+
),
|
|
168
|
+
isPartial=cs.is_partial,
|
|
169
|
+
categories=tuple(
|
|
170
|
+
[JsonCategory.from_model(c) for c in cs.categories]
|
|
171
|
+
),
|
|
172
|
+
)
|
|
173
|
+
|
|
89
174
|
|
|
90
|
-
class JsonCategorySchemes(Struct, frozen=True):
|
|
175
|
+
class JsonCategorySchemes(Struct, frozen=True, omit_defaults=True):
|
|
91
176
|
"""SDMX-JSON payload for the list of category schemes."""
|
|
92
177
|
|
|
93
178
|
categorySchemes: Sequence[JsonCategoryScheme]
|
|
@@ -95,7 +180,7 @@ class JsonCategorySchemes(Struct, frozen=True):
|
|
|
95
180
|
dataflows: Sequence[JsonDataflow] = ()
|
|
96
181
|
|
|
97
182
|
|
|
98
|
-
class JsonCategorySchemeMessage(Struct, frozen=True):
|
|
183
|
+
class JsonCategorySchemeMessage(Struct, frozen=True, omit_defaults=True):
|
|
99
184
|
"""SDMX-JSON payload for /categoryscheme queries."""
|
|
100
185
|
|
|
101
186
|
data: JsonCategorySchemes
|
|
@@ -139,13 +224,13 @@ class JsonCategorySchemeMessage(Struct, frozen=True):
|
|
|
139
224
|
return cs
|
|
140
225
|
|
|
141
226
|
|
|
142
|
-
class JsonCategorisations(Struct, frozen=True):
|
|
227
|
+
class JsonCategorisations(Struct, frozen=True, omit_defaults=True):
|
|
143
228
|
"""SDMX-JSON payload for the list of categorisations."""
|
|
144
229
|
|
|
145
230
|
categorisations: Sequence[JsonCategorisation]
|
|
146
231
|
|
|
147
232
|
|
|
148
|
-
class JsonCategorisationMessage(Struct, frozen=True):
|
|
233
|
+
class JsonCategorisationMessage(Struct, frozen=True, omit_defaults=True):
|
|
149
234
|
"""SDMX-JSON payload for /categorisation queries."""
|
|
150
235
|
|
|
151
236
|
data: JsonCategorisations
|
|
@@ -6,6 +6,7 @@ from typing import Optional, Sequence, Tuple
|
|
|
6
6
|
|
|
7
7
|
from msgspec import Struct
|
|
8
8
|
|
|
9
|
+
from pysdmx import errors
|
|
9
10
|
from pysdmx.io.json.sdmxjson2.messages.core import (
|
|
10
11
|
ItemSchemeType,
|
|
11
12
|
JsonAnnotation,
|
|
@@ -14,6 +15,8 @@ from pysdmx.io.json.sdmxjson2.messages.core import (
|
|
|
14
15
|
NameableType,
|
|
15
16
|
)
|
|
16
17
|
from pysdmx.model import (
|
|
18
|
+
Agency,
|
|
19
|
+
Annotation,
|
|
17
20
|
Code,
|
|
18
21
|
Codelist,
|
|
19
22
|
HierarchicalCode,
|
|
@@ -22,14 +25,16 @@ from pysdmx.model import (
|
|
|
22
25
|
)
|
|
23
26
|
from pysdmx.util import find_by_urn, parse_item_urn
|
|
24
27
|
|
|
28
|
+
_VAL_FMT = "%Y-%m-%dT%H:%M:%S%z"
|
|
25
29
|
|
|
26
|
-
|
|
30
|
+
|
|
31
|
+
class JsonCode(NameableType, frozen=True, omit_defaults=True):
|
|
27
32
|
"""SDMX-JSON payload for codes."""
|
|
28
33
|
|
|
29
34
|
parent: Optional[str] = None
|
|
30
35
|
|
|
31
36
|
def __handle_date(self, datestr: str) -> datetime:
|
|
32
|
-
return datetime.strptime(datestr,
|
|
37
|
+
return datetime.strptime(datestr, _VAL_FMT) # noqa
|
|
33
38
|
|
|
34
39
|
def __get_val(
|
|
35
40
|
self, a: JsonAnnotation
|
|
@@ -44,46 +49,135 @@ class JsonCode(NameableType, frozen=True):
|
|
|
44
49
|
|
|
45
50
|
def to_model(self) -> Code:
|
|
46
51
|
"""Converts a JsonCode to a standard code."""
|
|
52
|
+
# Pre-filter annotations once outside the tuple creation
|
|
53
|
+
vf, vt = None, None
|
|
54
|
+
|
|
47
55
|
if self.annotations:
|
|
48
|
-
|
|
49
|
-
|
|
56
|
+
# Get validity period info
|
|
57
|
+
vp = next(
|
|
58
|
+
(
|
|
59
|
+
a
|
|
60
|
+
for a in self.annotations
|
|
61
|
+
if a.type == "FR_VALIDITY_PERIOD"
|
|
62
|
+
),
|
|
63
|
+
None,
|
|
64
|
+
)
|
|
65
|
+
if vp:
|
|
66
|
+
vf, vt = self.__get_val(vp)
|
|
67
|
+
|
|
68
|
+
# Pre-filter non-validity period annotations
|
|
69
|
+
filtered_annotations = [
|
|
70
|
+
a.to_model()
|
|
71
|
+
for a in self.annotations
|
|
72
|
+
if a.type != "FR_VALIDITY_PERIOD"
|
|
50
73
|
]
|
|
51
74
|
else:
|
|
52
|
-
|
|
53
|
-
|
|
75
|
+
filtered_annotations = []
|
|
76
|
+
|
|
54
77
|
return Code(
|
|
55
78
|
id=self.id,
|
|
56
79
|
name=self.name,
|
|
57
80
|
description=self.description,
|
|
58
81
|
valid_from=vf,
|
|
59
82
|
valid_to=vt,
|
|
60
|
-
annotations=
|
|
83
|
+
annotations=tuple(filtered_annotations),
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_model(self, code: Code) -> "JsonCode":
|
|
88
|
+
"""Converts a pysdmx code to an SDMX-JSON one."""
|
|
89
|
+
if not code.name:
|
|
90
|
+
raise errors.Invalid(
|
|
91
|
+
"Invalid input",
|
|
92
|
+
"SDMX-JSON codes must have a name",
|
|
93
|
+
{"code": code.id},
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
annotations = [JsonAnnotation.from_model(a) for a in code.annotations]
|
|
97
|
+
if code.valid_from and code.valid_to:
|
|
98
|
+
vp = (
|
|
99
|
+
f"{datetime.strftime(code.valid_from, _VAL_FMT)}/"
|
|
100
|
+
f"{datetime.strftime(code.valid_to, _VAL_FMT)}"
|
|
101
|
+
)
|
|
102
|
+
elif code.valid_from:
|
|
103
|
+
vp = f"{datetime.strftime(code.valid_from, _VAL_FMT)}/"
|
|
104
|
+
elif code.valid_to:
|
|
105
|
+
vp = f"/{datetime.strftime(code.valid_to, _VAL_FMT)}"
|
|
106
|
+
else:
|
|
107
|
+
vp = ""
|
|
108
|
+
if vp:
|
|
109
|
+
annotations.append(
|
|
110
|
+
JsonAnnotation(title=vp, type="FR_VALIDITY_PERIOD")
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
return JsonCode(
|
|
114
|
+
id=code.id,
|
|
115
|
+
name=code.name,
|
|
116
|
+
description=code.description,
|
|
117
|
+
annotations=tuple(annotations),
|
|
61
118
|
)
|
|
62
119
|
|
|
63
120
|
|
|
64
|
-
class JsonCodelist(ItemSchemeType, frozen=True):
|
|
121
|
+
class JsonCodelist(ItemSchemeType, frozen=True, omit_defaults=True):
|
|
65
122
|
"""SDMX-JSON payload for a codelist."""
|
|
66
123
|
|
|
67
124
|
codes: Sequence[JsonCode] = ()
|
|
68
125
|
|
|
69
126
|
def to_model(self) -> Codelist:
|
|
70
127
|
"""Converts a JsonCodelist to a standard codelist."""
|
|
128
|
+
# Process codes in batches to reduce memory pressure
|
|
129
|
+
batch_size = 10000
|
|
130
|
+
all_codes = []
|
|
131
|
+
|
|
132
|
+
# Process in batches
|
|
133
|
+
for i in range(0, len(self.codes), batch_size):
|
|
134
|
+
batch = self.codes[i : i + batch_size]
|
|
135
|
+
batch_models = [code.to_model() for code in batch]
|
|
136
|
+
all_codes.extend(batch_models)
|
|
137
|
+
|
|
71
138
|
return Codelist(
|
|
72
139
|
id=self.id,
|
|
73
140
|
name=self.name,
|
|
74
141
|
agency=self.agency,
|
|
75
142
|
description=self.description,
|
|
76
143
|
version=self.version,
|
|
77
|
-
items=
|
|
78
|
-
annotations=[a.to_model() for a in self.annotations],
|
|
144
|
+
items=tuple(all_codes),
|
|
145
|
+
annotations=tuple([a.to_model() for a in self.annotations]),
|
|
79
146
|
is_external_reference=self.isExternalReference,
|
|
80
147
|
is_partial=self.isPartial,
|
|
81
148
|
valid_from=self.validFrom,
|
|
82
149
|
valid_to=self.validTo,
|
|
83
150
|
)
|
|
84
151
|
|
|
152
|
+
@classmethod
|
|
153
|
+
def from_model(self, cl: Codelist) -> "JsonCodelist":
|
|
154
|
+
"""Converts a pysdmx codelist to an SDMX-JSON one."""
|
|
155
|
+
if not cl.name:
|
|
156
|
+
raise errors.Invalid(
|
|
157
|
+
"Invalid input",
|
|
158
|
+
"SDMX-JSON codelists must have a name",
|
|
159
|
+
{"codelist": cl.id},
|
|
160
|
+
)
|
|
161
|
+
return JsonCodelist(
|
|
162
|
+
id=cl.id,
|
|
163
|
+
name=cl.name,
|
|
164
|
+
agency=(
|
|
165
|
+
cl.agency.id if isinstance(cl.agency, Agency) else cl.agency
|
|
166
|
+
),
|
|
167
|
+
description=cl.description,
|
|
168
|
+
version=cl.version,
|
|
169
|
+
codes=tuple([JsonCode.from_model(i) for i in cl.items]),
|
|
170
|
+
annotations=tuple(
|
|
171
|
+
[JsonAnnotation.from_model(a) for a in cl.annotations]
|
|
172
|
+
),
|
|
173
|
+
isExternalReference=cl.is_external_reference,
|
|
174
|
+
isPartial=cl.is_partial,
|
|
175
|
+
validFrom=cl.valid_from,
|
|
176
|
+
validTo=cl.valid_to,
|
|
177
|
+
)
|
|
85
178
|
|
|
86
|
-
|
|
179
|
+
|
|
180
|
+
class JsonValuelist(ItemSchemeType, frozen=True, omit_defaults=True):
|
|
87
181
|
"""SDMX-JSON payload for a valuelist."""
|
|
88
182
|
|
|
89
183
|
valueItems: Sequence[JsonCode] = ()
|
|
@@ -105,15 +199,42 @@ class JsonValuelist(ItemSchemeType, frozen=True):
|
|
|
105
199
|
sdmx_type="valuelist",
|
|
106
200
|
)
|
|
107
201
|
|
|
202
|
+
@classmethod
|
|
203
|
+
def from_model(self, cl: Codelist) -> "JsonValuelist":
|
|
204
|
+
"""Converts a pysdmx codelist to an SDMX-JSON valuelist."""
|
|
205
|
+
if not cl.name:
|
|
206
|
+
raise errors.Invalid(
|
|
207
|
+
"Invalid input",
|
|
208
|
+
"SDMX-JSON valuelists must have a name",
|
|
209
|
+
{"valuelist": cl.id},
|
|
210
|
+
)
|
|
211
|
+
return JsonValuelist(
|
|
212
|
+
id=cl.id,
|
|
213
|
+
name=cl.name,
|
|
214
|
+
agency=(
|
|
215
|
+
cl.agency.id if isinstance(cl.agency, Agency) else cl.agency
|
|
216
|
+
),
|
|
217
|
+
description=cl.description,
|
|
218
|
+
version=cl.version,
|
|
219
|
+
valueItems=tuple([JsonCode.from_model(i) for i in cl.items]),
|
|
220
|
+
annotations=tuple(
|
|
221
|
+
[JsonAnnotation.from_model(a) for a in cl.annotations]
|
|
222
|
+
),
|
|
223
|
+
isExternalReference=cl.is_external_reference,
|
|
224
|
+
isPartial=cl.is_partial,
|
|
225
|
+
validFrom=cl.valid_from,
|
|
226
|
+
validTo=cl.valid_to,
|
|
227
|
+
)
|
|
228
|
+
|
|
108
229
|
|
|
109
|
-
class JsonCodelists(Struct, frozen=True):
|
|
230
|
+
class JsonCodelists(Struct, frozen=True, omit_defaults=True):
|
|
110
231
|
"""SDMX-JSON payload for lists of codes."""
|
|
111
232
|
|
|
112
233
|
codelists: Sequence[JsonCodelist] = ()
|
|
113
234
|
valuelists: Sequence[JsonValuelist] = ()
|
|
114
235
|
|
|
115
236
|
|
|
116
|
-
class JsonCodelistMessage(Struct, frozen=True):
|
|
237
|
+
class JsonCodelistMessage(Struct, frozen=True, omit_defaults=True):
|
|
117
238
|
"""SDMX-JSON payload for /codelist queries."""
|
|
118
239
|
|
|
119
240
|
data: JsonCodelists
|
|
@@ -126,7 +247,7 @@ class JsonCodelistMessage(Struct, frozen=True):
|
|
|
126
247
|
return self.data.valuelists[0].to_model()
|
|
127
248
|
|
|
128
249
|
|
|
129
|
-
class JsonHierarchicalCode(Struct, frozen=True):
|
|
250
|
+
class JsonHierarchicalCode(Struct, frozen=True, omit_defaults=True):
|
|
130
251
|
"""Fusion-JSON payload for hierarchical codes."""
|
|
131
252
|
|
|
132
253
|
id: str
|
|
@@ -163,6 +284,11 @@ class JsonHierarchicalCode(Struct, frozen=True):
|
|
|
163
284
|
codes = [c.to_model(codelists) for c in self.hierarchicalCodes]
|
|
164
285
|
vf = self.validFrom.replace(tzinfo=tz.utc) if self.validFrom else None
|
|
165
286
|
vt = self.validTo.replace(tzinfo=tz.utc) if self.validTo else None
|
|
287
|
+
if self.id != code.id:
|
|
288
|
+
a = Annotation(id="hcode", type="pysdmx", text=self.id)
|
|
289
|
+
annotations = [a]
|
|
290
|
+
else:
|
|
291
|
+
annotations = []
|
|
166
292
|
return HierarchicalCode(
|
|
167
293
|
code.id,
|
|
168
294
|
name,
|
|
@@ -172,10 +298,45 @@ class JsonHierarchicalCode(Struct, frozen=True):
|
|
|
172
298
|
vf,
|
|
173
299
|
vt,
|
|
174
300
|
codes,
|
|
301
|
+
tuple(annotations),
|
|
302
|
+
self.code,
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
@classmethod
|
|
306
|
+
def from_model(self, code: HierarchicalCode) -> "JsonHierarchicalCode":
|
|
307
|
+
"""Converts a pysdmx hierarchical code to an SDMX-JSON one."""
|
|
308
|
+
if not code.urn:
|
|
309
|
+
raise errors.Invalid(
|
|
310
|
+
"Invalid input",
|
|
311
|
+
"SDMX-JSON hierarchical codes must have the code urn.",
|
|
312
|
+
{"code": code.id},
|
|
313
|
+
)
|
|
314
|
+
|
|
315
|
+
annotations = [
|
|
316
|
+
JsonAnnotation.from_model(a)
|
|
317
|
+
for a in code.annotations
|
|
318
|
+
if a.type != "pysdmx"
|
|
319
|
+
]
|
|
320
|
+
id_ano = [
|
|
321
|
+
a
|
|
322
|
+
for a in code.annotations
|
|
323
|
+
if a.type == "pysdmx" and a.id == "hcode"
|
|
324
|
+
]
|
|
325
|
+
hid = id_ano[0].value if len(id_ano) > 0 else code.id
|
|
326
|
+
|
|
327
|
+
return JsonHierarchicalCode(
|
|
328
|
+
id=hid, # type: ignore[arg-type]
|
|
329
|
+
code=code.urn,
|
|
330
|
+
validFrom=code.rel_valid_from,
|
|
331
|
+
validTo=code.rel_valid_to,
|
|
332
|
+
annotations=tuple(annotations),
|
|
333
|
+
hierarchicalCodes=[
|
|
334
|
+
JsonHierarchicalCode.from_model(c) for c in code.codes
|
|
335
|
+
],
|
|
175
336
|
)
|
|
176
337
|
|
|
177
338
|
|
|
178
|
-
class JsonHierarchy(ItemSchemeType, frozen=True):
|
|
339
|
+
class JsonHierarchy(ItemSchemeType, frozen=True, omit_defaults=True):
|
|
179
340
|
"""SDMX-JSON payload for a hierarchy."""
|
|
180
341
|
|
|
181
342
|
hierarchicalCodes: Sequence[JsonHierarchicalCode] = ()
|
|
@@ -189,7 +350,7 @@ class JsonHierarchy(ItemSchemeType, frozen=True):
|
|
|
189
350
|
agency=self.agency,
|
|
190
351
|
description=self.description,
|
|
191
352
|
version=self.version,
|
|
192
|
-
annotations=[a.to_model() for a in self.annotations],
|
|
353
|
+
annotations=tuple([a.to_model() for a in self.annotations]),
|
|
193
354
|
is_external_reference=self.isExternalReference,
|
|
194
355
|
is_partial=self.isPartial,
|
|
195
356
|
valid_from=self.validFrom,
|
|
@@ -197,8 +358,35 @@ class JsonHierarchy(ItemSchemeType, frozen=True):
|
|
|
197
358
|
codes=[i.to_model(cls) for i in self.hierarchicalCodes],
|
|
198
359
|
)
|
|
199
360
|
|
|
361
|
+
@classmethod
|
|
362
|
+
def from_model(self, h: Hierarchy) -> "JsonHierarchy":
|
|
363
|
+
"""Converts a pysdmx hierarchy to an SDMX-JSON one."""
|
|
364
|
+
if not h.name:
|
|
365
|
+
raise errors.Invalid(
|
|
366
|
+
"Invalid input",
|
|
367
|
+
"SDMX-JSON hierarchy must have a name",
|
|
368
|
+
{"hierarchy": h.id},
|
|
369
|
+
)
|
|
370
|
+
return JsonHierarchy(
|
|
371
|
+
id=h.id,
|
|
372
|
+
name=h.name,
|
|
373
|
+
agency=(h.agency.id if isinstance(h.agency, Agency) else h.agency),
|
|
374
|
+
description=h.description,
|
|
375
|
+
version=h.version,
|
|
376
|
+
hierarchicalCodes=tuple(
|
|
377
|
+
[JsonHierarchicalCode.from_model(i) for i in h.codes]
|
|
378
|
+
),
|
|
379
|
+
annotations=tuple(
|
|
380
|
+
[JsonAnnotation.from_model(a) for a in h.annotations]
|
|
381
|
+
),
|
|
382
|
+
isExternalReference=h.is_external_reference,
|
|
383
|
+
isPartial=h.is_partial,
|
|
384
|
+
validFrom=h.valid_from,
|
|
385
|
+
validTo=h.valid_to,
|
|
386
|
+
)
|
|
387
|
+
|
|
200
388
|
|
|
201
|
-
class JsonHierarchies(Struct, frozen=True):
|
|
389
|
+
class JsonHierarchies(Struct, frozen=True, omit_defaults=True):
|
|
202
390
|
"""SDMX-JSON payload for hierarchies."""
|
|
203
391
|
|
|
204
392
|
codelists: Sequence[JsonCodelist] = ()
|
|
@@ -209,7 +397,9 @@ class JsonHierarchies(Struct, frozen=True):
|
|
|
209
397
|
return [h.to_model(self.codelists) for h in self.hierarchies]
|
|
210
398
|
|
|
211
399
|
|
|
212
|
-
class JsonHierarchyAssociation(
|
|
400
|
+
class JsonHierarchyAssociation(
|
|
401
|
+
MaintainableType, frozen=True, omit_defaults=True
|
|
402
|
+
):
|
|
213
403
|
"""SDMX-JSON payload for a hierarchy association."""
|
|
214
404
|
|
|
215
405
|
linkedHierarchy: str = ""
|
|
@@ -251,8 +441,61 @@ class JsonHierarchyAssociation(MaintainableType, frozen=True):
|
|
|
251
441
|
operator=lnk[0].urn if lnk else None,
|
|
252
442
|
)
|
|
253
443
|
|
|
444
|
+
@classmethod
|
|
445
|
+
def from_model(
|
|
446
|
+
self, ha: HierarchyAssociation
|
|
447
|
+
) -> "JsonHierarchyAssociation":
|
|
448
|
+
"""Converts a pysdmx hierarchy association to an SDMX-JSON one."""
|
|
449
|
+
if not ha.name:
|
|
450
|
+
raise errors.Invalid(
|
|
451
|
+
"Invalid input",
|
|
452
|
+
"SDMX-JSON hierarchy associations must have a name",
|
|
453
|
+
{"hierarchy_association": ha.id},
|
|
454
|
+
)
|
|
455
|
+
if ha.hierarchy is None:
|
|
456
|
+
raise errors.Invalid(
|
|
457
|
+
"Invalid input",
|
|
458
|
+
"SDMX-JSON hierarchy associations must reference a hierarchy",
|
|
459
|
+
{"hierarchy_association": ha.id},
|
|
460
|
+
)
|
|
461
|
+
if not ha.component_ref:
|
|
462
|
+
raise errors.Invalid(
|
|
463
|
+
"Invalid input",
|
|
464
|
+
"SDMX-JSON hierarchy associations must reference a component",
|
|
465
|
+
{"hierarchy_association": ha.id},
|
|
466
|
+
)
|
|
467
|
+
if isinstance(ha.hierarchy, Hierarchy):
|
|
468
|
+
base = "urn:sdmx:org.sdmx.infomodel.codelist."
|
|
469
|
+
href = f"{base}{ha.hierarchy.short_urn}"
|
|
470
|
+
else:
|
|
471
|
+
href = ha.hierarchy
|
|
472
|
+
if not ha.context_ref:
|
|
473
|
+
raise errors.Invalid(
|
|
474
|
+
"Invalid input",
|
|
475
|
+
"SDMX-JSON hierarchy associations must reference a context",
|
|
476
|
+
{"hierarchy_association": ha.id},
|
|
477
|
+
)
|
|
478
|
+
return JsonHierarchyAssociation(
|
|
479
|
+
agency=(
|
|
480
|
+
ha.agency.id if isinstance(ha.agency, Agency) else ha.agency
|
|
481
|
+
),
|
|
482
|
+
id=ha.id,
|
|
483
|
+
name=ha.name,
|
|
484
|
+
version=ha.version,
|
|
485
|
+
isExternalReference=ha.is_external_reference,
|
|
486
|
+
validFrom=ha.valid_from,
|
|
487
|
+
validTo=ha.valid_to,
|
|
488
|
+
description=ha.description,
|
|
489
|
+
annotations=tuple(
|
|
490
|
+
[JsonAnnotation.from_model(a) for a in ha.annotations]
|
|
491
|
+
),
|
|
492
|
+
linkedHierarchy=href,
|
|
493
|
+
linkedObject=ha.component_ref,
|
|
494
|
+
contextObject=ha.context_ref,
|
|
495
|
+
)
|
|
496
|
+
|
|
254
497
|
|
|
255
|
-
class JsonHierarchyMessage(Struct, frozen=True):
|
|
498
|
+
class JsonHierarchyMessage(Struct, frozen=True, omit_defaults=True):
|
|
256
499
|
"""SDMX-JSON payload for /hierarchy queries."""
|
|
257
500
|
|
|
258
501
|
data: JsonHierarchies
|
|
@@ -262,7 +505,7 @@ class JsonHierarchyMessage(Struct, frozen=True):
|
|
|
262
505
|
return self.data.to_model()[0]
|
|
263
506
|
|
|
264
507
|
|
|
265
|
-
class JsonHierarchiesMessage(Struct, frozen=True):
|
|
508
|
+
class JsonHierarchiesMessage(Struct, frozen=True, omit_defaults=True):
|
|
266
509
|
"""SDMX-JSON payload for /hierarchy queries."""
|
|
267
510
|
|
|
268
511
|
data: JsonHierarchies
|
|
@@ -272,7 +515,7 @@ class JsonHierarchiesMessage(Struct, frozen=True):
|
|
|
272
515
|
return self.data.to_model()
|
|
273
516
|
|
|
274
517
|
|
|
275
|
-
class JsonHierarchyAssociations(Struct, frozen=True):
|
|
518
|
+
class JsonHierarchyAssociations(Struct, frozen=True, omit_defaults=True):
|
|
276
519
|
"""SDMX-JSON payload for hierarchy associations."""
|
|
277
520
|
|
|
278
521
|
codelists: Sequence[JsonCodelist] = ()
|
|
@@ -287,7 +530,7 @@ class JsonHierarchyAssociations(Struct, frozen=True):
|
|
|
287
530
|
]
|
|
288
531
|
|
|
289
532
|
|
|
290
|
-
class JsonHierarchyAssociationMessage(Struct, frozen=True):
|
|
533
|
+
class JsonHierarchyAssociationMessage(Struct, frozen=True, omit_defaults=True):
|
|
291
534
|
"""SDMX-JSON payload for hierarchy associations messages."""
|
|
292
535
|
|
|
293
536
|
data: JsonHierarchyAssociations
|