sdmxlib 0.8.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdmxlib/__init__.py +155 -0
- sdmxlib/api/__init__.py +7 -0
- sdmxlib/api/client.py +246 -0
- sdmxlib/api/filters.py +67 -0
- sdmxlib/api/providers.py +66 -0
- sdmxlib/api/query.py +428 -0
- sdmxlib/api/registry.py +666 -0
- sdmxlib/api/session.py +124 -0
- sdmxlib/formats/__init__.py +166 -0
- sdmxlib/formats/sdmx_csv/__init__.py +5 -0
- sdmxlib/formats/sdmx_csv/reader.py +147 -0
- sdmxlib/formats/sdmx_json/__init__.py +1 -0
- sdmxlib/formats/sdmx_json/reader.py +897 -0
- sdmxlib/formats/sdmx_json/writer.py +698 -0
- sdmxlib/formats/sdmx_ml21/__init__.py +5 -0
- sdmxlib/formats/sdmx_ml21/namespaces.py +12 -0
- sdmxlib/formats/sdmx_ml21/reader.py +1147 -0
- sdmxlib/formats/sdmx_ml21/writer.py +709 -0
- sdmxlib/formats/sdmx_ml30/__init__.py +1 -0
- sdmxlib/formats/sdmx_ml30/namespaces.py +12 -0
- sdmxlib/formats/sdmx_ml30/reader.py +1138 -0
- sdmxlib/formats/sdmx_ml30/writer.py +730 -0
- sdmxlib/local/__init__.py +6 -0
- sdmxlib/local/_registry.py +368 -0
- sdmxlib/model/__init__.py +122 -0
- sdmxlib/model/annotations.py +103 -0
- sdmxlib/model/base.py +61 -0
- sdmxlib/model/category.py +88 -0
- sdmxlib/model/codelist.py +91 -0
- sdmxlib/model/collections.py +382 -0
- sdmxlib/model/concept.py +121 -0
- sdmxlib/model/constraint.py +92 -0
- sdmxlib/model/convert.py +268 -0
- sdmxlib/model/dataflow.py +49 -0
- sdmxlib/model/dataset.py +337 -0
- sdmxlib/model/datastructure.py +390 -0
- sdmxlib/model/expr.py +120 -0
- sdmxlib/model/hierarchy.py +111 -0
- sdmxlib/model/istring.py +81 -0
- sdmxlib/model/mapping.py +134 -0
- sdmxlib/model/message.py +70 -0
- sdmxlib/model/organisation.py +149 -0
- sdmxlib/model/provision.py +45 -0
- sdmxlib/model/ref.py +195 -0
- sdmxlib/model/registry.py +191 -0
- sdmxlib/model/representation.py +99 -0
- sdmxlib/model/urn.py +130 -0
- sdmxlib/model/validation.py +338 -0
- sdmxlib/polars.py +392 -0
- sdmxlib/py.typed +0 -0
- sdmxlib/storage/__init__.py +28 -0
- sdmxlib/storage/lazy.py +329 -0
- sdmxlib/storage/readers.py +310 -0
- sdmxlib/storage/schema.py +289 -0
- sdmxlib/storage/writers.py +503 -0
- sdmxlib-0.8.0.dist-info/METADATA +102 -0
- sdmxlib-0.8.0.dist-info/RECORD +58 -0
- sdmxlib-0.8.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
"""Write SDMX domain objects to DuckDB normalized tables.
|
|
2
|
+
|
|
3
|
+
Flattens domain objects into the normalized schema defined by
|
|
4
|
+
``storage.schema``. Language-specific columns are populated based
|
|
5
|
+
on the configured language list stored in ``_meta``.
|
|
6
|
+
|
|
7
|
+
Example::
|
|
8
|
+
|
|
9
|
+
from sdmxlib.storage import schema, writers
|
|
10
|
+
|
|
11
|
+
conn = duckdb.connect("store.duckdb")
|
|
12
|
+
schema.create_tables(conn, languages=["en", "fr"])
|
|
13
|
+
writers.put_codelist(conn, codelist)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import duckdb
|
|
20
|
+
|
|
21
|
+
from sdmxlib.model.annotations import Annotations
|
|
22
|
+
from sdmxlib.model.codelist import Codelist
|
|
23
|
+
from sdmxlib.model.concept import ConceptScheme
|
|
24
|
+
from sdmxlib.model.convert import to_dict
|
|
25
|
+
from sdmxlib.model.dataflow import Dataflow
|
|
26
|
+
from sdmxlib.model.datastructure import DataStructure
|
|
27
|
+
from sdmxlib.model.hierarchy import Hierarchy
|
|
28
|
+
from sdmxlib.model.istring import InternationalString
|
|
29
|
+
from sdmxlib.model.ref import HasUrn, Ref
|
|
30
|
+
from sdmxlib.model.registry import _artefact_urn
|
|
31
|
+
from sdmxlib.storage.schema import get_languages
|
|
32
|
+
|
|
33
|
+
# Map Python class → type_name string for artefact_type column
|
|
34
|
+
_CLASS_TO_NAME: dict[type, str] = {
|
|
35
|
+
Codelist: "Codelist",
|
|
36
|
+
ConceptScheme: "ConceptScheme",
|
|
37
|
+
DataStructure: "DataStructure",
|
|
38
|
+
Dataflow: "Dataflow",
|
|
39
|
+
Hierarchy: "Hierarchy",
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
# Try to import optional types that may or may not be used
|
|
43
|
+
try:
|
|
44
|
+
from sdmxlib.model.category import Categorisation, CategoryScheme
|
|
45
|
+
from sdmxlib.model.constraint import ContentConstraint
|
|
46
|
+
from sdmxlib.model.mapping import StructureSet
|
|
47
|
+
from sdmxlib.model.organisation import AgencyScheme, DataProviderScheme
|
|
48
|
+
from sdmxlib.model.provision import ProvisionAgreement
|
|
49
|
+
|
|
50
|
+
_CLASS_TO_NAME.update(
|
|
51
|
+
{
|
|
52
|
+
ContentConstraint: "ContentConstraint",
|
|
53
|
+
CategoryScheme: "CategoryScheme",
|
|
54
|
+
Categorisation: "Categorisation",
|
|
55
|
+
ProvisionAgreement: "ProvisionAgreement",
|
|
56
|
+
AgencyScheme: "AgencyScheme",
|
|
57
|
+
DataProviderScheme: "DataProviderScheme",
|
|
58
|
+
StructureSet: "StructureSet",
|
|
59
|
+
}
|
|
60
|
+
)
|
|
61
|
+
except ImportError:
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# ── Helpers ─────────────────────────────────────────────────────────────
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _istring_lang_values(
|
|
69
|
+
s: InternationalString | None,
|
|
70
|
+
languages: list[str],
|
|
71
|
+
) -> list[str | None]:
|
|
72
|
+
"""Extract values for each configured language from an InternationalString."""
|
|
73
|
+
if not s:
|
|
74
|
+
return [None] * len(languages)
|
|
75
|
+
return [s.get(lang) if lang in s else None for lang in languages]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _annotations_json(annotations: Annotations | None) -> str | None:
|
|
79
|
+
if not annotations:
|
|
80
|
+
return None
|
|
81
|
+
result = []
|
|
82
|
+
for a in annotations:
|
|
83
|
+
d: dict[str, Any] = {}
|
|
84
|
+
if a.id is not None:
|
|
85
|
+
d["id"] = a.id
|
|
86
|
+
if a.title is not None:
|
|
87
|
+
d["title"] = a.title
|
|
88
|
+
if a.type is not None:
|
|
89
|
+
d["type"] = a.type
|
|
90
|
+
if a.url is not None:
|
|
91
|
+
d["url"] = a.url
|
|
92
|
+
if a.text:
|
|
93
|
+
d["text"] = {lang: a.text[lang] for lang in a.text}
|
|
94
|
+
result.append(d)
|
|
95
|
+
return json.dumps(result)
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _lang_cols(prefix: str, languages: list[str]) -> list[str]:
|
|
99
|
+
"""Build column names like ['name_en', 'name_fr'] for a prefix."""
|
|
100
|
+
return [f"{prefix}_{lang}" for lang in languages]
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _execute_insert(
|
|
104
|
+
conn: duckdb.DuckDBPyConnection,
|
|
105
|
+
table: str,
|
|
106
|
+
cols: list[str],
|
|
107
|
+
vals: list[Any],
|
|
108
|
+
) -> None:
|
|
109
|
+
"""Execute a parameterised INSERT.
|
|
110
|
+
|
|
111
|
+
Column names come from our own schema definitions (language config),
|
|
112
|
+
never from user input — SQL injection is not a concern here.
|
|
113
|
+
"""
|
|
114
|
+
placeholders = ", ".join(["?"] * len(vals))
|
|
115
|
+
col_str = ", ".join(cols)
|
|
116
|
+
conn.execute(
|
|
117
|
+
f"""
|
|
118
|
+
INSERT INTO {table} ({col_str})
|
|
119
|
+
VALUES ({placeholders})
|
|
120
|
+
""", # noqa: S608
|
|
121
|
+
vals,
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _execute_upsert(
|
|
126
|
+
conn: duckdb.DuckDBPyConnection,
|
|
127
|
+
table: str,
|
|
128
|
+
cols: list[str],
|
|
129
|
+
vals: list[Any],
|
|
130
|
+
) -> None:
|
|
131
|
+
"""INSERT OR REPLACE for tables with a single unique/PK constraint."""
|
|
132
|
+
placeholders = ", ".join(["?"] * len(vals))
|
|
133
|
+
col_str = ", ".join(cols)
|
|
134
|
+
conn.execute(
|
|
135
|
+
f"""
|
|
136
|
+
INSERT OR REPLACE INTO {table} ({col_str})
|
|
137
|
+
VALUES ({placeholders})
|
|
138
|
+
""", # noqa: S608
|
|
139
|
+
vals,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
# ── Artefact table helpers ──────────────────────────────────────────────
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _put_artefact_row(
|
|
147
|
+
conn: duckdb.DuckDBPyConnection,
|
|
148
|
+
urn: str,
|
|
149
|
+
artefact_type: str,
|
|
150
|
+
artefact: Any,
|
|
151
|
+
doc: str,
|
|
152
|
+
languages: list[str],
|
|
153
|
+
) -> None:
|
|
154
|
+
"""Insert or replace a row in the artefacts table."""
|
|
155
|
+
name_vals = _istring_lang_values(getattr(artefact, "name", None), languages)
|
|
156
|
+
desc_vals = _istring_lang_values(getattr(artefact, "description", None), languages)
|
|
157
|
+
is_final = getattr(artefact, "is_final", False)
|
|
158
|
+
|
|
159
|
+
cols = ["urn", "artefact_type", "agency", "id", "version", "is_final"]
|
|
160
|
+
vals: list[Any] = [urn, artefact_type, artefact.agency_id, artefact.id, artefact.version, is_final]
|
|
161
|
+
|
|
162
|
+
cols.extend(_lang_cols("name", languages))
|
|
163
|
+
vals.extend(name_vals)
|
|
164
|
+
|
|
165
|
+
cols.extend(_lang_cols("description", languages))
|
|
166
|
+
vals.extend(desc_vals)
|
|
167
|
+
|
|
168
|
+
cols.append("_document")
|
|
169
|
+
vals.append(doc)
|
|
170
|
+
|
|
171
|
+
# artefacts has PK + UNIQUE constraint; DELETE+INSERT to avoid
|
|
172
|
+
# DuckDB's "multiple unique constraints" binder error.
|
|
173
|
+
conn.execute(
|
|
174
|
+
"""
|
|
175
|
+
DELETE FROM artefacts WHERE urn = $urn
|
|
176
|
+
""",
|
|
177
|
+
{"urn": urn},
|
|
178
|
+
)
|
|
179
|
+
_execute_insert(conn, "artefacts", cols, vals)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
# ── Public writers ──────────────────────────────────────────────────────
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def put_codelist(conn: duckdb.DuckDBPyConnection, cl: Codelist) -> None:
|
|
186
|
+
"""Write a Codelist to artefacts + codelists + codes tables."""
|
|
187
|
+
languages = get_languages(conn)
|
|
188
|
+
urn = str(_artefact_urn(cl))
|
|
189
|
+
|
|
190
|
+
d = to_dict(cl)
|
|
191
|
+
d.pop("codes", None)
|
|
192
|
+
doc = json.dumps(d)
|
|
193
|
+
|
|
194
|
+
# artefacts row
|
|
195
|
+
_put_artefact_row(conn, urn, "Codelist", cl, doc, languages)
|
|
196
|
+
|
|
197
|
+
# codelists row
|
|
198
|
+
cl_cols = ["urn", "agency", "id", "version"]
|
|
199
|
+
cl_vals: list[Any] = [urn, cl.agency_id, cl.id, cl.version]
|
|
200
|
+
cl_cols.extend(_lang_cols("name", languages))
|
|
201
|
+
cl_vals.extend(_istring_lang_values(cl.name, languages))
|
|
202
|
+
cl_cols.append("is_partial")
|
|
203
|
+
cl_vals.append(False)
|
|
204
|
+
_execute_upsert(conn, "codelists", cl_cols, cl_vals)
|
|
205
|
+
|
|
206
|
+
# codes rows — delete old, insert new
|
|
207
|
+
conn.execute(
|
|
208
|
+
"""
|
|
209
|
+
DELETE FROM codes WHERE codelist_urn = $urn
|
|
210
|
+
""",
|
|
211
|
+
{"urn": urn},
|
|
212
|
+
)
|
|
213
|
+
for position, code in enumerate(cl.codes):
|
|
214
|
+
code_cols = ["codelist_urn", "code_id", "parent_code_id", "position"]
|
|
215
|
+
code_vals: list[Any] = [urn, code.id, code.parent_id, position]
|
|
216
|
+
code_cols.extend(_lang_cols("label", languages))
|
|
217
|
+
code_vals.extend(_istring_lang_values(code.name, languages))
|
|
218
|
+
code_cols.extend(_lang_cols("description", languages))
|
|
219
|
+
code_vals.extend(_istring_lang_values(code.description, languages))
|
|
220
|
+
code_cols.append("_annotations")
|
|
221
|
+
code_vals.append(_annotations_json(code.annotations))
|
|
222
|
+
_execute_insert(conn, "codes", code_cols, code_vals)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def put_data_structure(conn: duckdb.DuckDBPyConnection, dsd: DataStructure) -> None:
|
|
226
|
+
"""Write a DataStructure to artefacts + data_structures + dsd_components."""
|
|
227
|
+
languages = get_languages(conn)
|
|
228
|
+
urn = str(_artefact_urn(dsd))
|
|
229
|
+
doc = json.dumps(to_dict(dsd))
|
|
230
|
+
|
|
231
|
+
_put_artefact_row(conn, urn, "DataStructure", dsd, doc, languages)
|
|
232
|
+
|
|
233
|
+
# data_structures row
|
|
234
|
+
ds_cols = ["urn", "agency", "id", "version"]
|
|
235
|
+
ds_vals: list[Any] = [urn, dsd.agency_id, dsd.id, dsd.version]
|
|
236
|
+
ds_cols.extend(_lang_cols("name", languages))
|
|
237
|
+
ds_vals.extend(_istring_lang_values(dsd.name, languages))
|
|
238
|
+
_execute_upsert(conn, "data_structures", ds_cols, ds_vals)
|
|
239
|
+
|
|
240
|
+
# dsd_components — delete old, insert new
|
|
241
|
+
conn.execute(
|
|
242
|
+
"""
|
|
243
|
+
DELETE FROM dsd_components WHERE dsd_urn = $urn
|
|
244
|
+
""",
|
|
245
|
+
{"urn": urn},
|
|
246
|
+
)
|
|
247
|
+
|
|
248
|
+
components: list[tuple[str, Any]] = [("Dimension", dim) for dim in dsd.dimensions]
|
|
249
|
+
if dsd.time_dimension is not None:
|
|
250
|
+
components.append(("TimeDimension", dsd.time_dimension))
|
|
251
|
+
components.extend(("Attribute", attr) for attr in dsd.attributes)
|
|
252
|
+
components.extend(("Measure", m) for m in dsd.measures)
|
|
253
|
+
|
|
254
|
+
for position, (comp_type, comp) in enumerate(components):
|
|
255
|
+
concept_ref = getattr(comp, "concept", None)
|
|
256
|
+
concept_id = None
|
|
257
|
+
concept_urn_str = None
|
|
258
|
+
concept_name: InternationalString | None = None
|
|
259
|
+
if isinstance(concept_ref, Ref):
|
|
260
|
+
c_urn = concept_ref.urn
|
|
261
|
+
concept_urn_str = str(c_urn)
|
|
262
|
+
concept_id = getattr(c_urn, "item_id", None) or getattr(c_urn, "id", None)
|
|
263
|
+
if concept_ref.resolved:
|
|
264
|
+
concept_name = concept_ref().name
|
|
265
|
+
|
|
266
|
+
rep = getattr(comp, "representation", None)
|
|
267
|
+
codelist_urn = str(rep.urn) if isinstance(rep, Ref) else None
|
|
268
|
+
|
|
269
|
+
attachment = getattr(comp, "attachment", None)
|
|
270
|
+
attachment_str = attachment.value if attachment is not None else None
|
|
271
|
+
|
|
272
|
+
comp_cols = ["dsd_urn", "component_id", "component_type", "position", "concept_id", "concept_urn"]
|
|
273
|
+
comp_vals: list[Any] = [urn, comp.id, comp_type, position, concept_id, concept_urn_str]
|
|
274
|
+
comp_cols.extend(_lang_cols("concept_name", languages))
|
|
275
|
+
comp_vals.extend(_istring_lang_values(concept_name, languages))
|
|
276
|
+
comp_cols.extend(["codelist_urn", "attachment_level"])
|
|
277
|
+
comp_vals.extend([codelist_urn, attachment_str])
|
|
278
|
+
_execute_insert(conn, "dsd_components", comp_cols, comp_vals)
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def put_dataflow(conn: duckdb.DuckDBPyConnection, flow: Dataflow) -> None:
|
|
282
|
+
"""Write a Dataflow to artefacts + dataflows tables."""
|
|
283
|
+
languages = get_languages(conn)
|
|
284
|
+
urn = str(_artefact_urn(flow))
|
|
285
|
+
doc = json.dumps(to_dict(flow))
|
|
286
|
+
|
|
287
|
+
_put_artefact_row(conn, urn, "Dataflow", flow, doc, languages)
|
|
288
|
+
|
|
289
|
+
dsd_urn = str(flow.structure.urn) if flow.structure is not None else None
|
|
290
|
+
df_cols = ["urn", "agency", "id", "version"]
|
|
291
|
+
df_vals: list[Any] = [urn, flow.agency_id, flow.id, flow.version]
|
|
292
|
+
df_cols.extend(_lang_cols("name", languages))
|
|
293
|
+
df_vals.extend(_istring_lang_values(flow.name, languages))
|
|
294
|
+
df_cols.extend(_lang_cols("description", languages))
|
|
295
|
+
df_vals.extend(_istring_lang_values(flow.description, languages))
|
|
296
|
+
df_cols.append("dsd_urn")
|
|
297
|
+
df_vals.append(dsd_urn)
|
|
298
|
+
_execute_upsert(conn, "dataflows", df_cols, df_vals)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def put_concept_scheme(conn: duckdb.DuckDBPyConnection, cs: ConceptScheme) -> None:
|
|
302
|
+
"""Write a ConceptScheme to artefacts + concept_schemes + concepts tables."""
|
|
303
|
+
languages = get_languages(conn)
|
|
304
|
+
urn = str(_artefact_urn(cs))
|
|
305
|
+
doc = json.dumps(to_dict(cs))
|
|
306
|
+
|
|
307
|
+
_put_artefact_row(conn, urn, "ConceptScheme", cs, doc, languages)
|
|
308
|
+
|
|
309
|
+
# concept_schemes row
|
|
310
|
+
cs_cols = ["urn", "agency", "id", "version"]
|
|
311
|
+
cs_vals: list[Any] = [urn, cs.agency_id, cs.id, cs.version]
|
|
312
|
+
cs_cols.extend(_lang_cols("name", languages))
|
|
313
|
+
cs_vals.extend(_istring_lang_values(cs.name, languages))
|
|
314
|
+
_execute_upsert(conn, "concept_schemes", cs_cols, cs_vals)
|
|
315
|
+
|
|
316
|
+
# concepts — delete old, insert new
|
|
317
|
+
conn.execute(
|
|
318
|
+
"""
|
|
319
|
+
DELETE FROM concepts WHERE scheme_urn = $urn
|
|
320
|
+
""",
|
|
321
|
+
{"urn": urn},
|
|
322
|
+
)
|
|
323
|
+
for concept in cs.concepts:
|
|
324
|
+
c_cols = ["scheme_urn", "concept_id"]
|
|
325
|
+
c_vals: list[Any] = [urn, concept.id]
|
|
326
|
+
c_cols.extend(_lang_cols("name", languages))
|
|
327
|
+
c_vals.extend(_istring_lang_values(concept.name, languages))
|
|
328
|
+
c_cols.extend(_lang_cols("description", languages))
|
|
329
|
+
c_vals.extend(_istring_lang_values(concept.description, languages))
|
|
330
|
+
_execute_insert(conn, "concepts", c_cols, c_vals)
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def put_hierarchy(conn: duckdb.DuckDBPyConnection, h: Hierarchy) -> None:
|
|
334
|
+
"""Write a Hierarchy to artefacts + hierarchy_nodes tables."""
|
|
335
|
+
languages = get_languages(conn)
|
|
336
|
+
urn = str(_artefact_urn(h))
|
|
337
|
+
|
|
338
|
+
d = to_dict(h)
|
|
339
|
+
d.pop("tree", None)
|
|
340
|
+
doc = json.dumps(d)
|
|
341
|
+
|
|
342
|
+
_put_artefact_row(conn, urn, "Hierarchy", h, doc, languages)
|
|
343
|
+
|
|
344
|
+
conn.execute(
|
|
345
|
+
"""
|
|
346
|
+
DELETE FROM hierarchy_nodes WHERE hierarchy_urn = $urn
|
|
347
|
+
""",
|
|
348
|
+
{"urn": urn},
|
|
349
|
+
)
|
|
350
|
+
|
|
351
|
+
rows: list[list[Any]] = []
|
|
352
|
+
_flatten_hierarchy_nodes(urn, to_dict(h).get("tree", []), None, rows, languages)
|
|
353
|
+
|
|
354
|
+
n_cols = ["hierarchy_urn", "node_id", "parent_node_id", "code_urn"]
|
|
355
|
+
n_cols.extend(_lang_cols("name", languages))
|
|
356
|
+
n_cols.extend(_lang_cols("description", languages))
|
|
357
|
+
n_cols.extend(["_annotations", "valid_from", "valid_to"])
|
|
358
|
+
for row in rows:
|
|
359
|
+
_execute_insert(conn, "hierarchy_nodes", n_cols, row)
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _flatten_hierarchy_nodes(
|
|
363
|
+
urn: str,
|
|
364
|
+
nodes: list[dict[str, Any]],
|
|
365
|
+
parent_id: str | None,
|
|
366
|
+
rows: list[list[Any]],
|
|
367
|
+
languages: list[str],
|
|
368
|
+
) -> None:
|
|
369
|
+
"""Recursively flatten hierarchy tree into flat rows."""
|
|
370
|
+
for node in nodes:
|
|
371
|
+
code_urn = node.get("code")
|
|
372
|
+
name_dict = node.get("name")
|
|
373
|
+
desc_dict = node.get("description")
|
|
374
|
+
|
|
375
|
+
name_vals = [name_dict.get(lang) if name_dict else None for lang in languages]
|
|
376
|
+
desc_vals = [desc_dict.get(lang) if desc_dict else None for lang in languages]
|
|
377
|
+
anns_json = json.dumps(node["annotations"]) if node.get("annotations") else None
|
|
378
|
+
|
|
379
|
+
row: list[Any] = [urn, node["id"], parent_id, code_urn]
|
|
380
|
+
row.extend(name_vals)
|
|
381
|
+
row.extend(desc_vals)
|
|
382
|
+
row.extend([anns_json, node.get("valid_from"), node.get("valid_to")])
|
|
383
|
+
rows.append(row)
|
|
384
|
+
|
|
385
|
+
children = node.get("children", [])
|
|
386
|
+
if children:
|
|
387
|
+
_flatten_hierarchy_nodes(urn, children, node["id"], rows, languages)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def put_artefact(conn: duckdb.DuckDBPyConnection, artefact: Any) -> None:
|
|
391
|
+
"""Write any supported artefact, dispatching to the appropriate writer.
|
|
392
|
+
|
|
393
|
+
For Codelists, DataStructures, Dataflows, ConceptSchemes, and
|
|
394
|
+
Hierarchies, writes to both the artefacts table and the relevant
|
|
395
|
+
normalized tables. For all other types, writes only to the artefacts
|
|
396
|
+
table with a JSON document.
|
|
397
|
+
"""
|
|
398
|
+
if not isinstance(artefact, HasUrn):
|
|
399
|
+
msg = (
|
|
400
|
+
f"{type(artefact).__name__} does not satisfy HasUrn — "
|
|
401
|
+
"it needs sdmx_package, sdmx_class, agency_id, id, version"
|
|
402
|
+
)
|
|
403
|
+
raise TypeError(msg)
|
|
404
|
+
|
|
405
|
+
if isinstance(artefact, Codelist):
|
|
406
|
+
put_codelist(conn, artefact)
|
|
407
|
+
elif isinstance(artefact, DataStructure):
|
|
408
|
+
put_data_structure(conn, artefact)
|
|
409
|
+
elif isinstance(artefact, Dataflow):
|
|
410
|
+
put_dataflow(conn, artefact)
|
|
411
|
+
elif isinstance(artefact, ConceptScheme):
|
|
412
|
+
put_concept_scheme(conn, artefact)
|
|
413
|
+
elif isinstance(artefact, Hierarchy):
|
|
414
|
+
put_hierarchy(conn, artefact)
|
|
415
|
+
else:
|
|
416
|
+
_put_blob(conn, artefact)
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _put_blob(conn: duckdb.DuckDBPyConnection, artefact: Any) -> None:
|
|
420
|
+
"""Fallback: write artefact as JSON blob only (no normalized tables)."""
|
|
421
|
+
languages = get_languages(conn)
|
|
422
|
+
urn = str(_artefact_urn(artefact))
|
|
423
|
+
type_name = _CLASS_TO_NAME.get(type(artefact))
|
|
424
|
+
if type_name is None:
|
|
425
|
+
msg = f"Unsupported artefact type: {type(artefact).__name__}"
|
|
426
|
+
raise TypeError(msg)
|
|
427
|
+
doc = json.dumps(to_dict(artefact))
|
|
428
|
+
_put_artefact_row(conn, urn, type_name, artefact, doc, languages)
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def drop_artefact(conn: duckdb.DuckDBPyConnection, urn: str) -> bool:
|
|
432
|
+
"""Delete an artefact and its child rows (manual cascade).
|
|
433
|
+
|
|
434
|
+
Returns True if the artefact existed and was deleted.
|
|
435
|
+
"""
|
|
436
|
+
params = {"urn": urn}
|
|
437
|
+
conn.execute(
|
|
438
|
+
"""
|
|
439
|
+
DELETE FROM codes WHERE codelist_urn = $urn
|
|
440
|
+
""",
|
|
441
|
+
params,
|
|
442
|
+
)
|
|
443
|
+
conn.execute(
|
|
444
|
+
"""
|
|
445
|
+
DELETE FROM codelists WHERE urn = $urn
|
|
446
|
+
""",
|
|
447
|
+
params,
|
|
448
|
+
)
|
|
449
|
+
conn.execute(
|
|
450
|
+
"""
|
|
451
|
+
DELETE FROM dsd_components WHERE dsd_urn = $urn
|
|
452
|
+
""",
|
|
453
|
+
params,
|
|
454
|
+
)
|
|
455
|
+
conn.execute(
|
|
456
|
+
"""
|
|
457
|
+
DELETE FROM data_structures WHERE urn = $urn
|
|
458
|
+
""",
|
|
459
|
+
params,
|
|
460
|
+
)
|
|
461
|
+
conn.execute(
|
|
462
|
+
"""
|
|
463
|
+
DELETE FROM dataflows WHERE urn = $urn
|
|
464
|
+
""",
|
|
465
|
+
params,
|
|
466
|
+
)
|
|
467
|
+
conn.execute(
|
|
468
|
+
"""
|
|
469
|
+
DELETE FROM concepts WHERE scheme_urn = $urn
|
|
470
|
+
""",
|
|
471
|
+
params,
|
|
472
|
+
)
|
|
473
|
+
conn.execute(
|
|
474
|
+
"""
|
|
475
|
+
DELETE FROM concept_schemes WHERE urn = $urn
|
|
476
|
+
""",
|
|
477
|
+
params,
|
|
478
|
+
)
|
|
479
|
+
conn.execute(
|
|
480
|
+
"""
|
|
481
|
+
DELETE FROM hierarchy_nodes WHERE hierarchy_urn = $urn
|
|
482
|
+
""",
|
|
483
|
+
params,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
result = conn.execute(
|
|
487
|
+
"""
|
|
488
|
+
DELETE FROM artefacts WHERE urn = $urn RETURNING urn
|
|
489
|
+
""",
|
|
490
|
+
params,
|
|
491
|
+
).fetchone()
|
|
492
|
+
return result is not None
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def ingest_registry(conn: duckdb.DuckDBPyConnection, registry: Any) -> None:
|
|
496
|
+
"""Bulk-import all artefacts from a Registry into DuckDB storage.
|
|
497
|
+
|
|
498
|
+
Iterates over the registry's internal store and writes each artefact
|
|
499
|
+
using :func:`put_artefact`.
|
|
500
|
+
"""
|
|
501
|
+
store: dict[str, Any] = getattr(registry, "_store", {})
|
|
502
|
+
for artefact in store.values():
|
|
503
|
+
put_artefact(conn, artefact)
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: sdmxlib
|
|
3
|
+
Version: 0.8.0
|
|
4
|
+
Summary: SDMX structural metadata library for Python
|
|
5
|
+
Keywords: sdmx,statistics,metadata,datastructure
|
|
6
|
+
Author: gabrielgellner
|
|
7
|
+
Author-email: gabrielgellner <gabrielgellner@gmail.com>
|
|
8
|
+
License: Apache-2.0
|
|
9
|
+
Classifier: Development Status :: 4 - Beta
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
14
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
15
|
+
Requires-Dist: attrs>=26.1.0
|
|
16
|
+
Requires-Dist: duckdb>=1.0.0
|
|
17
|
+
Requires-Dist: httpx>=0.28.1
|
|
18
|
+
Requires-Dist: lxml>=6.0.2
|
|
19
|
+
Requires-Dist: polars>=1.39.3
|
|
20
|
+
Requires-Python: >=3.12
|
|
21
|
+
Project-URL: Homepage, https://gitlab.com/gabrielgellner/sdmxlib
|
|
22
|
+
Project-URL: Repository, https://gitlab.com/gabrielgellner/sdmxlib
|
|
23
|
+
Project-URL: Documentation, https://gabrielgellner.gitlab.io/sdmxlib
|
|
24
|
+
Project-URL: Changelog, https://gitlab.com/gabrielgellner/sdmxlib/-/blob/main/CHANGELOG.md
|
|
25
|
+
Description-Content-Type: text/markdown
|
|
26
|
+
|
|
27
|
+
# sdmxlib
|
|
28
|
+
|
|
29
|
+
> **Note:** This library has been written extensively with AI assistance
|
|
30
|
+
> (Claude Code). Users who are not comfortable with AI-generated code should
|
|
31
|
+
> take that into account before adopting it.
|
|
32
|
+
|
|
33
|
+
SDMX structural metadata library for Python. Fetch and navigate Data Structure
|
|
34
|
+
Definitions, Codelists, Dataflows, and related artefacts from any SDMX REST
|
|
35
|
+
endpoint — with plain attribute access, no XML or URN chasing required.
|
|
36
|
+
|
|
37
|
+
## Quick start
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
import sdmxlib as sl
|
|
41
|
+
|
|
42
|
+
with sl.RestRegistry(sl.Provider.BIS) as reg:
|
|
43
|
+
dsd = reg.get(sl.DataStructure, agency="BIS", id="WS_CBPOL").resolve()
|
|
44
|
+
|
|
45
|
+
for dim in dsd.dimensions:
|
|
46
|
+
if dim.is_enumerated:
|
|
47
|
+
cl = dim.representation()
|
|
48
|
+
print(f"{dim.id}: {cl.id} ({len(cl.codes)} codes)")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
FREQ: CL_FREQ (7 codes)
|
|
53
|
+
REF_AREA: CL_BIS_IF_REF_AREA (44 codes)
|
|
54
|
+
INSTR_ASSET: CL_INSTR_ASSET (12 codes)
|
|
55
|
+
BORROWER_CTY: CL_BIS_IF_REF_AREA (44 codes)
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Installation
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
uv add "sdmxlib @ git+https://gitlab.com/gabrielgellner/sdmxlib.git"
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## What it does
|
|
65
|
+
|
|
66
|
+
- Connects to any SDMX 2.1 or 3.0 REST endpoint; built-in shortcuts for ECB,
|
|
67
|
+
BIS, ESTAT, IMF, OECD, and the SDMX Global Registry via `sl.Provider`
|
|
68
|
+
- Three resolution modes: `browse()` for catalogue scanning, `fetch()` for
|
|
69
|
+
unresolved artefacts, `resolve()` for the full reference graph (with optional
|
|
70
|
+
constraint-trimming via `partial=True/False`)
|
|
71
|
+
- Returns fully resolved domain objects with plain attribute access — no XML,
|
|
72
|
+
no URN chasing
|
|
73
|
+
- Filter collections with lambdas or the `item()` expression API
|
|
74
|
+
- Build SDMX artefacts locally and serialise to SDMX-ML or SDMX-JSON
|
|
75
|
+
- `LocalRegistry` -- a DuckDB-backed persistent store for offline access
|
|
76
|
+
with lazy codelist loading, automatic Ref resolution, and direct SQL
|
|
77
|
+
code maps; supports shared connections and configurable language columns
|
|
78
|
+
- Integrates with [Polars](https://pola.rs) for schema generation and data
|
|
79
|
+
validation
|
|
80
|
+
- Fully typed — IDE completion and static analysis work throughout
|
|
81
|
+
|
|
82
|
+
## Documentation
|
|
83
|
+
|
|
84
|
+
See the [docs](https://gabrielgellner.gitlab.io/sdmxlib/) for the full cookbook,
|
|
85
|
+
including:
|
|
86
|
+
|
|
87
|
+
- Connecting to registries and choosing a resolution mode
|
|
88
|
+
- Persisting artefacts in a local DuckDB store with `LocalRegistry`
|
|
89
|
+
- Fetching observation data with `sl.dim()` filters and pivoting to wide format
|
|
90
|
+
- Inspecting DSDs and building code maps for ETL pipelines
|
|
91
|
+
- Generating Polars schemas from SDMX Data Structure Definitions
|
|
92
|
+
- Browsing data catalogs and filtering dataflows
|
|
93
|
+
- Working with constraints, hierarchies, and serialisation
|
|
94
|
+
|
|
95
|
+
## Development
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
just check # lint + typecheck
|
|
99
|
+
just test # unit tests
|
|
100
|
+
just test-integration # live endpoint tests (requires network)
|
|
101
|
+
just ci # full pre-commit gate
|
|
102
|
+
```
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
sdmxlib/__init__.py,sha256=kywSlc7M0H-YIDn7H7re2iebQs5KKtzMN_Az0qgGLDg,3125
|
|
2
|
+
sdmxlib/api/__init__.py,sha256=yNPFuM9w5OEAuXqWzpV9ZxKPOrfg75iF7AwMnTF3wtE,301
|
|
3
|
+
sdmxlib/api/client.py,sha256=hKWcG3Vy5JWjYxNowrI8wiLkAq-naqSJ-w-7kcUd8Lo,8761
|
|
4
|
+
sdmxlib/api/filters.py,sha256=0oH_hFJwGpOQxjnVsbxI3GO-IbxGuIxdR_JlBXFccpo,1966
|
|
5
|
+
sdmxlib/api/providers.py,sha256=8Jxf2WOOIMLl91iwRrOOXmW3VQe34cx7uA4e4FkT9qk,2201
|
|
6
|
+
sdmxlib/api/query.py,sha256=GqjBhYA7iESV97DJnxMOT1DVTAuccxYz3hobFDCco9k,15801
|
|
7
|
+
sdmxlib/api/registry.py,sha256=hAs_0x5plGZ1GfNiMICbrQlb6XRKeOijRAHAtdcOFVQ,25619
|
|
8
|
+
sdmxlib/api/session.py,sha256=8dipduGvKAx19Fd5W4U67_79c_l6V0RL67inUFRmfd0,5288
|
|
9
|
+
sdmxlib/formats/__init__.py,sha256=acm1AHbZwKXbgFD13Ien6yK_X-vuaxRXzT9W6oJ5JCw,5772
|
|
10
|
+
sdmxlib/formats/sdmx_csv/__init__.py,sha256=0_N-eOgYfNrkfakZ1NUwgfxV0EbKZhAmhBy3O8iX9ck,114
|
|
11
|
+
sdmxlib/formats/sdmx_csv/reader.py,sha256=SjMgqc0CYjXprzcJwKkGOOO06aBs8myPFRiMyFyc5jw,4856
|
|
12
|
+
sdmxlib/formats/sdmx_json/__init__.py,sha256=7RrczMcD9X1PRRll5CvYef79PZ88v0JYeLZ_gkusED0,68
|
|
13
|
+
sdmxlib/formats/sdmx_json/reader.py,sha256=5XhRNvYUC91FrTJNJlTQNEfOUqNWfIM2WH07on36xmU,35657
|
|
14
|
+
sdmxlib/formats/sdmx_json/writer.py,sha256=t2cIvMCN1nBsLsxNfVzw73YX17BFxLJkpjKICdRxmoE,25327
|
|
15
|
+
sdmxlib/formats/sdmx_ml21/__init__.py,sha256=sjCtD5QNb6VZrAoq3Tu0yc9lvuXBLrnisk78lFnwUT0,104
|
|
16
|
+
sdmxlib/formats/sdmx_ml21/namespaces.py,sha256=IjlTboUJxrUrSYxUE8fkvQTsHwJsmmwkcOEQEDaGfTE,512
|
|
17
|
+
sdmxlib/formats/sdmx_ml21/reader.py,sha256=REf51p9UxKeZDkjENPkS2bZkP6EpsiQUm3tKKosdVQ0,44203
|
|
18
|
+
sdmxlib/formats/sdmx_ml21/writer.py,sha256=VzDNP4P2jTo1kIZablfcrIefscbgvoZT_MF1GmU3ikw,27998
|
|
19
|
+
sdmxlib/formats/sdmx_ml30/__init__.py,sha256=qngL-lSQued5Qmy-Sfr3X-CRq8tjHAFSCu5oqU7tDMg,33
|
|
20
|
+
sdmxlib/formats/sdmx_ml30/namespaces.py,sha256=orsdl9Il2AiU-tIe6xjoB2nMJyDCsF5WTghqUhHeSII,512
|
|
21
|
+
sdmxlib/formats/sdmx_ml30/reader.py,sha256=FkBh2tNoCSv25Qm_jyTYIocMBBomsTfQu7AUDcFy8FI,43886
|
|
22
|
+
sdmxlib/formats/sdmx_ml30/writer.py,sha256=7e6twXGErIP-ttnWsRkeaD3KyiJqxkZNwb9a50j-RZA,28759
|
|
23
|
+
sdmxlib/local/__init__.py,sha256=ChpMTPfDW1umZRmG5zkc4i3_ahZlDgTBv32WtH9QvsM,233
|
|
24
|
+
sdmxlib/local/_registry.py,sha256=OkYptstdlMEnvjZQAxj2YZg9NreepNS0cCPkcHvGSI8,14280
|
|
25
|
+
sdmxlib/model/__init__.py,sha256=orY-ju8-l8A4h3c_ZF89j_w7PLToPiMDCExU1Z4qUos,3137
|
|
26
|
+
sdmxlib/model/annotations.py,sha256=AqChfYEeZ5ZTgArSi6E9vlGG6U7hHumiRDAqLFoV2aE,3697
|
|
27
|
+
sdmxlib/model/base.py,sha256=ieXvUbilBHnlfczh7OSIBGPWgR9rWN4OiXaGHXLUvmc,1956
|
|
28
|
+
sdmxlib/model/category.py,sha256=KzbJ6QFoXjjtirTZlHYkwbA7yawrpsf_eKdoJpN7GgQ,3649
|
|
29
|
+
sdmxlib/model/codelist.py,sha256=fwTknAnLQCF0WHxwdbdsh5pOkLF7TU1kmMGjXjmTtfI,3488
|
|
30
|
+
sdmxlib/model/collections.py,sha256=TX-vLZ7coJd1HtIxVD7Z6BXXID-K0YV9EwVBuX9BwwM,15486
|
|
31
|
+
sdmxlib/model/concept.py,sha256=BbYRZXPaGrYWUV8iTK0b8ZBQwktaSqzwr1oIqiWs1yE,4606
|
|
32
|
+
sdmxlib/model/constraint.py,sha256=pwL5D05bV2eAIsZ9GivaDAyQtghh4Ws6NaFHMQtaZsI,2920
|
|
33
|
+
sdmxlib/model/convert.py,sha256=5SkonvpiVxeALNHocyWj8k7o7UB3JZ5PAWij6CMqbcc,9436
|
|
34
|
+
sdmxlib/model/dataflow.py,sha256=z_zHg7XCgosK04u5rLUop057A--UqkuWliQXxdnVcYo,1780
|
|
35
|
+
sdmxlib/model/dataset.py,sha256=LLH1oOh92mTWkK9MtxzN8704sMqGosKXPJISFyHjCqQ,14753
|
|
36
|
+
sdmxlib/model/datastructure.py,sha256=X0f7OpsmWc0YeNBljMV9hJDLbksttCLP9mFwOfQMVro,15549
|
|
37
|
+
sdmxlib/model/expr.py,sha256=0a88sKBRUC_sUvvKL066kkX2PFVXXrmJYN24qs3sjps,4299
|
|
38
|
+
sdmxlib/model/hierarchy.py,sha256=qMCYCfanURzAELlfI3dCl16adL-GHi2yP-xTgM1trpY,4165
|
|
39
|
+
sdmxlib/model/istring.py,sha256=9956ucw2730ByddDBTj4SKM0ynvI13nXAJnisvkx8YM,2748
|
|
40
|
+
sdmxlib/model/mapping.py,sha256=yyBDvZy1DSji5PfmC1XamUHF7G3YFm16GK_ZTgduuiY,4335
|
|
41
|
+
sdmxlib/model/message.py,sha256=lwRKRLWwZrgzEelD9-9qpwVG_PVaJwKwzR-tPUaT9vc,3199
|
|
42
|
+
sdmxlib/model/organisation.py,sha256=TCC_YmtGFy3UYr_FZ83b8tuwFcQpvbakKJnjc-0RxL8,6047
|
|
43
|
+
sdmxlib/model/provision.py,sha256=FxMoNFYoVlfuqrUtqm_DXeR8Ey4itEhowh1mDsqMAGc,1653
|
|
44
|
+
sdmxlib/model/ref.py,sha256=zdG6j5z-TyBvKxPxe6HkQ5337b0OkuAjEz-T3PW93XQ,6978
|
|
45
|
+
sdmxlib/model/registry.py,sha256=hQ7dL3BWquYiimidjwddknZiEzHX6dAcnTDrKCy8z4o,7492
|
|
46
|
+
sdmxlib/model/representation.py,sha256=W_5R0_WZjnXy5_7Uc5e0Y4YQGfw-M4Kx9uDQcAV6g_0,3018
|
|
47
|
+
sdmxlib/model/urn.py,sha256=h16i_xSjZKxYwDbBcwGKS9Do-zg5fl5Zc5dRzRevbVM,4621
|
|
48
|
+
sdmxlib/model/validation.py,sha256=KlaC03RNbMLpz56P8PxBci9ndKQVwe-PdhRi5K4Q98c,12991
|
|
49
|
+
sdmxlib/polars.py,sha256=lWO6gwYiS73UKVeofMDWadHCsgd6I_0Vqwsl4xWb-UU,14769
|
|
50
|
+
sdmxlib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
|
+
sdmxlib/storage/__init__.py,sha256=1TT_oYx3EjtlxBMrJPIHjaJWbTz-X-wSTLEkg6UHLMQ,771
|
|
52
|
+
sdmxlib/storage/lazy.py,sha256=TouG-6UgQ1_z6SSnUr7hbNOaps96MUBcT7ZNG6gJhgA,10637
|
|
53
|
+
sdmxlib/storage/readers.py,sha256=9WfybYgvYV1rTNS1PHkZj0Fd5J0W7vgjbEfvb05d2Xw,10052
|
|
54
|
+
sdmxlib/storage/schema.py,sha256=f65jf2eSaP4Lr5PVGtt7wzk4YWP_Hx844IOI2MWvkZY,9496
|
|
55
|
+
sdmxlib/storage/writers.py,sha256=STbXpol4OtYEifqXF6AtFNCEHpABDLlPWrAOue_da-Q,17182
|
|
56
|
+
sdmxlib-0.8.0.dist-info/WHEEL,sha256=s_zqWxHFEH8b58BCtf46hFCqPaISurdB9R1XJ8za6XI,80
|
|
57
|
+
sdmxlib-0.8.0.dist-info/METADATA,sha256=Y0Je7YwuyR9fbp_TGn618D1shjAvknU0rMG0VMttI1E,3824
|
|
58
|
+
sdmxlib-0.8.0.dist-info/RECORD,,
|