sdmxlib 0.8.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. sdmxlib/__init__.py +155 -0
  2. sdmxlib/api/__init__.py +7 -0
  3. sdmxlib/api/client.py +246 -0
  4. sdmxlib/api/filters.py +67 -0
  5. sdmxlib/api/providers.py +66 -0
  6. sdmxlib/api/query.py +428 -0
  7. sdmxlib/api/registry.py +666 -0
  8. sdmxlib/api/session.py +124 -0
  9. sdmxlib/formats/__init__.py +166 -0
  10. sdmxlib/formats/sdmx_csv/__init__.py +5 -0
  11. sdmxlib/formats/sdmx_csv/reader.py +147 -0
  12. sdmxlib/formats/sdmx_json/__init__.py +1 -0
  13. sdmxlib/formats/sdmx_json/reader.py +897 -0
  14. sdmxlib/formats/sdmx_json/writer.py +698 -0
  15. sdmxlib/formats/sdmx_ml21/__init__.py +5 -0
  16. sdmxlib/formats/sdmx_ml21/namespaces.py +12 -0
  17. sdmxlib/formats/sdmx_ml21/reader.py +1147 -0
  18. sdmxlib/formats/sdmx_ml21/writer.py +709 -0
  19. sdmxlib/formats/sdmx_ml30/__init__.py +1 -0
  20. sdmxlib/formats/sdmx_ml30/namespaces.py +12 -0
  21. sdmxlib/formats/sdmx_ml30/reader.py +1138 -0
  22. sdmxlib/formats/sdmx_ml30/writer.py +730 -0
  23. sdmxlib/local/__init__.py +6 -0
  24. sdmxlib/local/_registry.py +368 -0
  25. sdmxlib/model/__init__.py +122 -0
  26. sdmxlib/model/annotations.py +103 -0
  27. sdmxlib/model/base.py +61 -0
  28. sdmxlib/model/category.py +88 -0
  29. sdmxlib/model/codelist.py +91 -0
  30. sdmxlib/model/collections.py +382 -0
  31. sdmxlib/model/concept.py +121 -0
  32. sdmxlib/model/constraint.py +92 -0
  33. sdmxlib/model/convert.py +268 -0
  34. sdmxlib/model/dataflow.py +49 -0
  35. sdmxlib/model/dataset.py +337 -0
  36. sdmxlib/model/datastructure.py +390 -0
  37. sdmxlib/model/expr.py +120 -0
  38. sdmxlib/model/hierarchy.py +111 -0
  39. sdmxlib/model/istring.py +81 -0
  40. sdmxlib/model/mapping.py +134 -0
  41. sdmxlib/model/message.py +70 -0
  42. sdmxlib/model/organisation.py +149 -0
  43. sdmxlib/model/provision.py +45 -0
  44. sdmxlib/model/ref.py +195 -0
  45. sdmxlib/model/registry.py +191 -0
  46. sdmxlib/model/representation.py +99 -0
  47. sdmxlib/model/urn.py +130 -0
  48. sdmxlib/model/validation.py +338 -0
  49. sdmxlib/polars.py +392 -0
  50. sdmxlib/py.typed +0 -0
  51. sdmxlib/storage/__init__.py +28 -0
  52. sdmxlib/storage/lazy.py +329 -0
  53. sdmxlib/storage/readers.py +310 -0
  54. sdmxlib/storage/schema.py +289 -0
  55. sdmxlib/storage/writers.py +503 -0
  56. sdmxlib-0.8.0.dist-info/METADATA +102 -0
  57. sdmxlib-0.8.0.dist-info/RECORD +58 -0
  58. sdmxlib-0.8.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,503 @@
1
+ """Write SDMX domain objects to DuckDB normalized tables.
2
+
3
+ Flattens domain objects into the normalized schema defined by
4
+ ``storage.schema``. Language-specific columns are populated based
5
+ on the configured language list stored in ``_meta``.
6
+
7
+ Example::
8
+
9
+ from sdmxlib.storage import schema, writers
10
+
11
+ conn = duckdb.connect("store.duckdb")
12
+ schema.create_tables(conn, languages=["en", "fr"])
13
+ writers.put_codelist(conn, codelist)
14
+ """
15
+
16
+ import json
17
+ from typing import Any
18
+
19
+ import duckdb
20
+
21
+ from sdmxlib.model.annotations import Annotations
22
+ from sdmxlib.model.codelist import Codelist
23
+ from sdmxlib.model.concept import ConceptScheme
24
+ from sdmxlib.model.convert import to_dict
25
+ from sdmxlib.model.dataflow import Dataflow
26
+ from sdmxlib.model.datastructure import DataStructure
27
+ from sdmxlib.model.hierarchy import Hierarchy
28
+ from sdmxlib.model.istring import InternationalString
29
+ from sdmxlib.model.ref import HasUrn, Ref
30
+ from sdmxlib.model.registry import _artefact_urn
31
+ from sdmxlib.storage.schema import get_languages
32
+
33
+ # Map Python class → type_name string for artefact_type column
34
+ _CLASS_TO_NAME: dict[type, str] = {
35
+ Codelist: "Codelist",
36
+ ConceptScheme: "ConceptScheme",
37
+ DataStructure: "DataStructure",
38
+ Dataflow: "Dataflow",
39
+ Hierarchy: "Hierarchy",
40
+ }
41
+
42
+ # Try to import optional types that may or may not be used
43
+ try:
44
+ from sdmxlib.model.category import Categorisation, CategoryScheme
45
+ from sdmxlib.model.constraint import ContentConstraint
46
+ from sdmxlib.model.mapping import StructureSet
47
+ from sdmxlib.model.organisation import AgencyScheme, DataProviderScheme
48
+ from sdmxlib.model.provision import ProvisionAgreement
49
+
50
+ _CLASS_TO_NAME.update(
51
+ {
52
+ ContentConstraint: "ContentConstraint",
53
+ CategoryScheme: "CategoryScheme",
54
+ Categorisation: "Categorisation",
55
+ ProvisionAgreement: "ProvisionAgreement",
56
+ AgencyScheme: "AgencyScheme",
57
+ DataProviderScheme: "DataProviderScheme",
58
+ StructureSet: "StructureSet",
59
+ }
60
+ )
61
+ except ImportError:
62
+ pass
63
+
64
+
65
+ # ── Helpers ─────────────────────────────────────────────────────────────
66
+
67
+
68
+ def _istring_lang_values(
69
+ s: InternationalString | None,
70
+ languages: list[str],
71
+ ) -> list[str | None]:
72
+ """Extract values for each configured language from an InternationalString."""
73
+ if not s:
74
+ return [None] * len(languages)
75
+ return [s.get(lang) if lang in s else None for lang in languages]
76
+
77
+
78
+ def _annotations_json(annotations: Annotations | None) -> str | None:
79
+ if not annotations:
80
+ return None
81
+ result = []
82
+ for a in annotations:
83
+ d: dict[str, Any] = {}
84
+ if a.id is not None:
85
+ d["id"] = a.id
86
+ if a.title is not None:
87
+ d["title"] = a.title
88
+ if a.type is not None:
89
+ d["type"] = a.type
90
+ if a.url is not None:
91
+ d["url"] = a.url
92
+ if a.text:
93
+ d["text"] = {lang: a.text[lang] for lang in a.text}
94
+ result.append(d)
95
+ return json.dumps(result)
96
+
97
+
98
+ def _lang_cols(prefix: str, languages: list[str]) -> list[str]:
99
+ """Build column names like ['name_en', 'name_fr'] for a prefix."""
100
+ return [f"{prefix}_{lang}" for lang in languages]
101
+
102
+
103
+ def _execute_insert(
104
+ conn: duckdb.DuckDBPyConnection,
105
+ table: str,
106
+ cols: list[str],
107
+ vals: list[Any],
108
+ ) -> None:
109
+ """Execute a parameterised INSERT.
110
+
111
+ Column names come from our own schema definitions (language config),
112
+ never from user input — SQL injection is not a concern here.
113
+ """
114
+ placeholders = ", ".join(["?"] * len(vals))
115
+ col_str = ", ".join(cols)
116
+ conn.execute(
117
+ f"""
118
+ INSERT INTO {table} ({col_str})
119
+ VALUES ({placeholders})
120
+ """, # noqa: S608
121
+ vals,
122
+ )
123
+
124
+
125
+ def _execute_upsert(
126
+ conn: duckdb.DuckDBPyConnection,
127
+ table: str,
128
+ cols: list[str],
129
+ vals: list[Any],
130
+ ) -> None:
131
+ """INSERT OR REPLACE for tables with a single unique/PK constraint."""
132
+ placeholders = ", ".join(["?"] * len(vals))
133
+ col_str = ", ".join(cols)
134
+ conn.execute(
135
+ f"""
136
+ INSERT OR REPLACE INTO {table} ({col_str})
137
+ VALUES ({placeholders})
138
+ """, # noqa: S608
139
+ vals,
140
+ )
141
+
142
+
143
+ # ── Artefact table helpers ──────────────────────────────────────────────
144
+
145
+
146
+ def _put_artefact_row(
147
+ conn: duckdb.DuckDBPyConnection,
148
+ urn: str,
149
+ artefact_type: str,
150
+ artefact: Any,
151
+ doc: str,
152
+ languages: list[str],
153
+ ) -> None:
154
+ """Insert or replace a row in the artefacts table."""
155
+ name_vals = _istring_lang_values(getattr(artefact, "name", None), languages)
156
+ desc_vals = _istring_lang_values(getattr(artefact, "description", None), languages)
157
+ is_final = getattr(artefact, "is_final", False)
158
+
159
+ cols = ["urn", "artefact_type", "agency", "id", "version", "is_final"]
160
+ vals: list[Any] = [urn, artefact_type, artefact.agency_id, artefact.id, artefact.version, is_final]
161
+
162
+ cols.extend(_lang_cols("name", languages))
163
+ vals.extend(name_vals)
164
+
165
+ cols.extend(_lang_cols("description", languages))
166
+ vals.extend(desc_vals)
167
+
168
+ cols.append("_document")
169
+ vals.append(doc)
170
+
171
+ # artefacts has PK + UNIQUE constraint; DELETE+INSERT to avoid
172
+ # DuckDB's "multiple unique constraints" binder error.
173
+ conn.execute(
174
+ """
175
+ DELETE FROM artefacts WHERE urn = $urn
176
+ """,
177
+ {"urn": urn},
178
+ )
179
+ _execute_insert(conn, "artefacts", cols, vals)
180
+
181
+
182
+ # ── Public writers ──────────────────────────────────────────────────────
183
+
184
+
185
+ def put_codelist(conn: duckdb.DuckDBPyConnection, cl: Codelist) -> None:
186
+ """Write a Codelist to artefacts + codelists + codes tables."""
187
+ languages = get_languages(conn)
188
+ urn = str(_artefact_urn(cl))
189
+
190
+ d = to_dict(cl)
191
+ d.pop("codes", None)
192
+ doc = json.dumps(d)
193
+
194
+ # artefacts row
195
+ _put_artefact_row(conn, urn, "Codelist", cl, doc, languages)
196
+
197
+ # codelists row
198
+ cl_cols = ["urn", "agency", "id", "version"]
199
+ cl_vals: list[Any] = [urn, cl.agency_id, cl.id, cl.version]
200
+ cl_cols.extend(_lang_cols("name", languages))
201
+ cl_vals.extend(_istring_lang_values(cl.name, languages))
202
+ cl_cols.append("is_partial")
203
+ cl_vals.append(False)
204
+ _execute_upsert(conn, "codelists", cl_cols, cl_vals)
205
+
206
+ # codes rows — delete old, insert new
207
+ conn.execute(
208
+ """
209
+ DELETE FROM codes WHERE codelist_urn = $urn
210
+ """,
211
+ {"urn": urn},
212
+ )
213
+ for position, code in enumerate(cl.codes):
214
+ code_cols = ["codelist_urn", "code_id", "parent_code_id", "position"]
215
+ code_vals: list[Any] = [urn, code.id, code.parent_id, position]
216
+ code_cols.extend(_lang_cols("label", languages))
217
+ code_vals.extend(_istring_lang_values(code.name, languages))
218
+ code_cols.extend(_lang_cols("description", languages))
219
+ code_vals.extend(_istring_lang_values(code.description, languages))
220
+ code_cols.append("_annotations")
221
+ code_vals.append(_annotations_json(code.annotations))
222
+ _execute_insert(conn, "codes", code_cols, code_vals)
223
+
224
+
225
+ def put_data_structure(conn: duckdb.DuckDBPyConnection, dsd: DataStructure) -> None:
226
+ """Write a DataStructure to artefacts + data_structures + dsd_components."""
227
+ languages = get_languages(conn)
228
+ urn = str(_artefact_urn(dsd))
229
+ doc = json.dumps(to_dict(dsd))
230
+
231
+ _put_artefact_row(conn, urn, "DataStructure", dsd, doc, languages)
232
+
233
+ # data_structures row
234
+ ds_cols = ["urn", "agency", "id", "version"]
235
+ ds_vals: list[Any] = [urn, dsd.agency_id, dsd.id, dsd.version]
236
+ ds_cols.extend(_lang_cols("name", languages))
237
+ ds_vals.extend(_istring_lang_values(dsd.name, languages))
238
+ _execute_upsert(conn, "data_structures", ds_cols, ds_vals)
239
+
240
+ # dsd_components — delete old, insert new
241
+ conn.execute(
242
+ """
243
+ DELETE FROM dsd_components WHERE dsd_urn = $urn
244
+ """,
245
+ {"urn": urn},
246
+ )
247
+
248
+ components: list[tuple[str, Any]] = [("Dimension", dim) for dim in dsd.dimensions]
249
+ if dsd.time_dimension is not None:
250
+ components.append(("TimeDimension", dsd.time_dimension))
251
+ components.extend(("Attribute", attr) for attr in dsd.attributes)
252
+ components.extend(("Measure", m) for m in dsd.measures)
253
+
254
+ for position, (comp_type, comp) in enumerate(components):
255
+ concept_ref = getattr(comp, "concept", None)
256
+ concept_id = None
257
+ concept_urn_str = None
258
+ concept_name: InternationalString | None = None
259
+ if isinstance(concept_ref, Ref):
260
+ c_urn = concept_ref.urn
261
+ concept_urn_str = str(c_urn)
262
+ concept_id = getattr(c_urn, "item_id", None) or getattr(c_urn, "id", None)
263
+ if concept_ref.resolved:
264
+ concept_name = concept_ref().name
265
+
266
+ rep = getattr(comp, "representation", None)
267
+ codelist_urn = str(rep.urn) if isinstance(rep, Ref) else None
268
+
269
+ attachment = getattr(comp, "attachment", None)
270
+ attachment_str = attachment.value if attachment is not None else None
271
+
272
+ comp_cols = ["dsd_urn", "component_id", "component_type", "position", "concept_id", "concept_urn"]
273
+ comp_vals: list[Any] = [urn, comp.id, comp_type, position, concept_id, concept_urn_str]
274
+ comp_cols.extend(_lang_cols("concept_name", languages))
275
+ comp_vals.extend(_istring_lang_values(concept_name, languages))
276
+ comp_cols.extend(["codelist_urn", "attachment_level"])
277
+ comp_vals.extend([codelist_urn, attachment_str])
278
+ _execute_insert(conn, "dsd_components", comp_cols, comp_vals)
279
+
280
+
281
+ def put_dataflow(conn: duckdb.DuckDBPyConnection, flow: Dataflow) -> None:
282
+ """Write a Dataflow to artefacts + dataflows tables."""
283
+ languages = get_languages(conn)
284
+ urn = str(_artefact_urn(flow))
285
+ doc = json.dumps(to_dict(flow))
286
+
287
+ _put_artefact_row(conn, urn, "Dataflow", flow, doc, languages)
288
+
289
+ dsd_urn = str(flow.structure.urn) if flow.structure is not None else None
290
+ df_cols = ["urn", "agency", "id", "version"]
291
+ df_vals: list[Any] = [urn, flow.agency_id, flow.id, flow.version]
292
+ df_cols.extend(_lang_cols("name", languages))
293
+ df_vals.extend(_istring_lang_values(flow.name, languages))
294
+ df_cols.extend(_lang_cols("description", languages))
295
+ df_vals.extend(_istring_lang_values(flow.description, languages))
296
+ df_cols.append("dsd_urn")
297
+ df_vals.append(dsd_urn)
298
+ _execute_upsert(conn, "dataflows", df_cols, df_vals)
299
+
300
+
301
+ def put_concept_scheme(conn: duckdb.DuckDBPyConnection, cs: ConceptScheme) -> None:
302
+ """Write a ConceptScheme to artefacts + concept_schemes + concepts tables."""
303
+ languages = get_languages(conn)
304
+ urn = str(_artefact_urn(cs))
305
+ doc = json.dumps(to_dict(cs))
306
+
307
+ _put_artefact_row(conn, urn, "ConceptScheme", cs, doc, languages)
308
+
309
+ # concept_schemes row
310
+ cs_cols = ["urn", "agency", "id", "version"]
311
+ cs_vals: list[Any] = [urn, cs.agency_id, cs.id, cs.version]
312
+ cs_cols.extend(_lang_cols("name", languages))
313
+ cs_vals.extend(_istring_lang_values(cs.name, languages))
314
+ _execute_upsert(conn, "concept_schemes", cs_cols, cs_vals)
315
+
316
+ # concepts — delete old, insert new
317
+ conn.execute(
318
+ """
319
+ DELETE FROM concepts WHERE scheme_urn = $urn
320
+ """,
321
+ {"urn": urn},
322
+ )
323
+ for concept in cs.concepts:
324
+ c_cols = ["scheme_urn", "concept_id"]
325
+ c_vals: list[Any] = [urn, concept.id]
326
+ c_cols.extend(_lang_cols("name", languages))
327
+ c_vals.extend(_istring_lang_values(concept.name, languages))
328
+ c_cols.extend(_lang_cols("description", languages))
329
+ c_vals.extend(_istring_lang_values(concept.description, languages))
330
+ _execute_insert(conn, "concepts", c_cols, c_vals)
331
+
332
+
333
+ def put_hierarchy(conn: duckdb.DuckDBPyConnection, h: Hierarchy) -> None:
334
+ """Write a Hierarchy to artefacts + hierarchy_nodes tables."""
335
+ languages = get_languages(conn)
336
+ urn = str(_artefact_urn(h))
337
+
338
+ d = to_dict(h)
339
+ d.pop("tree", None)
340
+ doc = json.dumps(d)
341
+
342
+ _put_artefact_row(conn, urn, "Hierarchy", h, doc, languages)
343
+
344
+ conn.execute(
345
+ """
346
+ DELETE FROM hierarchy_nodes WHERE hierarchy_urn = $urn
347
+ """,
348
+ {"urn": urn},
349
+ )
350
+
351
+ rows: list[list[Any]] = []
352
+ _flatten_hierarchy_nodes(urn, to_dict(h).get("tree", []), None, rows, languages)
353
+
354
+ n_cols = ["hierarchy_urn", "node_id", "parent_node_id", "code_urn"]
355
+ n_cols.extend(_lang_cols("name", languages))
356
+ n_cols.extend(_lang_cols("description", languages))
357
+ n_cols.extend(["_annotations", "valid_from", "valid_to"])
358
+ for row in rows:
359
+ _execute_insert(conn, "hierarchy_nodes", n_cols, row)
360
+
361
+
362
+ def _flatten_hierarchy_nodes(
363
+ urn: str,
364
+ nodes: list[dict[str, Any]],
365
+ parent_id: str | None,
366
+ rows: list[list[Any]],
367
+ languages: list[str],
368
+ ) -> None:
369
+ """Recursively flatten hierarchy tree into flat rows."""
370
+ for node in nodes:
371
+ code_urn = node.get("code")
372
+ name_dict = node.get("name")
373
+ desc_dict = node.get("description")
374
+
375
+ name_vals = [name_dict.get(lang) if name_dict else None for lang in languages]
376
+ desc_vals = [desc_dict.get(lang) if desc_dict else None for lang in languages]
377
+ anns_json = json.dumps(node["annotations"]) if node.get("annotations") else None
378
+
379
+ row: list[Any] = [urn, node["id"], parent_id, code_urn]
380
+ row.extend(name_vals)
381
+ row.extend(desc_vals)
382
+ row.extend([anns_json, node.get("valid_from"), node.get("valid_to")])
383
+ rows.append(row)
384
+
385
+ children = node.get("children", [])
386
+ if children:
387
+ _flatten_hierarchy_nodes(urn, children, node["id"], rows, languages)
388
+
389
+
390
+ def put_artefact(conn: duckdb.DuckDBPyConnection, artefact: Any) -> None:
391
+ """Write any supported artefact, dispatching to the appropriate writer.
392
+
393
+ For Codelists, DataStructures, Dataflows, ConceptSchemes, and
394
+ Hierarchies, writes to both the artefacts table and the relevant
395
+ normalized tables. For all other types, writes only to the artefacts
396
+ table with a JSON document.
397
+ """
398
+ if not isinstance(artefact, HasUrn):
399
+ msg = (
400
+ f"{type(artefact).__name__} does not satisfy HasUrn — "
401
+ "it needs sdmx_package, sdmx_class, agency_id, id, version"
402
+ )
403
+ raise TypeError(msg)
404
+
405
+ if isinstance(artefact, Codelist):
406
+ put_codelist(conn, artefact)
407
+ elif isinstance(artefact, DataStructure):
408
+ put_data_structure(conn, artefact)
409
+ elif isinstance(artefact, Dataflow):
410
+ put_dataflow(conn, artefact)
411
+ elif isinstance(artefact, ConceptScheme):
412
+ put_concept_scheme(conn, artefact)
413
+ elif isinstance(artefact, Hierarchy):
414
+ put_hierarchy(conn, artefact)
415
+ else:
416
+ _put_blob(conn, artefact)
417
+
418
+
419
+ def _put_blob(conn: duckdb.DuckDBPyConnection, artefact: Any) -> None:
420
+ """Fallback: write artefact as JSON blob only (no normalized tables)."""
421
+ languages = get_languages(conn)
422
+ urn = str(_artefact_urn(artefact))
423
+ type_name = _CLASS_TO_NAME.get(type(artefact))
424
+ if type_name is None:
425
+ msg = f"Unsupported artefact type: {type(artefact).__name__}"
426
+ raise TypeError(msg)
427
+ doc = json.dumps(to_dict(artefact))
428
+ _put_artefact_row(conn, urn, type_name, artefact, doc, languages)
429
+
430
+
431
+ def drop_artefact(conn: duckdb.DuckDBPyConnection, urn: str) -> bool:
432
+ """Delete an artefact and its child rows (manual cascade).
433
+
434
+ Returns True if the artefact existed and was deleted.
435
+ """
436
+ params = {"urn": urn}
437
+ conn.execute(
438
+ """
439
+ DELETE FROM codes WHERE codelist_urn = $urn
440
+ """,
441
+ params,
442
+ )
443
+ conn.execute(
444
+ """
445
+ DELETE FROM codelists WHERE urn = $urn
446
+ """,
447
+ params,
448
+ )
449
+ conn.execute(
450
+ """
451
+ DELETE FROM dsd_components WHERE dsd_urn = $urn
452
+ """,
453
+ params,
454
+ )
455
+ conn.execute(
456
+ """
457
+ DELETE FROM data_structures WHERE urn = $urn
458
+ """,
459
+ params,
460
+ )
461
+ conn.execute(
462
+ """
463
+ DELETE FROM dataflows WHERE urn = $urn
464
+ """,
465
+ params,
466
+ )
467
+ conn.execute(
468
+ """
469
+ DELETE FROM concepts WHERE scheme_urn = $urn
470
+ """,
471
+ params,
472
+ )
473
+ conn.execute(
474
+ """
475
+ DELETE FROM concept_schemes WHERE urn = $urn
476
+ """,
477
+ params,
478
+ )
479
+ conn.execute(
480
+ """
481
+ DELETE FROM hierarchy_nodes WHERE hierarchy_urn = $urn
482
+ """,
483
+ params,
484
+ )
485
+
486
+ result = conn.execute(
487
+ """
488
+ DELETE FROM artefacts WHERE urn = $urn RETURNING urn
489
+ """,
490
+ params,
491
+ ).fetchone()
492
+ return result is not None
493
+
494
+
495
+ def ingest_registry(conn: duckdb.DuckDBPyConnection, registry: Any) -> None:
496
+ """Bulk-import all artefacts from a Registry into DuckDB storage.
497
+
498
+ Iterates over the registry's internal store and writes each artefact
499
+ using :func:`put_artefact`.
500
+ """
501
+ store: dict[str, Any] = getattr(registry, "_store", {})
502
+ for artefact in store.values():
503
+ put_artefact(conn, artefact)
@@ -0,0 +1,102 @@
1
+ Metadata-Version: 2.3
2
+ Name: sdmxlib
3
+ Version: 0.8.0
4
+ Summary: SDMX structural metadata library for Python
5
+ Keywords: sdmx,statistics,metadata,datastructure
6
+ Author: gabrielgellner
7
+ Author-email: gabrielgellner <gabrielgellner@gmail.com>
8
+ License: Apache-2.0
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.12
14
+ Classifier: Topic :: Scientific/Engineering :: Information Analysis
15
+ Requires-Dist: attrs>=26.1.0
16
+ Requires-Dist: duckdb>=1.0.0
17
+ Requires-Dist: httpx>=0.28.1
18
+ Requires-Dist: lxml>=6.0.2
19
+ Requires-Dist: polars>=1.39.3
20
+ Requires-Python: >=3.12
21
+ Project-URL: Homepage, https://gitlab.com/gabrielgellner/sdmxlib
22
+ Project-URL: Repository, https://gitlab.com/gabrielgellner/sdmxlib
23
+ Project-URL: Documentation, https://gabrielgellner.gitlab.io/sdmxlib
24
+ Project-URL: Changelog, https://gitlab.com/gabrielgellner/sdmxlib/-/blob/main/CHANGELOG.md
25
+ Description-Content-Type: text/markdown
26
+
27
+ # sdmxlib
28
+
29
+ > **Note:** This library has been written extensively with AI assistance
30
+ > (Claude Code). Users who are not comfortable with AI-generated code should
31
+ > take that into account before adopting it.
32
+
33
+ SDMX structural metadata library for Python. Fetch and navigate Data Structure
34
+ Definitions, Codelists, Dataflows, and related artefacts from any SDMX REST
35
+ endpoint — with plain attribute access, no XML or URN chasing required.
36
+
37
+ ## Quick start
38
+
39
+ ```python
40
+ import sdmxlib as sl
41
+
42
+ with sl.RestRegistry(sl.Provider.BIS) as reg:
43
+ dsd = reg.get(sl.DataStructure, agency="BIS", id="WS_CBPOL").resolve()
44
+
45
+ for dim in dsd.dimensions:
46
+ if dim.is_enumerated:
47
+ cl = dim.representation()
48
+ print(f"{dim.id}: {cl.id} ({len(cl.codes)} codes)")
49
+ ```
50
+
51
+ ```
52
+ FREQ: CL_FREQ (7 codes)
53
+ REF_AREA: CL_BIS_IF_REF_AREA (44 codes)
54
+ INSTR_ASSET: CL_INSTR_ASSET (12 codes)
55
+ BORROWER_CTY: CL_BIS_IF_REF_AREA (44 codes)
56
+ ```
57
+
58
+ ## Installation
59
+
60
+ ```bash
61
+ uv add "sdmxlib @ git+https://gitlab.com/gabrielgellner/sdmxlib.git"
62
+ ```
63
+
64
+ ## What it does
65
+
66
+ - Connects to any SDMX 2.1 or 3.0 REST endpoint; built-in shortcuts for ECB,
67
+ BIS, ESTAT, IMF, OECD, and the SDMX Global Registry via `sl.Provider`
68
+ - Three resolution modes: `browse()` for catalogue scanning, `fetch()` for
69
+ unresolved artefacts, `resolve()` for the full reference graph (with optional
70
+ constraint-trimming via `partial=True/False`)
71
+ - Returns fully resolved domain objects with plain attribute access — no XML,
72
+ no URN chasing
73
+ - Filter collections with lambdas or the `item()` expression API
74
+ - Build SDMX artefacts locally and serialise to SDMX-ML or SDMX-JSON
75
+ - `LocalRegistry` -- a DuckDB-backed persistent store for offline access
76
+ with lazy codelist loading, automatic Ref resolution, and direct SQL
77
+ code maps; supports shared connections and configurable language columns
78
+ - Integrates with [Polars](https://pola.rs) for schema generation and data
79
+ validation
80
+ - Fully typed — IDE completion and static analysis work throughout
81
+
82
+ ## Documentation
83
+
84
+ See the [docs](https://gabrielgellner.gitlab.io/sdmxlib/) for the full cookbook,
85
+ including:
86
+
87
+ - Connecting to registries and choosing a resolution mode
88
+ - Persisting artefacts in a local DuckDB store with `LocalRegistry`
89
+ - Fetching observation data with `sl.dim()` filters and pivoting to wide format
90
+ - Inspecting DSDs and building code maps for ETL pipelines
91
+ - Generating Polars schemas from SDMX Data Structure Definitions
92
+ - Browsing data catalogs and filtering dataflows
93
+ - Working with constraints, hierarchies, and serialisation
94
+
95
+ ## Development
96
+
97
+ ```bash
98
+ just check # lint + typecheck
99
+ just test # unit tests
100
+ just test-integration # live endpoint tests (requires network)
101
+ just ci # full pre-commit gate
102
+ ```
@@ -0,0 +1,58 @@
1
+ sdmxlib/__init__.py,sha256=kywSlc7M0H-YIDn7H7re2iebQs5KKtzMN_Az0qgGLDg,3125
2
+ sdmxlib/api/__init__.py,sha256=yNPFuM9w5OEAuXqWzpV9ZxKPOrfg75iF7AwMnTF3wtE,301
3
+ sdmxlib/api/client.py,sha256=hKWcG3Vy5JWjYxNowrI8wiLkAq-naqSJ-w-7kcUd8Lo,8761
4
+ sdmxlib/api/filters.py,sha256=0oH_hFJwGpOQxjnVsbxI3GO-IbxGuIxdR_JlBXFccpo,1966
5
+ sdmxlib/api/providers.py,sha256=8Jxf2WOOIMLl91iwRrOOXmW3VQe34cx7uA4e4FkT9qk,2201
6
+ sdmxlib/api/query.py,sha256=GqjBhYA7iESV97DJnxMOT1DVTAuccxYz3hobFDCco9k,15801
7
+ sdmxlib/api/registry.py,sha256=hAs_0x5plGZ1GfNiMICbrQlb6XRKeOijRAHAtdcOFVQ,25619
8
+ sdmxlib/api/session.py,sha256=8dipduGvKAx19Fd5W4U67_79c_l6V0RL67inUFRmfd0,5288
9
+ sdmxlib/formats/__init__.py,sha256=acm1AHbZwKXbgFD13Ien6yK_X-vuaxRXzT9W6oJ5JCw,5772
10
+ sdmxlib/formats/sdmx_csv/__init__.py,sha256=0_N-eOgYfNrkfakZ1NUwgfxV0EbKZhAmhBy3O8iX9ck,114
11
+ sdmxlib/formats/sdmx_csv/reader.py,sha256=SjMgqc0CYjXprzcJwKkGOOO06aBs8myPFRiMyFyc5jw,4856
12
+ sdmxlib/formats/sdmx_json/__init__.py,sha256=7RrczMcD9X1PRRll5CvYef79PZ88v0JYeLZ_gkusED0,68
13
+ sdmxlib/formats/sdmx_json/reader.py,sha256=5XhRNvYUC91FrTJNJlTQNEfOUqNWfIM2WH07on36xmU,35657
14
+ sdmxlib/formats/sdmx_json/writer.py,sha256=t2cIvMCN1nBsLsxNfVzw73YX17BFxLJkpjKICdRxmoE,25327
15
+ sdmxlib/formats/sdmx_ml21/__init__.py,sha256=sjCtD5QNb6VZrAoq3Tu0yc9lvuXBLrnisk78lFnwUT0,104
16
+ sdmxlib/formats/sdmx_ml21/namespaces.py,sha256=IjlTboUJxrUrSYxUE8fkvQTsHwJsmmwkcOEQEDaGfTE,512
17
+ sdmxlib/formats/sdmx_ml21/reader.py,sha256=REf51p9UxKeZDkjENPkS2bZkP6EpsiQUm3tKKosdVQ0,44203
18
+ sdmxlib/formats/sdmx_ml21/writer.py,sha256=VzDNP4P2jTo1kIZablfcrIefscbgvoZT_MF1GmU3ikw,27998
19
+ sdmxlib/formats/sdmx_ml30/__init__.py,sha256=qngL-lSQued5Qmy-Sfr3X-CRq8tjHAFSCu5oqU7tDMg,33
20
+ sdmxlib/formats/sdmx_ml30/namespaces.py,sha256=orsdl9Il2AiU-tIe6xjoB2nMJyDCsF5WTghqUhHeSII,512
21
+ sdmxlib/formats/sdmx_ml30/reader.py,sha256=FkBh2tNoCSv25Qm_jyTYIocMBBomsTfQu7AUDcFy8FI,43886
22
+ sdmxlib/formats/sdmx_ml30/writer.py,sha256=7e6twXGErIP-ttnWsRkeaD3KyiJqxkZNwb9a50j-RZA,28759
23
+ sdmxlib/local/__init__.py,sha256=ChpMTPfDW1umZRmG5zkc4i3_ahZlDgTBv32WtH9QvsM,233
24
+ sdmxlib/local/_registry.py,sha256=OkYptstdlMEnvjZQAxj2YZg9NreepNS0cCPkcHvGSI8,14280
25
+ sdmxlib/model/__init__.py,sha256=orY-ju8-l8A4h3c_ZF89j_w7PLToPiMDCExU1Z4qUos,3137
26
+ sdmxlib/model/annotations.py,sha256=AqChfYEeZ5ZTgArSi6E9vlGG6U7hHumiRDAqLFoV2aE,3697
27
+ sdmxlib/model/base.py,sha256=ieXvUbilBHnlfczh7OSIBGPWgR9rWN4OiXaGHXLUvmc,1956
28
+ sdmxlib/model/category.py,sha256=KzbJ6QFoXjjtirTZlHYkwbA7yawrpsf_eKdoJpN7GgQ,3649
29
+ sdmxlib/model/codelist.py,sha256=fwTknAnLQCF0WHxwdbdsh5pOkLF7TU1kmMGjXjmTtfI,3488
30
+ sdmxlib/model/collections.py,sha256=TX-vLZ7coJd1HtIxVD7Z6BXXID-K0YV9EwVBuX9BwwM,15486
31
+ sdmxlib/model/concept.py,sha256=BbYRZXPaGrYWUV8iTK0b8ZBQwktaSqzwr1oIqiWs1yE,4606
32
+ sdmxlib/model/constraint.py,sha256=pwL5D05bV2eAIsZ9GivaDAyQtghh4Ws6NaFHMQtaZsI,2920
33
+ sdmxlib/model/convert.py,sha256=5SkonvpiVxeALNHocyWj8k7o7UB3JZ5PAWij6CMqbcc,9436
34
+ sdmxlib/model/dataflow.py,sha256=z_zHg7XCgosK04u5rLUop057A--UqkuWliQXxdnVcYo,1780
35
+ sdmxlib/model/dataset.py,sha256=LLH1oOh92mTWkK9MtxzN8704sMqGosKXPJISFyHjCqQ,14753
36
+ sdmxlib/model/datastructure.py,sha256=X0f7OpsmWc0YeNBljMV9hJDLbksttCLP9mFwOfQMVro,15549
37
+ sdmxlib/model/expr.py,sha256=0a88sKBRUC_sUvvKL066kkX2PFVXXrmJYN24qs3sjps,4299
38
+ sdmxlib/model/hierarchy.py,sha256=qMCYCfanURzAELlfI3dCl16adL-GHi2yP-xTgM1trpY,4165
39
+ sdmxlib/model/istring.py,sha256=9956ucw2730ByddDBTj4SKM0ynvI13nXAJnisvkx8YM,2748
40
+ sdmxlib/model/mapping.py,sha256=yyBDvZy1DSji5PfmC1XamUHF7G3YFm16GK_ZTgduuiY,4335
41
+ sdmxlib/model/message.py,sha256=lwRKRLWwZrgzEelD9-9qpwVG_PVaJwKwzR-tPUaT9vc,3199
42
+ sdmxlib/model/organisation.py,sha256=TCC_YmtGFy3UYr_FZ83b8tuwFcQpvbakKJnjc-0RxL8,6047
43
+ sdmxlib/model/provision.py,sha256=FxMoNFYoVlfuqrUtqm_DXeR8Ey4itEhowh1mDsqMAGc,1653
44
+ sdmxlib/model/ref.py,sha256=zdG6j5z-TyBvKxPxe6HkQ5337b0OkuAjEz-T3PW93XQ,6978
45
+ sdmxlib/model/registry.py,sha256=hQ7dL3BWquYiimidjwddknZiEzHX6dAcnTDrKCy8z4o,7492
46
+ sdmxlib/model/representation.py,sha256=W_5R0_WZjnXy5_7Uc5e0Y4YQGfw-M4Kx9uDQcAV6g_0,3018
47
+ sdmxlib/model/urn.py,sha256=h16i_xSjZKxYwDbBcwGKS9Do-zg5fl5Zc5dRzRevbVM,4621
48
+ sdmxlib/model/validation.py,sha256=KlaC03RNbMLpz56P8PxBci9ndKQVwe-PdhRi5K4Q98c,12991
49
+ sdmxlib/polars.py,sha256=lWO6gwYiS73UKVeofMDWadHCsgd6I_0Vqwsl4xWb-UU,14769
50
+ sdmxlib/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
51
+ sdmxlib/storage/__init__.py,sha256=1TT_oYx3EjtlxBMrJPIHjaJWbTz-X-wSTLEkg6UHLMQ,771
52
+ sdmxlib/storage/lazy.py,sha256=TouG-6UgQ1_z6SSnUr7hbNOaps96MUBcT7ZNG6gJhgA,10637
53
+ sdmxlib/storage/readers.py,sha256=9WfybYgvYV1rTNS1PHkZj0Fd5J0W7vgjbEfvb05d2Xw,10052
54
+ sdmxlib/storage/schema.py,sha256=f65jf2eSaP4Lr5PVGtt7wzk4YWP_Hx844IOI2MWvkZY,9496
55
+ sdmxlib/storage/writers.py,sha256=STbXpol4OtYEifqXF6AtFNCEHpABDLlPWrAOue_da-Q,17182
56
+ sdmxlib-0.8.0.dist-info/WHEEL,sha256=s_zqWxHFEH8b58BCtf46hFCqPaISurdB9R1XJ8za6XI,80
57
+ sdmxlib-0.8.0.dist-info/METADATA,sha256=Y0Je7YwuyR9fbp_TGn618D1shjAvknU0rMG0VMttI1E,3824
58
+ sdmxlib-0.8.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: uv 0.11.6
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any