statcatlib 0.10.0__tar.gz → 0.11.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {statcatlib-0.10.0 → statcatlib-0.11.0}/PKG-INFO +20 -14
- {statcatlib-0.10.0 → statcatlib-0.11.0}/README.md +19 -13
- {statcatlib-0.10.0 → statcatlib-0.11.0}/pyproject.toml +19 -38
- {statcatlib-0.10.0 → statcatlib-0.11.0}/pyproject.toml.orig +33 -71
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/expr.py +1 -1
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/dataset.py +1 -1
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/skos.py +7 -1
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/query.py +63 -12
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/sources/sdmx.py +62 -37
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/lance.py +9 -9
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/scopes.py +28 -7
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/sentinel.py +4 -2
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/store.py +71 -22
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/urn.py +38 -9
- {statcatlib-0.10.0 → statcatlib-0.11.0}/LICENSE +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/__init__.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/bundle.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/cli/__init__.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/__init__.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/agents.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/catalog.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/classification.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/distribution.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/licence.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/quality.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/spatial.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/temporal.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/model/types.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/py.typed +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/sources/__init__.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/sources/ckan.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/__init__.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/row.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/schema.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/store/xhtml.py +0 -0
- {statcatlib-0.10.0 → statcatlib-0.11.0}/src/statcatlib/testing.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: statcatlib
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.11.0
|
|
4
4
|
Summary: Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX
|
|
5
5
|
Keywords: sdmx,dcat,open-data,catalog,duckdb,statistical-data,ckan
|
|
6
6
|
Author: gabrielgellner
|
|
@@ -39,7 +39,7 @@ Description-Content-Type: text/markdown
|
|
|
39
39
|
> take that into account before adopting it.
|
|
40
40
|
|
|
41
41
|
**statcatlib** is a Python library for building, managing, and querying
|
|
42
|
-
statistical metadata catalogs.
|
|
42
|
+
statistical metadata catalogs.
|
|
43
43
|
|
|
44
44
|
**Documentation:** <https://statcat-suite.github.io/statcatlib/>
|
|
45
45
|
|
|
@@ -61,9 +61,9 @@ statcatlib materializes remote catalog and structural metadata into a local
|
|
|
61
61
|
DuckDB database and exposes a fluent Python API for discovery queries —
|
|
62
62
|
filtering by publisher, theme, dimension, code, free text, or provenance lineage.
|
|
63
63
|
|
|
64
|
-
**The catalog is a materialized graph.**
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
**The catalog is a materialized graph.** Ingest pulls the full structural model
|
|
65
|
+
into one database, SQL JOINs are the graph traversal, and a query makes no
|
|
66
|
+
network calls.
|
|
67
67
|
|
|
68
68
|
## Installation
|
|
69
69
|
|
|
@@ -85,7 +85,7 @@ import statcatlib as sc
|
|
|
85
85
|
import statcatlib.query as q
|
|
86
86
|
import sdmxlib as sl
|
|
87
87
|
|
|
88
|
-
# Creates my_catalog/
|
|
88
|
+
# Creates my_catalog/ with meta.duckdb, lake.ducklake and parquet/
|
|
89
89
|
with sc.Catalog("my_catalog").open() as store:
|
|
90
90
|
with sl.RestRegistry(sl.Provider.ESTAT) as reg:
|
|
91
91
|
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id="NAMA_10_GDP"), resolve=True)
|
|
@@ -142,8 +142,13 @@ with sc.Catalog("my_catalog").open() as store:
|
|
|
142
142
|
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id=df_id), resolve=True)
|
|
143
143
|
sc.ingest_sdmx(store, reg.registry)
|
|
144
144
|
|
|
145
|
-
#
|
|
145
|
+
# Write observations to the bundle's DuckLake and bind them to the dataflow
|
|
146
146
|
sc.ingest_data(store, reg, "ESTAT:NAMA_10_GDP(latest)", measure_dim="na_item")
|
|
147
|
+
|
|
148
|
+
# Read them back through the co-located sdmxlib registry
|
|
149
|
+
ds = store.get(sc.AggregateDataset, "ESTAT:NAMA_10_GDP(1.0)")
|
|
150
|
+
obs = store.registry.obs(sl.Dataflow.parse_urn(ds.sdmx_dataflow_urn))
|
|
151
|
+
wide = obs.pivot(on=ds.measure_dim) # a polars.DataFrame
|
|
147
152
|
```
|
|
148
153
|
|
|
149
154
|
### CKAN (open.canada.ca, data.gov, ...)
|
|
@@ -285,7 +290,7 @@ Use `.include()` to declare exactly which relationships to batch-load — the re
|
|
|
285
290
|
are set to an `UNLOADED` sentinel:
|
|
286
291
|
|
|
287
292
|
```python
|
|
288
|
-
#
|
|
293
|
+
# Load publisher, themes and keywords only: 3 queries instead of ~17
|
|
289
294
|
results = (
|
|
290
295
|
store.query(sc.BaseDataset)
|
|
291
296
|
.filter(status="published")
|
|
@@ -295,7 +300,7 @@ results = (
|
|
|
295
300
|
.all()
|
|
296
301
|
)
|
|
297
302
|
|
|
298
|
-
# Explicit full hydration
|
|
303
|
+
# Explicit full hydration shows the cost at the call site
|
|
299
304
|
results = store.query(sc.BaseDataset).filter(...).full().all()
|
|
300
305
|
|
|
301
306
|
# get() always loads everything — no include() needed
|
|
@@ -398,12 +403,13 @@ Three layers share one DuckDB database:
|
|
|
398
403
|
```
|
|
399
404
|
Discovery layer — Catalog (dataset, agent, concept_scheme, concept, distribution, lineage)
|
|
400
405
|
Structural layer — sdmxlib tables (dataflows, dsd_components, codes, codelists)
|
|
401
|
-
Observation layer —
|
|
406
|
+
Observation layer — DuckLake tables (actual time-series data, under parquet/)
|
|
402
407
|
```
|
|
403
408
|
|
|
404
409
|
statcatlib owns the discovery layer. sdmxlib owns the structural layer. Both write
|
|
405
|
-
to the same DuckDB connection
|
|
406
|
-
|
|
410
|
+
to the same DuckDB connection, and queries JOIN freely across both. `ingest_data()`
|
|
411
|
+
writes each dataflow's observations to a DuckLake table and registers an sdmxlib
|
|
412
|
+
`DataflowBinding`, through which `store.registry.obs(flow)` reads them.
|
|
407
413
|
|
|
408
414
|
## Development
|
|
409
415
|
|
|
@@ -415,12 +421,12 @@ just hooks-install
|
|
|
415
421
|
```
|
|
416
422
|
|
|
417
423
|
```bash
|
|
418
|
-
just ci # full gate: index guard + lint + format + docstrings + Any ratchet +
|
|
424
|
+
just ci # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
|
|
419
425
|
just test # unit tests
|
|
420
426
|
just test-integration # integration tests (no network)
|
|
421
427
|
just test-live # live tests against real SDMX endpoints
|
|
422
428
|
just lint # ruff check
|
|
423
|
-
just typecheck #
|
|
429
|
+
just typecheck # pyrefly
|
|
424
430
|
just check-indexes # only public PyPI is named anywhere in the repo
|
|
425
431
|
just docs # local docs server
|
|
426
432
|
```
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
> take that into account before adopting it.
|
|
6
6
|
|
|
7
7
|
**statcatlib** is a Python library for building, managing, and querying
|
|
8
|
-
statistical metadata catalogs.
|
|
8
|
+
statistical metadata catalogs.
|
|
9
9
|
|
|
10
10
|
**Documentation:** <https://statcat-suite.github.io/statcatlib/>
|
|
11
11
|
|
|
@@ -27,9 +27,9 @@ statcatlib materializes remote catalog and structural metadata into a local
|
|
|
27
27
|
DuckDB database and exposes a fluent Python API for discovery queries —
|
|
28
28
|
filtering by publisher, theme, dimension, code, free text, or provenance lineage.
|
|
29
29
|
|
|
30
|
-
**The catalog is a materialized graph.**
|
|
31
|
-
|
|
32
|
-
|
|
30
|
+
**The catalog is a materialized graph.** Ingest pulls the full structural model
|
|
31
|
+
into one database, SQL JOINs are the graph traversal, and a query makes no
|
|
32
|
+
network calls.
|
|
33
33
|
|
|
34
34
|
## Installation
|
|
35
35
|
|
|
@@ -51,7 +51,7 @@ import statcatlib as sc
|
|
|
51
51
|
import statcatlib.query as q
|
|
52
52
|
import sdmxlib as sl
|
|
53
53
|
|
|
54
|
-
# Creates my_catalog/
|
|
54
|
+
# Creates my_catalog/ with meta.duckdb, lake.ducklake and parquet/
|
|
55
55
|
with sc.Catalog("my_catalog").open() as store:
|
|
56
56
|
with sl.RestRegistry(sl.Provider.ESTAT) as reg:
|
|
57
57
|
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id="NAMA_10_GDP"), resolve=True)
|
|
@@ -108,8 +108,13 @@ with sc.Catalog("my_catalog").open() as store:
|
|
|
108
108
|
reg.get(sl.Dataflow.urn_for(agency="ESTAT", id=df_id), resolve=True)
|
|
109
109
|
sc.ingest_sdmx(store, reg.registry)
|
|
110
110
|
|
|
111
|
-
#
|
|
111
|
+
# Write observations to the bundle's DuckLake and bind them to the dataflow
|
|
112
112
|
sc.ingest_data(store, reg, "ESTAT:NAMA_10_GDP(latest)", measure_dim="na_item")
|
|
113
|
+
|
|
114
|
+
# Read them back through the co-located sdmxlib registry
|
|
115
|
+
ds = store.get(sc.AggregateDataset, "ESTAT:NAMA_10_GDP(1.0)")
|
|
116
|
+
obs = store.registry.obs(sl.Dataflow.parse_urn(ds.sdmx_dataflow_urn))
|
|
117
|
+
wide = obs.pivot(on=ds.measure_dim) # a polars.DataFrame
|
|
113
118
|
```
|
|
114
119
|
|
|
115
120
|
### CKAN (open.canada.ca, data.gov, ...)
|
|
@@ -251,7 +256,7 @@ Use `.include()` to declare exactly which relationships to batch-load — the re
|
|
|
251
256
|
are set to an `UNLOADED` sentinel:
|
|
252
257
|
|
|
253
258
|
```python
|
|
254
|
-
#
|
|
259
|
+
# Load publisher, themes and keywords only: 3 queries instead of ~17
|
|
255
260
|
results = (
|
|
256
261
|
store.query(sc.BaseDataset)
|
|
257
262
|
.filter(status="published")
|
|
@@ -261,7 +266,7 @@ results = (
|
|
|
261
266
|
.all()
|
|
262
267
|
)
|
|
263
268
|
|
|
264
|
-
# Explicit full hydration
|
|
269
|
+
# Explicit full hydration shows the cost at the call site
|
|
265
270
|
results = store.query(sc.BaseDataset).filter(...).full().all()
|
|
266
271
|
|
|
267
272
|
# get() always loads everything — no include() needed
|
|
@@ -364,12 +369,13 @@ Three layers share one DuckDB database:
|
|
|
364
369
|
```
|
|
365
370
|
Discovery layer — Catalog (dataset, agent, concept_scheme, concept, distribution, lineage)
|
|
366
371
|
Structural layer — sdmxlib tables (dataflows, dsd_components, codes, codelists)
|
|
367
|
-
Observation layer —
|
|
372
|
+
Observation layer — DuckLake tables (actual time-series data, under parquet/)
|
|
368
373
|
```
|
|
369
374
|
|
|
370
375
|
statcatlib owns the discovery layer. sdmxlib owns the structural layer. Both write
|
|
371
|
-
to the same DuckDB connection
|
|
372
|
-
|
|
376
|
+
to the same DuckDB connection, and queries JOIN freely across both. `ingest_data()`
|
|
377
|
+
writes each dataflow's observations to a DuckLake table and registers an sdmxlib
|
|
378
|
+
`DataflowBinding`, through which `store.registry.obs(flow)` reads them.
|
|
373
379
|
|
|
374
380
|
## Development
|
|
375
381
|
|
|
@@ -381,12 +387,12 @@ just hooks-install
|
|
|
381
387
|
```
|
|
382
388
|
|
|
383
389
|
```bash
|
|
384
|
-
just ci # full gate: index guard + lint + format + docstrings + Any ratchet +
|
|
390
|
+
just ci # full gate: index guard + lint + format + docstrings + prose + Any ratchet + pyrefly + unit and integration tests + strict docs build
|
|
385
391
|
just test # unit tests
|
|
386
392
|
just test-integration # integration tests (no network)
|
|
387
393
|
just test-live # live tests against real SDMX endpoints
|
|
388
394
|
just lint # ruff check
|
|
389
|
-
just typecheck #
|
|
395
|
+
just typecheck # pyrefly
|
|
390
396
|
just check-indexes # only public PyPI is named anywhere in the repo
|
|
391
397
|
just docs # local docs server
|
|
392
398
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "statcatlib"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.11.0"
|
|
4
4
|
description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
|
|
5
5
|
license = "Apache-2.0"
|
|
6
6
|
license-files = ["LICENSE"]
|
|
@@ -71,59 +71,40 @@ allow-init-docstring = true
|
|
|
71
71
|
|
|
72
72
|
[tool.pytest.ini_options]
|
|
73
73
|
testpaths = ["tests"]
|
|
74
|
+
strict = true
|
|
75
|
+
filterwarnings = ["error"]
|
|
74
76
|
markers = [
|
|
75
77
|
"unit: fast, in-process tests with no network I/O",
|
|
76
|
-
"docs: doctests and example-script tests",
|
|
77
78
|
"integration: tests that use real DuckDB and sdmxlib but no network",
|
|
78
79
|
"live: tests that make real HTTP calls to external SDMX endpoints",
|
|
79
80
|
]
|
|
80
81
|
|
|
81
|
-
[tool.
|
|
82
|
-
|
|
83
|
-
typeCheckingMode = "standard"
|
|
84
|
-
venvPath = "."
|
|
85
|
-
venv = ".venv"
|
|
86
|
-
exclude = [
|
|
87
|
-
"scripts",
|
|
88
|
-
".venv",
|
|
89
|
-
]
|
|
90
|
-
reportMissingTypeArgument = "error"
|
|
91
|
-
reportMissingParameterType = "error"
|
|
92
|
-
reportUnnecessaryCast = "error"
|
|
93
|
-
reportInvalidCast = "error"
|
|
94
|
-
reportPropertyTypeMismatch = "error"
|
|
95
|
-
reportInconsistentConstructor = "error"
|
|
96
|
-
reportUnnecessaryContains = "error"
|
|
97
|
-
reportDeprecated = "error"
|
|
98
|
-
reportUnsafeMultipleInheritance = "error"
|
|
99
|
-
reportTypeCommentUsage = "error"
|
|
100
|
-
reportSelfClsParameterName = "error"
|
|
101
|
-
reportUnnecessaryTypeIgnoreComment = "error"
|
|
102
|
-
reportAny = "none"
|
|
103
|
-
reportExplicitAny = "none"
|
|
104
|
-
reportUnknownVariableType = "none"
|
|
105
|
-
|
|
106
|
-
[[tool.basedpyright.executionEnvironments]]
|
|
107
|
-
root = "tests"
|
|
108
|
-
extraPaths = ["."]
|
|
109
|
-
reportMissingParameterType = "none"
|
|
110
|
-
reportTypedDictNotRequiredAccess = "none"
|
|
82
|
+
[tool.coverage.run]
|
|
83
|
+
source = ["src/statcatlib"]
|
|
111
84
|
|
|
112
|
-
[
|
|
113
|
-
|
|
85
|
+
[tool.coverage.report]
|
|
86
|
+
show_missing = true
|
|
87
|
+
skip_covered = false
|
|
88
|
+
fail_under = 85
|
|
89
|
+
exclude_lines = [
|
|
90
|
+
"pragma: no cover",
|
|
91
|
+
"if TYPE_CHECKING:",
|
|
92
|
+
"raise NotImplementedError",
|
|
93
|
+
"@overload",
|
|
94
|
+
]
|
|
114
95
|
|
|
115
96
|
[dependency-groups]
|
|
116
97
|
dev = [
|
|
117
|
-
"basedpyright>=1.38.4",
|
|
118
98
|
"lancedb>=0.39",
|
|
119
99
|
"mkdocs-material>=9.7.6",
|
|
120
100
|
"polars>=1.39.3",
|
|
121
|
-
"pydoclint>=0.
|
|
122
|
-
"pyrefly>=1.
|
|
101
|
+
"pydoclint>=0.11.1,<0.12",
|
|
102
|
+
"pyrefly>=1.4.0,<1.5",
|
|
123
103
|
"pytest>=9.0.3",
|
|
104
|
+
"pytest-cov>=7.1.0",
|
|
124
105
|
"pytest-httpx>=0.35",
|
|
125
106
|
"rich>=15.0",
|
|
126
|
-
"ruff>=0.
|
|
107
|
+
"ruff>=0.17.1,<0.18",
|
|
127
108
|
"sentence-transformers>=3.0",
|
|
128
109
|
"typer>=0.15",
|
|
129
110
|
"watchfiles>=1.1.1",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "statcatlib"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.11.0"
|
|
4
4
|
description = "Queryable open data catalog engine for DCAT-AP, StatDCAT-AP, CKAN, and SDMX"
|
|
5
5
|
license = "Apache-2.0"
|
|
6
6
|
license-files = ["LICENSE"]
|
|
@@ -74,91 +74,53 @@ allow-init-docstring = true
|
|
|
74
74
|
|
|
75
75
|
[tool.pytest.ini_options]
|
|
76
76
|
testpaths = ["tests"]
|
|
77
|
+
# `strict` makes an unregistered marker a collection error: a typo such as
|
|
78
|
+
# `pytest.mark.uint` would otherwise drop a whole file out of `just ci`, which
|
|
79
|
+
# selects by marker (GH-8). It also rejects duplicate parametrize ids.
|
|
80
|
+
strict = true
|
|
81
|
+
# A warning that cannot fail is a warning nobody reads. The suite raised seven
|
|
82
|
+
# when this went on, all statcatlib's own: each test now asserts the warning
|
|
83
|
+
# its setup provokes, or avoids it as a real caller would. No ignore list.
|
|
84
|
+
filterwarnings = ["error"]
|
|
77
85
|
markers = [
|
|
78
86
|
"unit: fast, in-process tests with no network I/O",
|
|
79
|
-
"docs: doctests and example-script tests",
|
|
80
87
|
"integration: tests that use real DuckDB and sdmxlib but no network",
|
|
81
88
|
"live: tests that make real HTTP calls to external SDMX endpoints",
|
|
82
89
|
]
|
|
83
90
|
|
|
91
|
+
[tool.coverage.run]
|
|
92
|
+
source = ["src/statcatlib"]
|
|
93
|
+
|
|
94
|
+
[tool.coverage.report]
|
|
95
|
+
show_missing = true
|
|
96
|
+
skip_covered = false
|
|
97
|
+
# 88% when this was set (2026-10-23, `just coverage`). A floor, a little under
|
|
98
|
+
# that, so a change that drops a module's tests fails here. `sources/sdmx.py`
|
|
99
|
+
# is the lowest at 54%: `ingest_data` is reached only by the `live` tier.
|
|
100
|
+
fail_under = 85
|
|
101
|
+
exclude_lines = [
|
|
102
|
+
"pragma: no cover",
|
|
103
|
+
"if TYPE_CHECKING:",
|
|
104
|
+
"raise NotImplementedError",
|
|
105
|
+
"@overload",
|
|
106
|
+
]
|
|
107
|
+
|
|
84
108
|
[dependency-groups]
|
|
85
109
|
dev = [
|
|
86
|
-
"basedpyright>=1.38.4",
|
|
87
110
|
"lancedb>=0.39",
|
|
88
111
|
"mkdocs-material>=9.7.6",
|
|
89
112
|
"polars>=1.39.3",
|
|
90
|
-
|
|
91
|
-
|
|
113
|
+
# The checkers are capped, as sdmxlib's are: uv.lock is gitignored, so an
|
|
114
|
+
# uncapped checker's next release would change what `just ci` reports
|
|
115
|
+
# without a commit. A bump goes in its own PR, with its findings fixed.
|
|
116
|
+
"pydoclint>=0.11.1,<0.12",
|
|
117
|
+
"pyrefly>=1.4.0,<1.5",
|
|
92
118
|
"pytest>=9.0.3",
|
|
119
|
+
"pytest-cov>=7.1.0",
|
|
93
120
|
"pytest-httpx>=0.35",
|
|
94
121
|
"rich>=15.0",
|
|
95
|
-
"ruff>=0.
|
|
122
|
+
"ruff>=0.17.1,<0.18",
|
|
96
123
|
"sentence-transformers>=3.0",
|
|
97
124
|
"typer>=0.15",
|
|
98
125
|
"watchfiles>=1.1.1",
|
|
99
126
|
]
|
|
100
|
-
|
|
101
|
-
[tool.basedpyright]
|
|
102
|
-
pythonVersion = "3.14"
|
|
103
|
-
typeCheckingMode = "standard"
|
|
104
|
-
venvPath = "."
|
|
105
|
-
venv = ".venv"
|
|
106
|
-
exclude = ["scripts", ".venv"]
|
|
107
|
-
# Strict-mode rules that are clean on this codebase (0 findings each, measured
|
|
108
|
-
# on the commit that turned them on), enabled as regression guards rather than
|
|
109
|
-
# as a cleanup. Borrowed from sdmxlib, which runs the same set — see #106.
|
|
110
|
-
#
|
|
111
|
-
# The first two are basedpyright's Any-degradation catchers: a bare generic
|
|
112
|
-
# (`list` rather than `list[str]`) and an untyped parameter both silently
|
|
113
|
-
# become Any.
|
|
114
|
-
reportMissingTypeArgument = "error"
|
|
115
|
-
reportMissingParameterType = "error"
|
|
116
|
-
# Dead casts, getter/setter drift, __init__/__new__ disagreement, and
|
|
117
|
-
# always-true/false `in` checks. Pure hardening.
|
|
118
|
-
reportUnnecessaryCast = "error"
|
|
119
|
-
reportInvalidCast = "error"
|
|
120
|
-
reportPropertyTypeMismatch = "error"
|
|
121
|
-
reportInconsistentConstructor = "error"
|
|
122
|
-
reportUnnecessaryContains = "error"
|
|
123
|
-
reportDeprecated = "error"
|
|
124
|
-
reportUnsafeMultipleInheritance = "error"
|
|
125
|
-
reportTypeCommentUsage = "error"
|
|
126
|
-
reportSelfClsParameterName = "error"
|
|
127
|
-
# Suppression hygiene, and the rule that keeps the .claude suppression guard
|
|
128
|
-
# honest in the other direction: an ignore that outlived its defect is a lie
|
|
129
|
-
# the next reader believes. Found 6 the day it went on, none of which were
|
|
130
|
-
# hiding anything.
|
|
131
|
-
reportUnnecessaryTypeIgnoreComment = "error"
|
|
132
|
-
#
|
|
133
|
-
# NOT enabled: reportCallInDefaultInitializer (23 findings). `ruff.toml`
|
|
134
|
-
# already ignores B008, which is the same concern — turning this on would
|
|
135
|
-
# contradict a decision the project has already made. Recorded here so the
|
|
136
|
-
# next person diffing against sdmxlib's config does not re-litigate it.
|
|
137
|
-
reportAny = "none"
|
|
138
|
-
reportExplicitAny = "none"
|
|
139
|
-
reportUnknownVariableType = "none"
|
|
140
|
-
|
|
141
|
-
# `tests/` is type-checked too (#116) — the gap let a wrong `tmp_path`
|
|
142
|
-
# annotation stand behind 25 suppressions, and hid three dead ignores from
|
|
143
|
-
# `reportUnnecessaryTypeIgnoreComment`, which exists to catch exactly those.
|
|
144
|
-
#
|
|
145
|
-
# Annotation-completeness rules stay off there. `ruff.toml` already exempts
|
|
146
|
-
# tests from `ANN`; demanding parameter types here would contradict that
|
|
147
|
-
# decision rather than extend it. Measured: 285 of basedpyright's 416
|
|
148
|
-
# findings over `tests/` were this one rule, and neither bug the gap hid was
|
|
149
|
-
# an annotation-coverage finding.
|
|
150
|
-
[[tool.basedpyright.executionEnvironments]]
|
|
151
|
-
root = "tests"
|
|
152
|
-
# `from tests.conftest import …` is how the suite shares fixtures; without the
|
|
153
|
-
# repo root on the path the package does not resolve from inside `tests`.
|
|
154
|
-
extraPaths = ["."]
|
|
155
|
-
reportMissingParameterType = "none"
|
|
156
|
-
# 48 — a test collects rows of one kind and reads that kind's keys, which
|
|
157
|
-
# `SearchRow` cannot express: every extension key is `NotRequired` because it
|
|
158
|
-
# belongs to one row kind, deliberately and per its docstring. The rule is
|
|
159
|
-
# right in `src`, where the kind is not known locally, and cannot see the
|
|
160
|
-
# narrowing a test performs by construction.
|
|
161
|
-
reportTypedDictNotRequiredAccess = "none"
|
|
162
|
-
|
|
163
|
-
[[tool.basedpyright.executionEnvironments]]
|
|
164
|
-
root = "src"
|
|
@@ -119,7 +119,7 @@ class AggregateDataset(BaseDataset):
|
|
|
119
119
|
num_series: int | None = None
|
|
120
120
|
# SDMX link: FK into sdmxlib's dataflows.urn in the shared DuckDB
|
|
121
121
|
sdmx_dataflow_urn: str | None = None
|
|
122
|
-
#
|
|
122
|
+
# Default pivot axis for readers, recorded by ingest_data(). None = the DSD's MeasureDimension.
|
|
123
123
|
measure_dim: str | None = None
|
|
124
124
|
|
|
125
125
|
|
|
@@ -134,8 +134,14 @@ def is_concept_in_scheme(scheme: ConceptScheme, notation: str) -> bool:
|
|
|
134
134
|
Pure-Python — no DuckDB or any other storage involved.
|
|
135
135
|
Useful for validation pipelines that operate on in-memory domain objects.
|
|
136
136
|
|
|
137
|
-
|
|
137
|
+
Args:
|
|
138
|
+
scheme: The concept scheme.
|
|
139
|
+
notation: The concept's notation.
|
|
138
140
|
|
|
141
|
+
Returns:
|
|
142
|
+
Whether *scheme* lists *notation* among its concepts.
|
|
143
|
+
|
|
144
|
+
Example:
|
|
139
145
|
valid = is_concept_in_scheme(product_type_scheme, dataset.dataset_type.notation)
|
|
140
146
|
"""
|
|
141
147
|
return notation in scheme.concepts
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Composable filter objects for Catalog queries.
|
|
2
2
|
|
|
3
|
-
Usage
|
|
3
|
+
Usage:
|
|
4
4
|
|
|
5
5
|
import statcatlib.query as q
|
|
6
6
|
|
|
@@ -630,8 +630,7 @@ def within(
|
|
|
630
630
|
``q.hcode.ancestors``, which excludes the node itself.
|
|
631
631
|
codelist_urn: Optional codelist URN to disambiguate same-named dimensions across DSDs.
|
|
632
632
|
|
|
633
|
-
Example
|
|
634
|
-
|
|
633
|
+
Example:
|
|
635
634
|
store.query(sc.AggregateDataset).filter(
|
|
636
635
|
q.within("REF_AREA", "ON", hierarchy=h_geo_statistical_urn)
|
|
637
636
|
)
|
|
@@ -653,8 +652,13 @@ def has_dimensions(dims: list[str]) -> SdmxDimensionFilter | _NoOpFilter:
|
|
|
653
652
|
JOINs through ``sdmx.dataflow → sdmx.dsd_component``.
|
|
654
653
|
An empty list matches all datasets.
|
|
655
654
|
|
|
656
|
-
|
|
655
|
+
Args:
|
|
656
|
+
dims: The dimension ids every matching DSD must have.
|
|
657
|
+
|
|
658
|
+
Returns:
|
|
659
|
+
The filter, or one that matches everything when *dims* is empty.
|
|
657
660
|
|
|
661
|
+
Example:
|
|
658
662
|
store.query(sc.AggregateDataset).filter(q.has_dimensions(["GEO", "SEX"])).execute()
|
|
659
663
|
"""
|
|
660
664
|
if not dims:
|
|
@@ -676,8 +680,14 @@ def distribution(**kwargs: object) -> ExistsFilter:
|
|
|
676
680
|
.filter(q.distribution(format="CSV"))
|
|
677
681
|
.filter(q.distribution(format="JSON"))
|
|
678
682
|
|
|
679
|
-
|
|
683
|
+
Args:
|
|
684
|
+
**kwargs: Column values one distribution must all match: ``format``,
|
|
685
|
+
``media_type``, ``service_type`` or ``endpoint_url``.
|
|
686
|
+
|
|
687
|
+
Returns:
|
|
688
|
+
An EXISTS filter over the dataset's distributions.
|
|
680
689
|
|
|
690
|
+
Example:
|
|
681
691
|
store.query(sc.BaseDataset).filter(q.distribution(media_type="text/csv")).execute()
|
|
682
692
|
"""
|
|
683
693
|
service_keys = {"service_type", "endpoint_url"}
|
|
@@ -709,8 +719,13 @@ def has_service(**kwargs: object) -> ExistsFilter:
|
|
|
709
719
|
|
|
710
720
|
Supported keys: ``service_type``, ``endpoint_url``.
|
|
711
721
|
|
|
712
|
-
|
|
722
|
+
Args:
|
|
723
|
+
**kwargs: Column values the data service must all match.
|
|
724
|
+
|
|
725
|
+
Returns:
|
|
726
|
+
An EXISTS filter over the dataset's distributions and their services.
|
|
713
727
|
|
|
728
|
+
Example:
|
|
714
729
|
store.query(sc.BaseDataset).filter(
|
|
715
730
|
q.has_service(endpoint_url="https://example.com/sdmx")
|
|
716
731
|
).execute()
|
|
@@ -849,8 +864,14 @@ def publishes(*inner_filters: PositionalFilter, **kwargs: object) -> ExistsFilte
|
|
|
849
864
|
positional arguments, and/or dataset-level conditions as keyword arguments
|
|
850
865
|
(``kind``, ``status``).
|
|
851
866
|
|
|
852
|
-
|
|
867
|
+
Args:
|
|
868
|
+
*inner_filters: Filters a published dataset must match.
|
|
869
|
+
**kwargs: Dataset columns a published dataset must match.
|
|
853
870
|
|
|
871
|
+
Returns:
|
|
872
|
+
An EXISTS filter over the agent's datasets.
|
|
873
|
+
|
|
874
|
+
Example:
|
|
854
875
|
# Agents that publish aggregate datasets about Ontario
|
|
855
876
|
store.query(sc.Agent).filter(
|
|
856
877
|
q.publishes(q.has_code("GEO", "ON"), kind="aggregate")
|
|
@@ -865,8 +886,14 @@ def serves(*inner_filters: PositionalFilter, **kwargs: object) -> ExistsFilter:
|
|
|
865
886
|
Pass ``ExistsFilter`` objects as positional arguments and/or dataset-level
|
|
866
887
|
conditions as keyword arguments (``kind``, ``status``).
|
|
867
888
|
|
|
868
|
-
|
|
889
|
+
Args:
|
|
890
|
+
*inner_filters: Filters a served dataset must match.
|
|
891
|
+
**kwargs: Dataset columns a served dataset must match.
|
|
892
|
+
|
|
893
|
+
Returns:
|
|
894
|
+
An EXISTS filter over the service's datasets.
|
|
869
895
|
|
|
896
|
+
Example:
|
|
870
897
|
# DataServices that serve datasets with a GEO dimension
|
|
871
898
|
store.query(sc.DataService).filter(q.serves(q.has_dimensions(["GEO"]))).all()
|
|
872
899
|
"""
|
|
@@ -881,8 +908,14 @@ def serves(*inner_filters: PositionalFilter, **kwargs: object) -> ExistsFilter:
|
|
|
881
908
|
def has_spatial(label: str, *, lang: str = "en") -> ExistsFilter:
|
|
882
909
|
"""Filter datasets that have a spatial coverage matching *label*.
|
|
883
910
|
|
|
884
|
-
|
|
911
|
+
Args:
|
|
912
|
+
label: The spatial coverage label, matched exactly.
|
|
913
|
+
lang: The language of *label*.
|
|
885
914
|
|
|
915
|
+
Returns:
|
|
916
|
+
An EXISTS filter over the dataset's spatial coverage.
|
|
917
|
+
|
|
918
|
+
Example:
|
|
886
919
|
store.query(sc.BaseDataset).filter(q.has_spatial("Canada")).execute()
|
|
887
920
|
"""
|
|
888
921
|
return ExistsFilter(
|
|
@@ -894,8 +927,14 @@ def has_spatial(label: str, *, lang: str = "en") -> ExistsFilter:
|
|
|
894
927
|
def has_keyword(keyword: str, *, lang: str = "en") -> ExistsFilter:
|
|
895
928
|
"""Filter datasets that have a keyword matching *keyword*.
|
|
896
929
|
|
|
897
|
-
|
|
930
|
+
Args:
|
|
931
|
+
keyword: The keyword, matched exactly.
|
|
932
|
+
lang: The language of *keyword*.
|
|
933
|
+
|
|
934
|
+
Returns:
|
|
935
|
+
An EXISTS filter over the dataset's keywords.
|
|
898
936
|
|
|
937
|
+
Example:
|
|
899
938
|
store.query(sc.BaseDataset).filter(q.has_keyword("employment")).execute()
|
|
900
939
|
"""
|
|
901
940
|
return ExistsFilter(
|
|
@@ -907,8 +946,14 @@ def has_keyword(keyword: str, *, lang: str = "en") -> ExistsFilter:
|
|
|
907
946
|
def title_contains(text: str, *, lang: str = "en") -> MapFilter:
|
|
908
947
|
"""Filter datasets whose title contains *text* (case-insensitive).
|
|
909
948
|
|
|
910
|
-
|
|
949
|
+
Args:
|
|
950
|
+
text: The text to find, anywhere in the title.
|
|
951
|
+
lang: Which language's title to search.
|
|
952
|
+
|
|
953
|
+
Returns:
|
|
954
|
+
A filter on the dataset's title in *lang*.
|
|
911
955
|
|
|
956
|
+
Example:
|
|
912
957
|
store.query(sc.BaseDataset).filter(q.title_contains("GDP")).execute()
|
|
913
958
|
"""
|
|
914
959
|
return MapFilter(column="title", key=lang, value=text, op="ilike")
|
|
@@ -917,8 +962,14 @@ def title_contains(text: str, *, lang: str = "en") -> MapFilter:
|
|
|
917
962
|
def description_contains(text: str, *, lang: str = "en") -> MapFilter:
|
|
918
963
|
"""Filter datasets whose description contains *text* (case-insensitive).
|
|
919
964
|
|
|
920
|
-
|
|
965
|
+
Args:
|
|
966
|
+
text: The text to find, anywhere in the description.
|
|
967
|
+
lang: Which language's description to search.
|
|
968
|
+
|
|
969
|
+
Returns:
|
|
970
|
+
A filter on the dataset's description in *lang*.
|
|
921
971
|
|
|
972
|
+
Example:
|
|
922
973
|
store.query(sc.BaseDataset).filter(q.description_contains("exchange")).execute()
|
|
923
974
|
"""
|
|
924
975
|
return MapFilter(column="description", key=lang, value=text, op="ilike")
|
|
@@ -6,7 +6,7 @@ This is the only statcatlib module that imports sdmxlib beyond InternationalStri
|
|
|
6
6
|
from __future__ import annotations
|
|
7
7
|
|
|
8
8
|
import re
|
|
9
|
-
from typing import TYPE_CHECKING
|
|
9
|
+
from typing import TYPE_CHECKING, Protocol
|
|
10
10
|
|
|
11
11
|
from sdmxlib import InternationalString
|
|
12
12
|
from sdmxlib.model.dataflow import Dataflow
|
|
@@ -16,14 +16,16 @@ from statcatlib.model.agents import Agent
|
|
|
16
16
|
from statcatlib.model.dataset import AggregateDataset
|
|
17
17
|
|
|
18
18
|
if TYPE_CHECKING:
|
|
19
|
-
from sdmxlib
|
|
19
|
+
from sdmxlib import ObsQuery
|
|
20
20
|
from sdmxlib.local.registry import LocalRegistry
|
|
21
21
|
from sdmxlib.model.in_memory_registry import InMemoryRegistry
|
|
22
|
+
from sdmxlib.model.urn import SdmxUrn
|
|
22
23
|
|
|
23
24
|
from statcatlib.model.skos import ConceptRef
|
|
24
25
|
from statcatlib.store.store import Catalog
|
|
25
26
|
|
|
26
27
|
__all__ = [
|
|
28
|
+
"ObservationSource",
|
|
27
29
|
"dataset_from_dataflow",
|
|
28
30
|
"ingest_data",
|
|
29
31
|
"ingest_sdmx",
|
|
@@ -34,6 +36,17 @@ __all__ = [
|
|
|
34
36
|
_IDENTIFIER_RE = re.compile(r"^([^:]+):([^(]+)\(([^)]+)\)$")
|
|
35
37
|
|
|
36
38
|
|
|
39
|
+
class ObservationSource(Protocol):
|
|
40
|
+
"""Where `ingest_data` reads observations from.
|
|
41
|
+
|
|
42
|
+
`sdmxlib.RestRegistry` is the usual one; `sdmxlib.LocalRegistry` fits too.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def obs(self, flow: SdmxUrn[Dataflow], /) -> ObsQuery:
|
|
46
|
+
"""A lazy query over *flow*'s observations."""
|
|
47
|
+
...
|
|
48
|
+
|
|
49
|
+
|
|
37
50
|
def _resolve_latest(store: Catalog, identifier: str) -> str:
|
|
38
51
|
"""Resolve ``(latest)`` to the highest stored version for that agency:id."""
|
|
39
52
|
if not identifier.endswith("(latest)"):
|
|
@@ -237,31 +250,40 @@ def _query_dimension_names(conn: object, dataflow_urn: str) -> list[Internationa
|
|
|
237
250
|
|
|
238
251
|
def ingest_data(
|
|
239
252
|
store: Catalog,
|
|
240
|
-
reg:
|
|
253
|
+
reg: ObservationSource,
|
|
241
254
|
identifier: str,
|
|
242
255
|
*,
|
|
243
256
|
measure_dim: str | None = None,
|
|
244
257
|
) -> str:
|
|
245
258
|
"""Materialise observation data for one dataflow into the bundle's lake.
|
|
246
259
|
|
|
247
|
-
Fetches the observations
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
260
|
+
Fetches the observations from *reg* in the long layout, one row per
|
|
261
|
+
observation with a column per dimension and measure, and writes them into
|
|
262
|
+
``lake.<agency>.df_<id>_<version>`` through the bundle's DuckLake-backed
|
|
263
|
+
data store. A re-run replaces the table. It then registers a
|
|
264
|
+
`sdmxlib.DataflowBinding` for that table, so the dataflow reads back
|
|
265
|
+
through the co-located registry::
|
|
266
|
+
|
|
267
|
+
obs = cat.registry.obs(flow_urn)
|
|
268
|
+
obs.filter(obs.col("GEO") == "CA").frame()
|
|
269
|
+
obs.pivot(on=ds.measure_dim)
|
|
252
270
|
|
|
253
271
|
Requires that ``ingest_sdmx`` has already been called for this
|
|
254
|
-
identifier so the catalog record
|
|
255
|
-
folder-mode `Catalog` (a DuckLake is ATTACHed on
|
|
272
|
+
identifier, so the catalog record and the DSD are stored, and that
|
|
273
|
+
``store`` is a folder-mode `Catalog` (a DuckLake is ATTACHed on
|
|
274
|
+
``open()``).
|
|
256
275
|
|
|
257
276
|
Args:
|
|
258
277
|
store: The catalog holding the `AggregateDataset` for *identifier*.
|
|
259
278
|
Must be folder-mode — a DuckLake is ATTACHed on ``open()``.
|
|
260
|
-
reg:
|
|
279
|
+
reg: Where to fetch observations from, usually a
|
|
280
|
+
`sdmxlib.RestRegistry`.
|
|
261
281
|
identifier: Catalog identifier in ``AGENCY:ID(VERSION)`` form.
|
|
262
282
|
``(latest)`` resolves to the highest stored version.
|
|
263
|
-
measure_dim:
|
|
264
|
-
|
|
283
|
+
measure_dim: The dimension readers pivot on, recorded as
|
|
284
|
+
`AggregateDataset.measure_dim`. It does not change what is
|
|
285
|
+
stored. ``None`` (default) records none, and ``pivot()`` then
|
|
286
|
+
uses the DSD's ``MeasureDimension``.
|
|
265
287
|
|
|
266
288
|
Returns:
|
|
267
289
|
The fully-qualified table location (``<schema>.<table>``).
|
|
@@ -271,9 +293,12 @@ def ingest_data(
|
|
|
271
293
|
``ingest_sdmx`` first.
|
|
272
294
|
ValueError: The dataset carries no ``sdmx_dataflow_urn``, so there is
|
|
273
295
|
nothing to fetch observations for; *identifier* is not in
|
|
274
|
-
``AGENCY:ID(VERSION)`` form;
|
|
275
|
-
|
|
296
|
+
``AGENCY:ID(VERSION)`` form; the dataflow's DSD is not stored, so
|
|
297
|
+
no binding can describe the table; or *measure_dim* is not one of
|
|
298
|
+
the DSD's dimensions.
|
|
276
299
|
"""
|
|
300
|
+
import sdmxlib as sl # noqa: PLC0415
|
|
301
|
+
|
|
277
302
|
identifier = _resolve_latest(store, identifier)
|
|
278
303
|
|
|
279
304
|
ds = store.get(AggregateDataset, identifier)
|
|
@@ -290,31 +315,31 @@ def ingest_data(
|
|
|
290
315
|
raise ValueError(msg)
|
|
291
316
|
agency, flow_id, version = m.group(1), m.group(2), m.group(3)
|
|
292
317
|
|
|
293
|
-
schema = agency.lower()
|
|
294
|
-
table = f"df_{flow_id}_{version.replace('.', '_')}".lower()
|
|
295
|
-
|
|
296
|
-
# Pull the wide dataframe and write through the bundle's lake. ``pivot()``
|
|
297
|
-
# returns an eager DataFrame; with ``on=None`` it pivots on the DSD's
|
|
298
|
-
# MeasureDimension, as ``reg.data(...).pivot()`` did.
|
|
299
318
|
flow_urn = Dataflow.parse_urn(ds.sdmx_dataflow_urn)
|
|
300
|
-
|
|
301
|
-
|
|
319
|
+
dsd = store.registry.dsd(flow_urn)
|
|
320
|
+
if dsd is None:
|
|
321
|
+
msg = f"The DSD of {ds.sdmx_dataflow_urn} is not stored — run ingest_sdmx with resolve=True first"
|
|
322
|
+
raise ValueError(msg)
|
|
323
|
+
dimensions = [d.id for d in dsd.dimensions]
|
|
324
|
+
if measure_dim is not None and measure_dim not in dimensions:
|
|
325
|
+
msg = f"measure_dim {measure_dim!r} is not a dimension of {dsd.id}: {', '.join(dimensions)}"
|
|
326
|
+
raise ValueError(msg)
|
|
302
327
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
import sdmxlib as sl # noqa: PLC0415
|
|
328
|
+
schema = agency.lower()
|
|
329
|
+
table = f"df_{flow_id}_{version.replace('.', '_')}".lower()
|
|
306
330
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
331
|
+
# Store the long layout. The binding below maps each DSD component to a
|
|
332
|
+
# column of the same name, which is the long layout; a pivoted table has a
|
|
333
|
+
# column per code instead and cannot be read through it (GH-29).
|
|
334
|
+
store.data.write_observations(schema=schema, table=table, source=reg.obs(flow_urn).frame())
|
|
335
|
+
store.registry.add(
|
|
336
|
+
sl.DataflowBinding.from_dsd(
|
|
337
|
+
dsd,
|
|
338
|
+
dataflow=sl.Ref[Dataflow].unresolved(flow_urn),
|
|
339
|
+
schema=schema,
|
|
340
|
+
table=table,
|
|
341
|
+
)
|
|
342
|
+
)
|
|
318
343
|
|
|
319
344
|
ds.measure_dim = measure_dim
|
|
320
345
|
store.put(ds)
|
|
@@ -933,18 +933,18 @@ class _LanguageIndex:
|
|
|
933
933
|
) -> list[Bm25HitRow]:
|
|
934
934
|
"""Run a BM25 query, returning raw result rows.
|
|
935
935
|
|
|
936
|
-
The `#
|
|
937
|
-
suppression, and is the shape sdmxlib's
|
|
938
|
-
gradual-typing boundary: the one unsafe
|
|
939
|
-
that performs it. `to_list()` is declared `List[dict[Unknown, Unknown]]`
|
|
936
|
+
The `# pyrefly: ignore[bad-return]` below (and its twin in `vector`)
|
|
937
|
+
is this module's only suppression, and is the shape sdmxlib's
|
|
938
|
+
CLAUDE.md prescribes for a gradual-typing boundary: the one unsafe
|
|
939
|
+
step, kept to the accessor that performs it. `to_list()` is declared `List[dict[Unknown, Unknown]]`
|
|
940
940
|
— LanceDB hands back rows of the schema *this module declared* and
|
|
941
941
|
says nothing about their shape. Validating 19 keys per row on every
|
|
942
942
|
search to recover what `make_schema` already guarantees would cost
|
|
943
943
|
more than it proves.
|
|
944
944
|
|
|
945
|
-
Not `# type: ignore`: that form
|
|
946
|
-
|
|
947
|
-
`
|
|
945
|
+
Not `# type: ignore`: that form names no checker and no error kind,
|
|
946
|
+
so a dead one cannot be told from a live one, and pyrefly's
|
|
947
|
+
`unused-ignore` stops being trustworthy.
|
|
948
948
|
"""
|
|
949
949
|
if self._tbl is None:
|
|
950
950
|
return []
|
|
@@ -957,7 +957,7 @@ class _LanguageIndex:
|
|
|
957
957
|
where = self._where(*filters)
|
|
958
958
|
if where:
|
|
959
959
|
search = search.where(where, prefilter=True)
|
|
960
|
-
return search.limit(limit).select(select_cols).to_list() #
|
|
960
|
+
return search.limit(limit).select(select_cols).to_list() # pyrefly: ignore[bad-return]
|
|
961
961
|
except Exception: # noqa: BLE001
|
|
962
962
|
return []
|
|
963
963
|
|
|
@@ -978,7 +978,7 @@ class _LanguageIndex:
|
|
|
978
978
|
where = self._where(*filters)
|
|
979
979
|
if where:
|
|
980
980
|
search = search.where(where, prefilter=True)
|
|
981
|
-
return search.limit(limit).select(select_cols).to_list() #
|
|
981
|
+
return search.limit(limit).select(select_cols).to_list() # pyrefly: ignore[bad-return]
|
|
982
982
|
except Exception: # noqa: BLE001
|
|
983
983
|
return []
|
|
984
984
|
|
|
@@ -4,7 +4,7 @@ Scopes represent positions in the catalog graph. Navigation builds new
|
|
|
4
4
|
scopes without executing SQL; only terminal methods (``.collect()``,
|
|
5
5
|
``.count()``) hit the database.
|
|
6
6
|
|
|
7
|
-
Usage
|
|
7
|
+
Usage:
|
|
8
8
|
|
|
9
9
|
store.themes["https://example.com/themes"].collect() # ItemList[ConceptRef]
|
|
10
10
|
store.themes["https://example.com/themes"]["13"].collect() # Concept
|
|
@@ -143,9 +143,10 @@ class _SdmxEdge(TypedDict):
|
|
|
143
143
|
#:
|
|
144
144
|
#: Discriminated on an explicit ``kind`` rather than on key presence. Testing
|
|
145
145
|
#: ``"custom_count_sql" in edge`` to conclude ``custom_exists_sql`` also exists
|
|
146
|
-
#: is sound given the TypedDicts, but
|
|
147
|
-
#:
|
|
148
|
-
#: whether a branch is safe is reason enough to say
|
|
146
|
+
#: is sound given the TypedDicts, but pyrefly reports reading the second key as
|
|
147
|
+
#: an error, where basedpyright (run beside it until GH-7) narrowed it. Two
|
|
148
|
+
#: checkers disagreeing about whether a branch is safe is reason enough to say
|
|
149
|
+
#: which branch it is.
|
|
149
150
|
type _Edge = _BridgeEdge | _SdmxEdge
|
|
150
151
|
|
|
151
152
|
|
|
@@ -1131,8 +1132,16 @@ class DatasetScope:
|
|
|
1131
1132
|
|
|
1132
1133
|
Supported edges: ``"codelist"``.
|
|
1133
1134
|
|
|
1134
|
-
|
|
1135
|
+
Args:
|
|
1136
|
+
edge: The kind of entity the datasets share.
|
|
1137
|
+
|
|
1138
|
+
Returns:
|
|
1139
|
+
A query over the datasets linked to this one through *edge*.
|
|
1135
1140
|
|
|
1141
|
+
Raises:
|
|
1142
|
+
ValueError: *edge* is not a supported edge type.
|
|
1143
|
+
|
|
1144
|
+
Example:
|
|
1136
1145
|
store.dataset("id").related("codelist").include("publisher").limit(10).collect()
|
|
1137
1146
|
"""
|
|
1138
1147
|
from statcatlib.model.dataset import BaseDataset # noqa: PLC0415
|
|
@@ -1163,8 +1172,14 @@ class DatasetScope:
|
|
|
1163
1172
|
|
|
1164
1173
|
Gremlin equivalent: ``g.V(id).repeat(in('derivedFrom')).times(depth)``
|
|
1165
1174
|
|
|
1166
|
-
|
|
1175
|
+
Args:
|
|
1176
|
+
relationship: Follow only lineage edges of this relationship; all of them when ``None``.
|
|
1177
|
+
depth: How many lineage hops to follow.
|
|
1178
|
+
|
|
1179
|
+
Returns:
|
|
1180
|
+
A query over this dataset's ancestors, to *depth* hops.
|
|
1167
1181
|
|
|
1182
|
+
Example:
|
|
1168
1183
|
store.dataset("id").upstream(depth=3).include("publisher").collect()
|
|
1169
1184
|
"""
|
|
1170
1185
|
from statcatlib.model.dataset import BaseDataset # noqa: PLC0415
|
|
@@ -1207,8 +1222,14 @@ class DatasetScope:
|
|
|
1207
1222
|
|
|
1208
1223
|
Gremlin equivalent: ``g.V(id).repeat(out('derivedFrom')).times(depth)``
|
|
1209
1224
|
|
|
1210
|
-
|
|
1225
|
+
Args:
|
|
1226
|
+
relationship: Follow only lineage edges of this relationship; all of them when ``None``.
|
|
1227
|
+
depth: How many lineage hops to follow.
|
|
1228
|
+
|
|
1229
|
+
Returns:
|
|
1230
|
+
A query over this dataset's descendants, to *depth* hops.
|
|
1211
1231
|
|
|
1232
|
+
Example:
|
|
1212
1233
|
store.dataset("id").downstream().include("publisher").collect()
|
|
1213
1234
|
"""
|
|
1214
1235
|
from statcatlib.model.dataset import BaseDataset # noqa: PLC0415
|
|
@@ -18,8 +18,10 @@ _cache: dict[str, _UnloadedType] = {}
|
|
|
18
18
|
class NotLoadedError(AttributeError):
|
|
19
19
|
"""Raised when accessing a relationship that was not loaded in this query.
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
Args:
|
|
22
|
+
field: The relationship that was not loaded, as `include` names it.
|
|
22
23
|
|
|
24
|
+
Example:
|
|
23
25
|
ds = store.query(BaseDataset).include("publisher").first()
|
|
24
26
|
ds.themes # NotLoadedError: 'themes' was not loaded.
|
|
25
27
|
# Add 'themes' to .include() or call .full().
|
|
@@ -86,7 +88,7 @@ def make_unloaded(field: str) -> Any:
|
|
|
86
88
|
|
|
87
89
|
(No `# noqa: ANN401` yet — ANN401 is still ignored project-wide until
|
|
88
90
|
#109 step 3, and a suppression that suppresses nothing is the thing
|
|
89
|
-
`
|
|
91
|
+
pyrefly's `unused-ignore` exists to catch. Add it with the
|
|
90
92
|
rule.)
|
|
91
93
|
"""
|
|
92
94
|
if field not in _cache:
|
|
@@ -36,7 +36,7 @@ from sdmxlib import InternationalString
|
|
|
36
36
|
from sdmxlib.model.registry import artefact_urn
|
|
37
37
|
from sdmxlib.storage.schema import SCHEMA_VERSION as _SDMX_SCHEMA_VERSION
|
|
38
38
|
|
|
39
|
-
from statcatlib.bundle import Embedded
|
|
39
|
+
from statcatlib.bundle import Embedded, normalize_lake_data_path, read_current_data_path
|
|
40
40
|
from statcatlib.model.agents import Agent, ContactPoint
|
|
41
41
|
from statcatlib.model.classification import Standard
|
|
42
42
|
from statcatlib.model.dataset import (
|
|
@@ -543,9 +543,10 @@ class _EntitySpec[R: NamedTuple, E]:
|
|
|
543
543
|
|
|
544
544
|
Bounding ``R`` by `NamedTuple` is what makes `columns` derivable —
|
|
545
545
|
it is the one spelling of "a row type" that basedpyright *and* pyrefly
|
|
546
|
-
both
|
|
547
|
-
`ClassVar` (which it truly is) basedpyright
|
|
548
|
-
declared as an instance attribute pyrefly rejects
|
|
546
|
+
both accepted while both ran (until GH-7). A `Protocol` carrying
|
|
547
|
+
``_fields`` did not: declared `ClassVar` (which it truly is) basedpyright
|
|
548
|
+
rejected ``type[R]``; declared as an instance attribute pyrefly rejects
|
|
549
|
+
the specialization.
|
|
549
550
|
|
|
550
551
|
Attributes:
|
|
551
552
|
table: The DuckDB table name, unqualified.
|
|
@@ -590,8 +591,8 @@ class EmbeddingModel(Protocol):
|
|
|
590
591
|
``inputs`` (older versions: ``sentences``), and declares a return union
|
|
591
592
|
wide enough to include ``dict[str, Tensor]``. Promising
|
|
592
593
|
``-> Iterable[float]`` here was the first attempt and no real encoder
|
|
593
|
-
satisfied it; worse, narrowing to it with `isinstance` is unsound,
|
|
594
|
-
basedpyright
|
|
594
|
+
satisfied it; worse, narrowing to it with `isinstance` is unsound, as
|
|
595
|
+
basedpyright reported — a runtime_checkable Protocol checks method
|
|
595
596
|
*presence*, never signatures.
|
|
596
597
|
|
|
597
598
|
So: two overloads, one per shape statcatlib passes, `object` out, and
|
|
@@ -1157,7 +1158,7 @@ class _SortSpec:
|
|
|
1157
1158
|
#:
|
|
1158
1159
|
#: A union rather than `Hit[object]` because attrs generics are invariant:
|
|
1159
1160
|
#: `Hit` is frozen and `SearchResult.items` could be a `Sequence`, and
|
|
1160
|
-
#: basedpyright
|
|
1161
|
+
#: basedpyright reported the parameter invariant, since the synthesised
|
|
1161
1162
|
#: `__init__` puts `T` in a parameter position. `Hit[CodeHit]` is therefore not
|
|
1162
1163
|
#: a `Hit[object]`, and no amount of freezing changes that.
|
|
1163
1164
|
type AnyHit = Hit[BaseDataset] | Hit[CodeHit] | Hit[ConceptHit] | Hit[ReferenceMetadataHit]
|
|
@@ -2847,8 +2848,8 @@ class Catalog:
|
|
|
2847
2848
|
# abfss:// URI) via ``data_path``.
|
|
2848
2849
|
if self._bundle_dir is not None:
|
|
2849
2850
|
attach_url = f"'ducklake:{self._bundle_dir / 'lake.ducklake'}'"
|
|
2850
|
-
data_path_literal = self._resolve_data_path(self._bundle_dir)
|
|
2851
|
-
attach_options = self._attach_options(data_path_literal)
|
|
2851
|
+
data_path_literal, override = self._resolve_data_path(self._bundle_dir)
|
|
2852
|
+
attach_options = self._attach_options(data_path_literal, override=override)
|
|
2852
2853
|
|
|
2853
2854
|
self._data = sdmxlib.DuckDBDataStore.from_registry(
|
|
2854
2855
|
self._registry,
|
|
@@ -2914,7 +2915,11 @@ class Catalog:
|
|
|
2914
2915
|
msg = "data_path is not supported with an in-memory catalog (':memory:')"
|
|
2915
2916
|
raise ValueError(msg)
|
|
2916
2917
|
return duckdb.connect(":memory:"), None
|
|
2917
|
-
|
|
2918
|
+
# Absolute, so the stamp DuckLake records on first open is too: a
|
|
2919
|
+
# relative one is resolved against whatever the working directory is
|
|
2920
|
+
# when the lake is read (GH-23). `absolute()`, not `resolve()`, which
|
|
2921
|
+
# follows symlinks and would respell a catalog created by absolute path.
|
|
2922
|
+
path = Path(self._uri).absolute()
|
|
2918
2923
|
if self._read_only:
|
|
2919
2924
|
meta_path = path / "meta.duckdb"
|
|
2920
2925
|
if not meta_path.exists():
|
|
@@ -2929,7 +2934,7 @@ class Catalog:
|
|
|
2929
2934
|
path.mkdir(parents=True, exist_ok=True)
|
|
2930
2935
|
return duckdb.connect(str(path / "meta.duckdb")), path
|
|
2931
2936
|
|
|
2932
|
-
def _attach_options(self, data_path_literal: str | None) -> str:
|
|
2937
|
+
def _attach_options(self, data_path_literal: str | None, *, override: bool = False) -> str:
|
|
2933
2938
|
"""Render the DuckLake ATTACH option list.
|
|
2934
2939
|
|
|
2935
2940
|
A ``None`` literal means honour the lake's own stamp: DuckLake only
|
|
@@ -2944,31 +2949,73 @@ class Catalog:
|
|
|
2944
2949
|
allowed under ``READ_ONLY``. The default (``data_path=None``) stays
|
|
2945
2950
|
a checked ``<bundle>/parquet/``, so a plain ``Catalog(path)`` on a
|
|
2946
2951
|
split bundle still fails at ATTACH rather than reading an empty
|
|
2947
|
-
local directory.
|
|
2952
|
+
local directory. *override* is the one exception, for a read-only
|
|
2953
|
+
open of a catalog stamped with a relative path (`_resolve_data_path`).
|
|
2948
2954
|
"""
|
|
2949
2955
|
opts = [] if data_path_literal is None else [f"DATA_PATH '{data_path_literal}'"]
|
|
2950
|
-
if data_path_literal is not None and self._data_path is not None:
|
|
2956
|
+
if data_path_literal is not None and (self._data_path is not None or override):
|
|
2951
2957
|
opts.append("OVERRIDE_DATA_PATH true")
|
|
2952
2958
|
if self._read_only:
|
|
2953
2959
|
opts.append("READ_ONLY")
|
|
2954
2960
|
return f"({', '.join(opts)})" if opts else ""
|
|
2955
2961
|
|
|
2956
|
-
def _resolve_data_path(self, bundle_dir: Path) -> str | None:
|
|
2957
|
-
"""Return the DuckLake DATA_PATH literal, with trailing slash.
|
|
2962
|
+
def _resolve_data_path(self, bundle_dir: Path) -> tuple[str | None, bool]:
|
|
2963
|
+
"""Return the DuckLake DATA_PATH literal, with trailing slash, and whether to override.
|
|
2958
2964
|
|
|
2959
2965
|
Remote URIs (``abfss://``, ``s3://``, ``gs://``, etc.) are passed
|
|
2960
2966
|
through verbatim; local paths are rendered as ``<bundle>/parquet/``
|
|
2961
|
-
when no override is set
|
|
2967
|
+
when no override is set, reconciled with the lake's stamp by
|
|
2968
|
+
`_colocated_data_path`.
|
|
2962
2969
|
|
|
2963
2970
|
Returns `None` for `statcatlib.bundle.Embedded` — meaning "omit the
|
|
2964
2971
|
option entirely and let the lake's own stamp stand".
|
|
2965
2972
|
"""
|
|
2966
2973
|
if isinstance(self._data_path, Embedded):
|
|
2967
|
-
return None
|
|
2974
|
+
return None, False
|
|
2968
2975
|
if self._data_path is None:
|
|
2969
|
-
return
|
|
2976
|
+
return self._colocated_data_path(bundle_dir)
|
|
2970
2977
|
raw = str(self._data_path)
|
|
2971
|
-
return raw if raw.endswith("/") else f"{raw}/"
|
|
2978
|
+
return (raw if raw.endswith("/") else f"{raw}/"), False
|
|
2979
|
+
|
|
2980
|
+
def _colocated_data_path(self, bundle_dir: Path) -> tuple[str, bool]:
|
|
2981
|
+
"""The DATA_PATH for a default open: ``<bundle>/parquet/``, as the lake spells it.
|
|
2982
|
+
|
|
2983
|
+
DuckLake compares DATA_PATH with its stamp as strings, so the same
|
|
2984
|
+
directory spelled differently was refused (GH-23). The stamp is read
|
|
2985
|
+
first:
|
|
2986
|
+
|
|
2987
|
+
- the same directory, spelled any way (a relative open, a symlink):
|
|
2988
|
+
passed back verbatim, which DuckLake accepts unchanged;
|
|
2989
|
+
- relative and ending in ``<bundle name>/parquet``, as
|
|
2990
|
+
``Catalog("relative")`` stamped it before GH-23: DuckLake resolves
|
|
2991
|
+
it against the working directory, so it is replaced by the absolute
|
|
2992
|
+
path. A read-write open rewrites the stamp, which repairs the
|
|
2993
|
+
catalog; a read-only one overrides it for this attach only;
|
|
2994
|
+
- anything else, a remote URI or another directory: the absolute
|
|
2995
|
+
path, unchanged, so a split bundle still fails at ATTACH rather than
|
|
2996
|
+
reading an empty local directory.
|
|
2997
|
+
"""
|
|
2998
|
+
parquet = bundle_dir / "parquet"
|
|
2999
|
+
target = f"{parquet}/"
|
|
3000
|
+
lake = bundle_dir / "lake.ducklake"
|
|
3001
|
+
if not lake.exists():
|
|
3002
|
+
return target, False
|
|
3003
|
+
try:
|
|
3004
|
+
stamp = read_current_data_path(lake)
|
|
3005
|
+
except duckdb.Error, FileNotFoundError:
|
|
3006
|
+
return target, False
|
|
3007
|
+
stamped = Path(stamp)
|
|
3008
|
+
if "://" in stamp or stamp == target:
|
|
3009
|
+
return target, False
|
|
3010
|
+
if stamped.is_absolute():
|
|
3011
|
+
return (stamp, False) if stamped.resolve() == parquet.resolve() else (target, False)
|
|
3012
|
+
colocated = stamped.name == "parquet" and stamped.parent.name in {"", bundle_dir.name}
|
|
3013
|
+
if not colocated:
|
|
3014
|
+
return target, False
|
|
3015
|
+
if self._read_only:
|
|
3016
|
+
return target, True
|
|
3017
|
+
normalize_lake_data_path(lake, target)
|
|
3018
|
+
return target, False
|
|
2972
3019
|
|
|
2973
3020
|
@classmethod
|
|
2974
3021
|
def from_connection(
|
|
@@ -3228,8 +3275,10 @@ class Catalog:
|
|
|
3228
3275
|
the caller's responsibility — the encoder name is opaque to
|
|
3229
3276
|
statcatlib.
|
|
3230
3277
|
|
|
3231
|
-
|
|
3278
|
+
Args:
|
|
3279
|
+
model: The encoder, the one `embedding_model_name` names.
|
|
3232
3280
|
|
|
3281
|
+
Example:
|
|
3233
3282
|
store = Catalog("data/catalog").open()
|
|
3234
3283
|
if store.embedding_model_name:
|
|
3235
3284
|
from sentence_transformers import SentenceTransformer
|
|
@@ -6391,7 +6440,7 @@ class Catalog:
|
|
|
6391
6440
|
# arg 2 must be a type`. Found by trying it; five tests failed.
|
|
6392
6441
|
# Entity queries never see the alias, which is why the same fix holds
|
|
6393
6442
|
# there.
|
|
6394
|
-
items: list[T] = [ #
|
|
6443
|
+
items: list[T] = [ # pyrefly: ignore[bad-assignment]
|
|
6395
6444
|
ds_by_id[ident] for ident in qr.rows if ident in ds_by_id
|
|
6396
6445
|
]
|
|
6397
6446
|
|
|
@@ -6447,7 +6496,7 @@ class Catalog:
|
|
|
6447
6496
|
# returns `Any` here, and the `isinstance` is what soundly recovers `T`
|
|
6448
6497
|
# — the bridge back from runtime dispatch to the declared return type,
|
|
6449
6498
|
# not a redundant assertion. It is also why this path needs no
|
|
6450
|
-
#
|
|
6499
|
+
# suppression while the dataset path above does.
|
|
6451
6500
|
items: list[T] = []
|
|
6452
6501
|
for raw in rows:
|
|
6453
6502
|
item = spec.build(raw)
|
|
@@ -48,8 +48,15 @@ def _build(package: str, artefact_class: str, agency: str, artefact_id: str, ver
|
|
|
48
48
|
def codelist(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
|
|
49
49
|
"""Build a codelist URN string.
|
|
50
50
|
|
|
51
|
-
|
|
51
|
+
Args:
|
|
52
|
+
agency: The maintenance agency's id.
|
|
53
|
+
id: The codelist's id.
|
|
54
|
+
version: Its version.
|
|
52
55
|
|
|
56
|
+
Returns:
|
|
57
|
+
The URN, as SDMX spells it.
|
|
58
|
+
|
|
59
|
+
Example:
|
|
53
60
|
sc.urn.codelist("SDMX", "CL_GEO", "1.0")
|
|
54
61
|
# → "urn:sdmx:org.sdmx.infomodel.codelist.Codelist=SDMX:CL_GEO(1.0)"
|
|
55
62
|
"""
|
|
@@ -59,8 +66,15 @@ def codelist(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
|
|
|
59
66
|
def dataflow(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
|
|
60
67
|
"""Build a dataflow URN string.
|
|
61
68
|
|
|
62
|
-
|
|
69
|
+
Args:
|
|
70
|
+
agency: The maintenance agency's id.
|
|
71
|
+
id: The dataflow's id.
|
|
72
|
+
version: Its version.
|
|
73
|
+
|
|
74
|
+
Returns:
|
|
75
|
+
The URN, as SDMX spells it.
|
|
63
76
|
|
|
77
|
+
Example:
|
|
64
78
|
sc.urn.dataflow("ESTAT", "NAMA_10_GDP", "1.0")
|
|
65
79
|
# → "urn:sdmx:org.sdmx.infomodel.datastructure.Dataflow=ESTAT:NAMA_10_GDP(1.0)"
|
|
66
80
|
"""
|
|
@@ -70,8 +84,15 @@ def dataflow(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
|
|
|
70
84
|
def data_structure(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
|
|
71
85
|
"""Build a data structure definition URN string.
|
|
72
86
|
|
|
73
|
-
|
|
87
|
+
Args:
|
|
88
|
+
agency: The maintenance agency's id.
|
|
89
|
+
id: The data structure definition's id.
|
|
90
|
+
version: Its version.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
The URN, as SDMX spells it.
|
|
74
94
|
|
|
95
|
+
Example:
|
|
75
96
|
sc.urn.data_structure("ESTAT", "DSD_NAMA_10_GDP", "1.0")
|
|
76
97
|
# → "urn:sdmx:org.sdmx.infomodel.datastructure.DataStructure=ESTAT:DSD_NAMA_10_GDP(1.0)"
|
|
77
98
|
"""
|
|
@@ -81,8 +102,15 @@ def data_structure(agency: str, id: str, version: str = "1.0") -> str: # noqa:
|
|
|
81
102
|
def concept_scheme(agency: str, id: str, version: str = "1.0") -> str: # noqa: A002
|
|
82
103
|
"""Build a concept scheme URN string.
|
|
83
104
|
|
|
84
|
-
|
|
105
|
+
Args:
|
|
106
|
+
agency: The maintenance agency's id.
|
|
107
|
+
id: The concept scheme's id.
|
|
108
|
+
version: Its version.
|
|
85
109
|
|
|
110
|
+
Returns:
|
|
111
|
+
The URN, as SDMX spells it.
|
|
112
|
+
|
|
113
|
+
Example:
|
|
86
114
|
sc.urn.concept_scheme("SDMX", "CS_FREQ", "1.0")
|
|
87
115
|
# → "urn:sdmx:org.sdmx.infomodel.conceptscheme.ConceptScheme=SDMX:CS_FREQ(1.0)"
|
|
88
116
|
"""
|
|
@@ -92,17 +120,18 @@ def concept_scheme(agency: str, id: str, version: str = "1.0") -> str: # noqa:
|
|
|
92
120
|
def parse(urn: str) -> SdmxUrn[object]:
|
|
93
121
|
"""Parse a URN string into its components.
|
|
94
122
|
|
|
95
|
-
|
|
96
|
-
|
|
123
|
+
Args:
|
|
124
|
+
urn: An SDMX URN string. One that is not valid raises `ValueError`.
|
|
97
125
|
|
|
98
|
-
|
|
126
|
+
Returns:
|
|
127
|
+
The parsed URN, with ``.agency``, ``.id``, ``.version``, ``.package``
|
|
128
|
+
and ``.artefact_class``.
|
|
99
129
|
|
|
130
|
+
Example:
|
|
100
131
|
parsed = sc.urn.parse("urn:sdmx:org.sdmx.infomodel.codelist.Codelist=SDMX:CL_GEO(1.0)")
|
|
101
132
|
parsed.agency # "SDMX"
|
|
102
133
|
parsed.id # "CL_GEO"
|
|
103
134
|
parsed.version # "1.0"
|
|
104
135
|
str(parsed) # round-trips to original URN string
|
|
105
|
-
|
|
106
|
-
Raises ``ValueError`` if the string is not a valid SDMX URN.
|
|
107
136
|
"""
|
|
108
137
|
return SdmxUrn.parse(urn)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|